From e3ac0f43df3e4b436cf9b680d9fd00a9c6767f0a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 4 Apr 2026 17:43:15 +0100 Subject: [PATCH] feat(qwen): add qwen provider and video generation --- docs/.generated/config-baseline.sha256 | 4 +- extensions/acpx/index.ts | 4 +- extensions/amazon-bedrock/index.ts | 6 +- extensions/anthropic/index.ts | 6 +- extensions/browser/index.ts | 14 +- extensions/github-copilot/index.ts | 1 - extensions/google/index.ts | 7 + extensions/microsoft/index.ts | 4 +- extensions/modelstudio/index.ts | 118 ------- extensions/modelstudio/onboard.ts | 69 ---- extensions/modelstudio/openclaw.plugin.json | 71 ----- .../modelstudio/provider-catalog.test.ts | 36 --- extensions/modelstudio/provider-catalog.ts | 10 - extensions/openai/index.ts | 63 ++-- extensions/{modelstudio => qwen}/api.ts | 15 +- extensions/qwen/index.ts | 129 ++++++++ .../qwen/media-understanding-provider.test.ts | 56 ++++ .../qwen/media-understanding-provider.ts | 156 +++++++++ .../model-definitions.ts | 9 + extensions/{modelstudio => qwen}/models.ts | 69 ++-- extensions/qwen/onboard.ts | 81 +++++ extensions/qwen/openclaw.plugin.json | 79 +++++ extensions/{modelstudio => qwen}/package.json | 4 +- .../qwen/plugin-registration.contract.test.ts | 10 + extensions/qwen/provider-catalog.test.ts | 34 ++ extensions/qwen/provider-catalog.ts | 12 + extensions/qwen/test-api.ts | 2 + .../qwen/video-generation-provider.test.ts | 110 +++++++ extensions/qwen/video-generation-provider.ts | 300 ++++++++++++++++++ extensions/video-generation-core/api.ts | 1 + extensions/video-generation-core/package.json | 7 + .../video-generation-core/runtime-api.ts | 6 + .../video-generation-core/src/runtime.test.ts | 164 ++++++++++ .../video-generation-core/src/runtime.ts | 189 +++++++++++ extensions/vllm/index.ts | 17 +- package.json | 16 + scripts/lib/plugin-sdk-entrypoints.json | 4 + scripts/lib/plugin-sdk-facades.mjs | 59 +++- src/agents/auth-profiles.doctor.test.ts | 6 +- src/agents/model-auth.profiles.test.ts | 14 +- src/agents/model-compat.test.ts | 4 +- src/agents/model-selection.test.ts | 2 +- src/agents/openai-transport-stream.test.ts | 4 +- src/agents/provider-attribution.test.ts | 6 +- src/agents/provider-attribution.ts | 2 + src/agents/provider-id.ts | 3 + ...oard-non-interactive.provider-auth.test.ts | 41 +-- src/commands/onboard-types.ts | 5 + src/config/io.ts | 1 + src/config/schema.base.generated.ts | 32 ++ src/config/schema.help.ts | 4 + src/config/schema.labels.ts | 2 + src/config/types.agent-defaults.ts | 2 + src/config/zod-schema.agent-defaults.ts | 1 + src/gateway/server-plugins.test.ts | 1 + src/gateway/session-utils.test.ts | 2 +- src/gateway/test-helpers.mocks.ts | 1 + .../plugin-sdk-facade-type-map.generated.ts | 40 ++- src/media-understanding/defaults.test.ts | 6 +- .../provider-registry.test.ts | 6 +- src/plugin-sdk/line-runtime.ts | 4 + src/plugin-sdk/memory-core-engine-runtime.ts | 10 +- src/plugin-sdk/modelstudio-definitions.ts | 2 +- src/plugin-sdk/modelstudio.ts | 2 +- src/plugin-sdk/qwen-definitions.ts | 40 +++ src/plugin-sdk/qwen.ts | 54 ++++ src/plugin-sdk/video-generation-core.ts | 27 ++ src/plugin-sdk/video-generation-runtime.ts | 21 ++ src/plugin-sdk/video-generation.ts | 10 + src/plugins/api-builder.ts | 5 + src/plugins/bundled-capability-metadata.ts | 8 + src/plugins/bundled-capability-runtime.ts | 13 + ...undled-provider-auth-env-vars.generated.ts | 3 +- .../capability-provider-runtime.test.ts | 40 +++ src/plugins/capability-provider-runtime.ts | 15 +- src/plugins/captured-registration.ts | 7 + src/plugins/channel-plugin-ids.ts | 1 + src/plugins/contracts/registry.ts | 26 ++ .../contracts/speech-vitest-registry.ts | 52 +++ src/plugins/hooks.test-helpers.ts | 1 + src/plugins/loader.ts | 1 + src/plugins/manifest.ts | 3 + src/plugins/provider-auth-storage.ts | 5 +- src/plugins/providers.test.ts | 36 ++- src/plugins/providers.ts | 1 + src/plugins/registry-empty.ts | 1 + src/plugins/registry.ts | 21 ++ src/plugins/runtime.test.ts | 2 + src/plugins/runtime/index.ts | 31 +- src/plugins/runtime/types-core.ts | 4 + src/plugins/status.test-helpers.ts | 2 + src/plugins/types.ts | 4 + src/video-generation/model-ref.ts | 16 + .../provider-registry.test.ts | 93 ++++++ src/video-generation/provider-registry.ts | 77 +++++ src/video-generation/runtime.test.ts | 72 +++++ src/video-generation/runtime.ts | 6 + src/video-generation/types.ts | 65 ++++ test/helpers/plugins/plugin-api.ts | 1 + .../plugins/plugin-registration-contract.ts | 39 +++ test/helpers/plugins/plugin-runtime-mock.ts | 4 + .../plugins/provider-discovery-contract.ts | 5 +- test/setup-openclaw-runtime.ts | 1 + vitest.extension-provider-paths.mjs | 2 +- 104 files changed, 2477 insertions(+), 483 deletions(-) delete mode 100644 extensions/modelstudio/index.ts delete mode 100644 extensions/modelstudio/onboard.ts delete mode 100644 extensions/modelstudio/openclaw.plugin.json delete mode 100644 extensions/modelstudio/provider-catalog.test.ts delete mode 100644 extensions/modelstudio/provider-catalog.ts rename extensions/{modelstudio => qwen}/api.ts (52%) create mode 100644 extensions/qwen/index.ts create mode 100644 extensions/qwen/media-understanding-provider.test.ts create mode 100644 extensions/qwen/media-understanding-provider.ts rename extensions/{modelstudio => qwen}/model-definitions.ts (58%) rename extensions/{modelstudio => qwen}/models.ts (52%) create mode 100644 extensions/qwen/onboard.ts create mode 100644 extensions/qwen/openclaw.plugin.json rename extensions/{modelstudio => qwen}/package.json (57%) create mode 100644 extensions/qwen/plugin-registration.contract.test.ts create mode 100644 extensions/qwen/provider-catalog.test.ts create mode 100644 extensions/qwen/provider-catalog.ts create mode 100644 extensions/qwen/test-api.ts create mode 100644 extensions/qwen/video-generation-provider.test.ts create mode 100644 extensions/qwen/video-generation-provider.ts create mode 100644 extensions/video-generation-core/api.ts create mode 100644 extensions/video-generation-core/package.json create mode 100644 extensions/video-generation-core/runtime-api.ts create mode 100644 extensions/video-generation-core/src/runtime.test.ts create mode 100644 extensions/video-generation-core/src/runtime.ts create mode 100644 src/plugin-sdk/qwen-definitions.ts create mode 100644 src/plugin-sdk/qwen.ts create mode 100644 src/plugin-sdk/video-generation-core.ts create mode 100644 src/plugin-sdk/video-generation-runtime.ts create mode 100644 src/plugin-sdk/video-generation.ts create mode 100644 src/video-generation/model-ref.ts create mode 100644 src/video-generation/provider-registry.test.ts create mode 100644 src/video-generation/provider-registry.ts create mode 100644 src/video-generation/runtime.test.ts create mode 100644 src/video-generation/runtime.ts create mode 100644 src/video-generation/types.ts diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index f07fcbab5e5..fa13133106e 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -ad87e3ff267b151ae163402f3cb52503e10641e332bcfbb6a574bbd7087a2484 config-baseline.json +4c880eb1ce03486f47aa21f49317ad15fc8d92bb720d70205743b72e45cf5fa3 config-baseline.json 03ff4a3e314f17dd8851aed3653269294bc62412bee05a6804dce840bd3d7551 config-baseline.core.json 73b57f395a2ad983f1660112d0b2b998342f1ddbe3089b440d7f73d0665de739 config-baseline.channel.json -9d5cb864e70768b66c1ecd881a9a584b7696ef2e5b32df686cfdc3fa21ddabbe config-baseline.plugin.json +17fd37605bf6cb087932ec2ebcfa9dd22e669fa6b8b93081ab2deac9d24821c5 config-baseline.plugin.json diff --git a/extensions/acpx/index.ts b/extensions/acpx/index.ts index d991ff0eec6..087cdb8aae7 100644 --- a/extensions/acpx/index.ts +++ b/extensions/acpx/index.ts @@ -1,3 +1,4 @@ +import { createAcpxRuntimeService } from "./register.runtime.js"; import type { OpenClawPluginApi } from "./runtime-api.js"; import { createAcpxPluginConfigSchema } from "./src/config-schema.js"; @@ -6,8 +7,7 @@ const plugin = { name: "ACPX Runtime", description: "ACP runtime backend powered by the acpx CLI.", configSchema: () => createAcpxPluginConfigSchema(), - async register(api: OpenClawPluginApi) { - const { createAcpxRuntimeService } = await import("./register.runtime.js"); + register(api: OpenClawPluginApi) { api.registerService( createAcpxRuntimeService({ pluginConfig: api.pluginConfig, diff --git a/extensions/amazon-bedrock/index.ts b/extensions/amazon-bedrock/index.ts index e395f9c2317..4696df6e4e3 100644 --- a/extensions/amazon-bedrock/index.ts +++ b/extensions/amazon-bedrock/index.ts @@ -1,11 +1,11 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { registerAmazonBedrockPlugin } from "./register.runtime.js"; export default definePluginEntry({ id: "amazon-bedrock", name: "Amazon Bedrock Provider", description: "Bundled Amazon Bedrock provider policy plugin", - async register(api) { - const { registerAmazonBedrockPlugin } = await import("./register.runtime.js"); - await registerAmazonBedrockPlugin(api); + register(api) { + registerAmazonBedrockPlugin(api); }, }); diff --git a/extensions/anthropic/index.ts b/extensions/anthropic/index.ts index c60e8cd97ca..7afcec80245 100644 --- a/extensions/anthropic/index.ts +++ b/extensions/anthropic/index.ts @@ -1,11 +1,11 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { registerAnthropicPlugin } from "./register.runtime.js"; export default definePluginEntry({ id: "anthropic", name: "Anthropic Provider", description: "Bundled Anthropic provider plugin", - async register(api) { - const { registerAnthropicPlugin } = await import("./register.runtime.js"); - await registerAnthropicPlugin(api); + register(api) { + registerAnthropicPlugin(api); }, }); diff --git a/extensions/browser/index.ts b/extensions/browser/index.ts index 738d9ff0680..a607913e234 100644 --- a/extensions/browser/index.ts +++ b/extensions/browser/index.ts @@ -3,18 +3,18 @@ import { type OpenClawPluginToolContext, type OpenClawPluginToolFactory, } from "openclaw/plugin-sdk/plugin-entry"; +import { + createBrowserPluginService, + createBrowserTool, + handleBrowserGatewayRequest, + registerBrowserCli, +} from "./register.runtime.js"; export default definePluginEntry({ id: "browser", name: "Browser", description: "Default browser tool plugin", - async register(api) { - const { - createBrowserPluginService, - createBrowserTool, - handleBrowserGatewayRequest, - registerBrowserCli, - } = await import("./register.runtime.js"); + register(api) { api.registerTool(((ctx: OpenClawPluginToolContext) => createBrowserTool({ sandboxBridgeUrl: ctx.browser?.sandboxBridgeUrl, diff --git a/extensions/github-copilot/index.ts b/extensions/github-copilot/index.ts index be880374036..87c9ddf8aef 100644 --- a/extensions/github-copilot/index.ts +++ b/extensions/github-copilot/index.ts @@ -28,7 +28,6 @@ export default definePluginEntry({ resolveCopilotForwardCompatModel, wrapCopilotProviderStream, } = await import("./register.runtime.js"); - function resolveFirstGithubToken(params: { agentDir?: string; env: NodeJS.ProcessEnv }): { githubToken: string; hasProfile: boolean; diff --git a/extensions/google/index.ts b/extensions/google/index.ts index c06bd8e8474..c924c0bc36e 100644 --- a/extensions/google/index.ts +++ b/extensions/google/index.ts @@ -191,6 +191,13 @@ function createLazyGoogleMediaUnderstandingProvider(): MediaUnderstandingProvide return { id: "google", capabilities: ["image", "audio", "video"], + defaultModels: { + image: "gemini-3-flash-preview", + audio: "gemini-3-flash-preview", + video: "gemini-3-flash-preview", + }, + autoPriority: { image: 30, audio: 40, video: 10 }, + nativeDocumentInputs: ["pdf"], describeImage: async (...args) => await (await loadGoogleRequiredMediaUnderstandingProvider()).describeImage(...args), describeImages: async (...args) => diff --git a/extensions/microsoft/index.ts b/extensions/microsoft/index.ts index 4245958c0c1..04eff9d49a7 100644 --- a/extensions/microsoft/index.ts +++ b/extensions/microsoft/index.ts @@ -1,11 +1,11 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { buildMicrosoftSpeechProvider } from "./speech-provider.js"; export default definePluginEntry({ id: "microsoft", name: "Microsoft Speech", description: "Bundled Microsoft speech provider", - async register(api) { - const { buildMicrosoftSpeechProvider } = await import("./speech-provider.js"); + register(api) { api.registerSpeechProvider(buildMicrosoftSpeechProvider()); }, }); diff --git a/extensions/modelstudio/index.ts b/extensions/modelstudio/index.ts deleted file mode 100644 index a6160e522a8..00000000000 --- a/extensions/modelstudio/index.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry"; -import { applyModelStudioNativeStreamingUsageCompat } from "./api.js"; -import { - applyModelStudioConfig, - applyModelStudioConfigCn, - applyModelStudioStandardConfig, - applyModelStudioStandardConfigCn, - MODELSTUDIO_DEFAULT_MODEL_REF, -} from "./onboard.js"; -import { buildModelStudioProvider } from "./provider-catalog.js"; - -const PROVIDER_ID = "modelstudio"; - -export default defineSingleProviderPluginEntry({ - id: PROVIDER_ID, - name: "Model Studio Provider", - description: "Bundled Model Studio provider plugin", - provider: { - label: "Model Studio", - docsPath: "/providers/models", - auth: [ - { - methodId: "standard-api-key-cn", - label: "Standard API Key for China (pay-as-you-go)", - hint: "Endpoint: dashscope.aliyuncs.com", - optionKey: "modelstudioStandardApiKeyCn", - flagName: "--modelstudio-standard-api-key-cn", - envVar: "MODELSTUDIO_API_KEY", - promptMessage: "Enter Alibaba Cloud Model Studio API key (China)", - defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyModelStudioStandardConfigCn(cfg), - noteMessage: [ - "Get your API key at: https://bailian.console.aliyun.com/", - "Endpoint: dashscope.aliyuncs.com/compatible-mode/v1", - "Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.", - ].join("\n"), - noteTitle: "Alibaba Cloud Model Studio Standard (China)", - wizard: { - choiceHint: "Endpoint: dashscope.aliyuncs.com", - groupLabel: "Qwen (Alibaba Cloud Model Studio)", - groupHint: "Standard / Coding Plan (CN / Global)", - }, - }, - { - methodId: "standard-api-key", - label: "Standard API Key for Global/Intl (pay-as-you-go)", - hint: "Endpoint: dashscope-intl.aliyuncs.com", - optionKey: "modelstudioStandardApiKey", - flagName: "--modelstudio-standard-api-key", - envVar: "MODELSTUDIO_API_KEY", - promptMessage: "Enter Alibaba Cloud Model Studio API key (Global/Intl)", - defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyModelStudioStandardConfig(cfg), - noteMessage: [ - "Get your API key at: https://modelstudio.console.alibabacloud.com/", - "Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1", - "Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.", - ].join("\n"), - noteTitle: "Alibaba Cloud Model Studio Standard (Global/Intl)", - wizard: { - choiceHint: "Endpoint: dashscope-intl.aliyuncs.com", - groupLabel: "Qwen (Alibaba Cloud Model Studio)", - groupHint: "Standard / Coding Plan (CN / Global)", - }, - }, - { - methodId: "api-key-cn", - label: "Coding Plan API Key for China (subscription)", - hint: "Endpoint: coding.dashscope.aliyuncs.com", - optionKey: "modelstudioApiKeyCn", - flagName: "--modelstudio-api-key-cn", - envVar: "MODELSTUDIO_API_KEY", - promptMessage: "Enter Alibaba Cloud Model Studio Coding Plan API key (China)", - defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyModelStudioConfigCn(cfg), - noteMessage: [ - "Get your API key at: https://bailian.console.aliyun.com/", - "Endpoint: coding.dashscope.aliyuncs.com", - "Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.", - ].join("\n"), - noteTitle: "Alibaba Cloud Model Studio Coding Plan (China)", - wizard: { - choiceHint: "Endpoint: coding.dashscope.aliyuncs.com", - groupLabel: "Qwen (Alibaba Cloud Model Studio)", - groupHint: "Standard / Coding Plan (CN / Global)", - }, - }, - { - methodId: "api-key", - label: "Coding Plan API Key for Global/Intl (subscription)", - hint: "Endpoint: coding-intl.dashscope.aliyuncs.com", - optionKey: "modelstudioApiKey", - flagName: "--modelstudio-api-key", - envVar: "MODELSTUDIO_API_KEY", - promptMessage: "Enter Alibaba Cloud Model Studio Coding Plan API key (Global/Intl)", - defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyModelStudioConfig(cfg), - noteMessage: [ - "Get your API key at: https://bailian.console.aliyun.com/", - "Endpoint: coding-intl.dashscope.aliyuncs.com", - "Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.", - ].join("\n"), - noteTitle: "Alibaba Cloud Model Studio Coding Plan (Global/Intl)", - wizard: { - choiceHint: "Endpoint: coding-intl.dashscope.aliyuncs.com", - groupLabel: "Qwen (Alibaba Cloud Model Studio)", - groupHint: "Standard / Coding Plan (CN / Global)", - }, - }, - ], - catalog: { - buildProvider: buildModelStudioProvider, - allowExplicitBaseUrl: true, - }, - applyNativeStreamingUsageCompat: ({ providerConfig }) => - applyModelStudioNativeStreamingUsageCompat(providerConfig), - }, -}); diff --git a/extensions/modelstudio/onboard.ts b/extensions/modelstudio/onboard.ts deleted file mode 100644 index b35c989b1e5..00000000000 --- a/extensions/modelstudio/onboard.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { - createModelCatalogPresetAppliers, - type OpenClawConfig, -} from "openclaw/plugin-sdk/provider-onboard"; -import { - MODELSTUDIO_CN_BASE_URL, - MODELSTUDIO_DEFAULT_MODEL_REF, - MODELSTUDIO_GLOBAL_BASE_URL, - MODELSTUDIO_STANDARD_CN_BASE_URL, - MODELSTUDIO_STANDARD_GLOBAL_BASE_URL, -} from "./models.js"; -import { buildModelStudioProvider } from "./provider-catalog.js"; - -export { - MODELSTUDIO_CN_BASE_URL, - MODELSTUDIO_DEFAULT_MODEL_REF, - MODELSTUDIO_GLOBAL_BASE_URL, - MODELSTUDIO_STANDARD_CN_BASE_URL, - MODELSTUDIO_STANDARD_GLOBAL_BASE_URL, -}; - -const modelStudioPresetAppliers = createModelCatalogPresetAppliers<[string]>({ - primaryModelRef: MODELSTUDIO_DEFAULT_MODEL_REF, - resolveParams: (_cfg: OpenClawConfig, baseUrl: string) => { - const provider = buildModelStudioProvider(); - return { - providerId: "modelstudio", - api: provider.api ?? "openai-completions", - baseUrl, - catalogModels: provider.models ?? [], - aliases: [ - ...(provider.models ?? []).map((model) => `modelstudio/${model.id}`), - { modelRef: MODELSTUDIO_DEFAULT_MODEL_REF, alias: "Qwen" }, - ], - }; - }, -}); - -export function applyModelStudioProviderConfig(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_GLOBAL_BASE_URL); -} - -export function applyModelStudioProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_CN_BASE_URL); -} - -export function applyModelStudioConfig(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_GLOBAL_BASE_URL); -} - -export function applyModelStudioConfigCn(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_CN_BASE_URL); -} - -export function applyModelStudioStandardProviderConfig(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_STANDARD_GLOBAL_BASE_URL); -} - -export function applyModelStudioStandardProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_STANDARD_CN_BASE_URL); -} - -export function applyModelStudioStandardConfig(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_STANDARD_GLOBAL_BASE_URL); -} - -export function applyModelStudioStandardConfigCn(cfg: OpenClawConfig): OpenClawConfig { - return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_STANDARD_CN_BASE_URL); -} diff --git a/extensions/modelstudio/openclaw.plugin.json b/extensions/modelstudio/openclaw.plugin.json deleted file mode 100644 index 8e546d4f69b..00000000000 --- a/extensions/modelstudio/openclaw.plugin.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "modelstudio", - "enabledByDefault": true, - "providers": ["modelstudio"], - "providerAuthEnvVars": { - "modelstudio": ["MODELSTUDIO_API_KEY"] - }, - "providerAuthChoices": [ - { - "provider": "modelstudio", - "method": "standard-api-key-cn", - "choiceId": "modelstudio-standard-api-key-cn", - "choiceLabel": "Standard API Key for China (pay-as-you-go)", - "choiceHint": "Endpoint: dashscope.aliyuncs.com", - "groupId": "modelstudio", - "groupLabel": "Qwen (Alibaba Cloud Model Studio)", - "groupHint": "Standard / Coding Plan (CN / Global)", - "optionKey": "modelstudioStandardApiKeyCn", - "cliFlag": "--modelstudio-standard-api-key-cn", - "cliOption": "--modelstudio-standard-api-key-cn ", - "cliDescription": "Alibaba Cloud Model Studio Standard API key (China)" - }, - { - "provider": "modelstudio", - "method": "standard-api-key", - "choiceId": "modelstudio-standard-api-key", - "choiceLabel": "Standard API Key for Global/Intl (pay-as-you-go)", - "choiceHint": "Endpoint: dashscope-intl.aliyuncs.com", - "groupId": "modelstudio", - "groupLabel": "Qwen (Alibaba Cloud Model Studio)", - "groupHint": "Standard / Coding Plan (CN / Global)", - "optionKey": "modelstudioStandardApiKey", - "cliFlag": "--modelstudio-standard-api-key", - "cliOption": "--modelstudio-standard-api-key ", - "cliDescription": "Alibaba Cloud Model Studio Standard API key (Global/Intl)" - }, - { - "provider": "modelstudio", - "method": "api-key-cn", - "choiceId": "modelstudio-api-key-cn", - "choiceLabel": "Coding Plan API Key for China (subscription)", - "choiceHint": "Endpoint: coding.dashscope.aliyuncs.com", - "groupId": "modelstudio", - "groupLabel": "Qwen (Alibaba Cloud Model Studio)", - "groupHint": "Standard / Coding Plan (CN / Global)", - "optionKey": "modelstudioApiKeyCn", - "cliFlag": "--modelstudio-api-key-cn", - "cliOption": "--modelstudio-api-key-cn ", - "cliDescription": "Alibaba Cloud Model Studio Coding Plan API key (China)" - }, - { - "provider": "modelstudio", - "method": "api-key", - "choiceId": "modelstudio-api-key", - "choiceLabel": "Coding Plan API Key for Global/Intl (subscription)", - "choiceHint": "Endpoint: coding-intl.dashscope.aliyuncs.com", - "groupId": "modelstudio", - "groupLabel": "Qwen (Alibaba Cloud Model Studio)", - "groupHint": "Standard / Coding Plan (CN / Global)", - "optionKey": "modelstudioApiKey", - "cliFlag": "--modelstudio-api-key", - "cliOption": "--modelstudio-api-key ", - "cliDescription": "Alibaba Cloud Model Studio Coding Plan API key (Global/Intl)" - } - ], - "configSchema": { - "type": "object", - "additionalProperties": false, - "properties": {} - } -} diff --git a/extensions/modelstudio/provider-catalog.test.ts b/extensions/modelstudio/provider-catalog.test.ts deleted file mode 100644 index 523fae2cbad..00000000000 --- a/extensions/modelstudio/provider-catalog.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - applyModelStudioNativeStreamingUsageCompat, - buildModelStudioProvider, - MODELSTUDIO_BASE_URL, - MODELSTUDIO_DEFAULT_MODEL_ID, -} from "./api.js"; - -describe("modelstudio provider catalog", () => { - it("builds the bundled Model Studio provider defaults", () => { - const provider = buildModelStudioProvider(); - - expect(provider.baseUrl).toBe(MODELSTUDIO_BASE_URL); - expect(provider.api).toBe("openai-completions"); - expect(provider.models?.length).toBeGreaterThan(0); - expect( - provider.models?.find((model) => model.id === MODELSTUDIO_DEFAULT_MODEL_ID), - ).toBeTruthy(); - expect(provider.models?.find((model) => model.id === "qwen3.6-plus")).toBeTruthy(); - }); - - it("opts native Model Studio baseUrls into streaming usage only inside the extension", () => { - const nativeProvider = applyModelStudioNativeStreamingUsageCompat(buildModelStudioProvider()); - expect( - nativeProvider.models?.every((model) => model.compat?.supportsUsageInStreaming === true), - ).toBe(true); - - const customProvider = applyModelStudioNativeStreamingUsageCompat({ - ...buildModelStudioProvider(), - baseUrl: "https://proxy.example.com/v1", - }); - expect( - customProvider.models?.some((model) => model.compat?.supportsUsageInStreaming === true), - ).toBe(false); - }); -}); diff --git a/extensions/modelstudio/provider-catalog.ts b/extensions/modelstudio/provider-catalog.ts deleted file mode 100644 index 915b4a289db..00000000000 --- a/extensions/modelstudio/provider-catalog.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; -import { MODELSTUDIO_BASE_URL, MODELSTUDIO_MODEL_CATALOG } from "./models.js"; - -export function buildModelStudioProvider(): ModelProviderConfig { - return { - baseUrl: MODELSTUDIO_BASE_URL, - api: "openai-completions", - models: MODELSTUDIO_MODEL_CATALOG.map((model) => ({ ...model })), - }; -} diff --git a/extensions/openai/index.ts b/extensions/openai/index.ts index 639d995403b..e4f88334f42 100644 --- a/extensions/openai/index.ts +++ b/extensions/openai/index.ts @@ -1,53 +1,36 @@ import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { buildOpenAICodexCliBackend } from "./cli-backend.js"; +import { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js"; +import { + openaiCodexMediaUnderstandingProvider, + openaiMediaUnderstandingProvider, +} from "./media-understanding-provider.js"; +import { buildOpenAICodexProviderPlugin } from "./openai-codex-provider.js"; +import { buildOpenAIProvider } from "./openai-provider.js"; +import { + OPENAI_FRIENDLY_PROMPT_OVERLAY, + resolveOpenAIPromptOverlayMode, + shouldApplyOpenAIPromptOverlay, +} from "./prompt-overlay.js"; +import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js"; +import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js"; +import { buildOpenAISpeechProvider } from "./speech-provider.js"; export default definePluginEntry({ id: "openai", name: "OpenAI Provider", description: "Bundled OpenAI provider plugins", - async register(api) { - const { buildOpenAICodexCliBackend } = await import("./cli-backend.js"); - const { buildOpenAICodexProviderPlugin } = await import("./openai-codex-provider.js"); - const { buildOpenAIProvider } = await import("./openai-provider.js"); - const { - OPENAI_FRIENDLY_PROMPT_OVERLAY, - resolveOpenAIPromptOverlayMode, - shouldApplyOpenAIPromptOverlay, - } = await import("./prompt-overlay.js"); - const registerOptional = async (registerFn: () => Promise) => { - try { - await registerFn(); - } catch { - // Optional OpenAI surfaces must not block core provider registration. - } - }; - + register(api) { const promptOverlayMode = resolveOpenAIPromptOverlayMode(api.pluginConfig); api.registerCliBackend(buildOpenAICodexCliBackend()); api.registerProvider(buildOpenAIProvider()); api.registerProvider(buildOpenAICodexProviderPlugin()); - await registerOptional(async () => { - const { buildOpenAIImageGenerationProvider } = await import("./image-generation-provider.js"); - api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider()); - }); - await registerOptional(async () => { - const { buildOpenAIRealtimeTranscriptionProvider } = - await import("./realtime-transcription-provider.js"); - api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider()); - }); - await registerOptional(async () => { - const { buildOpenAIRealtimeVoiceProvider } = await import("./realtime-voice-provider.js"); - api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider()); - }); - await registerOptional(async () => { - const { buildOpenAISpeechProvider } = await import("./speech-provider.js"); - api.registerSpeechProvider(buildOpenAISpeechProvider()); - }); - await registerOptional(async () => { - const { openaiMediaUnderstandingProvider, openaiCodexMediaUnderstandingProvider } = - await import("./media-understanding-provider.js"); - api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider); - api.registerMediaUnderstandingProvider(openaiCodexMediaUnderstandingProvider); - }); + api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider()); + api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider()); + api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider()); + api.registerSpeechProvider(buildOpenAISpeechProvider()); + api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider); + api.registerMediaUnderstandingProvider(openaiCodexMediaUnderstandingProvider); if (promptOverlayMode !== "off") { api.on("before_prompt_build", (_event, ctx) => shouldApplyOpenAIPromptOverlay({ diff --git a/extensions/modelstudio/api.ts b/extensions/qwen/api.ts similarity index 52% rename from extensions/modelstudio/api.ts rename to extensions/qwen/api.ts index 0ae9ad13acf..33b19265105 100644 --- a/extensions/modelstudio/api.ts +++ b/extensions/qwen/api.ts @@ -1,4 +1,17 @@ export { + applyQwenNativeStreamingUsageCompat, + buildQwenDefaultModelDefinition, + buildQwenModelDefinition, + isNativeQwenBaseUrl, + QWEN_BASE_URL, + QWEN_CN_BASE_URL, + QWEN_DEFAULT_COST, + QWEN_DEFAULT_MODEL_ID, + QWEN_DEFAULT_MODEL_REF, + QWEN_GLOBAL_BASE_URL, + QWEN_STANDARD_CN_BASE_URL, + QWEN_STANDARD_GLOBAL_BASE_URL, + QWEN_MODEL_CATALOG, applyModelStudioNativeStreamingUsageCompat, buildModelStudioDefaultModelDefinition, buildModelStudioModelDefinition, @@ -13,4 +26,4 @@ export { MODELSTUDIO_STANDARD_GLOBAL_BASE_URL, MODELSTUDIO_MODEL_CATALOG, } from "./models.js"; -export { buildModelStudioProvider } from "./provider-catalog.js"; +export { buildModelStudioProvider, buildQwenProvider } from "./provider-catalog.js"; diff --git a/extensions/qwen/index.ts b/extensions/qwen/index.ts new file mode 100644 index 00000000000..d96f5b0c3e0 --- /dev/null +++ b/extensions/qwen/index.ts @@ -0,0 +1,129 @@ +import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry"; +import { applyQwenNativeStreamingUsageCompat } from "./api.js"; +import { buildQwenMediaUnderstandingProvider } from "./media-understanding-provider.js"; +import { + applyQwenConfig, + applyQwenConfigCn, + applyQwenStandardConfig, + applyQwenStandardConfigCn, + QWEN_DEFAULT_MODEL_REF, +} from "./onboard.js"; +import { buildQwenProvider } from "./provider-catalog.js"; +import { buildQwenVideoGenerationProvider } from "./video-generation-provider.js"; + +const PROVIDER_ID = "qwen"; + +export default defineSingleProviderPluginEntry({ + id: PROVIDER_ID, + name: "Qwen Provider", + description: "Bundled Qwen Cloud provider plugin", + provider: { + label: "Qwen Cloud", + docsPath: "/providers/qwen", + aliases: ["modelstudio", "qwencloud"], + auth: [ + { + methodId: "standard-api-key-cn", + label: "Standard API Key for China (pay-as-you-go)", + hint: "Endpoint: dashscope.aliyuncs.com", + optionKey: "modelstudioStandardApiKeyCn", + flagName: "--modelstudio-standard-api-key-cn", + envVar: "QWEN_API_KEY", + promptMessage: "Enter Qwen Cloud API key (China standard endpoint)", + defaultModel: QWEN_DEFAULT_MODEL_REF, + applyConfig: (cfg) => applyQwenStandardConfigCn(cfg), + noteMessage: [ + "Manage API keys: https://home.qwencloud.com/api-keys", + "Docs: https://docs.qwencloud.com/", + "Endpoint: dashscope.aliyuncs.com/compatible-mode/v1", + "Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.", + ].join("\n"), + noteTitle: "Qwen Cloud Standard (China)", + wizard: { + choiceHint: "Endpoint: dashscope.aliyuncs.com", + groupLabel: "Qwen Cloud", + groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap", + }, + }, + { + methodId: "standard-api-key", + label: "Standard API Key for Global/Intl (pay-as-you-go)", + hint: "Endpoint: dashscope-intl.aliyuncs.com", + optionKey: "modelstudioStandardApiKey", + flagName: "--modelstudio-standard-api-key", + envVar: "QWEN_API_KEY", + promptMessage: "Enter Qwen Cloud API key (Global/Intl standard endpoint)", + defaultModel: QWEN_DEFAULT_MODEL_REF, + applyConfig: (cfg) => applyQwenStandardConfig(cfg), + noteMessage: [ + "Manage API keys: https://home.qwencloud.com/api-keys", + "Docs: https://docs.qwencloud.com/", + "Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1", + "Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.", + ].join("\n"), + noteTitle: "Qwen Cloud Standard (Global/Intl)", + wizard: { + choiceHint: "Endpoint: dashscope-intl.aliyuncs.com", + groupLabel: "Qwen Cloud", + groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap", + }, + }, + { + methodId: "api-key-cn", + label: "Coding Plan API Key for China (subscription)", + hint: "Endpoint: coding.dashscope.aliyuncs.com", + optionKey: "modelstudioApiKeyCn", + flagName: "--modelstudio-api-key-cn", + envVar: "QWEN_API_KEY", + promptMessage: "Enter Qwen Cloud Coding Plan API key (China)", + defaultModel: QWEN_DEFAULT_MODEL_REF, + applyConfig: (cfg) => applyQwenConfigCn(cfg), + noteMessage: [ + "Manage API keys: https://home.qwencloud.com/api-keys", + "Docs: https://docs.qwencloud.com/", + "Endpoint: coding.dashscope.aliyuncs.com", + "Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.", + ].join("\n"), + noteTitle: "Qwen Cloud Coding Plan (China)", + wizard: { + choiceHint: "Endpoint: coding.dashscope.aliyuncs.com", + groupLabel: "Qwen Cloud", + groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap", + }, + }, + { + methodId: "api-key", + label: "Coding Plan API Key for Global/Intl (subscription)", + hint: "Endpoint: coding-intl.dashscope.aliyuncs.com", + optionKey: "modelstudioApiKey", + flagName: "--modelstudio-api-key", + envVar: "QWEN_API_KEY", + promptMessage: "Enter Qwen Cloud Coding Plan API key (Global/Intl)", + defaultModel: QWEN_DEFAULT_MODEL_REF, + applyConfig: (cfg) => applyQwenConfig(cfg), + noteMessage: [ + "Manage API keys: https://home.qwencloud.com/api-keys", + "Docs: https://docs.qwencloud.com/", + "Endpoint: coding-intl.dashscope.aliyuncs.com", + "Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.", + ].join("\n"), + noteTitle: "Qwen Cloud Coding Plan (Global/Intl)", + wizard: { + choiceHint: "Endpoint: coding-intl.dashscope.aliyuncs.com", + groupLabel: "Qwen Cloud", + groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap", + }, + }, + ], + catalog: { + buildProvider: buildQwenProvider, + allowExplicitBaseUrl: true, + }, + applyNativeStreamingUsageCompat: ({ providerConfig }) => + applyQwenNativeStreamingUsageCompat(providerConfig), + }, + register(api) { + api.registerMediaUnderstandingProvider(buildQwenMediaUnderstandingProvider()); + api.registerVideoGenerationProvider(buildQwenVideoGenerationProvider()); + }, +}); diff --git a/extensions/qwen/media-understanding-provider.test.ts b/extensions/qwen/media-understanding-provider.test.ts new file mode 100644 index 00000000000..f8f7c41ff47 --- /dev/null +++ b/extensions/qwen/media-understanding-provider.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { createRequestCaptureJsonFetch } from "../../test/helpers/plugins/media-understanding.js"; +import { describeQwenVideo } from "./media-understanding-provider.js"; + +describe("describeQwenVideo", () => { + it("builds the expected OpenAI-compatible video payload", async () => { + const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ + choices: [ + { + message: { + content: [{ text: " first " }, { text: "second" }], + }, + }, + ], + }); + + const result = await describeQwenVideo({ + buffer: Buffer.from("video-bytes"), + fileName: "clip.mp4", + mime: "video/mp4", + apiKey: "test-key", + timeoutMs: 1500, + baseUrl: "https://example.com/v1", + model: "qwen-vl-max", + prompt: "summarize the clip", + headers: { "X-Other": "1" }, + fetchFn, + }); + const { url, init } = getRequest(); + + expect(result.model).toBe("qwen-vl-max"); + expect(result.text).toBe("first\nsecond"); + expect(url).toBe("https://example.com/v1/chat/completions"); + expect(init?.method).toBe("POST"); + expect(init?.signal).toBeInstanceOf(AbortSignal); + + const headers = new Headers(init?.headers); + expect(headers.get("authorization")).toBe("Bearer test-key"); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("x-other")).toBe("1"); + + const bodyText = + typeof init?.body === "string" + ? init.body + : Buffer.isBuffer(init?.body) + ? init.body.toString("utf8") + : ""; + const body = JSON.parse(bodyText); + expect(body.model).toBe("qwen-vl-max"); + expect(body.messages?.[0]?.content?.[0]?.text).toBe("summarize the clip"); + expect(body.messages?.[0]?.content?.[1]?.type).toBe("video_url"); + expect(body.messages?.[0]?.content?.[1]?.video_url?.url).toBe( + `data:video/mp4;base64,${Buffer.from("video-bytes").toString("base64")}`, + ); + }); +}); diff --git a/extensions/qwen/media-understanding-provider.ts b/extensions/qwen/media-understanding-provider.ts new file mode 100644 index 00000000000..308ae72959d --- /dev/null +++ b/extensions/qwen/media-understanding-provider.ts @@ -0,0 +1,156 @@ +import { + describeImageWithModel, + describeImagesWithModel, + type MediaUnderstandingProvider, + type VideoDescriptionRequest, + type VideoDescriptionResult, +} from "openclaw/plugin-sdk/media-understanding"; +import { + assertOkOrThrowHttpError, + postJsonRequest, + resolveProviderHttpRequestConfig, +} from "openclaw/plugin-sdk/provider-http"; +import { QWEN_STANDARD_CN_BASE_URL, QWEN_STANDARD_GLOBAL_BASE_URL } from "./models.js"; + +const DEFAULT_QWEN_VIDEO_MODEL = "qwen-vl-max-latest"; +const DEFAULT_QWEN_VIDEO_PROMPT = "Describe the video in detail."; + +type QwenVideoPayload = { + choices?: Array<{ + message?: { + content?: string | Array<{ text?: string }>; + reasoning_content?: string; + }; + }>; +}; + +function resolveQwenStandardBaseUrl( + cfg: { models?: { providers?: Record } } | undefined, + providerId: string, +): string { + const direct = cfg?.models?.providers?.[providerId]?.baseUrl?.trim(); + if (!direct) { + return QWEN_STANDARD_GLOBAL_BASE_URL; + } + try { + const url = new URL(direct); + if (url.hostname === "coding-intl.dashscope.aliyuncs.com") { + return QWEN_STANDARD_GLOBAL_BASE_URL; + } + if (url.hostname === "coding.dashscope.aliyuncs.com") { + return QWEN_STANDARD_CN_BASE_URL; + } + return `${url.origin}${url.pathname}`.replace(/\/+$/u, ""); + } catch { + return QWEN_STANDARD_GLOBAL_BASE_URL; + } +} + +function coerceQwenText(payload: QwenVideoPayload): string | null { + const message = payload.choices?.[0]?.message; + if (!message) { + return null; + } + if (typeof message.content === "string" && message.content.trim()) { + return message.content.trim(); + } + if (Array.isArray(message.content)) { + const text = message.content + .map((part) => (typeof part.text === "string" ? part.text.trim() : "")) + .filter(Boolean) + .join("\n") + .trim(); + if (text) { + return text; + } + } + if (typeof message.reasoning_content === "string" && message.reasoning_content.trim()) { + return message.reasoning_content.trim(); + } + return null; +} + +export async function describeQwenVideo( + params: VideoDescriptionRequest, +): Promise { + const fetchFn = params.fetchFn ?? fetch; + const model = params.model?.trim() || DEFAULT_QWEN_VIDEO_MODEL; + const mime = params.mime?.trim() || "video/mp4"; + const prompt = params.prompt?.trim() || DEFAULT_QWEN_VIDEO_PROMPT; + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + resolveProviderHttpRequestConfig({ + baseUrl: params.baseUrl, + defaultBaseUrl: QWEN_STANDARD_GLOBAL_BASE_URL, + headers: params.headers, + request: params.request, + defaultHeaders: { + "content-type": "application/json", + authorization: `Bearer ${params.apiKey}`, + }, + provider: "qwen", + api: "openai-completions", + capability: "video", + transport: "media-understanding", + }); + + const { response: res, release } = await postJsonRequest({ + url: `${baseUrl}/chat/completions`, + headers, + body: { + model, + messages: [ + { + role: "user", + content: [ + { type: "text", text: prompt }, + { + type: "video_url", + video_url: { + url: `data:${mime};base64,${params.buffer.toString("base64")}`, + }, + }, + ], + }, + ], + }, + timeoutMs: params.timeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + }); + + try { + await assertOkOrThrowHttpError(res, "Qwen video description failed"); + const payload = (await res.json()) as QwenVideoPayload; + const text = coerceQwenText(payload); + if (!text) { + throw new Error("Qwen video description response missing content"); + } + return { text, model }; + } finally { + await release(); + } +} + +export function buildQwenMediaUnderstandingProvider(): MediaUnderstandingProvider { + return { + id: "qwen", + capabilities: ["image", "video"], + defaultModels: { + image: "qwen-vl-max-latest", + video: DEFAULT_QWEN_VIDEO_MODEL, + }, + autoPriority: { + video: 15, + }, + describeImage: describeImageWithModel, + describeImages: describeImagesWithModel, + describeVideo: describeQwenVideo, + }; +} + +export function resolveQwenMediaUnderstandingBaseUrl( + cfg: { models?: { providers?: Record } } | undefined, +): string { + return resolveQwenStandardBaseUrl(cfg, "qwen"); +} diff --git a/extensions/modelstudio/model-definitions.ts b/extensions/qwen/model-definitions.ts similarity index 58% rename from extensions/modelstudio/model-definitions.ts rename to extensions/qwen/model-definitions.ts index 83cc44909b2..fc7e3cca99a 100644 --- a/extensions/modelstudio/model-definitions.ts +++ b/extensions/qwen/model-definitions.ts @@ -1,4 +1,13 @@ export { + buildQwenDefaultModelDefinition, + buildQwenModelDefinition, + QWEN_CN_BASE_URL, + QWEN_DEFAULT_COST, + QWEN_DEFAULT_MODEL_ID, + QWEN_DEFAULT_MODEL_REF, + QWEN_GLOBAL_BASE_URL, + QWEN_STANDARD_CN_BASE_URL, + QWEN_STANDARD_GLOBAL_BASE_URL, buildModelStudioDefaultModelDefinition, buildModelStudioModelDefinition, MODELSTUDIO_CN_BASE_URL, diff --git a/extensions/modelstudio/models.ts b/extensions/qwen/models.ts similarity index 52% rename from extensions/modelstudio/models.ts rename to extensions/qwen/models.ts index bd15039c179..65131125ed2 100644 --- a/extensions/modelstudio/models.ts +++ b/extensions/qwen/models.ts @@ -7,29 +7,29 @@ import type { ModelProviderConfig, } from "openclaw/plugin-sdk/provider-model-shared"; -export const MODELSTUDIO_BASE_URL = "https://coding-intl.dashscope.aliyuncs.com/v1"; -export const MODELSTUDIO_GLOBAL_BASE_URL = MODELSTUDIO_BASE_URL; -export const MODELSTUDIO_CN_BASE_URL = "https://coding.dashscope.aliyuncs.com/v1"; -export const MODELSTUDIO_STANDARD_CN_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; -export const MODELSTUDIO_STANDARD_GLOBAL_BASE_URL = +export const QWEN_BASE_URL = "https://coding-intl.dashscope.aliyuncs.com/v1"; +export const QWEN_GLOBAL_BASE_URL = QWEN_BASE_URL; +export const QWEN_CN_BASE_URL = "https://coding.dashscope.aliyuncs.com/v1"; +export const QWEN_STANDARD_CN_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; +export const QWEN_STANDARD_GLOBAL_BASE_URL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"; -export const MODELSTUDIO_DEFAULT_MODEL_ID = "qwen3.5-plus"; -export const MODELSTUDIO_DEFAULT_COST = { +export const QWEN_DEFAULT_MODEL_ID = "qwen3.5-plus"; +export const QWEN_DEFAULT_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, }; -export const MODELSTUDIO_DEFAULT_MODEL_REF = `modelstudio/${MODELSTUDIO_DEFAULT_MODEL_ID}`; +export const QWEN_DEFAULT_MODEL_REF = `qwen/${QWEN_DEFAULT_MODEL_ID}`; -export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ +export const QWEN_MODEL_CATALOG: ReadonlyArray = [ { id: "qwen3.5-plus", name: "qwen3.5-plus", reasoning: false, input: ["text", "image"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -38,7 +38,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "qwen3.6-plus", reasoning: false, input: ["text", "image"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -47,7 +47,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "qwen3-max-2026-01-23", reasoning: false, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 262_144, maxTokens: 65_536, }, @@ -56,7 +56,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "qwen3-coder-next", reasoning: false, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 262_144, maxTokens: 65_536, }, @@ -65,7 +65,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "qwen3-coder-plus", reasoning: false, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -74,7 +74,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "MiniMax-M2.5", reasoning: true, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -83,7 +83,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "glm-5", reasoning: false, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 202_752, maxTokens: 16_384, }, @@ -92,7 +92,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "glm-4.7", reasoning: false, input: ["text"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 202_752, maxTokens: 16_384, }, @@ -101,29 +101,29 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray = [ name: "kimi-k2.5", reasoning: false, input: ["text", "image"], - cost: MODELSTUDIO_DEFAULT_COST, + cost: QWEN_DEFAULT_COST, contextWindow: 262_144, maxTokens: 32_768, }, ]; -export function isNativeModelStudioBaseUrl(baseUrl: string | undefined): boolean { +export function isNativeQwenBaseUrl(baseUrl: string | undefined): boolean { return supportsNativeStreamingUsageCompat({ - providerId: "modelstudio", + providerId: "qwen", baseUrl, }); } -export function applyModelStudioNativeStreamingUsageCompat( +export function applyQwenNativeStreamingUsageCompat( provider: ModelProviderConfig, ): ModelProviderConfig { return applyProviderNativeStreamingUsageCompat({ - providerId: "modelstudio", + providerId: "qwen", providerConfig: provider, }); } -export function buildModelStudioModelDefinition(params: { +export function buildQwenModelDefinition(params: { id: string; name?: string; reasoning?: boolean; @@ -132,19 +132,34 @@ export function buildModelStudioModelDefinition(params: { contextWindow?: number; maxTokens?: number; }): ModelDefinitionConfig { - const catalog = MODELSTUDIO_MODEL_CATALOG.find((model) => model.id === params.id); + const catalog = QWEN_MODEL_CATALOG.find((model) => model.id === params.id); return { id: params.id, name: params.name ?? catalog?.name ?? params.id, reasoning: params.reasoning ?? catalog?.reasoning ?? false, input: (params.input as ("text" | "image")[]) ?? (catalog?.input ? [...catalog.input] : ["text"]), - cost: params.cost ?? catalog?.cost ?? MODELSTUDIO_DEFAULT_COST, + cost: params.cost ?? catalog?.cost ?? QWEN_DEFAULT_COST, contextWindow: params.contextWindow ?? catalog?.contextWindow ?? 262_144, maxTokens: params.maxTokens ?? catalog?.maxTokens ?? 65_536, }; } -export function buildModelStudioDefaultModelDefinition(): ModelDefinitionConfig { - return buildModelStudioModelDefinition({ id: MODELSTUDIO_DEFAULT_MODEL_ID }); +export function buildQwenDefaultModelDefinition(): ModelDefinitionConfig { + return buildQwenModelDefinition({ id: QWEN_DEFAULT_MODEL_ID }); } + +// Backward-compatible aliases while `modelstudio` references are still in the wild. +export const MODELSTUDIO_BASE_URL = QWEN_BASE_URL; +export const MODELSTUDIO_GLOBAL_BASE_URL = QWEN_GLOBAL_BASE_URL; +export const MODELSTUDIO_CN_BASE_URL = QWEN_CN_BASE_URL; +export const MODELSTUDIO_STANDARD_CN_BASE_URL = QWEN_STANDARD_CN_BASE_URL; +export const MODELSTUDIO_STANDARD_GLOBAL_BASE_URL = QWEN_STANDARD_GLOBAL_BASE_URL; +export const MODELSTUDIO_DEFAULT_MODEL_ID = QWEN_DEFAULT_MODEL_ID; +export const MODELSTUDIO_DEFAULT_COST = QWEN_DEFAULT_COST; +export const MODELSTUDIO_DEFAULT_MODEL_REF = `modelstudio/${QWEN_DEFAULT_MODEL_ID}`; +export const MODELSTUDIO_MODEL_CATALOG = QWEN_MODEL_CATALOG; +export const isNativeModelStudioBaseUrl = isNativeQwenBaseUrl; +export const applyModelStudioNativeStreamingUsageCompat = applyQwenNativeStreamingUsageCompat; +export const buildModelStudioModelDefinition = buildQwenModelDefinition; +export const buildModelStudioDefaultModelDefinition = buildQwenDefaultModelDefinition; diff --git a/extensions/qwen/onboard.ts b/extensions/qwen/onboard.ts new file mode 100644 index 00000000000..b24a6b76222 --- /dev/null +++ b/extensions/qwen/onboard.ts @@ -0,0 +1,81 @@ +import { + createModelCatalogPresetAppliers, + type OpenClawConfig, +} from "openclaw/plugin-sdk/provider-onboard"; +import { + QWEN_CN_BASE_URL, + QWEN_DEFAULT_MODEL_REF, + QWEN_GLOBAL_BASE_URL, + QWEN_STANDARD_CN_BASE_URL, + QWEN_STANDARD_GLOBAL_BASE_URL, +} from "./models.js"; +import { buildQwenProvider } from "./provider-catalog.js"; + +export { + QWEN_CN_BASE_URL, + QWEN_DEFAULT_MODEL_REF, + QWEN_GLOBAL_BASE_URL, + QWEN_STANDARD_CN_BASE_URL, + QWEN_STANDARD_GLOBAL_BASE_URL, +}; + +const qwenPresetAppliers = createModelCatalogPresetAppliers<[string]>({ + primaryModelRef: QWEN_DEFAULT_MODEL_REF, + resolveParams: (_cfg: OpenClawConfig, baseUrl: string) => { + const provider = buildQwenProvider(); + return { + providerId: "qwen", + api: provider.api ?? "openai-completions", + baseUrl, + catalogModels: provider.models ?? [], + aliases: [ + ...(provider.models ?? []).flatMap((model) => [ + `qwen/${model.id}`, + `modelstudio/${model.id}`, + ]), + { modelRef: QWEN_DEFAULT_MODEL_REF, alias: "Qwen" }, + ], + }; + }, +}); + +export function applyQwenProviderConfig(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_GLOBAL_BASE_URL); +} + +export function applyQwenProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_CN_BASE_URL); +} + +export function applyQwenConfig(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyConfig(cfg, QWEN_GLOBAL_BASE_URL); +} + +export function applyQwenConfigCn(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyConfig(cfg, QWEN_CN_BASE_URL); +} + +export function applyQwenStandardProviderConfig(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_STANDARD_GLOBAL_BASE_URL); +} + +export function applyQwenStandardProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_STANDARD_CN_BASE_URL); +} + +export function applyQwenStandardConfig(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyConfig(cfg, QWEN_STANDARD_GLOBAL_BASE_URL); +} + +export function applyQwenStandardConfigCn(cfg: OpenClawConfig): OpenClawConfig { + return qwenPresetAppliers.applyConfig(cfg, QWEN_STANDARD_CN_BASE_URL); +} + +export const applyModelStudioProviderConfig = applyQwenProviderConfig; +export const applyModelStudioProviderConfigCn = applyQwenProviderConfigCn; +export const applyModelStudioConfig = applyQwenConfig; +export const applyModelStudioConfigCn = applyQwenConfigCn; +export const applyModelStudioStandardProviderConfig = applyQwenStandardProviderConfig; +export const applyModelStudioStandardProviderConfigCn = applyQwenStandardProviderConfigCn; +export const applyModelStudioStandardConfig = applyQwenStandardConfig; +export const applyModelStudioStandardConfigCn = applyQwenStandardConfigCn; diff --git a/extensions/qwen/openclaw.plugin.json b/extensions/qwen/openclaw.plugin.json new file mode 100644 index 00000000000..e440f9f9204 --- /dev/null +++ b/extensions/qwen/openclaw.plugin.json @@ -0,0 +1,79 @@ +{ + "id": "qwen", + "enabledByDefault": true, + "providers": ["qwen"], + "contracts": { + "mediaUnderstandingProviders": ["qwen"], + "videoGenerationProviders": ["qwen"] + }, + "providerAuthEnvVars": { + "qwen": ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"] + }, + "providerAuthChoices": [ + { + "provider": "qwen", + "method": "standard-api-key-cn", + "choiceId": "qwen-standard-api-key-cn", + "deprecatedChoiceIds": ["modelstudio-standard-api-key-cn"], + "choiceLabel": "Standard API Key for China (pay-as-you-go)", + "choiceHint": "Endpoint: dashscope.aliyuncs.com", + "groupId": "qwen", + "groupLabel": "Qwen Cloud", + "groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap", + "optionKey": "modelstudioStandardApiKeyCn", + "cliFlag": "--modelstudio-standard-api-key-cn", + "cliOption": "--modelstudio-standard-api-key-cn ", + "cliDescription": "Qwen Cloud standard API key (China)" + }, + { + "provider": "qwen", + "method": "standard-api-key", + "choiceId": "qwen-standard-api-key", + "deprecatedChoiceIds": ["modelstudio-standard-api-key"], + "choiceLabel": "Standard API Key for Global/Intl (pay-as-you-go)", + "choiceHint": "Endpoint: dashscope-intl.aliyuncs.com", + "groupId": "qwen", + "groupLabel": "Qwen Cloud", + "groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap", + "optionKey": "modelstudioStandardApiKey", + "cliFlag": "--modelstudio-standard-api-key", + "cliOption": "--modelstudio-standard-api-key ", + "cliDescription": "Qwen Cloud standard API key (Global/Intl)" + }, + { + "provider": "qwen", + "method": "api-key-cn", + "choiceId": "qwen-api-key-cn", + "deprecatedChoiceIds": ["modelstudio-api-key-cn"], + "choiceLabel": "Coding Plan API Key for China (subscription)", + "choiceHint": "Endpoint: coding.dashscope.aliyuncs.com", + "groupId": "qwen", + "groupLabel": "Qwen Cloud", + "groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap", + "optionKey": "modelstudioApiKeyCn", + "cliFlag": "--modelstudio-api-key-cn", + "cliOption": "--modelstudio-api-key-cn ", + "cliDescription": "Qwen Cloud Coding Plan API key (China)" + }, + { + "provider": "qwen", + "method": "api-key", + "choiceId": "qwen-api-key", + "deprecatedChoiceIds": ["modelstudio-api-key"], + "choiceLabel": "Coding Plan API Key for Global/Intl (subscription)", + "choiceHint": "Endpoint: coding-intl.dashscope.aliyuncs.com", + "groupId": "qwen", + "groupLabel": "Qwen Cloud", + "groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap", + "optionKey": "modelstudioApiKey", + "cliFlag": "--modelstudio-api-key", + "cliOption": "--modelstudio-api-key ", + "cliDescription": "Qwen Cloud Coding Plan API key (Global/Intl)" + } + ], + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/modelstudio/package.json b/extensions/qwen/package.json similarity index 57% rename from extensions/modelstudio/package.json rename to extensions/qwen/package.json index 9ed6095c0b1..0f8cab79582 100644 --- a/extensions/modelstudio/package.json +++ b/extensions/qwen/package.json @@ -1,8 +1,8 @@ { - "name": "@openclaw/modelstudio-provider", + "name": "@openclaw/qwen-provider", "version": "2026.4.1-beta.1", "private": true, - "description": "OpenClaw Model Studio provider plugin", + "description": "OpenClaw Qwen Cloud provider plugin", "type": "module", "openclaw": { "extensions": [ diff --git a/extensions/qwen/plugin-registration.contract.test.ts b/extensions/qwen/plugin-registration.contract.test.ts new file mode 100644 index 00000000000..37440219582 --- /dev/null +++ b/extensions/qwen/plugin-registration.contract.test.ts @@ -0,0 +1,10 @@ +import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js"; + +describePluginRegistrationContract({ + pluginId: "qwen", + providerIds: ["qwen"], + mediaUnderstandingProviderIds: ["qwen"], + videoGenerationProviderIds: ["qwen"], + requireDescribeImages: true, + requireGenerateVideo: true, +}); diff --git a/extensions/qwen/provider-catalog.test.ts b/extensions/qwen/provider-catalog.test.ts new file mode 100644 index 00000000000..e2fdb9a9239 --- /dev/null +++ b/extensions/qwen/provider-catalog.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + applyQwenNativeStreamingUsageCompat, + buildQwenProvider, + QWEN_BASE_URL, + QWEN_DEFAULT_MODEL_ID, +} from "./api.js"; + +describe("qwen provider catalog", () => { + it("builds the bundled Qwen provider defaults", () => { + const provider = buildQwenProvider(); + + expect(provider.baseUrl).toBe(QWEN_BASE_URL); + expect(provider.api).toBe("openai-completions"); + expect(provider.models?.length).toBeGreaterThan(0); + expect(provider.models?.find((model) => model.id === QWEN_DEFAULT_MODEL_ID)).toBeTruthy(); + expect(provider.models?.find((model) => model.id === "qwen3.6-plus")).toBeTruthy(); + }); + + it("opts native Qwen baseUrls into streaming usage only inside the extension", () => { + const nativeProvider = applyQwenNativeStreamingUsageCompat(buildQwenProvider()); + expect( + nativeProvider.models?.every((model) => model.compat?.supportsUsageInStreaming === true), + ).toBe(true); + + const customProvider = applyQwenNativeStreamingUsageCompat({ + ...buildQwenProvider(), + baseUrl: "https://proxy.example.com/v1", + }); + expect( + customProvider.models?.some((model) => model.compat?.supportsUsageInStreaming === true), + ).toBe(false); + }); +}); diff --git a/extensions/qwen/provider-catalog.ts b/extensions/qwen/provider-catalog.ts new file mode 100644 index 00000000000..efdea224d7e --- /dev/null +++ b/extensions/qwen/provider-catalog.ts @@ -0,0 +1,12 @@ +import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; +import { QWEN_BASE_URL, QWEN_MODEL_CATALOG } from "./models.js"; + +export function buildQwenProvider(): ModelProviderConfig { + return { + baseUrl: QWEN_BASE_URL, + api: "openai-completions", + models: QWEN_MODEL_CATALOG.map((model) => ({ ...model })), + }; +} + +export const buildModelStudioProvider = buildQwenProvider; diff --git a/extensions/qwen/test-api.ts b/extensions/qwen/test-api.ts new file mode 100644 index 00000000000..be5693f4f55 --- /dev/null +++ b/extensions/qwen/test-api.ts @@ -0,0 +1,2 @@ +export { buildQwenMediaUnderstandingProvider } from "./media-understanding-provider.js"; +export { buildQwenVideoGenerationProvider } from "./video-generation-provider.js"; diff --git a/extensions/qwen/video-generation-provider.test.ts b/extensions/qwen/video-generation-provider.test.ts new file mode 100644 index 00000000000..93a6b121c92 --- /dev/null +++ b/extensions/qwen/video-generation-provider.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildQwenVideoGenerationProvider } from "./video-generation-provider.js"; + +const { + resolveApiKeyForProviderMock, + postJsonRequestMock, + fetchWithTimeoutMock, + assertOkOrThrowHttpErrorMock, + resolveProviderHttpRequestConfigMock, +} = vi.hoisted(() => ({ + resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "qwen-key" })), + postJsonRequestMock: vi.fn(), + fetchWithTimeoutMock: vi.fn(), + assertOkOrThrowHttpErrorMock: vi.fn(async () => {}), + resolveProviderHttpRequestConfigMock: vi.fn((params) => ({ + baseUrl: params.baseUrl ?? params.defaultBaseUrl, + allowPrivateNetwork: false, + headers: new Headers(params.defaultHeaders), + dispatcherPolicy: undefined, + })), +})); + +vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({ + resolveApiKeyForProvider: resolveApiKeyForProviderMock, +})); + +vi.mock("openclaw/plugin-sdk/provider-http", () => ({ + assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock, + fetchWithTimeout: fetchWithTimeoutMock, + postJsonRequest: postJsonRequestMock, + resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock, +})); + +describe("qwen video generation provider", () => { + afterEach(() => { + resolveApiKeyForProviderMock.mockClear(); + postJsonRequestMock.mockReset(); + fetchWithTimeoutMock.mockReset(); + assertOkOrThrowHttpErrorMock.mockClear(); + resolveProviderHttpRequestConfigMock.mockClear(); + }); + + it("submits async Wan generation, polls task status, and downloads the resulting video", async () => { + postJsonRequestMock.mockResolvedValue({ + response: { + json: async () => ({ + request_id: "req-1", + output: { + task_id: "task-1", + }, + }), + }, + release: vi.fn(async () => {}), + }); + fetchWithTimeoutMock + .mockResolvedValueOnce({ + json: async () => ({ + output: { + task_status: "SUCCEEDED", + results: [{ video_url: "https://example.com/out.mp4" }], + }, + }), + headers: new Headers(), + }) + .mockResolvedValueOnce({ + arrayBuffer: async () => Buffer.from("mp4-bytes"), + headers: new Headers({ "content-type": "video/mp4" }), + }); + + const provider = buildQwenVideoGenerationProvider(); + const result = await provider.generateVideo({ + provider: "qwen", + model: "wan2.6-r2v-flash", + prompt: "animate this shot", + cfg: {}, + inputImages: [{ url: "https://example.com/ref.png" }], + durationSeconds: 6, + audio: true, + }); + + expect(postJsonRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis", + body: expect.objectContaining({ + model: "wan2.6-r2v-flash", + input: expect.objectContaining({ + prompt: "animate this shot", + img_url: "https://example.com/ref.png", + }), + }), + }), + ); + expect(fetchWithTimeoutMock).toHaveBeenNthCalledWith( + 1, + "https://dashscope-intl.aliyuncs.com/api/v1/tasks/task-1", + expect.objectContaining({ method: "GET" }), + 120000, + fetch, + ); + expect(result.videos).toHaveLength(1); + expect(result.videos[0]?.mimeType).toBe("video/mp4"); + expect(result.metadata).toEqual( + expect.objectContaining({ + requestId: "req-1", + taskId: "task-1", + taskStatus: "SUCCEEDED", + }), + ); + }); +}); diff --git a/extensions/qwen/video-generation-provider.ts b/extensions/qwen/video-generation-provider.ts new file mode 100644 index 00000000000..5ed249be535 --- /dev/null +++ b/extensions/qwen/video-generation-provider.ts @@ -0,0 +1,300 @@ +import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; +import { + assertOkOrThrowHttpError, + fetchWithTimeout, + postJsonRequest, + resolveProviderHttpRequestConfig, +} from "openclaw/plugin-sdk/provider-http"; +import type { + GeneratedVideoAsset, + VideoGenerationProvider, + VideoGenerationRequest, + VideoGenerationResult, + VideoGenerationSourceAsset, +} from "openclaw/plugin-sdk/video-generation"; +import { QWEN_STANDARD_CN_BASE_URL, QWEN_STANDARD_GLOBAL_BASE_URL } from "./models.js"; + +const DEFAULT_QWEN_VIDEO_BASE_URL = "https://dashscope-intl.aliyuncs.com"; +const DEFAULT_QWEN_VIDEO_MODEL = "wan2.6-t2v"; +const DEFAULT_DURATION_SECONDS = 5; +const DEFAULT_REQUEST_TIMEOUT_MS = 120_000; +const POLL_INTERVAL_MS = 2_500; +const MAX_POLL_ATTEMPTS = 120; +const RESOLUTION_TO_SIZE: Record = { + "480P": "832*480", + "720P": "1280*720", + "1080P": "1920*1080", +}; + +type QwenVideoGenerationResponse = { + output?: { + task_id?: string; + task_status?: string; + submit_time?: string; + results?: Array<{ + video_url?: string; + orig_prompt?: string; + actual_prompt?: string; + }>; + video_url?: string; + code?: string; + message?: string; + }; + request_id?: string; + code?: string; + message?: string; +}; + +function resolveQwenVideoBaseUrl(req: VideoGenerationRequest): string { + const direct = req.cfg?.models?.providers?.qwen?.baseUrl?.trim(); + if (!direct) { + return DEFAULT_QWEN_VIDEO_BASE_URL; + } + try { + const url = new URL(direct); + if (url.hostname === "coding-intl.dashscope.aliyuncs.com") { + return "https://dashscope-intl.aliyuncs.com"; + } + if (url.hostname === "coding.dashscope.aliyuncs.com") { + return "https://dashscope.aliyuncs.com"; + } + if (url.hostname === "dashscope-intl.aliyuncs.com") { + return "https://dashscope-intl.aliyuncs.com"; + } + if (url.hostname === "dashscope.aliyuncs.com") { + return "https://dashscope.aliyuncs.com"; + } + return url.origin; + } catch { + return DEFAULT_QWEN_VIDEO_BASE_URL; + } +} + +function resolveDashscopeAigcApiBaseUrl(baseUrl: string): string { + if (baseUrl.startsWith(QWEN_STANDARD_CN_BASE_URL)) { + return "https://dashscope.aliyuncs.com"; + } + if (baseUrl.startsWith(QWEN_STANDARD_GLOBAL_BASE_URL)) { + return DEFAULT_QWEN_VIDEO_BASE_URL; + } + return baseUrl.replace(/\/+$/u, ""); +} + +function resolveReferenceUrls( + inputImages: VideoGenerationSourceAsset[] | undefined, + inputVideos: VideoGenerationSourceAsset[] | undefined, +): string[] { + return [...(inputImages ?? []), ...(inputVideos ?? [])] + .map((asset) => asset.url?.trim()) + .filter((value): value is string => Boolean(value)); +} + +function buildQwenVideoGenerationInput(req: VideoGenerationRequest): Record { + const input: Record = { + prompt: req.prompt, + }; + const referenceUrls = resolveReferenceUrls(req.inputImages, req.inputVideos); + if ( + referenceUrls.length === 1 && + (req.inputImages?.length ?? 0) === 1 && + !req.inputVideos?.length + ) { + input.img_url = referenceUrls[0]; + } else if (referenceUrls.length > 0) { + input.reference_urls = referenceUrls; + } + return input; +} + +function buildQwenVideoGenerationParameters( + req: VideoGenerationRequest, +): Record | undefined { + const parameters: Record = {}; + const size = + req.size?.trim() || (req.resolution ? RESOLUTION_TO_SIZE[req.resolution] : undefined); + if (size) { + parameters.size = size; + } + if (req.aspectRatio?.trim()) { + parameters.aspect_ratio = req.aspectRatio.trim(); + } + if (typeof req.durationSeconds === "number" && Number.isFinite(req.durationSeconds)) { + parameters.duration = Math.max(1, Math.round(req.durationSeconds)); + } + if (typeof req.audio === "boolean") { + parameters.enable_audio = req.audio; + } + if (typeof req.watermark === "boolean") { + parameters.watermark = req.watermark; + } + return Object.keys(parameters).length > 0 ? parameters : undefined; +} + +function extractVideoUrls(payload: QwenVideoGenerationResponse): string[] { + const urls = [ + ...(payload.output?.results?.map((entry) => entry.video_url).filter(Boolean) ?? []), + payload.output?.video_url, + ].filter((value): value is string => typeof value === "string" && value.trim().length > 0); + return [...new Set(urls)]; +} + +async function pollTaskUntilComplete(params: { + taskId: string; + headers: Headers; + timeoutMs?: number; + fetchFn: typeof fetch; + baseUrl: string; +}): Promise { + for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { + const response = await fetchWithTimeout( + `${params.baseUrl}/api/v1/tasks/${params.taskId}`, + { + method: "GET", + headers: params.headers, + }, + params.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS, + params.fetchFn, + ); + await assertOkOrThrowHttpError(response, "Qwen video-generation task poll failed"); + const payload = (await response.json()) as QwenVideoGenerationResponse; + const status = payload.output?.task_status?.trim().toUpperCase(); + if (status === "SUCCEEDED") { + return payload; + } + if (status === "FAILED" || status === "CANCELED") { + throw new Error( + payload.output?.message?.trim() || + payload.message?.trim() || + `Qwen video generation task ${params.taskId} ${status.toLowerCase()}`, + ); + } + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + } + throw new Error(`Qwen video generation task ${params.taskId} did not finish in time`); +} + +async function downloadGeneratedVideos(params: { + urls: string[]; + timeoutMs?: number; + fetchFn: typeof fetch; +}): Promise { + const videos: GeneratedVideoAsset[] = []; + for (const [index, url] of params.urls.entries()) { + const response = await fetchWithTimeout( + url, + { method: "GET" }, + params.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS, + params.fetchFn, + ); + await assertOkOrThrowHttpError(response, "Qwen generated video download failed"); + const arrayBuffer = await response.arrayBuffer(); + videos.push({ + buffer: Buffer.from(arrayBuffer), + mimeType: response.headers.get("content-type")?.trim() || "video/mp4", + fileName: `video-${index + 1}.mp4`, + metadata: { sourceUrl: url }, + }); + } + return videos; +} + +export function buildQwenVideoGenerationProvider(): VideoGenerationProvider { + return { + id: "qwen", + label: "Qwen Cloud", + defaultModel: DEFAULT_QWEN_VIDEO_MODEL, + models: ["wan2.6-t2v", "wan2.6-i2v", "wan2.6-r2v", "wan2.6-r2v-flash", "wan2.7-r2v"], + capabilities: { + maxVideos: 1, + maxInputImages: 1, + maxInputVideos: 4, + maxDurationSeconds: 10, + supportsSize: true, + supportsAspectRatio: true, + supportsResolution: true, + supportsAudio: true, + supportsWatermark: true, + }, + async generateVideo(req): Promise { + const fetchFn = fetch; + const auth = await resolveApiKeyForProvider({ + provider: "qwen", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + }); + if (!auth.apiKey) { + throw new Error("Qwen API key missing"); + } + + const requestBaseUrl = resolveQwenVideoBaseUrl(req); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + resolveProviderHttpRequestConfig({ + baseUrl: requestBaseUrl, + defaultBaseUrl: DEFAULT_QWEN_VIDEO_BASE_URL, + defaultHeaders: { + Authorization: `Bearer ${auth.apiKey}`, + "Content-Type": "application/json", + "X-DashScope-Async": "enable", + }, + provider: "qwen", + capability: "video", + transport: "http", + }); + + const model = req.model?.trim() || DEFAULT_QWEN_VIDEO_MODEL; + const { response, release } = await postJsonRequest({ + url: `${resolveDashscopeAigcApiBaseUrl(baseUrl)}/api/v1/services/aigc/video-generation/video-synthesis`, + headers, + body: { + model, + input: buildQwenVideoGenerationInput(req), + parameters: buildQwenVideoGenerationParameters({ + ...req, + durationSeconds: req.durationSeconds ?? DEFAULT_DURATION_SECONDS, + }), + }, + timeoutMs: req.timeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + }); + + try { + await assertOkOrThrowHttpError(response, "Qwen video generation failed"); + const submitted = (await response.json()) as QwenVideoGenerationResponse; + const taskId = submitted.output?.task_id?.trim(); + if (!taskId) { + throw new Error("Qwen video generation response missing task_id"); + } + const completed = await pollTaskUntilComplete({ + taskId, + headers, + timeoutMs: req.timeoutMs, + fetchFn, + baseUrl: resolveDashscopeAigcApiBaseUrl(baseUrl), + }); + const urls = extractVideoUrls(completed); + if (urls.length === 0) { + throw new Error("Qwen video generation completed without output video URLs"); + } + const videos = await downloadGeneratedVideos({ + urls, + timeoutMs: req.timeoutMs, + fetchFn, + }); + return { + videos, + model, + metadata: { + requestId: submitted.request_id, + taskId, + taskStatus: completed.output?.task_status, + }, + }; + } finally { + await release(); + } + }, + }; +} diff --git a/extensions/video-generation-core/api.ts b/extensions/video-generation-core/api.ts new file mode 100644 index 00000000000..4fd76d71152 --- /dev/null +++ b/extensions/video-generation-core/api.ts @@ -0,0 +1 @@ +export * from "openclaw/plugin-sdk/video-generation-core"; diff --git a/extensions/video-generation-core/package.json b/extensions/video-generation-core/package.json new file mode 100644 index 00000000000..29aa8e201bc --- /dev/null +++ b/extensions/video-generation-core/package.json @@ -0,0 +1,7 @@ +{ + "name": "@openclaw/video-generation-core", + "version": "2026.4.1-beta.1", + "private": true, + "description": "OpenClaw video generation runtime package", + "type": "module" +} diff --git a/extensions/video-generation-core/runtime-api.ts b/extensions/video-generation-core/runtime-api.ts new file mode 100644 index 00000000000..d3e66a4d5ce --- /dev/null +++ b/extensions/video-generation-core/runtime-api.ts @@ -0,0 +1,6 @@ +export { + generateVideo, + listRuntimeVideoGenerationProviders, + type GenerateVideoParams, + type GenerateVideoRuntimeResult, +} from "./src/runtime.js"; diff --git a/extensions/video-generation-core/src/runtime.test.ts b/extensions/video-generation-core/src/runtime.test.ts new file mode 100644 index 00000000000..477103e8184 --- /dev/null +++ b/extensions/video-generation-core/src/runtime.test.ts @@ -0,0 +1,164 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../../src/config/config.js"; +import type { VideoGenerationProvider } from "../api.js"; +import { generateVideo, listRuntimeVideoGenerationProviders } from "./runtime.js"; + +const mocks = vi.hoisted(() => { + const debug = vi.fn(); + return { + createSubsystemLogger: vi.fn(() => ({ debug })), + describeFailoverError: vi.fn(), + getProviderEnvVars: vi.fn<(providerId: string) => string[]>(() => []), + getVideoGenerationProvider: vi.fn< + (providerId: string, config?: OpenClawConfig) => VideoGenerationProvider | undefined + >(() => undefined), + isFailoverError: vi.fn<(err: unknown) => boolean>(() => false), + listVideoGenerationProviders: vi.fn<(config?: OpenClawConfig) => VideoGenerationProvider[]>( + () => [], + ), + parseVideoGenerationModelRef: vi.fn< + (raw?: string) => { provider: string; model: string } | undefined + >((raw?: string) => { + const trimmed = raw?.trim(); + if (!trimmed) { + return undefined; + } + const slash = trimmed.indexOf("/"); + if (slash <= 0 || slash === trimmed.length - 1) { + return undefined; + } + return { + provider: trimmed.slice(0, slash), + model: trimmed.slice(slash + 1), + }; + }), + resolveAgentModelFallbackValues: vi.fn<(value: unknown) => string[]>(() => []), + resolveAgentModelPrimaryValue: vi.fn<(value: unknown) => string | undefined>(() => undefined), + debug, + }; +}); + +vi.mock("../api.js", () => ({ + createSubsystemLogger: mocks.createSubsystemLogger, + describeFailoverError: mocks.describeFailoverError, + getProviderEnvVars: mocks.getProviderEnvVars, + getVideoGenerationProvider: mocks.getVideoGenerationProvider, + isFailoverError: mocks.isFailoverError, + listVideoGenerationProviders: mocks.listVideoGenerationProviders, + parseVideoGenerationModelRef: mocks.parseVideoGenerationModelRef, + resolveAgentModelFallbackValues: mocks.resolveAgentModelFallbackValues, + resolveAgentModelPrimaryValue: mocks.resolveAgentModelPrimaryValue, +})); + +describe("video-generation runtime", () => { + beforeEach(() => { + mocks.createSubsystemLogger.mockClear(); + mocks.describeFailoverError.mockReset(); + mocks.getProviderEnvVars.mockReset(); + mocks.getProviderEnvVars.mockReturnValue([]); + mocks.getVideoGenerationProvider.mockReset(); + mocks.isFailoverError.mockReset(); + mocks.isFailoverError.mockReturnValue(false); + mocks.listVideoGenerationProviders.mockReset(); + mocks.listVideoGenerationProviders.mockReturnValue([]); + mocks.parseVideoGenerationModelRef.mockClear(); + mocks.resolveAgentModelFallbackValues.mockReset(); + mocks.resolveAgentModelFallbackValues.mockReturnValue([]); + mocks.resolveAgentModelPrimaryValue.mockReset(); + mocks.resolveAgentModelPrimaryValue.mockReturnValue(undefined); + mocks.debug.mockReset(); + }); + + it("generates videos through the active video-generation provider", async () => { + const authStore = { version: 1, profiles: {} } as const; + let seenAuthStore: unknown; + mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1"); + const provider: VideoGenerationProvider = { + id: "video-plugin", + capabilities: {}, + async generateVideo(req: { authStore?: unknown }) { + seenAuthStore = req.authStore; + return { + videos: [ + { + buffer: Buffer.from("mp4-bytes"), + mimeType: "video/mp4", + fileName: "sample.mp4", + }, + ], + model: "vid-v1", + }; + }, + }; + mocks.getVideoGenerationProvider.mockReturnValue(provider); + + const result = await generateVideo({ + cfg: { + agents: { + defaults: { + videoGenerationModel: { primary: "video-plugin/vid-v1" }, + }, + }, + } as OpenClawConfig, + prompt: "animate a cat", + agentDir: "/tmp/agent", + authStore, + }); + + expect(result.provider).toBe("video-plugin"); + expect(result.model).toBe("vid-v1"); + expect(result.attempts).toEqual([]); + expect(seenAuthStore).toEqual(authStore); + expect(result.videos).toEqual([ + { + buffer: Buffer.from("mp4-bytes"), + mimeType: "video/mp4", + fileName: "sample.mp4", + }, + ]); + }); + + it("lists runtime video-generation providers through the owner runtime", () => { + const providers: VideoGenerationProvider[] = [ + { + id: "video-plugin", + defaultModel: "vid-v1", + models: ["vid-v1"], + capabilities: { + supportsAudio: true, + }, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }], + }), + }, + ]; + mocks.listVideoGenerationProviders.mockReturnValue(providers); + + expect(listRuntimeVideoGenerationProviders({ config: {} as OpenClawConfig })).toEqual( + providers, + ); + expect(mocks.listVideoGenerationProviders).toHaveBeenCalledWith({} as OpenClawConfig); + }); + + it("explains native video-generation config and provider auth when no model is configured", async () => { + mocks.listVideoGenerationProviders.mockReturnValue([ + { + id: "qwen", + defaultModel: "wan2.6-t2v", + capabilities: {}, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }], + }), + }, + ]); + mocks.getProviderEnvVars.mockReturnValue(["QWEN_API_KEY"]); + + const promise = generateVideo({ cfg: {} as OpenClawConfig, prompt: "animate a cat" }); + + await expect(promise).rejects.toThrow("No video-generation model configured."); + await expect(promise).rejects.toThrow( + 'Set agents.defaults.videoGenerationModel.primary to a provider/model like "', + ); + await expect(promise).rejects.toThrow("qwen: QWEN_API_KEY"); + }); +}); diff --git a/extensions/video-generation-core/src/runtime.ts b/extensions/video-generation-core/src/runtime.ts new file mode 100644 index 00000000000..573eb14ea9c --- /dev/null +++ b/extensions/video-generation-core/src/runtime.ts @@ -0,0 +1,189 @@ +import { + createSubsystemLogger, + describeFailoverError, + getProviderEnvVars, + getVideoGenerationProvider, + isFailoverError, + listVideoGenerationProviders, + parseVideoGenerationModelRef, + resolveAgentModelFallbackValues, + resolveAgentModelPrimaryValue, + type AuthProfileStore, + type FallbackAttempt, + type GeneratedVideoAsset, + type OpenClawConfig, + type VideoGenerationResolution, + type VideoGenerationResult, + type VideoGenerationSourceAsset, +} from "../api.js"; + +const log = createSubsystemLogger("video-generation"); + +export type GenerateVideoParams = { + cfg: OpenClawConfig; + prompt: string; + agentDir?: string; + authStore?: AuthProfileStore; + modelOverride?: string; + size?: string; + aspectRatio?: string; + resolution?: VideoGenerationResolution; + durationSeconds?: number; + audio?: boolean; + watermark?: boolean; + inputImages?: VideoGenerationSourceAsset[]; + inputVideos?: VideoGenerationSourceAsset[]; +}; + +export type GenerateVideoRuntimeResult = { + videos: GeneratedVideoAsset[]; + provider: string; + model: string; + attempts: FallbackAttempt[]; + metadata?: Record; +}; + +function resolveVideoGenerationCandidates(params: { + cfg: OpenClawConfig; + modelOverride?: string; +}): Array<{ provider: string; model: string }> { + const candidates: Array<{ provider: string; model: string }> = []; + const seen = new Set(); + const add = (raw: string | undefined) => { + const parsed = parseVideoGenerationModelRef(raw); + if (!parsed) { + return; + } + const key = `${parsed.provider}/${parsed.model}`; + if (seen.has(key)) { + return; + } + seen.add(key); + candidates.push(parsed); + }; + + add(params.modelOverride); + add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.videoGenerationModel)); + for (const fallback of resolveAgentModelFallbackValues( + params.cfg.agents?.defaults?.videoGenerationModel, + )) { + add(fallback); + } + return candidates; +} + +function throwVideoGenerationFailure(params: { + attempts: FallbackAttempt[]; + lastError: unknown; +}): never { + if (params.attempts.length <= 1 && params.lastError) { + throw params.lastError; + } + const summary = + params.attempts.length > 0 + ? params.attempts + .map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`) + .join(" | ") + : "unknown"; + throw new Error(`All video generation models failed (${params.attempts.length}): ${summary}`, { + cause: params.lastError instanceof Error ? params.lastError : undefined, + }); +} + +function buildNoVideoGenerationModelConfiguredMessage(cfg: OpenClawConfig): string { + const providers = listVideoGenerationProviders(cfg); + const sampleModel = + providers.find((provider) => provider.defaultModel) ?? + ({ id: "qwen", defaultModel: "wan2.6-t2v" } as const); + const authHints = providers + .flatMap((provider) => { + const envVars = getProviderEnvVars(provider.id); + if (envVars.length === 0) { + return []; + } + return [`${provider.id}: ${envVars.join(" / ")}`]; + }) + .slice(0, 3); + return [ + `No video-generation model configured. Set agents.defaults.videoGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`, + authHints.length > 0 + ? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).` + : "If you want a specific provider, also configure that provider's auth/API key first.", + ].join(" "); +} + +export function listRuntimeVideoGenerationProviders(params?: { config?: OpenClawConfig }) { + return listVideoGenerationProviders(params?.config); +} + +export async function generateVideo( + params: GenerateVideoParams, +): Promise { + const candidates = resolveVideoGenerationCandidates({ + cfg: params.cfg, + modelOverride: params.modelOverride, + }); + if (candidates.length === 0) { + throw new Error(buildNoVideoGenerationModelConfiguredMessage(params.cfg)); + } + + const attempts: FallbackAttempt[] = []; + let lastError: unknown; + + for (const candidate of candidates) { + const provider = getVideoGenerationProvider(candidate.provider, params.cfg); + if (!provider) { + const error = `No video-generation provider registered for ${candidate.provider}`; + attempts.push({ + provider: candidate.provider, + model: candidate.model, + error, + }); + lastError = new Error(error); + continue; + } + + try { + const result: VideoGenerationResult = await provider.generateVideo({ + provider: candidate.provider, + model: candidate.model, + prompt: params.prompt, + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + size: params.size, + aspectRatio: params.aspectRatio, + resolution: params.resolution, + durationSeconds: params.durationSeconds, + audio: params.audio, + watermark: params.watermark, + inputImages: params.inputImages, + inputVideos: params.inputVideos, + }); + if (!Array.isArray(result.videos) || result.videos.length === 0) { + throw new Error("Video generation provider returned no videos."); + } + return { + videos: result.videos, + provider: candidate.provider, + model: result.model ?? candidate.model, + attempts, + metadata: result.metadata, + }; + } catch (err) { + lastError = err; + const described = isFailoverError(err) ? describeFailoverError(err) : undefined; + attempts.push({ + provider: candidate.provider, + model: candidate.model, + error: described?.message ?? (err instanceof Error ? err.message : String(err)), + reason: described?.reason, + status: described?.status, + code: described?.code, + }); + log.debug(`video-generation candidate failed: ${candidate.provider}/${candidate.model}`); + } + } + + throwVideoGenerationFailure({ attempts, lastError }); +} diff --git a/extensions/vllm/index.ts b/extensions/vllm/index.ts index d944d09fa3d..dcc0f4e2dbf 100644 --- a/extensions/vllm/index.ts +++ b/extensions/vllm/index.ts @@ -3,6 +3,13 @@ import { type OpenClawPluginApi, type ProviderAuthMethodNonInteractiveContext, } from "openclaw/plugin-sdk/plugin-entry"; +import { + buildVllmProvider, + VLLM_DEFAULT_API_KEY_ENV_VAR, + VLLM_DEFAULT_BASE_URL, + VLLM_MODEL_PLACEHOLDER, + VLLM_PROVIDER_LABEL, +} from "./api.js"; const PROVIDER_ID = "vllm"; @@ -14,15 +21,7 @@ export default definePluginEntry({ id: "vllm", name: "vLLM Provider", description: "Bundled vLLM provider plugin", - async register(api: OpenClawPluginApi) { - const { - buildVllmProvider, - VLLM_DEFAULT_API_KEY_ENV_VAR, - VLLM_DEFAULT_BASE_URL, - VLLM_MODEL_PLACEHOLDER, - VLLM_PROVIDER_LABEL, - } = await import("./register.runtime.js"); - + register(api: OpenClawPluginApi) { api.registerProvider({ id: PROVIDER_ID, label: "vLLM", diff --git a/package.json b/package.json index c3269968b82..41d323648f8 100644 --- a/package.json +++ b/package.json @@ -567,6 +567,14 @@ "types": "./dist/plugin-sdk/image-generation-core.d.ts", "default": "./dist/plugin-sdk/image-generation-core.js" }, + "./plugin-sdk/video-generation": { + "types": "./dist/plugin-sdk/video-generation.d.ts", + "default": "./dist/plugin-sdk/video-generation.js" + }, + "./plugin-sdk/video-generation-core": { + "types": "./dist/plugin-sdk/video-generation-core.d.ts", + "default": "./dist/plugin-sdk/video-generation-core.js" + }, "./plugin-sdk/irc": { "types": "./dist/plugin-sdk/irc.d.ts", "default": "./dist/plugin-sdk/irc.js" @@ -727,6 +735,14 @@ "types": "./dist/plugin-sdk/minimax.d.ts", "default": "./dist/plugin-sdk/minimax.js" }, + "./plugin-sdk/qwen": { + "types": "./dist/plugin-sdk/qwen.d.ts", + "default": "./dist/plugin-sdk/qwen.js" + }, + "./plugin-sdk/qwen-definitions": { + "types": "./dist/plugin-sdk/qwen-definitions.d.ts", + "default": "./dist/plugin-sdk/qwen-definitions.js" + }, "./plugin-sdk/modelstudio": { "types": "./dist/plugin-sdk/modelstudio.d.ts", "default": "./dist/plugin-sdk/modelstudio.js" diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index d83eacc6dff..12dcf8b0257 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -130,6 +130,8 @@ "googlechat", "image-generation", "image-generation-core", + "video-generation", + "video-generation-core", "irc", "irc-surface", "kimi-coding", @@ -171,6 +173,8 @@ "memory-core-host-runtime-files", "memory-lancedb", "minimax", + "qwen", + "qwen-definitions", "modelstudio", "modelstudio-definitions", "moonshot", diff --git a/scripts/lib/plugin-sdk-facades.mjs b/scripts/lib/plugin-sdk-facades.mjs index e6df0da2bf7..fc4cfa8f813 100644 --- a/scripts/lib/plugin-sdk-facades.mjs +++ b/scripts/lib/plugin-sdk-facades.mjs @@ -315,6 +315,18 @@ export const GENERATED_PLUGIN_SDK_FACADES = [ ], typeExports: ["GenerateImageParams", "GenerateImageRuntimeResult"], }, + { + subpath: "video-generation-runtime", + source: pluginSource("video-generation-core", "runtime-api.js"), + loadPolicy: "activated", + exports: [ + "generateVideo", + "listRuntimeVideoGenerationProviders", + "GenerateVideoParams", + "GenerateVideoRuntimeResult", + ], + typeExports: ["GenerateVideoParams", "GenerateVideoRuntimeResult"], + }, { subpath: "kimi-coding", source: pluginSource("kimi-coding", "api.js"), @@ -370,13 +382,19 @@ export const GENERATED_PLUGIN_SDK_FACADES = [ source: pluginSource("memory-core", "runtime-api.js"), loadPolicy: "activated", exports: [ + "auditShortTermPromotionArtifacts", "BuiltinMemoryEmbeddingProviderDoctorMetadata", "getBuiltinMemoryEmbeddingProviderDoctorMetadata", "getMemorySearchManager", "listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata", "MemoryIndexManager", + "repairShortTermPromotionArtifacts", + ], + typeExports: [ + "BuiltinMemoryEmbeddingProviderDoctorMetadata", + "RepairShortTermPromotionArtifactsResult", + "ShortTermAuditSummary", ], - typeExports: ["BuiltinMemoryEmbeddingProviderDoctorMetadata"], }, { subpath: "mattermost-policy", @@ -532,9 +550,44 @@ export const GENERATED_PLUGIN_SDK_FACADES = [ "MINIMAX_TEXT_MODEL_REFS", ], }, + { + subpath: "qwen", + source: pluginSource("qwen", "api.js"), + exports: [ + "applyQwenNativeStreamingUsageCompat", + "buildQwenDefaultModelDefinition", + "buildQwenModelDefinition", + "QWEN_BASE_URL", + "QWEN_CN_BASE_URL", + "QWEN_DEFAULT_COST", + "QWEN_DEFAULT_MODEL_ID", + "QWEN_DEFAULT_MODEL_REF", + "QWEN_GLOBAL_BASE_URL", + "QWEN_STANDARD_CN_BASE_URL", + "QWEN_STANDARD_GLOBAL_BASE_URL", + "QWEN_MODEL_CATALOG", + "isNativeQwenBaseUrl", + "buildQwenProvider", + ], + }, + { + subpath: "qwen-definitions", + source: pluginSource("qwen", "api.js"), + exports: [ + "buildQwenDefaultModelDefinition", + "buildQwenModelDefinition", + "QWEN_CN_BASE_URL", + "QWEN_DEFAULT_COST", + "QWEN_DEFAULT_MODEL_ID", + "QWEN_DEFAULT_MODEL_REF", + "QWEN_GLOBAL_BASE_URL", + "QWEN_STANDARD_CN_BASE_URL", + "QWEN_STANDARD_GLOBAL_BASE_URL", + ], + }, { subpath: "modelstudio", - source: pluginSource("modelstudio", "api.js"), + source: pluginSource("qwen", "api.js"), exports: [ "applyModelStudioNativeStreamingUsageCompat", "buildModelStudioDefaultModelDefinition", @@ -554,7 +607,7 @@ export const GENERATED_PLUGIN_SDK_FACADES = [ }, { subpath: "modelstudio-definitions", - source: pluginSource("modelstudio", "api.js"), + source: pluginSource("qwen", "api.js"), exports: [ "buildModelStudioDefaultModelDefinition", "buildModelStudioModelDefinition", diff --git a/src/agents/auth-profiles.doctor.test.ts b/src/agents/auth-profiles.doctor.test.ts index debf2e31f6a..4b6d2dd6fd9 100644 --- a/src/agents/auth-profiles.doctor.test.ts +++ b/src/agents/auth-profiles.doctor.test.ts @@ -14,8 +14,8 @@ describe("formatAuthDoctorHint", () => { provider: "qwen-portal", }); - expect(hint).toContain("openclaw onboard --auth-choice modelstudio-api-key"); - expect(hint).toContain("modelstudio-api-key-cn"); - expect(hint).not.toContain("--provider modelstudio"); + expect(hint).toContain("openclaw onboard --auth-choice qwen-api-key"); + expect(hint).toContain("qwen-api-key-cn"); + expect(hint).not.toContain("--provider qwen"); }); }); diff --git a/src/agents/model-auth.profiles.test.ts b/src/agents/model-auth.profiles.test.ts index 4406893e070..b7db25f7d7d 100644 --- a/src/agents/model-auth.profiles.test.ts +++ b/src/agents/model-auth.profiles.test.ts @@ -13,7 +13,15 @@ import { } from "./model-auth.js"; vi.mock("../plugins/provider-runtime.js", () => ({ - buildProviderMissingAuthMessageWithPlugin: () => undefined, + buildProviderMissingAuthMessageWithPlugin: (params: { + provider: string; + context: { listProfileIds: (providerId: string) => string[] }; + }) => { + if (params.provider === "openai" && params.context.listProfileIds("openai-codex").length > 0) { + return 'No API key found for provider "openai". Use openai-codex/gpt-5.4.'; + } + return undefined; + }, formatProviderAuthProfileApiKeyWithPlugin: async () => undefined, refreshProviderOAuthCredentialWithPlugin: async () => null, resolveProviderSyntheticAuthWithPlugin: (params: { @@ -311,13 +319,13 @@ describe("getApiKeyForModel", () => { }); }); - it("resolves Model Studio API key from env", async () => { + it("resolves Qwen API key from env", async () => { await withEnvAsync( { [envVar("MODELSTUDIO", "API", "KEY")]: "modelstudio-test-key" }, async () => { // pragma: allowlist secret const resolved = await resolveApiKeyForProvider({ - provider: "modelstudio", + provider: "qwen", store: { version: 1, profiles: {} }, }); expect(resolved.apiKey).toBe("modelstudio-test-key"); diff --git a/src/agents/model-compat.test.ts b/src/agents/model-compat.test.ts index 1f9370f71d6..c2d573da5ee 100644 --- a/src/agents/model-compat.test.ts +++ b/src/agents/model-compat.test.ts @@ -157,10 +157,10 @@ describe("normalizeModelCompat", () => { }); }); - it("keeps supportsUsageInStreaming on for native ModelStudio endpoints", () => { + it("keeps supportsUsageInStreaming on for native Qwen endpoints", () => { const model = { ...baseModel(), - provider: "modelstudio", + provider: "qwen", baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", }; delete (model as { compat?: unknown }).compat; diff --git a/src/agents/model-selection.test.ts b/src/agents/model-selection.test.ts index 74fb6dcc1da..f012c0655be 100644 --- a/src/agents/model-selection.test.ts +++ b/src/agents/model-selection.test.ts @@ -419,7 +419,7 @@ describe("model-selection", () => { "qwen-dashscope": { models: [{ id: "qwen-max" }], }, - modelstudio: { + qwen: { models: [{ id: "qwen-max" }], }, }, diff --git a/src/agents/openai-transport-stream.test.ts b/src/agents/openai-transport-stream.test.ts index 1dabd1bdf0f..bee07b42a28 100644 --- a/src/agents/openai-transport-stream.test.ts +++ b/src/agents/openai-transport-stream.test.ts @@ -834,13 +834,13 @@ describe("openai transport stream", () => { expect(params.messages?.[0]?.content).toBe("Stable prefix\nDynamic suffix"); }); - it("uses system role and streaming usage compat for native ModelStudio completions providers", () => { + it("uses system role and streaming usage compat for native Qwen completions providers", () => { const params = buildOpenAICompletionsParams( { id: "qwen3.6-plus", name: "Qwen 3.6 Plus", api: "openai-completions", - provider: "modelstudio", + provider: "qwen", baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", reasoning: true, input: ["text"], diff --git a/src/agents/provider-attribution.test.ts b/src/agents/provider-attribution.test.ts index 4d36cd4103f..641ac98f8f5 100644 --- a/src/agents/provider-attribution.test.ts +++ b/src/agents/provider-attribution.test.ts @@ -561,7 +561,7 @@ describe("provider attribution", () => { expect( resolveProviderRequestCapabilities({ - provider: "modelstudio", + provider: "qwen", api: "openai-completions", baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", capability: "llm", @@ -745,9 +745,9 @@ describe("provider attribution", () => { }, }, { - name: "native ModelStudio completions", + name: "native Qwen completions", input: { - provider: "modelstudio", + provider: "qwen", api: "openai-completions", baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", capability: "llm" as const, diff --git a/src/agents/provider-attribution.ts b/src/agents/provider-attribution.ts index 3b80881218a..9739a510eab 100644 --- a/src/agents/provider-attribution.ts +++ b/src/agents/provider-attribution.ts @@ -295,6 +295,8 @@ function resolveKnownProviderFamily(provider: string | undefined): string { case "moonshot": case "kimi": return "moonshot"; + case "qwen": + case "qwencloud": case "modelstudio": case "dashscope": return "modelstudio"; diff --git a/src/agents/provider-id.ts b/src/agents/provider-id.ts index b774785fc45..86c5f32387c 100644 --- a/src/agents/provider-id.ts +++ b/src/agents/provider-id.ts @@ -1,5 +1,8 @@ export function normalizeProviderId(provider: string): string { const normalized = provider.trim().toLowerCase(); + if (normalized === "modelstudio" || normalized === "qwencloud") { + return "qwen"; + } if (normalized === "z.ai" || normalized === "z-ai") { return "zai"; } diff --git a/src/commands/onboard-non-interactive.provider-auth.test.ts b/src/commands/onboard-non-interactive.provider-auth.test.ts index fbf137a1da9..a364983c573 100644 --- a/src/commands/onboard-non-interactive.provider-auth.test.ts +++ b/src/commands/onboard-non-interactive.provider-auth.test.ts @@ -537,17 +537,17 @@ vi.mock("./onboard-non-interactive/local/auth-choice.plugin-providers.js", async }), ], [ - "modelstudio-api-key", + "qwen-api-key", createApiKeyChoice({ - providerId: "modelstudio", - label: "Model Studio", - choiceId: "modelstudio-api-key", + providerId: "qwen", + label: "Qwen Cloud", + choiceId: "qwen-api-key", optionKey: "modelstudioApiKey", flagName: "--modelstudio-api-key", - envVar: "MODELSTUDIO_API_KEY", - defaultModel: "modelstudio/qwen3.5-plus", + envVar: "QWEN_API_KEY", + defaultModel: "qwen/qwen3.5-plus", applyConfig: (cfg) => - withProviderConfig(cfg, "modelstudio", { + withProviderConfig(cfg, "qwen", { baseUrl: "https://coding-intl.dashscope.aliyuncs.com/v1", api: "openai-completions", models: [buildTestProviderModel("qwen3.5-plus")], @@ -1071,18 +1071,9 @@ describe("onboard (non-interactive): provider auth", () => { token, tokenProfileId: "anthropic:default", }), - ).rejects.toThrow("Process exited with code 1"); + ).rejects.toThrow('Auth choice "token" is no longer supported for Anthropic onboarding.'); - expect(runtime.error).toHaveBeenCalledWith( - [ - 'Auth choice "token" is no longer supported for Anthropic onboarding.', - "Existing Anthropic token profiles still run if they are already configured.", - 'Use "--auth-choice anthropic-cli" or "--auth-choice apiKey" instead.', - ].join("\n"), - ); - - const cfg = await readJsonFile(configPath); - expect(cfg.auth?.profiles?.["anthropic:default"]).toBeUndefined(); + await expect(fs.access(configPath)).rejects.toMatchObject({ code: "ENOENT" }); expect(ensureAuthProfileStore().profiles["anthropic:default"]).toBeUndefined(); }); }); @@ -1365,21 +1356,21 @@ describe("onboard (non-interactive): provider auth", () => { }); }); - it("infers Model Studio auth choice from --modelstudio-api-key and sets default model", async () => { + it("infers Qwen auth choice from --modelstudio-api-key and sets default model", async () => { await withOnboardEnv("openclaw-onboard-modelstudio-infer-", async (env) => { const cfg = await runOnboardingAndReadConfig(env, { modelstudioApiKey: "modelstudio-test-key", // pragma: allowlist secret }); - expect(cfg.auth?.profiles?.["modelstudio:default"]?.provider).toBe("modelstudio"); - expect(cfg.auth?.profiles?.["modelstudio:default"]?.mode).toBe("api_key"); - expect(cfg.models?.providers?.modelstudio?.baseUrl).toBe( + expect(cfg.auth?.profiles?.["qwen:default"]?.provider).toBe("qwen"); + expect(cfg.auth?.profiles?.["qwen:default"]?.mode).toBe("api_key"); + expect(cfg.models?.providers?.qwen?.baseUrl).toBe( "https://coding-intl.dashscope.aliyuncs.com/v1", ); - expect(cfg.agents?.defaults?.model?.primary).toBe("modelstudio/qwen3.5-plus"); + expect(cfg.agents?.defaults?.model?.primary).toBe("qwen/qwen3.5-plus"); await expectApiKeyProfile({ - profileId: "modelstudio:default", - provider: "modelstudio", + profileId: "qwen:default", + provider: "qwen", key: "modelstudio-test-key", }); }); diff --git a/src/commands/onboard-types.ts b/src/commands/onboard-types.ts index 9497c277e45..55aa28c9b77 100644 --- a/src/commands/onboard-types.ts +++ b/src/commands/onboard-types.ts @@ -46,6 +46,10 @@ export type BuiltInAuthChoice = | "volcengine-api-key" | "byteplus-api-key" | "qianfan-api-key" + | "qwen-standard-api-key-cn" + | "qwen-standard-api-key" + | "qwen-api-key-cn" + | "qwen-api-key" | "modelstudio-standard-api-key-cn" | "modelstudio-standard-api-key" | "modelstudio-api-key-cn" @@ -77,6 +81,7 @@ export type BuiltInAuthChoiceGroupId = | "together" | "huggingface" | "qianfan" + | "qwen" | "modelstudio" | "xai" | "volcengine" diff --git a/src/config/io.ts b/src/config/io.ts index c31ad20013b..94bde4f10e7 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -80,6 +80,7 @@ const SHELL_ENV_EXPECTED_KEYS = [ "OPENROUTER_API_KEY", "AI_GATEWAY_API_KEY", "MINIMAX_API_KEY", + "QWEN_API_KEY", "MODELSTUDIO_API_KEY", "SYNTHETIC_API_KEY", "KILOCODE_API_KEY", diff --git a/src/config/schema.base.generated.ts b/src/config/schema.base.generated.ts index ec032404da4..50f3f2f7333 100644 --- a/src/config/schema.base.generated.ts +++ b/src/config/schema.base.generated.ts @@ -2573,6 +2573,28 @@ export const GENERATED_BASE_CONFIG_SCHEMA = { }, ], }, + videoGenerationModel: { + anyOf: [ + { + type: "string", + }, + { + type: "object", + properties: { + primary: { + type: "string", + }, + fallbacks: { + type: "array", + items: { + type: "string", + }, + }, + }, + additionalProperties: false, + }, + ], + }, pdfModel: { anyOf: [ { @@ -22637,6 +22659,16 @@ export const GENERATED_BASE_CONFIG_SCHEMA = { help: "Ordered fallback image-generation models (provider/model).", tags: ["reliability", "media"], }, + "agents.defaults.videoGenerationModel.primary": { + label: "Video Generation Model", + help: "Optional video-generation model (provider/model) used by the shared video generation capability.", + tags: ["media"], + }, + "agents.defaults.videoGenerationModel.fallbacks": { + label: "Video Generation Model Fallbacks", + help: "Ordered fallback video-generation models (provider/model).", + tags: ["reliability", "media"], + }, "agents.defaults.pdfModel.primary": { label: "PDF Model", help: "Optional PDF model (provider/model) for the PDF analysis tool. Defaults to imageModel, then session model.", diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 93d6412a33a..38090eb3f6f 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -1098,6 +1098,10 @@ export const FIELD_HELP: Record = { "Optional image-generation model (provider/model) used by the shared image generation capability.", "agents.defaults.imageGenerationModel.fallbacks": "Ordered fallback image-generation models (provider/model).", + "agents.defaults.videoGenerationModel.primary": + "Optional video-generation model (provider/model) used by the shared video generation capability.", + "agents.defaults.videoGenerationModel.fallbacks": + "Ordered fallback video-generation models (provider/model).", "agents.defaults.pdfModel.primary": "Optional PDF model (provider/model) for the PDF analysis tool. Defaults to imageModel, then session model.", "agents.defaults.pdfModel.fallbacks": "Ordered fallback PDF models (provider/model).", diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 1547df4bff8..1c527252af1 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -499,6 +499,8 @@ export const FIELD_LABELS: Record = { "agents.defaults.imageModel.fallbacks": "Image Model Fallbacks", "agents.defaults.imageGenerationModel.primary": "Image Generation Model", "agents.defaults.imageGenerationModel.fallbacks": "Image Generation Model Fallbacks", + "agents.defaults.videoGenerationModel.primary": "Video Generation Model", + "agents.defaults.videoGenerationModel.fallbacks": "Video Generation Model Fallbacks", "agents.defaults.pdfModel.primary": "PDF Model", "agents.defaults.pdfModel.fallbacks": "PDF Model Fallbacks", "agents.defaults.pdfMaxBytesMb": "PDF Max Size (MB)", diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index cfed6dbb9f1..2c907324978 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -126,6 +126,8 @@ export type AgentDefaultsConfig = { imageModel?: AgentModelConfig; /** Optional image-generation model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */ imageGenerationModel?: AgentModelConfig; + /** Optional video-generation model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */ + videoGenerationModel?: AgentModelConfig; /** Optional PDF-capable model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */ pdfModel?: AgentModelConfig; /** Maximum PDF file size in megabytes (default: 10). */ diff --git a/src/config/zod-schema.agent-defaults.ts b/src/config/zod-schema.agent-defaults.ts index d0702615a1f..69a3481b44f 100644 --- a/src/config/zod-schema.agent-defaults.ts +++ b/src/config/zod-schema.agent-defaults.ts @@ -21,6 +21,7 @@ export const AgentDefaultsSchema = z model: AgentModelSchema.optional(), imageModel: AgentModelSchema.optional(), imageGenerationModel: AgentModelSchema.optional(), + videoGenerationModel: AgentModelSchema.optional(), pdfModel: AgentModelSchema.optional(), pdfMaxBytesMb: z.number().positive().optional(), pdfMaxPages: z.number().int().positive().optional(), diff --git a/src/gateway/server-plugins.test.ts b/src/gateway/server-plugins.test.ts index 0687a11983d..a42499cb839 100644 --- a/src/gateway/server-plugins.test.ts +++ b/src/gateway/server-plugins.test.ts @@ -73,6 +73,7 @@ const createRegistry = (diagnostics: PluginDiagnostic[]): PluginRegistry => ({ realtimeVoiceProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], gatewayHandlers: {}, diff --git a/src/gateway/session-utils.test.ts b/src/gateway/session-utils.test.ts index ad5cea52636..ab88f735cef 100644 --- a/src/gateway/session-utils.test.ts +++ b/src/gateway/session-utils.test.ts @@ -771,7 +771,7 @@ describe("resolveSessionModelIdentityRef", () => { "qwen-dashscope": { models: [{ id: "qwen-max" }], }, - modelstudio: { + qwen: { models: [{ id: "qwen-max" }], }, }, diff --git a/src/gateway/test-helpers.mocks.ts b/src/gateway/test-helpers.mocks.ts index 1b26cf62f23..b55efa9ba2b 100644 --- a/src/gateway/test-helpers.mocks.ts +++ b/src/gateway/test-helpers.mocks.ts @@ -205,6 +205,7 @@ const createStubPluginRegistry = (): PluginRegistry => ({ realtimeVoiceProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], gatewayHandlers: {}, diff --git a/src/generated/plugin-sdk-facade-type-map.generated.ts b/src/generated/plugin-sdk-facade-type-map.generated.ts index 6e03da1dd3a..9902fea0a39 100644 --- a/src/generated/plugin-sdk-facade-type-map.generated.ts +++ b/src/generated/plugin-sdk-facade-type-map.generated.ts @@ -166,6 +166,18 @@ export interface PluginSdkFacadeTypeMap { GenerateImageRuntimeResult: import("@openclaw/image-generation-core/runtime-api.js").GenerateImageRuntimeResult; }; }; + "video-generation-runtime": { + module: typeof import("@openclaw/video-generation-core/runtime-api.js"); + sourceModules: { + source1: { + module: typeof import("@openclaw/video-generation-core/runtime-api.js"); + }; + }; + types: { + GenerateVideoParams: import("@openclaw/video-generation-core/runtime-api.js").GenerateVideoParams; + GenerateVideoRuntimeResult: import("@openclaw/video-generation-core/runtime-api.js").GenerateVideoRuntimeResult; + }; + }; "kimi-coding": { module: typeof import("@openclaw/kimi-coding/api.js"); sourceModules: { @@ -337,20 +349,38 @@ export interface PluginSdkFacadeTypeMap { }; types: {}; }; - modelstudio: { - module: typeof import("@openclaw/modelstudio/api.js"); + qwen: { + module: typeof import("@openclaw/qwen/api.js"); sourceModules: { source1: { - module: typeof import("@openclaw/modelstudio/api.js"); + module: typeof import("@openclaw/qwen/api.js"); + }; + }; + types: {}; + }; + "qwen-definitions": { + module: typeof import("@openclaw/qwen/api.js"); + sourceModules: { + source1: { + module: typeof import("@openclaw/qwen/api.js"); + }; + }; + types: {}; + }; + modelstudio: { + module: typeof import("@openclaw/qwen/api.js"); + sourceModules: { + source1: { + module: typeof import("@openclaw/qwen/api.js"); }; }; types: {}; }; "modelstudio-definitions": { - module: typeof import("@openclaw/modelstudio/api.js"); + module: typeof import("@openclaw/qwen/api.js"); sourceModules: { source1: { - module: typeof import("@openclaw/modelstudio/api.js"); + module: typeof import("@openclaw/qwen/api.js"); }; }; types: {}; diff --git a/src/media-understanding/defaults.test.ts b/src/media-understanding/defaults.test.ts index 03da17225e5..407b1a0481f 100644 --- a/src/media-understanding/defaults.test.ts +++ b/src/media-understanding/defaults.test.ts @@ -51,7 +51,11 @@ describe("resolveAutoMediaKeyProviders", () => { }); it("keeps the bundled video fallback order", () => { - expect(resolveAutoMediaKeyProviders({ capability: "video" })).toEqual(["google", "moonshot"]); + expect(resolveAutoMediaKeyProviders({ capability: "video" })).toEqual([ + "google", + "qwen", + "moonshot", + ]); }); }); diff --git a/src/media-understanding/provider-registry.test.ts b/src/media-understanding/provider-registry.test.ts index bdaf58a1070..1abfa366545 100644 --- a/src/media-understanding/provider-registry.test.ts +++ b/src/media-understanding/provider-registry.test.ts @@ -11,10 +11,10 @@ describe("media-understanding provider registry", () => { setActivePluginRegistry(createEmptyPluginRegistry()); }); - it("returns no providers by default when no active registry is present", () => { + it("loads bundled providers by default when no active registry is present", () => { const registry = buildMediaUnderstandingRegistry(); - expect(getMediaUnderstandingProvider("groq", registry)).toBeUndefined(); - expect(getMediaUnderstandingProvider("deepgram", registry)).toBeUndefined(); + expect(getMediaUnderstandingProvider("groq", registry)?.id).toBe("groq"); + expect(getMediaUnderstandingProvider("deepgram", registry)?.id).toBe("deepgram"); }); it("merges plugin-registered media providers into the active registry", async () => { diff --git a/src/plugin-sdk/line-runtime.ts b/src/plugin-sdk/line-runtime.ts index 4d2b1c7d8cc..f5abf271981 100644 --- a/src/plugin-sdk/line-runtime.ts +++ b/src/plugin-sdk/line-runtime.ts @@ -78,6 +78,8 @@ export const getRichMenuIdOfUser: FacadeModule["getRichMenuIdOfUser"] = ((...arg loadFacadeModule()["getRichMenuIdOfUser"](...args)) as FacadeModule["getRichMenuIdOfUser"]; export const getRichMenuList: FacadeModule["getRichMenuList"] = ((...args) => loadFacadeModule()["getRichMenuList"](...args)) as FacadeModule["getRichMenuList"]; +export const hasLineDirectives: FacadeModule["hasLineDirectives"] = ((...args) => + loadFacadeModule()["hasLineDirectives"](...args)) as FacadeModule["hasLineDirectives"]; export const isSenderAllowed: FacadeModule["isSenderAllowed"] = ((...args) => loadFacadeModule()["isSenderAllowed"](...args)) as FacadeModule["isSenderAllowed"]; export const linkRichMenuToUser: FacadeModule["linkRichMenuToUser"] = ((...args) => @@ -96,6 +98,8 @@ export const normalizeDmAllowFromWithStore: FacadeModule["normalizeDmAllowFromWi loadFacadeModule()["normalizeDmAllowFromWithStore"]( ...args, )) as FacadeModule["normalizeDmAllowFromWithStore"]; +export const parseLineDirectives: FacadeModule["parseLineDirectives"] = ((...args) => + loadFacadeModule()["parseLineDirectives"](...args)) as FacadeModule["parseLineDirectives"]; export const postbackAction: FacadeModule["postbackAction"] = ((...args) => loadFacadeModule()["postbackAction"](...args)) as FacadeModule["postbackAction"]; export const probeLineBot: FacadeModule["probeLineBot"] = ((...args) => diff --git a/src/plugin-sdk/memory-core-engine-runtime.ts b/src/plugin-sdk/memory-core-engine-runtime.ts index 347b4fc856b..259c227f4dd 100644 --- a/src/plugin-sdk/memory-core-engine-runtime.ts +++ b/src/plugin-sdk/memory-core-engine-runtime.ts @@ -13,17 +13,17 @@ function loadFacadeModule(): FacadeModule { artifactBasename: "runtime-api.js", }); } -export const getBuiltinMemoryEmbeddingProviderDoctorMetadata: FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"] = - ((...args) => - loadFacadeModule()["getBuiltinMemoryEmbeddingProviderDoctorMetadata"]( - ...args, - )) as FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"]; export const auditShortTermPromotionArtifacts: FacadeModule["auditShortTermPromotionArtifacts"] = (( ...args ) => loadFacadeModule()["auditShortTermPromotionArtifacts"]( ...args, )) as FacadeModule["auditShortTermPromotionArtifacts"]; +export const getBuiltinMemoryEmbeddingProviderDoctorMetadata: FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"] = + ((...args) => + loadFacadeModule()["getBuiltinMemoryEmbeddingProviderDoctorMetadata"]( + ...args, + )) as FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"]; export const getMemorySearchManager: FacadeModule["getMemorySearchManager"] = ((...args) => loadFacadeModule()["getMemorySearchManager"](...args)) as FacadeModule["getMemorySearchManager"]; export const listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata: FacadeModule["listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata"] = diff --git a/src/plugin-sdk/modelstudio-definitions.ts b/src/plugin-sdk/modelstudio-definitions.ts index 52955b235d0..698b9c62550 100644 --- a/src/plugin-sdk/modelstudio-definitions.ts +++ b/src/plugin-sdk/modelstudio-definitions.ts @@ -9,7 +9,7 @@ import { function loadFacadeModule(): FacadeModule { return loadBundledPluginPublicSurfaceModuleSync({ - dirName: "modelstudio", + dirName: "qwen", artifactBasename: "api.js", }); } diff --git a/src/plugin-sdk/modelstudio.ts b/src/plugin-sdk/modelstudio.ts index 0aeaa4c8580..53279b3abd7 100644 --- a/src/plugin-sdk/modelstudio.ts +++ b/src/plugin-sdk/modelstudio.ts @@ -10,7 +10,7 @@ import { function loadFacadeModule(): FacadeModule { return loadBundledPluginPublicSurfaceModuleSync({ - dirName: "modelstudio", + dirName: "qwen", artifactBasename: "api.js", }); } diff --git a/src/plugin-sdk/qwen-definitions.ts b/src/plugin-sdk/qwen-definitions.ts new file mode 100644 index 00000000000..1918fa02c33 --- /dev/null +++ b/src/plugin-sdk/qwen-definitions.ts @@ -0,0 +1,40 @@ +// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually. +import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js"; +type FacadeEntry = PluginSdkFacadeTypeMap["qwen-definitions"]; +type FacadeModule = FacadeEntry["module"]; +import { + createLazyFacadeObjectValue, + loadBundledPluginPublicSurfaceModuleSync, +} from "./facade-runtime.js"; + +function loadFacadeModule(): FacadeModule { + return loadBundledPluginPublicSurfaceModuleSync({ + dirName: "qwen", + artifactBasename: "api.js", + }); +} +export const buildQwenDefaultModelDefinition: FacadeModule["buildQwenDefaultModelDefinition"] = (( + ...args +) => + loadFacadeModule()["buildQwenDefaultModelDefinition"]( + ...args, + )) as FacadeModule["buildQwenDefaultModelDefinition"]; +export const buildQwenModelDefinition: FacadeModule["buildQwenModelDefinition"] = ((...args) => + loadFacadeModule()["buildQwenModelDefinition"]( + ...args, + )) as FacadeModule["buildQwenModelDefinition"]; +export const QWEN_CN_BASE_URL: FacadeModule["QWEN_CN_BASE_URL"] = + loadFacadeModule()["QWEN_CN_BASE_URL"]; +export const QWEN_DEFAULT_COST: FacadeModule["QWEN_DEFAULT_COST"] = createLazyFacadeObjectValue( + () => loadFacadeModule()["QWEN_DEFAULT_COST"] as object, +) as FacadeModule["QWEN_DEFAULT_COST"]; +export const QWEN_DEFAULT_MODEL_ID: FacadeModule["QWEN_DEFAULT_MODEL_ID"] = + loadFacadeModule()["QWEN_DEFAULT_MODEL_ID"]; +export const QWEN_DEFAULT_MODEL_REF: FacadeModule["QWEN_DEFAULT_MODEL_REF"] = + loadFacadeModule()["QWEN_DEFAULT_MODEL_REF"]; +export const QWEN_GLOBAL_BASE_URL: FacadeModule["QWEN_GLOBAL_BASE_URL"] = + loadFacadeModule()["QWEN_GLOBAL_BASE_URL"]; +export const QWEN_STANDARD_CN_BASE_URL: FacadeModule["QWEN_STANDARD_CN_BASE_URL"] = + loadFacadeModule()["QWEN_STANDARD_CN_BASE_URL"]; +export const QWEN_STANDARD_GLOBAL_BASE_URL: FacadeModule["QWEN_STANDARD_GLOBAL_BASE_URL"] = + loadFacadeModule()["QWEN_STANDARD_GLOBAL_BASE_URL"]; diff --git a/src/plugin-sdk/qwen.ts b/src/plugin-sdk/qwen.ts new file mode 100644 index 00000000000..6ad0071a9f0 --- /dev/null +++ b/src/plugin-sdk/qwen.ts @@ -0,0 +1,54 @@ +// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually. +import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js"; +type FacadeEntry = PluginSdkFacadeTypeMap["qwen"]; +type FacadeModule = FacadeEntry["module"]; +import { + createLazyFacadeArrayValue, + createLazyFacadeObjectValue, + loadBundledPluginPublicSurfaceModuleSync, +} from "./facade-runtime.js"; + +function loadFacadeModule(): FacadeModule { + return loadBundledPluginPublicSurfaceModuleSync({ + dirName: "qwen", + artifactBasename: "api.js", + }); +} +export const applyQwenNativeStreamingUsageCompat: FacadeModule["applyQwenNativeStreamingUsageCompat"] = + ((...args) => + loadFacadeModule()["applyQwenNativeStreamingUsageCompat"]( + ...args, + )) as FacadeModule["applyQwenNativeStreamingUsageCompat"]; +export const buildQwenDefaultModelDefinition: FacadeModule["buildQwenDefaultModelDefinition"] = (( + ...args +) => + loadFacadeModule()["buildQwenDefaultModelDefinition"]( + ...args, + )) as FacadeModule["buildQwenDefaultModelDefinition"]; +export const buildQwenModelDefinition: FacadeModule["buildQwenModelDefinition"] = ((...args) => + loadFacadeModule()["buildQwenModelDefinition"]( + ...args, + )) as FacadeModule["buildQwenModelDefinition"]; +export const QWEN_BASE_URL: FacadeModule["QWEN_BASE_URL"] = loadFacadeModule()["QWEN_BASE_URL"]; +export const QWEN_CN_BASE_URL: FacadeModule["QWEN_CN_BASE_URL"] = + loadFacadeModule()["QWEN_CN_BASE_URL"]; +export const QWEN_DEFAULT_COST: FacadeModule["QWEN_DEFAULT_COST"] = createLazyFacadeObjectValue( + () => loadFacadeModule()["QWEN_DEFAULT_COST"] as object, +) as FacadeModule["QWEN_DEFAULT_COST"]; +export const QWEN_DEFAULT_MODEL_ID: FacadeModule["QWEN_DEFAULT_MODEL_ID"] = + loadFacadeModule()["QWEN_DEFAULT_MODEL_ID"]; +export const QWEN_DEFAULT_MODEL_REF: FacadeModule["QWEN_DEFAULT_MODEL_REF"] = + loadFacadeModule()["QWEN_DEFAULT_MODEL_REF"]; +export const QWEN_GLOBAL_BASE_URL: FacadeModule["QWEN_GLOBAL_BASE_URL"] = + loadFacadeModule()["QWEN_GLOBAL_BASE_URL"]; +export const QWEN_STANDARD_CN_BASE_URL: FacadeModule["QWEN_STANDARD_CN_BASE_URL"] = + loadFacadeModule()["QWEN_STANDARD_CN_BASE_URL"]; +export const QWEN_STANDARD_GLOBAL_BASE_URL: FacadeModule["QWEN_STANDARD_GLOBAL_BASE_URL"] = + loadFacadeModule()["QWEN_STANDARD_GLOBAL_BASE_URL"]; +export const QWEN_MODEL_CATALOG: FacadeModule["QWEN_MODEL_CATALOG"] = createLazyFacadeArrayValue( + () => loadFacadeModule()["QWEN_MODEL_CATALOG"] as unknown as readonly unknown[], +) as FacadeModule["QWEN_MODEL_CATALOG"]; +export const isNativeQwenBaseUrl: FacadeModule["isNativeQwenBaseUrl"] = ((...args) => + loadFacadeModule()["isNativeQwenBaseUrl"](...args)) as FacadeModule["isNativeQwenBaseUrl"]; +export const buildQwenProvider: FacadeModule["buildQwenProvider"] = ((...args) => + loadFacadeModule()["buildQwenProvider"](...args)) as FacadeModule["buildQwenProvider"]; diff --git a/src/plugin-sdk/video-generation-core.ts b/src/plugin-sdk/video-generation-core.ts new file mode 100644 index 00000000000..cb735cc7ceb --- /dev/null +++ b/src/plugin-sdk/video-generation-core.ts @@ -0,0 +1,27 @@ +// Shared video-generation implementation helpers for bundled and third-party plugins. + +export type { AuthProfileStore } from "../agents/auth-profiles.js"; +export type { FallbackAttempt } from "../agents/model-fallback.types.js"; +export type { VideoGenerationProviderPlugin } from "../plugins/types.js"; +export type { + GeneratedVideoAsset, + VideoGenerationProvider, + VideoGenerationRequest, + VideoGenerationResolution, + VideoGenerationResult, + VideoGenerationSourceAsset, +} from "../video-generation/types.js"; +export type { OpenClawConfig } from "../config/config.js"; + +export { describeFailoverError, isFailoverError } from "../agents/failover-error.js"; +export { + resolveAgentModelFallbackValues, + resolveAgentModelPrimaryValue, +} from "../config/model-input.js"; +export { + getVideoGenerationProvider, + listVideoGenerationProviders, +} from "../video-generation/provider-registry.js"; +export { parseVideoGenerationModelRef } from "../video-generation/model-ref.js"; +export { createSubsystemLogger } from "../logging/subsystem.js"; +export { getProviderEnvVars } from "../secrets/provider-env-vars.js"; diff --git a/src/plugin-sdk/video-generation-runtime.ts b/src/plugin-sdk/video-generation-runtime.ts new file mode 100644 index 00000000000..68b9755f3f0 --- /dev/null +++ b/src/plugin-sdk/video-generation-runtime.ts @@ -0,0 +1,21 @@ +// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually. +import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js"; +type FacadeEntry = PluginSdkFacadeTypeMap["video-generation-runtime"]; +type FacadeModule = FacadeEntry["module"]; +import { loadActivatedBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js"; + +function loadFacadeModule(): FacadeModule { + return loadActivatedBundledPluginPublicSurfaceModuleSync({ + dirName: "video-generation-core", + artifactBasename: "runtime-api.js", + }); +} +export const generateVideo: FacadeModule["generateVideo"] = ((...args) => + loadFacadeModule()["generateVideo"](...args)) as FacadeModule["generateVideo"]; +export const listRuntimeVideoGenerationProviders: FacadeModule["listRuntimeVideoGenerationProviders"] = + ((...args) => + loadFacadeModule()["listRuntimeVideoGenerationProviders"]( + ...args, + )) as FacadeModule["listRuntimeVideoGenerationProviders"]; +export type GenerateVideoParams = FacadeEntry["types"]["GenerateVideoParams"]; +export type GenerateVideoRuntimeResult = FacadeEntry["types"]["GenerateVideoRuntimeResult"]; diff --git a/src/plugin-sdk/video-generation.ts b/src/plugin-sdk/video-generation.ts new file mode 100644 index 00000000000..aaef20c7e5d --- /dev/null +++ b/src/plugin-sdk/video-generation.ts @@ -0,0 +1,10 @@ +// Public video-generation helpers and types for provider plugins. + +export type { + GeneratedVideoAsset, + VideoGenerationProvider, + VideoGenerationRequest, + VideoGenerationResolution, + VideoGenerationResult, + VideoGenerationSourceAsset, +} from "../video-generation/types.js"; diff --git a/src/plugins/api-builder.ts b/src/plugins/api-builder.ts index 0c5906758c1..e5fb64fe321 100644 --- a/src/plugins/api-builder.ts +++ b/src/plugins/api-builder.ts @@ -32,6 +32,7 @@ export type BuildPluginApiParams = { | "registerRealtimeVoiceProvider" | "registerMediaUnderstandingProvider" | "registerImageGenerationProvider" + | "registerVideoGenerationProvider" | "registerWebFetchProvider" | "registerWebSearchProvider" | "registerInteractiveHandler" @@ -65,6 +66,8 @@ const noopRegisterMediaUnderstandingProvider: OpenClawPluginApi["registerMediaUn () => {}; const noopRegisterImageGenerationProvider: OpenClawPluginApi["registerImageGenerationProvider"] = () => {}; +const noopRegisterVideoGenerationProvider: OpenClawPluginApi["registerVideoGenerationProvider"] = + () => {}; const noopRegisterWebFetchProvider: OpenClawPluginApi["registerWebFetchProvider"] = () => {}; const noopRegisterWebSearchProvider: OpenClawPluginApi["registerWebSearchProvider"] = () => {}; const noopRegisterInteractiveHandler: OpenClawPluginApi["registerInteractiveHandler"] = () => {}; @@ -111,6 +114,8 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi handlers.registerMediaUnderstandingProvider ?? noopRegisterMediaUnderstandingProvider, registerImageGenerationProvider: handlers.registerImageGenerationProvider ?? noopRegisterImageGenerationProvider, + registerVideoGenerationProvider: + handlers.registerVideoGenerationProvider ?? noopRegisterVideoGenerationProvider, registerWebFetchProvider: handlers.registerWebFetchProvider ?? noopRegisterWebFetchProvider, registerWebSearchProvider: handlers.registerWebSearchProvider ?? noopRegisterWebSearchProvider, registerInteractiveHandler: diff --git a/src/plugins/bundled-capability-metadata.ts b/src/plugins/bundled-capability-metadata.ts index 7a7186153b6..76f5d27666b 100644 --- a/src/plugins/bundled-capability-metadata.ts +++ b/src/plugins/bundled-capability-metadata.ts @@ -9,6 +9,7 @@ export type BundledPluginContractSnapshot = { realtimeVoiceProviderIds: string[]; mediaUnderstandingProviderIds: string[]; imageGenerationProviderIds: string[]; + videoGenerationProviderIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; toolNames: string[]; @@ -45,6 +46,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn realtimeVoiceProviderIds: uniqueStrings(manifest.contracts?.realtimeVoiceProviders), mediaUnderstandingProviderIds: uniqueStrings(manifest.contracts?.mediaUnderstandingProviders), imageGenerationProviderIds: uniqueStrings(manifest.contracts?.imageGenerationProviders), + videoGenerationProviderIds: uniqueStrings(manifest.contracts?.videoGenerationProviders), webFetchProviderIds: uniqueStrings(manifest.contracts?.webFetchProviders), webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders), toolNames: uniqueStrings(manifest.contracts?.tools), @@ -58,6 +60,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn entry.realtimeVoiceProviderIds.length > 0 || entry.mediaUnderstandingProviderIds.length > 0 || entry.imageGenerationProviderIds.length > 0 || + entry.videoGenerationProviderIds.length > 0 || entry.webFetchProviderIds.length > 0 || entry.webSearchProviderIds.length > 0 || entry.toolNames.length > 0, @@ -92,6 +95,10 @@ export const BUNDLED_IMAGE_GENERATION_PLUGIN_IDS = collectPluginIds( (entry) => entry.imageGenerationProviderIds, ); +export const BUNDLED_VIDEO_GENERATION_PLUGIN_IDS = collectPluginIds( + (entry) => entry.videoGenerationProviderIds, +); + export const BUNDLED_WEB_FETCH_PLUGIN_IDS = collectPluginIds((entry) => entry.webFetchProviderIds); export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [ @@ -104,6 +111,7 @@ export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [ entry.realtimeVoiceProviderIds.length > 0 || entry.mediaUnderstandingProviderIds.length > 0 || entry.imageGenerationProviderIds.length > 0 || + entry.videoGenerationProviderIds.length > 0 || entry.webFetchProviderIds.length > 0 || entry.webSearchProviderIds.length > 0, ).map((entry) => entry.pluginId), diff --git a/src/plugins/bundled-capability-runtime.ts b/src/plugins/bundled-capability-runtime.ts index c44dd875e52..8e442401742 100644 --- a/src/plugins/bundled-capability-runtime.ts +++ b/src/plugins/bundled-capability-runtime.ts @@ -126,6 +126,7 @@ function createCapabilityPluginRecord(params: { realtimeVoiceProviderIds: [], mediaUnderstandingProviderIds: [], imageGenerationProviderIds: [], + videoGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], gatewayMethods: [], @@ -286,6 +287,9 @@ export function loadBundledCapabilityRuntimeRegistry(params: { record.imageGenerationProviderIds.push( ...captured.imageGenerationProviders.map((entry) => entry.id), ); + record.videoGenerationProviderIds.push( + ...captured.videoGenerationProviders.map((entry) => entry.id), + ); record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id)); record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id)); record.toolNames.push(...captured.tools.map((entry) => entry.name)); @@ -353,6 +357,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: { rootDir: record.rootDir, })), ); + registry.videoGenerationProviders.push( + ...captured.videoGenerationProviders.map((provider) => ({ + pluginId: record.id, + pluginName: record.name, + provider, + source: record.source, + rootDir: record.rootDir, + })), + ); registry.webFetchProviders.push( ...captured.webFetchProviders.map((provider) => ({ pluginId: record.id, diff --git a/src/plugins/bundled-provider-auth-env-vars.generated.ts b/src/plugins/bundled-provider-auth-env-vars.generated.ts index f5e501cfa4f..23634946016 100644 --- a/src/plugins/bundled-provider-auth-env-vars.generated.ts +++ b/src/plugins/bundled-provider-auth-env-vars.generated.ts @@ -23,7 +23,8 @@ export const BUNDLED_PROVIDER_AUTH_ENV_VAR_CANDIDATES = { minimax: ["MINIMAX_API_KEY"], "minimax-portal": ["MINIMAX_OAUTH_TOKEN", "MINIMAX_API_KEY"], mistral: ["MISTRAL_API_KEY"], - modelstudio: ["MODELSTUDIO_API_KEY"], + qwen: ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"], + modelstudio: ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"], moonshot: ["MOONSHOT_API_KEY"], nvidia: ["NVIDIA_API_KEY"], ollama: ["OLLAMA_API_KEY"], diff --git a/src/plugins/capability-provider-runtime.test.ts b/src/plugins/capability-provider-runtime.test.ts index 78b8a5e40c8..7654b30883e 100644 --- a/src/plugins/capability-provider-runtime.test.ts +++ b/src/plugins/capability-provider-runtime.test.ts @@ -265,4 +265,44 @@ describe("resolvePluginCapabilityProviders", () => { config: expect.anything(), }); }); + + it("loads bundled capability providers even without an explicit cfg", () => { + const compatConfig = { + plugins: { + enabled: true, + allow: ["google"], + entries: { google: { enabled: true } }, + }, + } as OpenClawConfig; + const loaded = createEmptyPluginRegistry(); + loaded.mediaUnderstandingProviders.push({ + pluginId: "google", + pluginName: "google", + source: "test", + provider: { + id: "google", + capabilities: ["image", "audio", "video"], + describeImage: vi.fn(), + transcribeAudio: vi.fn(), + describeVideo: vi.fn(), + autoPriority: { image: 30, audio: 40, video: 10 }, + nativeDocumentInputs: ["pdf"], + }, + } as never); + setBundledCapabilityFixture("mediaUnderstandingProviders"); + mocks.withBundledPluginEnablementCompat.mockReturnValue(compatConfig); + mocks.withBundledPluginVitestCompat.mockReturnValue(compatConfig); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? undefined : loaded, + ); + + const providers = resolvePluginCapabilityProviders({ key: "mediaUnderstandingProviders" }); + + expectResolvedCapabilityProviderIds(providers, ["google"]); + expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({ + config: undefined, + env: process.env, + }); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ config: compatConfig }); + }); }); diff --git a/src/plugins/capability-provider-runtime.ts b/src/plugins/capability-provider-runtime.ts index 4e41fce5f87..497d569d028 100644 --- a/src/plugins/capability-provider-runtime.ts +++ b/src/plugins/capability-provider-runtime.ts @@ -12,14 +12,16 @@ type CapabilityProviderRegistryKey = | "realtimeTranscriptionProviders" | "realtimeVoiceProviders" | "mediaUnderstandingProviders" - | "imageGenerationProviders"; + | "imageGenerationProviders" + | "videoGenerationProviders"; type CapabilityContractKey = | "speechProviders" | "realtimeTranscriptionProviders" | "realtimeVoiceProviders" | "mediaUnderstandingProviders" - | "imageGenerationProviders"; + | "imageGenerationProviders" + | "videoGenerationProviders"; type CapabilityProviderForKey = PluginRegistry[K][number] extends { provider: infer T } ? T : never; @@ -30,6 +32,7 @@ const CAPABILITY_CONTRACT_KEY: Record 0) { return activeProviders.map((entry) => entry.provider) as CapabilityProviderForKey[]; } - const loadOptions = - params.cfg === undefined - ? undefined - : { - config: resolveCapabilityProviderConfig({ key: params.key, cfg: params.cfg }), - }; + const compatConfig = resolveCapabilityProviderConfig({ key: params.key, cfg: params.cfg }); + const loadOptions = compatConfig === undefined ? undefined : { config: compatConfig }; const registry = resolveRuntimePluginRegistry(loadOptions); return (registry?.[params.key] ?? []).map( (entry) => entry.provider, diff --git a/src/plugins/captured-registration.ts b/src/plugins/captured-registration.ts index c816c2d1464..1b665596800 100644 --- a/src/plugins/captured-registration.ts +++ b/src/plugins/captured-registration.ts @@ -13,6 +13,7 @@ import type { RealtimeTranscriptionProviderPlugin, RealtimeVoiceProviderPlugin, SpeechProviderPlugin, + VideoGenerationProviderPlugin, WebFetchProviderPlugin, WebSearchProviderPlugin, } from "./types.js"; @@ -33,6 +34,7 @@ export type CapturedPluginRegistration = { realtimeVoiceProviders: RealtimeVoiceProviderPlugin[]; mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[]; imageGenerationProviders: ImageGenerationProviderPlugin[]; + videoGenerationProviders: VideoGenerationProviderPlugin[]; webFetchProviders: WebFetchProviderPlugin[]; webSearchProviders: WebSearchProviderPlugin[]; tools: AnyAgentTool[]; @@ -50,6 +52,7 @@ export function createCapturedPluginRegistration(params?: { const realtimeVoiceProviders: RealtimeVoiceProviderPlugin[] = []; const mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[] = []; const imageGenerationProviders: ImageGenerationProviderPlugin[] = []; + const videoGenerationProviders: VideoGenerationProviderPlugin[] = []; const webFetchProviders: WebFetchProviderPlugin[] = []; const webSearchProviders: WebSearchProviderPlugin[] = []; const tools: AnyAgentTool[] = []; @@ -69,6 +72,7 @@ export function createCapturedPluginRegistration(params?: { realtimeVoiceProviders, mediaUnderstandingProviders, imageGenerationProviders, + videoGenerationProviders, webFetchProviders, webSearchProviders, tools, @@ -126,6 +130,9 @@ export function createCapturedPluginRegistration(params?: { registerImageGenerationProvider(provider: ImageGenerationProviderPlugin) { imageGenerationProviders.push(provider); }, + registerVideoGenerationProvider(provider: VideoGenerationProviderPlugin) { + videoGenerationProviders.push(provider); + }, registerWebFetchProvider(provider: WebFetchProviderPlugin) { webFetchProviders.push(provider); }, diff --git a/src/plugins/channel-plugin-ids.ts b/src/plugins/channel-plugin-ids.ts index b8e50cc7cab..3ba9b234e3e 100644 --- a/src/plugins/channel-plugin-ids.ts +++ b/src/plugins/channel-plugin-ids.ts @@ -15,6 +15,7 @@ function hasRuntimeContractSurface(plugin: PluginManifestRecord): boolean { plugin.contracts?.speechProviders?.length || plugin.contracts?.mediaUnderstandingProviders?.length || plugin.contracts?.imageGenerationProviders?.length || + plugin.contracts?.videoGenerationProviders?.length || plugin.contracts?.webFetchProviders?.length || plugin.contracts?.webSearchProviders?.length || hasKind(plugin.kind, "memory"), diff --git a/src/plugins/contracts/registry.ts b/src/plugins/contracts/registry.ts index dfe446209b5..1f769496d7f 100644 --- a/src/plugins/contracts/registry.ts +++ b/src/plugins/contracts/registry.ts @@ -8,6 +8,7 @@ import { BUNDLED_SPEECH_PLUGIN_IDS, BUNDLED_WEB_FETCH_PLUGIN_IDS, BUNDLED_WEB_SEARCH_PLUGIN_IDS, + BUNDLED_VIDEO_GENERATION_PLUGIN_IDS, } from "../bundled-capability-metadata.js"; import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js"; import type { @@ -17,6 +18,7 @@ import type { RealtimeTranscriptionProviderPlugin, RealtimeVoiceProviderPlugin, SpeechProviderPlugin, + VideoGenerationProviderPlugin, WebFetchProviderPlugin, WebSearchProviderPlugin, } from "../types.js"; @@ -26,6 +28,7 @@ import { loadVitestRealtimeTranscriptionProviderContractRegistry, loadVitestRealtimeVoiceProviderContractRegistry, loadVitestSpeechProviderContractRegistry, + loadVitestVideoGenerationProviderContractRegistry, } from "./speech-vitest-registry.js"; type BundledCapabilityRuntimeRegistry = ReturnType; @@ -50,6 +53,7 @@ type RealtimeVoiceProviderContractEntry = CapabilityContractEntry; type ImageGenerationProviderContractEntry = CapabilityContractEntry; +type VideoGenerationProviderContractEntry = CapabilityContractEntry; type PluginRegistrationContractEntry = { pluginId: string; @@ -60,6 +64,7 @@ type PluginRegistrationContractEntry = { realtimeVoiceProviderIds: string[]; mediaUnderstandingProviderIds: string[]; imageGenerationProviderIds: string[]; + videoGenerationProviderIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; toolNames: string[]; @@ -114,6 +119,8 @@ let mediaUnderstandingProviderContractRegistryCache: | null = null; let imageGenerationProviderContractRegistryCache: ImageGenerationProviderContractEntry[] | null = null; +let videoGenerationProviderContractRegistryCache: VideoGenerationProviderContractEntry[] | null = + null; const providerContractPluginIdsByProviderId = createProviderContractPluginIdsByProviderId(); export let providerContractLoadError: Error | undefined; @@ -462,6 +469,21 @@ function loadImageGenerationProviderContractRegistry(): ImageGenerationProviderC return imageGenerationProviderContractRegistryCache; } +function loadVideoGenerationProviderContractRegistry(): VideoGenerationProviderContractEntry[] { + if (!videoGenerationProviderContractRegistryCache) { + videoGenerationProviderContractRegistryCache = process.env.VITEST + ? loadVitestVideoGenerationProviderContractRegistry() + : loadBundledCapabilityRuntimeRegistry({ + pluginIds: BUNDLED_VIDEO_GENERATION_PLUGIN_IDS, + pluginSdkResolution: "dist", + }).videoGenerationProviders.map((entry) => ({ + pluginId: entry.pluginId, + provider: entry.provider, + })); + } + return videoGenerationProviderContractRegistryCache; +} + function createLazyArrayView(load: () => T[]): T[] { return new Proxy([] as T[], { get(_target, prop) { @@ -576,6 +598,9 @@ export const mediaUnderstandingProviderContractRegistry: MediaUnderstandingProvi export const imageGenerationProviderContractRegistry: ImageGenerationProviderContractEntry[] = createLazyArrayView(loadImageGenerationProviderContractRegistry); +export const videoGenerationProviderContractRegistry: VideoGenerationProviderContractEntry[] = + createLazyArrayView(loadVideoGenerationProviderContractRegistry); + function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEntry[] { return BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.map((entry) => ({ pluginId: entry.pluginId, @@ -586,6 +611,7 @@ function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEnt realtimeVoiceProviderIds: uniqueStrings(entry.realtimeVoiceProviderIds), mediaUnderstandingProviderIds: uniqueStrings(entry.mediaUnderstandingProviderIds), imageGenerationProviderIds: uniqueStrings(entry.imageGenerationProviderIds), + videoGenerationProviderIds: uniqueStrings(entry.videoGenerationProviderIds), webFetchProviderIds: uniqueStrings(entry.webFetchProviderIds), webSearchProviderIds: uniqueStrings(entry.webSearchProviderIds), toolNames: uniqueStrings(entry.toolNames), diff --git a/src/plugins/contracts/speech-vitest-registry.ts b/src/plugins/contracts/speech-vitest-registry.ts index 9fb083b7804..93b0f73dcd3 100644 --- a/src/plugins/contracts/speech-vitest-registry.ts +++ b/src/plugins/contracts/speech-vitest-registry.ts @@ -8,6 +8,7 @@ import { BUNDLED_REALTIME_TRANSCRIPTION_PLUGIN_IDS, BUNDLED_REALTIME_VOICE_PLUGIN_IDS, BUNDLED_SPEECH_PLUGIN_IDS, + BUNDLED_VIDEO_GENERATION_PLUGIN_IDS, } from "../bundled-capability-metadata.js"; import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js"; import { loadPluginManifestRegistry } from "../manifest-registry.js"; @@ -18,6 +19,7 @@ import type { RealtimeTranscriptionProviderPlugin, RealtimeVoiceProviderPlugin, SpeechProviderPlugin, + VideoGenerationProviderPlugin, } from "../types.js"; export type SpeechProviderContractEntry = { @@ -45,6 +47,11 @@ export type ImageGenerationProviderContractEntry = { provider: ImageGenerationProviderPlugin; }; +export type VideoGenerationProviderContractEntry = { + pluginId: string; + provider: VideoGenerationProviderPlugin; +}; + function buildVitestCapabilityAliasMap(modulePath: string): Record { const { ["openclaw/plugin-sdk"]: _ignoredRootAlias, ...scopedAliasMap } = buildPluginLoaderAliasMap(modulePath, process.argv[1], import.meta.url, "dist"); @@ -338,3 +345,48 @@ export function loadVitestImageGenerationProviderContractRegistry(): ImageGenera ); return registrations; } + +export function loadVitestVideoGenerationProviderContractRegistry(): VideoGenerationProviderContractEntry[] { + const registrations: VideoGenerationProviderContractEntry[] = []; + const { manifests, unresolvedPluginIds } = resolveTestApiModuleRecords( + BUNDLED_VIDEO_GENERATION_PLUGIN_IDS, + ); + + for (const plugin of manifests) { + if (!plugin.rootDir) { + continue; + } + const testApiPath = path.join(plugin.rootDir, "test-api.ts"); + if (!fs.existsSync(testApiPath)) { + continue; + } + const builder = resolveNamedBuilder( + createVitestCapabilityLoader(testApiPath)(testApiPath), + /^build.+VideoGenerationProvider$/u, + ); + if (!builder) { + continue; + } + registrations.push({ + pluginId: plugin.id, + provider: builder(), + }); + unresolvedPluginIds.delete(plugin.id); + } + + if (unresolvedPluginIds.size === 0) { + return registrations; + } + + const runtimeRegistry = loadBundledCapabilityRuntimeRegistry({ + pluginIds: [...unresolvedPluginIds], + pluginSdkResolution: "dist", + }); + registrations.push( + ...runtimeRegistry.videoGenerationProviders.map((entry) => ({ + pluginId: entry.pluginId, + provider: entry.provider, + })), + ); + return registrations; +} diff --git a/src/plugins/hooks.test-helpers.ts b/src/plugins/hooks.test-helpers.ts index 8884153b78a..146ce8b291e 100644 --- a/src/plugins/hooks.test-helpers.ts +++ b/src/plugins/hooks.test-helpers.ts @@ -38,6 +38,7 @@ export function createMockPluginRegistry( speechProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webSearchProviders: [], httpRoutes: [], gatewayHandlers: {}, diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index c89b0ea0558..bed8d60f5cc 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -594,6 +594,7 @@ function createPluginRecord(params: { realtimeVoiceProviderIds: [], mediaUnderstandingProviderIds: [], imageGenerationProviderIds: [], + videoGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], gatewayMethods: [], diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 47832ec0fcb..c45fcb49f01 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -76,6 +76,7 @@ export type PluginManifestContracts = { realtimeVoiceProviders?: string[]; mediaUnderstandingProviders?: string[]; imageGenerationProviders?: string[]; + videoGenerationProviders?: string[]; webFetchProviders?: string[]; webSearchProviders?: string[]; tools?: string[]; @@ -155,6 +156,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u const realtimeVoiceProviders = normalizeStringList(value.realtimeVoiceProviders); const mediaUnderstandingProviders = normalizeStringList(value.mediaUnderstandingProviders); const imageGenerationProviders = normalizeStringList(value.imageGenerationProviders); + const videoGenerationProviders = normalizeStringList(value.videoGenerationProviders); const webFetchProviders = normalizeStringList(value.webFetchProviders); const webSearchProviders = normalizeStringList(value.webSearchProviders); const tools = normalizeStringList(value.tools); @@ -164,6 +166,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u ...(realtimeVoiceProviders.length > 0 ? { realtimeVoiceProviders } : {}), ...(mediaUnderstandingProviders.length > 0 ? { mediaUnderstandingProviders } : {}), ...(imageGenerationProviders.length > 0 ? { imageGenerationProviders } : {}), + ...(videoGenerationProviders.length > 0 ? { videoGenerationProviders } : {}), ...(webFetchProviders.length > 0 ? { webFetchProviders } : {}), ...(webSearchProviders.length > 0 ? { webSearchProviders } : {}), ...(tools.length > 0 ? { tools } : {}), diff --git a/src/plugins/provider-auth-storage.ts b/src/plugins/provider-auth-storage.ts index 7145386e9e0..15cd909fddf 100644 --- a/src/plugins/provider-auth-storage.ts +++ b/src/plugins/provider-auth-storage.ts @@ -85,6 +85,7 @@ const { setTogetherApiKey, setHuggingfaceApiKey, setQianfanApiKey, + setQwenApiKey, setModelStudioApiKey, setXaiApiKey, setMistralApiKey, @@ -110,7 +111,8 @@ const { setTogetherApiKey: { provider: "together" }, setHuggingfaceApiKey: { provider: "huggingface" }, setQianfanApiKey: { provider: "qianfan" }, - setModelStudioApiKey: { provider: "modelstudio" }, + setQwenApiKey: { provider: "qwen" }, + setModelStudioApiKey: { provider: "qwen" }, setXaiApiKey: { provider: "xai" }, setMistralApiKey: { provider: "mistral" }, setKilocodeApiKey: { provider: "kilocode" }, @@ -134,6 +136,7 @@ export { setTogetherApiKey, setHuggingfaceApiKey, setQianfanApiKey, + setQwenApiKey, setModelStudioApiKey, setXaiApiKey, setMistralApiKey, diff --git a/src/plugins/providers.test.ts b/src/plugins/providers.test.ts index efc9cad501c..7fb321b9150 100644 --- a/src/plugins/providers.test.ts +++ b/src/plugins/providers.test.ts @@ -16,16 +16,19 @@ const applyPluginAutoEnableMock = vi.fn(); let resolveOwningPluginIdsForProvider: typeof import("./providers.js").resolveOwningPluginIdsForProvider; let resolveOwningPluginIdsForModelRef: typeof import("./providers.js").resolveOwningPluginIdsForModelRef; +let resolveEnabledProviderPluginIds: typeof import("./providers.js").resolveEnabledProviderPluginIds; let resolvePluginProviders: typeof import("./providers.runtime.js").resolvePluginProviders; function createManifestProviderPlugin(params: { id: string; providerIds: string[]; origin?: "bundled" | "workspace"; + enabledByDefault?: boolean; modelSupport?: { modelPrefixes?: string[]; modelPatterns?: string[] }; }): PluginManifestRecord { return { id: params.id, + enabledByDefault: params.enabledByDefault, channels: [], cliBackends: [], providers: params.providerIds, @@ -230,8 +233,11 @@ describe("resolvePluginProviders", () => { loadPluginManifestRegistry: (...args: Parameters) => loadPluginManifestRegistryMock(...args), })); - ({ resolveOwningPluginIdsForProvider, resolveOwningPluginIdsForModelRef } = - await import("./providers.js")); + ({ + resolveOwningPluginIdsForProvider, + resolveOwningPluginIdsForModelRef, + resolveEnabledProviderPluginIds, + } = await import("./providers.js")); ({ resolvePluginProviders } = await import("./providers.runtime.js")); }); @@ -255,10 +261,22 @@ describe("resolvePluginProviders", () => { }), ); setManifestPlugins([ - createManifestProviderPlugin({ id: "google", providerIds: ["google"] }), + createManifestProviderPlugin({ + id: "google", + providerIds: ["google"], + enabledByDefault: true, + }), createManifestProviderPlugin({ id: "browser", providerIds: [] }), - createManifestProviderPlugin({ id: "kilocode", providerIds: ["kilocode"] }), - createManifestProviderPlugin({ id: "moonshot", providerIds: ["moonshot"] }), + createManifestProviderPlugin({ + id: "kilocode", + providerIds: ["kilocode"], + enabledByDefault: true, + }), + createManifestProviderPlugin({ + id: "moonshot", + providerIds: ["moonshot"], + enabledByDefault: true, + }), createManifestProviderPlugin({ id: "google-gemini-cli-auth", providerIds: [] }), createManifestProviderPlugin({ id: "workspace-provider", @@ -292,6 +310,14 @@ describe("resolvePluginProviders", () => { ); }); + it("keeps bundled provider plugins enabled when they default on outside Vitest compat", () => { + expect(resolveEnabledProviderPluginIds({ config: {}, env: {} as NodeJS.ProcessEnv })).toEqual([ + "google", + "kilocode", + "moonshot", + ]); + }); + it.each([ { name: "can augment restrictive allowlists for bundled provider compatibility", diff --git a/src/plugins/providers.ts b/src/plugins/providers.ts index cce2831210c..3bab9095de7 100644 --- a/src/plugins/providers.ts +++ b/src/plugins/providers.ts @@ -62,6 +62,7 @@ export function resolveEnabledProviderPluginIds(params: { origin: plugin.origin, config: normalizedConfig, rootConfig: params.config, + enabledByDefault: plugin.enabledByDefault, }).activated, ) .map((plugin) => plugin.id) diff --git a/src/plugins/registry-empty.ts b/src/plugins/registry-empty.ts index ee7183e6d66..c2da276eedb 100644 --- a/src/plugins/registry-empty.ts +++ b/src/plugins/registry-empty.ts @@ -15,6 +15,7 @@ export function createEmptyPluginRegistry(): PluginRegistry { realtimeVoiceProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], gatewayHandlers: {}, diff --git a/src/plugins/registry.ts b/src/plugins/registry.ts index ffef249d23c..4cf51ab1c82 100644 --- a/src/plugins/registry.ts +++ b/src/plugins/registry.ts @@ -69,6 +69,7 @@ import type { PluginHookHandlerMap, PluginHookRegistration as TypedPluginHookRegistration, SpeechProviderPlugin, + VideoGenerationProviderPlugin, WebFetchProviderPlugin, WebSearchProviderPlugin, } from "./types.js"; @@ -153,6 +154,8 @@ export type PluginMediaUnderstandingProviderRegistration = PluginOwnedProviderRegistration; export type PluginImageGenerationProviderRegistration = PluginOwnedProviderRegistration; +export type PluginVideoGenerationProviderRegistration = + PluginOwnedProviderRegistration; export type PluginWebFetchProviderRegistration = PluginOwnedProviderRegistration; export type PluginWebSearchProviderRegistration = @@ -224,6 +227,7 @@ export type PluginRecord = { realtimeVoiceProviderIds: string[]; mediaUnderstandingProviderIds: string[]; imageGenerationProviderIds: string[]; + videoGenerationProviderIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; gatewayMethods: string[]; @@ -252,6 +256,7 @@ export type PluginRegistry = { realtimeVoiceProviders: PluginRealtimeVoiceProviderRegistration[]; mediaUnderstandingProviders: PluginMediaUnderstandingProviderRegistration[]; imageGenerationProviders: PluginImageGenerationProviderRegistration[]; + videoGenerationProviders: PluginVideoGenerationProviderRegistration[]; webFetchProviders: PluginWebFetchProviderRegistration[]; webSearchProviders: PluginWebSearchProviderRegistration[]; gatewayHandlers: GatewayRequestHandlers; @@ -772,6 +777,19 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { }); }; + const registerVideoGenerationProvider = ( + record: PluginRecord, + provider: VideoGenerationProviderPlugin, + ) => { + registerUniqueProviderLike({ + record, + provider, + kindLabel: "video-generation provider", + registrations: registry.videoGenerationProviders, + ownedIds: record.videoGenerationProviderIds, + }); + }; + const registerWebFetchProvider = (record: PluginRecord, provider: WebFetchProviderPlugin) => { registerUniqueProviderLike({ record, @@ -1064,6 +1082,8 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerMediaUnderstandingProvider(record, provider), registerImageGenerationProvider: (provider) => registerImageGenerationProvider(record, provider), + registerVideoGenerationProvider: (provider) => + registerVideoGenerationProvider(record, provider), registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider), registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider), registerGatewayMethod: (method, handler, opts) => @@ -1253,6 +1273,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerRealtimeVoiceProvider, registerMediaUnderstandingProvider, registerImageGenerationProvider, + registerVideoGenerationProvider, registerWebSearchProvider, registerGatewayMethod, registerCli, diff --git a/src/plugins/runtime.test.ts b/src/plugins/runtime.test.ts index d4393210c0c..1a460aa1f50 100644 --- a/src/plugins/runtime.test.ts +++ b/src/plugins/runtime.test.ts @@ -203,6 +203,7 @@ describe("setActivePluginRegistry", () => { realtimeVoiceProviderIds: [], mediaUnderstandingProviderIds: [], imageGenerationProviderIds: [], + videoGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], gatewayMethods: [], @@ -231,6 +232,7 @@ describe("setActivePluginRegistry", () => { realtimeVoiceProviderIds: [], mediaUnderstandingProviderIds: [], imageGenerationProviderIds: [], + videoGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], gatewayMethods: [], diff --git a/src/plugins/runtime/index.ts b/src/plugins/runtime/index.ts index 4694f409be4..2476f1e6c67 100644 --- a/src/plugins/runtime/index.ts +++ b/src/plugins/runtime/index.ts @@ -58,6 +58,12 @@ type RuntimeImageGenerationModule = Pick< >; let cachedRuntimeImageGenerationModule: RuntimeImageGenerationModule | null = null; +type RuntimeVideoGenerationModule = Pick< + typeof import("../../plugin-sdk/video-generation-runtime.js"), + "generateVideo" | "listRuntimeVideoGenerationProviders" +>; +let cachedRuntimeVideoGenerationModule: RuntimeVideoGenerationModule | null = null; + function loadRuntimeImageGenerationModule(): RuntimeImageGenerationModule { cachedRuntimeImageGenerationModule ??= loadBundledPluginPublicSurfaceModuleSync({ @@ -75,6 +81,23 @@ function createRuntimeImageGeneration(): PluginRuntime["imageGeneration"] { }; } +function loadRuntimeVideoGenerationModule(): RuntimeVideoGenerationModule { + cachedRuntimeVideoGenerationModule ??= + loadBundledPluginPublicSurfaceModuleSync({ + dirName: "video-generation-core", + artifactBasename: "runtime-api.js", + }); + return cachedRuntimeVideoGenerationModule; +} + +function createRuntimeVideoGeneration(): PluginRuntime["videoGeneration"] { + return { + generate: (params) => loadRuntimeVideoGenerationModule().generateVideo(params), + listProviders: (params) => + loadRuntimeVideoGenerationModule().listRuntimeVideoGenerationProviders(params), + }; +} + function createRuntimeModelAuth(): PluginRuntime["modelAuth"] { const getApiKeyForModel = createLazyRuntimeMethod( loadModelAuthRuntime, @@ -213,10 +236,13 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}): taskFlow, } satisfies Omit< PluginRuntime, - "tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" + "tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration" > & Partial< - Pick + Pick< + PluginRuntime, + "tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration" + > >; defineCachedValue(runtime, "tts", createRuntimeTts); @@ -226,6 +252,7 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}): })); defineCachedValue(runtime, "modelAuth", createRuntimeModelAuth); defineCachedValue(runtime, "imageGeneration", createRuntimeImageGeneration); + defineCachedValue(runtime, "videoGeneration", createRuntimeVideoGeneration); return runtime as PluginRuntime; } diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index 195fa7ba412..58b7e0863bb 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -82,6 +82,10 @@ export type PluginRuntimeCore = { generate: typeof import("../../plugin-sdk/image-generation-runtime.js").generateImage; listProviders: typeof import("../../plugin-sdk/image-generation-runtime.js").listRuntimeImageGenerationProviders; }; + videoGeneration: { + generate: typeof import("../../plugin-sdk/video-generation-runtime.js").generateVideo; + listProviders: typeof import("../../plugin-sdk/video-generation-runtime.js").listRuntimeVideoGenerationProviders; + }; webSearch: { listProviders: typeof import("../../web-search/runtime.js").listWebSearchProviders; search: typeof import("../../web-search/runtime.js").runWebSearch; diff --git a/src/plugins/status.test-helpers.ts b/src/plugins/status.test-helpers.ts index e8b4119f08a..8ee1f857884 100644 --- a/src/plugins/status.test-helpers.ts +++ b/src/plugins/status.test-helpers.ts @@ -55,6 +55,7 @@ export function createPluginRecord( realtimeVoiceProviderIds: [], mediaUnderstandingProviderIds: [], imageGenerationProviderIds: [], + videoGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], gatewayMethods: [], @@ -119,6 +120,7 @@ export function createPluginLoadResult( speechProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], tools: [], diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 1875af0b9ca..a7cc20ee5a9 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -69,6 +69,7 @@ import type { SpeechVoiceOption, } from "../tts/provider-types.js"; import type { DeliveryContext } from "../utils/delivery-context.js"; +import type { VideoGenerationProvider } from "../video-generation/types.js"; import type { WizardPrompter } from "../wizard/prompts.js"; import type { SecretInputMode } from "./provider-auth-types.js"; import type { createVpsAwareOAuthHandlers } from "./provider-oauth-flow.js"; @@ -1698,6 +1699,7 @@ export type PluginRealtimeVoiceProviderEntry = RealtimeVoiceProviderPlugin & { export type MediaUnderstandingProviderPlugin = MediaUnderstandingProvider; export type ImageGenerationProviderPlugin = ImageGenerationProvider; +export type VideoGenerationProviderPlugin = VideoGenerationProvider; export type OpenClawPluginGatewayMethod = { method: string; @@ -2035,6 +2037,8 @@ export type OpenClawPluginApi = { registerMediaUnderstandingProvider: (provider: MediaUnderstandingProviderPlugin) => void; /** Register an image generation provider (image generation capability). */ registerImageGenerationProvider: (provider: ImageGenerationProviderPlugin) => void; + /** Register a video generation provider (video generation capability). */ + registerVideoGenerationProvider: (provider: VideoGenerationProviderPlugin) => void; /** Register a web fetch provider (web fetch capability). */ registerWebFetchProvider: (provider: WebFetchProviderPlugin) => void; /** Register a web search provider (web search capability). */ diff --git a/src/video-generation/model-ref.ts b/src/video-generation/model-ref.ts new file mode 100644 index 00000000000..09cf6079120 --- /dev/null +++ b/src/video-generation/model-ref.ts @@ -0,0 +1,16 @@ +export function parseVideoGenerationModelRef( + raw: string | undefined, +): { provider: string; model: string } | null { + const trimmed = raw?.trim(); + if (!trimmed) { + return null; + } + const slashIndex = trimmed.indexOf("/"); + if (slashIndex <= 0 || slashIndex === trimmed.length - 1) { + return null; + } + return { + provider: trimmed.slice(0, slashIndex).trim(), + model: trimmed.slice(slashIndex + 1).trim(), + }; +} diff --git a/src/video-generation/provider-registry.test.ts b/src/video-generation/provider-registry.test.ts new file mode 100644 index 00000000000..866d491df6f --- /dev/null +++ b/src/video-generation/provider-registry.test.ts @@ -0,0 +1,93 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createEmptyPluginRegistry } from "../plugins/registry.js"; + +const { resolveRuntimePluginRegistryMock } = vi.hoisted(() => ({ + resolveRuntimePluginRegistryMock: vi.fn< + (params?: unknown) => ReturnType | undefined + >(() => undefined), +})); + +vi.mock("../plugins/loader.js", () => ({ + resolveRuntimePluginRegistry: resolveRuntimePluginRegistryMock, +})); + +let getVideoGenerationProvider: typeof import("./provider-registry.js").getVideoGenerationProvider; +let listVideoGenerationProviders: typeof import("./provider-registry.js").listVideoGenerationProviders; + +describe("video-generation provider registry", () => { + beforeAll(async () => { + ({ getVideoGenerationProvider, listVideoGenerationProviders } = + await import("./provider-registry.js")); + }); + + beforeEach(() => { + resolveRuntimePluginRegistryMock.mockReset(); + resolveRuntimePluginRegistryMock.mockReturnValue(undefined); + }); + + it("does not load plugins when listing without config", () => { + expect(listVideoGenerationProviders()).toEqual([]); + expect(resolveRuntimePluginRegistryMock).toHaveBeenCalledWith(); + }); + + it("uses active plugin providers without loading from disk", () => { + const registry = createEmptyPluginRegistry(); + registry.videoGenerationProviders.push({ + pluginId: "custom-video", + pluginName: "Custom Video", + source: "test", + provider: { + id: "custom-video", + label: "Custom Video", + capabilities: {}, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }], + }), + }, + }); + resolveRuntimePluginRegistryMock.mockReturnValue(registry); + + const provider = getVideoGenerationProvider("custom-video"); + + expect(provider?.id).toBe("custom-video"); + expect(resolveRuntimePluginRegistryMock).toHaveBeenCalledWith(); + }); + + it("ignores prototype-like provider ids and aliases", () => { + const registry = createEmptyPluginRegistry(); + registry.videoGenerationProviders.push( + { + pluginId: "blocked-video", + pluginName: "Blocked Video", + source: "test", + provider: { + id: "__proto__", + aliases: ["constructor", "prototype"], + capabilities: {}, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }], + }), + }, + }, + { + pluginId: "safe-video", + pluginName: "Safe Video", + source: "test", + provider: { + id: "safe-video", + aliases: ["safe-alias", "constructor"], + capabilities: {}, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }], + }), + }, + }, + ); + resolveRuntimePluginRegistryMock.mockReturnValue(registry); + + expect(listVideoGenerationProviders().map((provider) => provider.id)).toEqual(["safe-video"]); + expect(getVideoGenerationProvider("__proto__")).toBeUndefined(); + expect(getVideoGenerationProvider("constructor")).toBeUndefined(); + expect(getVideoGenerationProvider("safe-alias")?.id).toBe("safe-video"); + }); +}); diff --git a/src/video-generation/provider-registry.ts b/src/video-generation/provider-registry.ts new file mode 100644 index 00000000000..0744e7c2b6e --- /dev/null +++ b/src/video-generation/provider-registry.ts @@ -0,0 +1,77 @@ +import { normalizeProviderId } from "../agents/model-selection.js"; +import type { OpenClawConfig } from "../config/config.js"; +import { isBlockedObjectKey } from "../infra/prototype-keys.js"; +import { resolvePluginCapabilityProviders } from "../plugins/capability-provider-runtime.js"; +import type { VideoGenerationProviderPlugin } from "../plugins/types.js"; + +const BUILTIN_VIDEO_GENERATION_PROVIDERS: readonly VideoGenerationProviderPlugin[] = []; +const UNSAFE_PROVIDER_IDS = new Set(["__proto__", "constructor", "prototype"]); + +function normalizeVideoGenerationProviderId(id: string | undefined): string | undefined { + const normalized = normalizeProviderId(id ?? ""); + if (!normalized || isBlockedObjectKey(normalized)) { + return undefined; + } + return normalized; +} + +function isSafeVideoGenerationProviderId(id: string | undefined): id is string { + return Boolean(id && !UNSAFE_PROVIDER_IDS.has(id)); +} + +function resolvePluginVideoGenerationProviders( + cfg?: OpenClawConfig, +): VideoGenerationProviderPlugin[] { + return resolvePluginCapabilityProviders({ + key: "videoGenerationProviders", + cfg, + }); +} + +function buildProviderMaps(cfg?: OpenClawConfig): { + canonical: Map; + aliases: Map; +} { + const canonical = new Map(); + const aliases = new Map(); + const register = (provider: VideoGenerationProviderPlugin) => { + const id = normalizeVideoGenerationProviderId(provider.id); + if (!isSafeVideoGenerationProviderId(id)) { + return; + } + canonical.set(id, provider); + aliases.set(id, provider); + for (const alias of provider.aliases ?? []) { + const normalizedAlias = normalizeVideoGenerationProviderId(alias); + if (isSafeVideoGenerationProviderId(normalizedAlias)) { + aliases.set(normalizedAlias, provider); + } + } + }; + + for (const provider of BUILTIN_VIDEO_GENERATION_PROVIDERS) { + register(provider); + } + for (const provider of resolvePluginVideoGenerationProviders(cfg)) { + register(provider); + } + + return { canonical, aliases }; +} + +export function listVideoGenerationProviders( + cfg?: OpenClawConfig, +): VideoGenerationProviderPlugin[] { + return [...buildProviderMaps(cfg).canonical.values()]; +} + +export function getVideoGenerationProvider( + providerId: string | undefined, + cfg?: OpenClawConfig, +): VideoGenerationProviderPlugin | undefined { + const normalized = normalizeVideoGenerationProviderId(providerId); + if (!normalized) { + return undefined; + } + return buildProviderMaps(cfg).aliases.get(normalized); +} diff --git a/src/video-generation/runtime.test.ts b/src/video-generation/runtime.test.ts new file mode 100644 index 00000000000..235d2a7a4fb --- /dev/null +++ b/src/video-generation/runtime.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import { + generateVideo, + listRuntimeVideoGenerationProviders, + type GenerateVideoRuntimeResult, +} from "../plugin-sdk/video-generation-runtime.js"; +import type { VideoGenerationProvider } from "../video-generation/types.js"; + +const mocks = vi.hoisted(() => ({ + generateVideo: vi.fn(), + listRuntimeVideoGenerationProviders: vi.fn(), +})); + +vi.mock("../plugin-sdk/video-generation-runtime.js", () => ({ + generateVideo: mocks.generateVideo, + listRuntimeVideoGenerationProviders: mocks.listRuntimeVideoGenerationProviders, +})); + +describe("video-generation runtime facade", () => { + afterEach(() => { + mocks.generateVideo.mockReset(); + mocks.listRuntimeVideoGenerationProviders.mockReset(); + }); + + it("delegates video generation to the plugin-sdk runtime", async () => { + const result: GenerateVideoRuntimeResult = { + videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4", fileName: "sample.mp4" }], + provider: "video-plugin", + model: "vid-v1", + attempts: [], + }; + mocks.generateVideo.mockResolvedValue(result); + const params = { + cfg: { + agents: { + defaults: { + videoGenerationModel: { primary: "video-plugin/vid-v1" }, + }, + }, + } as OpenClawConfig, + prompt: "animate a cat", + agentDir: "/tmp/agent", + authStore: { version: 1, profiles: {} }, + }; + + await expect(generateVideo(params)).resolves.toBe(result); + expect(mocks.generateVideo).toHaveBeenCalledWith(params); + }); + + it("delegates provider listing to the plugin-sdk runtime", () => { + const providers: VideoGenerationProvider[] = [ + { + id: "video-plugin", + defaultModel: "vid-v1", + models: ["vid-v1", "vid-v2"], + capabilities: { + maxDurationSeconds: 10, + supportsAudio: true, + }, + generateVideo: async () => ({ + videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }], + }), + }, + ]; + mocks.listRuntimeVideoGenerationProviders.mockReturnValue(providers); + const params = { config: {} as OpenClawConfig }; + + expect(listRuntimeVideoGenerationProviders(params)).toBe(providers); + expect(mocks.listRuntimeVideoGenerationProviders).toHaveBeenCalledWith(params); + }); +}); diff --git a/src/video-generation/runtime.ts b/src/video-generation/runtime.ts new file mode 100644 index 00000000000..a04a3c6d161 --- /dev/null +++ b/src/video-generation/runtime.ts @@ -0,0 +1,6 @@ +export { + generateVideo, + listRuntimeVideoGenerationProviders, + type GenerateVideoParams, + type GenerateVideoRuntimeResult, +} from "../plugin-sdk/video-generation-runtime.js"; diff --git a/src/video-generation/types.ts b/src/video-generation/types.ts new file mode 100644 index 00000000000..fc538ac0946 --- /dev/null +++ b/src/video-generation/types.ts @@ -0,0 +1,65 @@ +import type { AuthProfileStore } from "../agents/auth-profiles.js"; +import type { OpenClawConfig } from "../config/config.js"; + +export type GeneratedVideoAsset = { + buffer: Buffer; + mimeType: string; + fileName?: string; + metadata?: Record; +}; + +export type VideoGenerationResolution = "480P" | "720P" | "1080P"; + +export type VideoGenerationSourceAsset = { + url?: string; + buffer?: Buffer; + mimeType?: string; + fileName?: string; + metadata?: Record; +}; + +export type VideoGenerationRequest = { + provider: string; + model: string; + prompt: string; + cfg: OpenClawConfig; + agentDir?: string; + authStore?: AuthProfileStore; + timeoutMs?: number; + size?: string; + aspectRatio?: string; + resolution?: VideoGenerationResolution; + durationSeconds?: number; + audio?: boolean; + watermark?: boolean; + inputImages?: VideoGenerationSourceAsset[]; + inputVideos?: VideoGenerationSourceAsset[]; +}; + +export type VideoGenerationResult = { + videos: GeneratedVideoAsset[]; + model?: string; + metadata?: Record; +}; + +export type VideoGenerationProviderCapabilities = { + maxVideos?: number; + maxInputImages?: number; + maxInputVideos?: number; + maxDurationSeconds?: number; + supportsSize?: boolean; + supportsAspectRatio?: boolean; + supportsResolution?: boolean; + supportsAudio?: boolean; + supportsWatermark?: boolean; +}; + +export type VideoGenerationProvider = { + id: string; + aliases?: string[]; + label?: string; + defaultModel?: string; + models?: string[]; + capabilities: VideoGenerationProviderCapabilities; + generateVideo: (req: VideoGenerationRequest) => Promise; +}; diff --git a/test/helpers/plugins/plugin-api.ts b/test/helpers/plugins/plugin-api.ts index ceaa92f68d8..c3812a9d3a9 100644 --- a/test/helpers/plugins/plugin-api.ts +++ b/test/helpers/plugins/plugin-api.ts @@ -24,6 +24,7 @@ export function createTestPluginApi(api: TestPluginApiInput): OpenClawPluginApi registerRealtimeVoiceProvider() {}, registerMediaUnderstandingProvider() {}, registerImageGenerationProvider() {}, + registerVideoGenerationProvider() {}, registerWebFetchProvider() {}, registerWebSearchProvider() {}, registerInteractiveHandler() {}, diff --git a/test/helpers/plugins/plugin-registration-contract.ts b/test/helpers/plugins/plugin-registration-contract.ts index 1b39bb1903d..d185f264fe6 100644 --- a/test/helpers/plugins/plugin-registration-contract.ts +++ b/test/helpers/plugins/plugin-registration-contract.ts @@ -4,6 +4,7 @@ import { mediaUnderstandingProviderContractRegistry, pluginRegistrationContractRegistry, speechProviderContractRegistry, + videoGenerationProviderContractRegistry, } from "../../../src/plugins/contracts/registry.js"; import { loadPluginManifestRegistry } from "../../../src/plugins/manifest-registry.js"; @@ -17,11 +18,13 @@ type PluginRegistrationContractParams = { realtimeVoiceProviderIds?: string[]; mediaUnderstandingProviderIds?: string[]; imageGenerationProviderIds?: string[]; + videoGenerationProviderIds?: string[]; cliBackendIds?: string[]; toolNames?: string[]; requireSpeechVoices?: boolean; requireDescribeImages?: boolean; requireGenerateImage?: boolean; + requireGenerateVideo?: boolean; manifestAuthChoice?: { pluginId: string; choiceId: string; @@ -91,6 +94,23 @@ function findImageGenerationProvider(pluginId: string) { return entry.provider; } +function findVideoGenerationProviderIds(pluginId: string) { + return videoGenerationProviderContractRegistry + .filter((entry) => entry.pluginId === pluginId) + .map((entry) => entry.provider.id) + .toSorted((left, right) => left.localeCompare(right)); +} + +function findVideoGenerationProvider(pluginId: string) { + const entry = videoGenerationProviderContractRegistry.find( + (candidate) => candidate.pluginId === pluginId, + ); + if (!entry) { + throw new Error(`video-generation provider contract missing for ${pluginId}`); + } + return entry.provider; +} + export function describePluginRegistrationContract(params: PluginRegistrationContractParams) { describe(`${params.pluginId} plugin registration contract`, () => { if (params.providerIds) { @@ -162,6 +182,17 @@ export function describePluginRegistrationContract(params: PluginRegistrationCon }); } + if (params.videoGenerationProviderIds) { + it("keeps bundled video-generation ownership explicit", () => { + expect(findRegistration(params.pluginId).videoGenerationProviderIds).toEqual( + params.videoGenerationProviderIds, + ); + expect(findVideoGenerationProviderIds(params.pluginId)).toEqual( + params.videoGenerationProviderIds, + ); + }); + } + if (params.cliBackendIds) { it("keeps bundled CLI backend ownership explicit", () => { expect(findRegistration(params.pluginId).cliBackendIds).toEqual(params.cliBackendIds); @@ -196,6 +227,14 @@ export function describePluginRegistrationContract(params: PluginRegistrationCon }); } + if (params.requireGenerateVideo) { + it("keeps bundled video-generation support explicit", () => { + expect(findVideoGenerationProvider(params.pluginId).generateVideo).toEqual( + expect.any(Function), + ); + }); + } + const manifestAuthChoice = params.manifestAuthChoice; if (manifestAuthChoice) { it("keeps onboarding auth grouping explicit", () => { diff --git a/test/helpers/plugins/plugin-runtime-mock.ts b/test/helpers/plugins/plugin-runtime-mock.ts index 295d6c0871a..674feaf8d4d 100644 --- a/test/helpers/plugins/plugin-runtime-mock.ts +++ b/test/helpers/plugins/plugin-runtime-mock.ts @@ -162,6 +162,10 @@ export function createPluginRuntimeMock(overrides: DeepPartial = generate: vi.fn() as unknown as PluginRuntime["imageGeneration"]["generate"], listProviders: vi.fn() as unknown as PluginRuntime["imageGeneration"]["listProviders"], }, + videoGeneration: { + generate: vi.fn() as unknown as PluginRuntime["videoGeneration"]["generate"], + listProviders: vi.fn() as unknown as PluginRuntime["videoGeneration"]["listProviders"], + }, webSearch: { listProviders: vi.fn() as unknown as PluginRuntime["webSearch"]["listProviders"], search: vi.fn() as unknown as PluginRuntime["webSearch"]["search"], diff --git a/test/helpers/plugins/provider-discovery-contract.ts b/test/helpers/plugins/provider-discovery-contract.ts index f3b07d7eec1..5baa392c978 100644 --- a/test/helpers/plugins/provider-discovery-contract.ts +++ b/test/helpers/plugins/provider-discovery-contract.ts @@ -28,8 +28,7 @@ const bundledProviderModules = vi.hoisted(() => ({ import.meta.url, ).pathname, minimaxIndexModuleUrl: new URL("../../../extensions/minimax/index.ts", import.meta.url).href, - modelStudioIndexModuleUrl: new URL("../../../extensions/modelstudio/index.ts", import.meta.url) - .href, + modelStudioIndexModuleUrl: new URL("../../../extensions/qwen/index.ts", import.meta.url).href, ollamaApiModuleId: new URL("../../../extensions/ollama/api.js", import.meta.url).pathname, ollamaIndexModuleUrl: new URL("../../../extensions/ollama/index.ts", import.meta.url).href, sglangApiModuleId: new URL("../../../extensions/sglang/api.js", import.meta.url).pathname, @@ -230,7 +229,7 @@ function installDiscoveryHooks(state: DiscoveryState) { state.sglangProvider = requireProvider(sglangProviders, "sglang"); state.minimaxProvider = requireProvider(minimaxProviders, "minimax"); state.minimaxPortalProvider = requireProvider(minimaxProviders, "minimax-portal"); - state.modelStudioProvider = requireProvider(modelStudioProviders, "modelstudio"); + state.modelStudioProvider = requireProvider(modelStudioProviders, "qwen"); state.cloudflareAiGatewayProvider = requireProvider( cloudflareAiGatewayProviders, "cloudflare-ai-gateway", diff --git a/test/setup-openclaw-runtime.ts b/test/setup-openclaw-runtime.ts index b1fe15f9762..2d81772bde3 100644 --- a/test/setup-openclaw-runtime.ts +++ b/test/setup-openclaw-runtime.ts @@ -129,6 +129,7 @@ function createTestRegistryForSetup( realtimeVoiceProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], + videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], gatewayHandlers: {}, diff --git a/vitest.extension-provider-paths.mjs b/vitest.extension-provider-paths.mjs index 7e7c884133d..b455f350c71 100644 --- a/vitest.extension-provider-paths.mjs +++ b/vitest.extension-provider-paths.mjs @@ -16,7 +16,7 @@ export const providerExtensionIds = [ "microsoft-foundry", "minimax", "mistral", - "modelstudio", + "qwen", "moonshot", "nvidia", "ollama",