feat(qwen): add qwen provider and video generation

This commit is contained in:
Peter Steinberger
2026-04-04 17:43:15 +01:00
parent 759373e887
commit e3ac0f43df
104 changed files with 2477 additions and 483 deletions

View File

@@ -1,4 +1,4 @@
ad87e3ff267b151ae163402f3cb52503e10641e332bcfbb6a574bbd7087a2484 config-baseline.json
4c880eb1ce03486f47aa21f49317ad15fc8d92bb720d70205743b72e45cf5fa3 config-baseline.json
03ff4a3e314f17dd8851aed3653269294bc62412bee05a6804dce840bd3d7551 config-baseline.core.json
73b57f395a2ad983f1660112d0b2b998342f1ddbe3089b440d7f73d0665de739 config-baseline.channel.json
9d5cb864e70768b66c1ecd881a9a584b7696ef2e5b32df686cfdc3fa21ddabbe config-baseline.plugin.json
17fd37605bf6cb087932ec2ebcfa9dd22e669fa6b8b93081ab2deac9d24821c5 config-baseline.plugin.json

View File

@@ -1,3 +1,4 @@
import { createAcpxRuntimeService } from "./register.runtime.js";
import type { OpenClawPluginApi } from "./runtime-api.js";
import { createAcpxPluginConfigSchema } from "./src/config-schema.js";
@@ -6,8 +7,7 @@ const plugin = {
name: "ACPX Runtime",
description: "ACP runtime backend powered by the acpx CLI.",
configSchema: () => createAcpxPluginConfigSchema(),
async register(api: OpenClawPluginApi) {
const { createAcpxRuntimeService } = await import("./register.runtime.js");
register(api: OpenClawPluginApi) {
api.registerService(
createAcpxRuntimeService({
pluginConfig: api.pluginConfig,

View File

@@ -1,11 +1,11 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { registerAmazonBedrockPlugin } from "./register.runtime.js";
export default definePluginEntry({
id: "amazon-bedrock",
name: "Amazon Bedrock Provider",
description: "Bundled Amazon Bedrock provider policy plugin",
async register(api) {
const { registerAmazonBedrockPlugin } = await import("./register.runtime.js");
await registerAmazonBedrockPlugin(api);
register(api) {
registerAmazonBedrockPlugin(api);
},
});

View File

@@ -1,11 +1,11 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { registerAnthropicPlugin } from "./register.runtime.js";
export default definePluginEntry({
id: "anthropic",
name: "Anthropic Provider",
description: "Bundled Anthropic provider plugin",
async register(api) {
const { registerAnthropicPlugin } = await import("./register.runtime.js");
await registerAnthropicPlugin(api);
register(api) {
registerAnthropicPlugin(api);
},
});

View File

@@ -3,18 +3,18 @@ import {
type OpenClawPluginToolContext,
type OpenClawPluginToolFactory,
} from "openclaw/plugin-sdk/plugin-entry";
import {
createBrowserPluginService,
createBrowserTool,
handleBrowserGatewayRequest,
registerBrowserCli,
} from "./register.runtime.js";
export default definePluginEntry({
id: "browser",
name: "Browser",
description: "Default browser tool plugin",
async register(api) {
const {
createBrowserPluginService,
createBrowserTool,
handleBrowserGatewayRequest,
registerBrowserCli,
} = await import("./register.runtime.js");
register(api) {
api.registerTool(((ctx: OpenClawPluginToolContext) =>
createBrowserTool({
sandboxBridgeUrl: ctx.browser?.sandboxBridgeUrl,

View File

@@ -28,7 +28,6 @@ export default definePluginEntry({
resolveCopilotForwardCompatModel,
wrapCopilotProviderStream,
} = await import("./register.runtime.js");
function resolveFirstGithubToken(params: { agentDir?: string; env: NodeJS.ProcessEnv }): {
githubToken: string;
hasProfile: boolean;

View File

@@ -191,6 +191,13 @@ function createLazyGoogleMediaUnderstandingProvider(): MediaUnderstandingProvide
return {
id: "google",
capabilities: ["image", "audio", "video"],
defaultModels: {
image: "gemini-3-flash-preview",
audio: "gemini-3-flash-preview",
video: "gemini-3-flash-preview",
},
autoPriority: { image: 30, audio: 40, video: 10 },
nativeDocumentInputs: ["pdf"],
describeImage: async (...args) =>
await (await loadGoogleRequiredMediaUnderstandingProvider()).describeImage(...args),
describeImages: async (...args) =>

View File

@@ -1,11 +1,11 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { buildMicrosoftSpeechProvider } from "./speech-provider.js";
export default definePluginEntry({
id: "microsoft",
name: "Microsoft Speech",
description: "Bundled Microsoft speech provider",
async register(api) {
const { buildMicrosoftSpeechProvider } = await import("./speech-provider.js");
register(api) {
api.registerSpeechProvider(buildMicrosoftSpeechProvider());
},
});

View File

@@ -1,118 +0,0 @@
import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";
import { applyModelStudioNativeStreamingUsageCompat } from "./api.js";
import {
applyModelStudioConfig,
applyModelStudioConfigCn,
applyModelStudioStandardConfig,
applyModelStudioStandardConfigCn,
MODELSTUDIO_DEFAULT_MODEL_REF,
} from "./onboard.js";
import { buildModelStudioProvider } from "./provider-catalog.js";
const PROVIDER_ID = "modelstudio";
export default defineSingleProviderPluginEntry({
id: PROVIDER_ID,
name: "Model Studio Provider",
description: "Bundled Model Studio provider plugin",
provider: {
label: "Model Studio",
docsPath: "/providers/models",
auth: [
{
methodId: "standard-api-key-cn",
label: "Standard API Key for China (pay-as-you-go)",
hint: "Endpoint: dashscope.aliyuncs.com",
optionKey: "modelstudioStandardApiKeyCn",
flagName: "--modelstudio-standard-api-key-cn",
envVar: "MODELSTUDIO_API_KEY",
promptMessage: "Enter Alibaba Cloud Model Studio API key (China)",
defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyModelStudioStandardConfigCn(cfg),
noteMessage: [
"Get your API key at: https://bailian.console.aliyun.com/",
"Endpoint: dashscope.aliyuncs.com/compatible-mode/v1",
"Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.",
].join("\n"),
noteTitle: "Alibaba Cloud Model Studio Standard (China)",
wizard: {
choiceHint: "Endpoint: dashscope.aliyuncs.com",
groupLabel: "Qwen (Alibaba Cloud Model Studio)",
groupHint: "Standard / Coding Plan (CN / Global)",
},
},
{
methodId: "standard-api-key",
label: "Standard API Key for Global/Intl (pay-as-you-go)",
hint: "Endpoint: dashscope-intl.aliyuncs.com",
optionKey: "modelstudioStandardApiKey",
flagName: "--modelstudio-standard-api-key",
envVar: "MODELSTUDIO_API_KEY",
promptMessage: "Enter Alibaba Cloud Model Studio API key (Global/Intl)",
defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyModelStudioStandardConfig(cfg),
noteMessage: [
"Get your API key at: https://modelstudio.console.alibabacloud.com/",
"Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1",
"Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.",
].join("\n"),
noteTitle: "Alibaba Cloud Model Studio Standard (Global/Intl)",
wizard: {
choiceHint: "Endpoint: dashscope-intl.aliyuncs.com",
groupLabel: "Qwen (Alibaba Cloud Model Studio)",
groupHint: "Standard / Coding Plan (CN / Global)",
},
},
{
methodId: "api-key-cn",
label: "Coding Plan API Key for China (subscription)",
hint: "Endpoint: coding.dashscope.aliyuncs.com",
optionKey: "modelstudioApiKeyCn",
flagName: "--modelstudio-api-key-cn",
envVar: "MODELSTUDIO_API_KEY",
promptMessage: "Enter Alibaba Cloud Model Studio Coding Plan API key (China)",
defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyModelStudioConfigCn(cfg),
noteMessage: [
"Get your API key at: https://bailian.console.aliyun.com/",
"Endpoint: coding.dashscope.aliyuncs.com",
"Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.",
].join("\n"),
noteTitle: "Alibaba Cloud Model Studio Coding Plan (China)",
wizard: {
choiceHint: "Endpoint: coding.dashscope.aliyuncs.com",
groupLabel: "Qwen (Alibaba Cloud Model Studio)",
groupHint: "Standard / Coding Plan (CN / Global)",
},
},
{
methodId: "api-key",
label: "Coding Plan API Key for Global/Intl (subscription)",
hint: "Endpoint: coding-intl.dashscope.aliyuncs.com",
optionKey: "modelstudioApiKey",
flagName: "--modelstudio-api-key",
envVar: "MODELSTUDIO_API_KEY",
promptMessage: "Enter Alibaba Cloud Model Studio Coding Plan API key (Global/Intl)",
defaultModel: MODELSTUDIO_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyModelStudioConfig(cfg),
noteMessage: [
"Get your API key at: https://bailian.console.aliyun.com/",
"Endpoint: coding-intl.dashscope.aliyuncs.com",
"Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.",
].join("\n"),
noteTitle: "Alibaba Cloud Model Studio Coding Plan (Global/Intl)",
wizard: {
choiceHint: "Endpoint: coding-intl.dashscope.aliyuncs.com",
groupLabel: "Qwen (Alibaba Cloud Model Studio)",
groupHint: "Standard / Coding Plan (CN / Global)",
},
},
],
catalog: {
buildProvider: buildModelStudioProvider,
allowExplicitBaseUrl: true,
},
applyNativeStreamingUsageCompat: ({ providerConfig }) =>
applyModelStudioNativeStreamingUsageCompat(providerConfig),
},
});

View File

@@ -1,69 +0,0 @@
import {
createModelCatalogPresetAppliers,
type OpenClawConfig,
} from "openclaw/plugin-sdk/provider-onboard";
import {
MODELSTUDIO_CN_BASE_URL,
MODELSTUDIO_DEFAULT_MODEL_REF,
MODELSTUDIO_GLOBAL_BASE_URL,
MODELSTUDIO_STANDARD_CN_BASE_URL,
MODELSTUDIO_STANDARD_GLOBAL_BASE_URL,
} from "./models.js";
import { buildModelStudioProvider } from "./provider-catalog.js";
export {
MODELSTUDIO_CN_BASE_URL,
MODELSTUDIO_DEFAULT_MODEL_REF,
MODELSTUDIO_GLOBAL_BASE_URL,
MODELSTUDIO_STANDARD_CN_BASE_URL,
MODELSTUDIO_STANDARD_GLOBAL_BASE_URL,
};
const modelStudioPresetAppliers = createModelCatalogPresetAppliers<[string]>({
primaryModelRef: MODELSTUDIO_DEFAULT_MODEL_REF,
resolveParams: (_cfg: OpenClawConfig, baseUrl: string) => {
const provider = buildModelStudioProvider();
return {
providerId: "modelstudio",
api: provider.api ?? "openai-completions",
baseUrl,
catalogModels: provider.models ?? [],
aliases: [
...(provider.models ?? []).map((model) => `modelstudio/${model.id}`),
{ modelRef: MODELSTUDIO_DEFAULT_MODEL_REF, alias: "Qwen" },
],
};
},
});
export function applyModelStudioProviderConfig(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_GLOBAL_BASE_URL);
}
export function applyModelStudioProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_CN_BASE_URL);
}
export function applyModelStudioConfig(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_GLOBAL_BASE_URL);
}
export function applyModelStudioConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_CN_BASE_URL);
}
export function applyModelStudioStandardProviderConfig(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_STANDARD_GLOBAL_BASE_URL);
}
export function applyModelStudioStandardProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyProviderConfig(cfg, MODELSTUDIO_STANDARD_CN_BASE_URL);
}
export function applyModelStudioStandardConfig(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_STANDARD_GLOBAL_BASE_URL);
}
export function applyModelStudioStandardConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return modelStudioPresetAppliers.applyConfig(cfg, MODELSTUDIO_STANDARD_CN_BASE_URL);
}

View File

@@ -1,71 +0,0 @@
{
"id": "modelstudio",
"enabledByDefault": true,
"providers": ["modelstudio"],
"providerAuthEnvVars": {
"modelstudio": ["MODELSTUDIO_API_KEY"]
},
"providerAuthChoices": [
{
"provider": "modelstudio",
"method": "standard-api-key-cn",
"choiceId": "modelstudio-standard-api-key-cn",
"choiceLabel": "Standard API Key for China (pay-as-you-go)",
"choiceHint": "Endpoint: dashscope.aliyuncs.com",
"groupId": "modelstudio",
"groupLabel": "Qwen (Alibaba Cloud Model Studio)",
"groupHint": "Standard / Coding Plan (CN / Global)",
"optionKey": "modelstudioStandardApiKeyCn",
"cliFlag": "--modelstudio-standard-api-key-cn",
"cliOption": "--modelstudio-standard-api-key-cn <key>",
"cliDescription": "Alibaba Cloud Model Studio Standard API key (China)"
},
{
"provider": "modelstudio",
"method": "standard-api-key",
"choiceId": "modelstudio-standard-api-key",
"choiceLabel": "Standard API Key for Global/Intl (pay-as-you-go)",
"choiceHint": "Endpoint: dashscope-intl.aliyuncs.com",
"groupId": "modelstudio",
"groupLabel": "Qwen (Alibaba Cloud Model Studio)",
"groupHint": "Standard / Coding Plan (CN / Global)",
"optionKey": "modelstudioStandardApiKey",
"cliFlag": "--modelstudio-standard-api-key",
"cliOption": "--modelstudio-standard-api-key <key>",
"cliDescription": "Alibaba Cloud Model Studio Standard API key (Global/Intl)"
},
{
"provider": "modelstudio",
"method": "api-key-cn",
"choiceId": "modelstudio-api-key-cn",
"choiceLabel": "Coding Plan API Key for China (subscription)",
"choiceHint": "Endpoint: coding.dashscope.aliyuncs.com",
"groupId": "modelstudio",
"groupLabel": "Qwen (Alibaba Cloud Model Studio)",
"groupHint": "Standard / Coding Plan (CN / Global)",
"optionKey": "modelstudioApiKeyCn",
"cliFlag": "--modelstudio-api-key-cn",
"cliOption": "--modelstudio-api-key-cn <key>",
"cliDescription": "Alibaba Cloud Model Studio Coding Plan API key (China)"
},
{
"provider": "modelstudio",
"method": "api-key",
"choiceId": "modelstudio-api-key",
"choiceLabel": "Coding Plan API Key for Global/Intl (subscription)",
"choiceHint": "Endpoint: coding-intl.dashscope.aliyuncs.com",
"groupId": "modelstudio",
"groupLabel": "Qwen (Alibaba Cloud Model Studio)",
"groupHint": "Standard / Coding Plan (CN / Global)",
"optionKey": "modelstudioApiKey",
"cliFlag": "--modelstudio-api-key",
"cliOption": "--modelstudio-api-key <key>",
"cliDescription": "Alibaba Cloud Model Studio Coding Plan API key (Global/Intl)"
}
],
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {}
}
}

View File

@@ -1,36 +0,0 @@
import { describe, expect, it } from "vitest";
import {
applyModelStudioNativeStreamingUsageCompat,
buildModelStudioProvider,
MODELSTUDIO_BASE_URL,
MODELSTUDIO_DEFAULT_MODEL_ID,
} from "./api.js";
describe("modelstudio provider catalog", () => {
it("builds the bundled Model Studio provider defaults", () => {
const provider = buildModelStudioProvider();
expect(provider.baseUrl).toBe(MODELSTUDIO_BASE_URL);
expect(provider.api).toBe("openai-completions");
expect(provider.models?.length).toBeGreaterThan(0);
expect(
provider.models?.find((model) => model.id === MODELSTUDIO_DEFAULT_MODEL_ID),
).toBeTruthy();
expect(provider.models?.find((model) => model.id === "qwen3.6-plus")).toBeTruthy();
});
it("opts native Model Studio baseUrls into streaming usage only inside the extension", () => {
const nativeProvider = applyModelStudioNativeStreamingUsageCompat(buildModelStudioProvider());
expect(
nativeProvider.models?.every((model) => model.compat?.supportsUsageInStreaming === true),
).toBe(true);
const customProvider = applyModelStudioNativeStreamingUsageCompat({
...buildModelStudioProvider(),
baseUrl: "https://proxy.example.com/v1",
});
expect(
customProvider.models?.some((model) => model.compat?.supportsUsageInStreaming === true),
).toBe(false);
});
});

View File

@@ -1,10 +0,0 @@
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import { MODELSTUDIO_BASE_URL, MODELSTUDIO_MODEL_CATALOG } from "./models.js";
export function buildModelStudioProvider(): ModelProviderConfig {
return {
baseUrl: MODELSTUDIO_BASE_URL,
api: "openai-completions",
models: MODELSTUDIO_MODEL_CATALOG.map((model) => ({ ...model })),
};
}

View File

@@ -1,53 +1,36 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { buildOpenAICodexCliBackend } from "./cli-backend.js";
import { buildOpenAIImageGenerationProvider } from "./image-generation-provider.js";
import {
openaiCodexMediaUnderstandingProvider,
openaiMediaUnderstandingProvider,
} from "./media-understanding-provider.js";
import { buildOpenAICodexProviderPlugin } from "./openai-codex-provider.js";
import { buildOpenAIProvider } from "./openai-provider.js";
import {
OPENAI_FRIENDLY_PROMPT_OVERLAY,
resolveOpenAIPromptOverlayMode,
shouldApplyOpenAIPromptOverlay,
} from "./prompt-overlay.js";
import { buildOpenAIRealtimeTranscriptionProvider } from "./realtime-transcription-provider.js";
import { buildOpenAIRealtimeVoiceProvider } from "./realtime-voice-provider.js";
import { buildOpenAISpeechProvider } from "./speech-provider.js";
export default definePluginEntry({
id: "openai",
name: "OpenAI Provider",
description: "Bundled OpenAI provider plugins",
async register(api) {
const { buildOpenAICodexCliBackend } = await import("./cli-backend.js");
const { buildOpenAICodexProviderPlugin } = await import("./openai-codex-provider.js");
const { buildOpenAIProvider } = await import("./openai-provider.js");
const {
OPENAI_FRIENDLY_PROMPT_OVERLAY,
resolveOpenAIPromptOverlayMode,
shouldApplyOpenAIPromptOverlay,
} = await import("./prompt-overlay.js");
const registerOptional = async (registerFn: () => Promise<void>) => {
try {
await registerFn();
} catch {
// Optional OpenAI surfaces must not block core provider registration.
}
};
register(api) {
const promptOverlayMode = resolveOpenAIPromptOverlayMode(api.pluginConfig);
api.registerCliBackend(buildOpenAICodexCliBackend());
api.registerProvider(buildOpenAIProvider());
api.registerProvider(buildOpenAICodexProviderPlugin());
await registerOptional(async () => {
const { buildOpenAIImageGenerationProvider } = await import("./image-generation-provider.js");
api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider());
});
await registerOptional(async () => {
const { buildOpenAIRealtimeTranscriptionProvider } =
await import("./realtime-transcription-provider.js");
api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider());
});
await registerOptional(async () => {
const { buildOpenAIRealtimeVoiceProvider } = await import("./realtime-voice-provider.js");
api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider());
});
await registerOptional(async () => {
const { buildOpenAISpeechProvider } = await import("./speech-provider.js");
api.registerSpeechProvider(buildOpenAISpeechProvider());
});
await registerOptional(async () => {
const { openaiMediaUnderstandingProvider, openaiCodexMediaUnderstandingProvider } =
await import("./media-understanding-provider.js");
api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider);
api.registerMediaUnderstandingProvider(openaiCodexMediaUnderstandingProvider);
});
api.registerImageGenerationProvider(buildOpenAIImageGenerationProvider());
api.registerRealtimeTranscriptionProvider(buildOpenAIRealtimeTranscriptionProvider());
api.registerRealtimeVoiceProvider(buildOpenAIRealtimeVoiceProvider());
api.registerSpeechProvider(buildOpenAISpeechProvider());
api.registerMediaUnderstandingProvider(openaiMediaUnderstandingProvider);
api.registerMediaUnderstandingProvider(openaiCodexMediaUnderstandingProvider);
if (promptOverlayMode !== "off") {
api.on("before_prompt_build", (_event, ctx) =>
shouldApplyOpenAIPromptOverlay({

View File

@@ -1,4 +1,17 @@
export {
applyQwenNativeStreamingUsageCompat,
buildQwenDefaultModelDefinition,
buildQwenModelDefinition,
isNativeQwenBaseUrl,
QWEN_BASE_URL,
QWEN_CN_BASE_URL,
QWEN_DEFAULT_COST,
QWEN_DEFAULT_MODEL_ID,
QWEN_DEFAULT_MODEL_REF,
QWEN_GLOBAL_BASE_URL,
QWEN_STANDARD_CN_BASE_URL,
QWEN_STANDARD_GLOBAL_BASE_URL,
QWEN_MODEL_CATALOG,
applyModelStudioNativeStreamingUsageCompat,
buildModelStudioDefaultModelDefinition,
buildModelStudioModelDefinition,
@@ -13,4 +26,4 @@ export {
MODELSTUDIO_STANDARD_GLOBAL_BASE_URL,
MODELSTUDIO_MODEL_CATALOG,
} from "./models.js";
export { buildModelStudioProvider } from "./provider-catalog.js";
export { buildModelStudioProvider, buildQwenProvider } from "./provider-catalog.js";

129
extensions/qwen/index.ts Normal file
View File

@@ -0,0 +1,129 @@
import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";
import { applyQwenNativeStreamingUsageCompat } from "./api.js";
import { buildQwenMediaUnderstandingProvider } from "./media-understanding-provider.js";
import {
applyQwenConfig,
applyQwenConfigCn,
applyQwenStandardConfig,
applyQwenStandardConfigCn,
QWEN_DEFAULT_MODEL_REF,
} from "./onboard.js";
import { buildQwenProvider } from "./provider-catalog.js";
import { buildQwenVideoGenerationProvider } from "./video-generation-provider.js";
const PROVIDER_ID = "qwen";
export default defineSingleProviderPluginEntry({
id: PROVIDER_ID,
name: "Qwen Provider",
description: "Bundled Qwen Cloud provider plugin",
provider: {
label: "Qwen Cloud",
docsPath: "/providers/qwen",
aliases: ["modelstudio", "qwencloud"],
auth: [
{
methodId: "standard-api-key-cn",
label: "Standard API Key for China (pay-as-you-go)",
hint: "Endpoint: dashscope.aliyuncs.com",
optionKey: "modelstudioStandardApiKeyCn",
flagName: "--modelstudio-standard-api-key-cn",
envVar: "QWEN_API_KEY",
promptMessage: "Enter Qwen Cloud API key (China standard endpoint)",
defaultModel: QWEN_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyQwenStandardConfigCn(cfg),
noteMessage: [
"Manage API keys: https://home.qwencloud.com/api-keys",
"Docs: https://docs.qwencloud.com/",
"Endpoint: dashscope.aliyuncs.com/compatible-mode/v1",
"Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.",
].join("\n"),
noteTitle: "Qwen Cloud Standard (China)",
wizard: {
choiceHint: "Endpoint: dashscope.aliyuncs.com",
groupLabel: "Qwen Cloud",
groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap",
},
},
{
methodId: "standard-api-key",
label: "Standard API Key for Global/Intl (pay-as-you-go)",
hint: "Endpoint: dashscope-intl.aliyuncs.com",
optionKey: "modelstudioStandardApiKey",
flagName: "--modelstudio-standard-api-key",
envVar: "QWEN_API_KEY",
promptMessage: "Enter Qwen Cloud API key (Global/Intl standard endpoint)",
defaultModel: QWEN_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyQwenStandardConfig(cfg),
noteMessage: [
"Manage API keys: https://home.qwencloud.com/api-keys",
"Docs: https://docs.qwencloud.com/",
"Endpoint: dashscope-intl.aliyuncs.com/compatible-mode/v1",
"Models: qwen3.6-plus, qwen3.5-plus, qwen3-coder-plus, etc.",
].join("\n"),
noteTitle: "Qwen Cloud Standard (Global/Intl)",
wizard: {
choiceHint: "Endpoint: dashscope-intl.aliyuncs.com",
groupLabel: "Qwen Cloud",
groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap",
},
},
{
methodId: "api-key-cn",
label: "Coding Plan API Key for China (subscription)",
hint: "Endpoint: coding.dashscope.aliyuncs.com",
optionKey: "modelstudioApiKeyCn",
flagName: "--modelstudio-api-key-cn",
envVar: "QWEN_API_KEY",
promptMessage: "Enter Qwen Cloud Coding Plan API key (China)",
defaultModel: QWEN_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyQwenConfigCn(cfg),
noteMessage: [
"Manage API keys: https://home.qwencloud.com/api-keys",
"Docs: https://docs.qwencloud.com/",
"Endpoint: coding.dashscope.aliyuncs.com",
"Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.",
].join("\n"),
noteTitle: "Qwen Cloud Coding Plan (China)",
wizard: {
choiceHint: "Endpoint: coding.dashscope.aliyuncs.com",
groupLabel: "Qwen Cloud",
groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap",
},
},
{
methodId: "api-key",
label: "Coding Plan API Key for Global/Intl (subscription)",
hint: "Endpoint: coding-intl.dashscope.aliyuncs.com",
optionKey: "modelstudioApiKey",
flagName: "--modelstudio-api-key",
envVar: "QWEN_API_KEY",
promptMessage: "Enter Qwen Cloud Coding Plan API key (Global/Intl)",
defaultModel: QWEN_DEFAULT_MODEL_REF,
applyConfig: (cfg) => applyQwenConfig(cfg),
noteMessage: [
"Manage API keys: https://home.qwencloud.com/api-keys",
"Docs: https://docs.qwencloud.com/",
"Endpoint: coding-intl.dashscope.aliyuncs.com",
"Models: qwen3.6-plus, glm-5, kimi-k2.5, MiniMax-M2.5, etc.",
].join("\n"),
noteTitle: "Qwen Cloud Coding Plan (Global/Intl)",
wizard: {
choiceHint: "Endpoint: coding-intl.dashscope.aliyuncs.com",
groupLabel: "Qwen Cloud",
groupHint: "Standard / Coding Plan (CN / Global) + multimodal roadmap",
},
},
],
catalog: {
buildProvider: buildQwenProvider,
allowExplicitBaseUrl: true,
},
applyNativeStreamingUsageCompat: ({ providerConfig }) =>
applyQwenNativeStreamingUsageCompat(providerConfig),
},
register(api) {
api.registerMediaUnderstandingProvider(buildQwenMediaUnderstandingProvider());
api.registerVideoGenerationProvider(buildQwenVideoGenerationProvider());
},
});

View File

@@ -0,0 +1,56 @@
import { describe, expect, it } from "vitest";
import { createRequestCaptureJsonFetch } from "../../test/helpers/plugins/media-understanding.js";
import { describeQwenVideo } from "./media-understanding-provider.js";
describe("describeQwenVideo", () => {
it("builds the expected OpenAI-compatible video payload", async () => {
const { fetchFn, getRequest } = createRequestCaptureJsonFetch({
choices: [
{
message: {
content: [{ text: " first " }, { text: "second" }],
},
},
],
});
const result = await describeQwenVideo({
buffer: Buffer.from("video-bytes"),
fileName: "clip.mp4",
mime: "video/mp4",
apiKey: "test-key",
timeoutMs: 1500,
baseUrl: "https://example.com/v1",
model: "qwen-vl-max",
prompt: "summarize the clip",
headers: { "X-Other": "1" },
fetchFn,
});
const { url, init } = getRequest();
expect(result.model).toBe("qwen-vl-max");
expect(result.text).toBe("first\nsecond");
expect(url).toBe("https://example.com/v1/chat/completions");
expect(init?.method).toBe("POST");
expect(init?.signal).toBeInstanceOf(AbortSignal);
const headers = new Headers(init?.headers);
expect(headers.get("authorization")).toBe("Bearer test-key");
expect(headers.get("content-type")).toBe("application/json");
expect(headers.get("x-other")).toBe("1");
const bodyText =
typeof init?.body === "string"
? init.body
: Buffer.isBuffer(init?.body)
? init.body.toString("utf8")
: "";
const body = JSON.parse(bodyText);
expect(body.model).toBe("qwen-vl-max");
expect(body.messages?.[0]?.content?.[0]?.text).toBe("summarize the clip");
expect(body.messages?.[0]?.content?.[1]?.type).toBe("video_url");
expect(body.messages?.[0]?.content?.[1]?.video_url?.url).toBe(
`data:video/mp4;base64,${Buffer.from("video-bytes").toString("base64")}`,
);
});
});

View File

@@ -0,0 +1,156 @@
import {
describeImageWithModel,
describeImagesWithModel,
type MediaUnderstandingProvider,
type VideoDescriptionRequest,
type VideoDescriptionResult,
} from "openclaw/plugin-sdk/media-understanding";
import {
assertOkOrThrowHttpError,
postJsonRequest,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import { QWEN_STANDARD_CN_BASE_URL, QWEN_STANDARD_GLOBAL_BASE_URL } from "./models.js";
const DEFAULT_QWEN_VIDEO_MODEL = "qwen-vl-max-latest";
const DEFAULT_QWEN_VIDEO_PROMPT = "Describe the video in detail.";
type QwenVideoPayload = {
choices?: Array<{
message?: {
content?: string | Array<{ text?: string }>;
reasoning_content?: string;
};
}>;
};
function resolveQwenStandardBaseUrl(
cfg: { models?: { providers?: Record<string, { baseUrl?: string } | undefined> } } | undefined,
providerId: string,
): string {
const direct = cfg?.models?.providers?.[providerId]?.baseUrl?.trim();
if (!direct) {
return QWEN_STANDARD_GLOBAL_BASE_URL;
}
try {
const url = new URL(direct);
if (url.hostname === "coding-intl.dashscope.aliyuncs.com") {
return QWEN_STANDARD_GLOBAL_BASE_URL;
}
if (url.hostname === "coding.dashscope.aliyuncs.com") {
return QWEN_STANDARD_CN_BASE_URL;
}
return `${url.origin}${url.pathname}`.replace(/\/+$/u, "");
} catch {
return QWEN_STANDARD_GLOBAL_BASE_URL;
}
}
function coerceQwenText(payload: QwenVideoPayload): string | null {
const message = payload.choices?.[0]?.message;
if (!message) {
return null;
}
if (typeof message.content === "string" && message.content.trim()) {
return message.content.trim();
}
if (Array.isArray(message.content)) {
const text = message.content
.map((part) => (typeof part.text === "string" ? part.text.trim() : ""))
.filter(Boolean)
.join("\n")
.trim();
if (text) {
return text;
}
}
if (typeof message.reasoning_content === "string" && message.reasoning_content.trim()) {
return message.reasoning_content.trim();
}
return null;
}
export async function describeQwenVideo(
params: VideoDescriptionRequest,
): Promise<VideoDescriptionResult> {
const fetchFn = params.fetchFn ?? fetch;
const model = params.model?.trim() || DEFAULT_QWEN_VIDEO_MODEL;
const mime = params.mime?.trim() || "video/mp4";
const prompt = params.prompt?.trim() || DEFAULT_QWEN_VIDEO_PROMPT;
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: params.baseUrl,
defaultBaseUrl: QWEN_STANDARD_GLOBAL_BASE_URL,
headers: params.headers,
request: params.request,
defaultHeaders: {
"content-type": "application/json",
authorization: `Bearer ${params.apiKey}`,
},
provider: "qwen",
api: "openai-completions",
capability: "video",
transport: "media-understanding",
});
const { response: res, release } = await postJsonRequest({
url: `${baseUrl}/chat/completions`,
headers,
body: {
model,
messages: [
{
role: "user",
content: [
{ type: "text", text: prompt },
{
type: "video_url",
video_url: {
url: `data:${mime};base64,${params.buffer.toString("base64")}`,
},
},
],
},
],
},
timeoutMs: params.timeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(res, "Qwen video description failed");
const payload = (await res.json()) as QwenVideoPayload;
const text = coerceQwenText(payload);
if (!text) {
throw new Error("Qwen video description response missing content");
}
return { text, model };
} finally {
await release();
}
}
export function buildQwenMediaUnderstandingProvider(): MediaUnderstandingProvider {
return {
id: "qwen",
capabilities: ["image", "video"],
defaultModels: {
image: "qwen-vl-max-latest",
video: DEFAULT_QWEN_VIDEO_MODEL,
},
autoPriority: {
video: 15,
},
describeImage: describeImageWithModel,
describeImages: describeImagesWithModel,
describeVideo: describeQwenVideo,
};
}
export function resolveQwenMediaUnderstandingBaseUrl(
cfg: { models?: { providers?: Record<string, { baseUrl?: string } | undefined> } } | undefined,
): string {
return resolveQwenStandardBaseUrl(cfg, "qwen");
}

View File

@@ -1,4 +1,13 @@
export {
buildQwenDefaultModelDefinition,
buildQwenModelDefinition,
QWEN_CN_BASE_URL,
QWEN_DEFAULT_COST,
QWEN_DEFAULT_MODEL_ID,
QWEN_DEFAULT_MODEL_REF,
QWEN_GLOBAL_BASE_URL,
QWEN_STANDARD_CN_BASE_URL,
QWEN_STANDARD_GLOBAL_BASE_URL,
buildModelStudioDefaultModelDefinition,
buildModelStudioModelDefinition,
MODELSTUDIO_CN_BASE_URL,

View File

@@ -7,29 +7,29 @@ import type {
ModelProviderConfig,
} from "openclaw/plugin-sdk/provider-model-shared";
export const MODELSTUDIO_BASE_URL = "https://coding-intl.dashscope.aliyuncs.com/v1";
export const MODELSTUDIO_GLOBAL_BASE_URL = MODELSTUDIO_BASE_URL;
export const MODELSTUDIO_CN_BASE_URL = "https://coding.dashscope.aliyuncs.com/v1";
export const MODELSTUDIO_STANDARD_CN_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
export const MODELSTUDIO_STANDARD_GLOBAL_BASE_URL =
export const QWEN_BASE_URL = "https://coding-intl.dashscope.aliyuncs.com/v1";
export const QWEN_GLOBAL_BASE_URL = QWEN_BASE_URL;
export const QWEN_CN_BASE_URL = "https://coding.dashscope.aliyuncs.com/v1";
export const QWEN_STANDARD_CN_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
export const QWEN_STANDARD_GLOBAL_BASE_URL =
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1";
export const MODELSTUDIO_DEFAULT_MODEL_ID = "qwen3.5-plus";
export const MODELSTUDIO_DEFAULT_COST = {
export const QWEN_DEFAULT_MODEL_ID = "qwen3.5-plus";
export const QWEN_DEFAULT_COST = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
};
export const MODELSTUDIO_DEFAULT_MODEL_REF = `modelstudio/${MODELSTUDIO_DEFAULT_MODEL_ID}`;
export const QWEN_DEFAULT_MODEL_REF = `qwen/${QWEN_DEFAULT_MODEL_ID}`;
export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
export const QWEN_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
{
id: "qwen3.5-plus",
name: "qwen3.5-plus",
reasoning: false,
input: ["text", "image"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 1_000_000,
maxTokens: 65_536,
},
@@ -38,7 +38,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "qwen3.6-plus",
reasoning: false,
input: ["text", "image"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 1_000_000,
maxTokens: 65_536,
},
@@ -47,7 +47,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "qwen3-max-2026-01-23",
reasoning: false,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 262_144,
maxTokens: 65_536,
},
@@ -56,7 +56,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "qwen3-coder-next",
reasoning: false,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 262_144,
maxTokens: 65_536,
},
@@ -65,7 +65,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "qwen3-coder-plus",
reasoning: false,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 1_000_000,
maxTokens: 65_536,
},
@@ -74,7 +74,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "MiniMax-M2.5",
reasoning: true,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 1_000_000,
maxTokens: 65_536,
},
@@ -83,7 +83,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "glm-5",
reasoning: false,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 202_752,
maxTokens: 16_384,
},
@@ -92,7 +92,7 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "glm-4.7",
reasoning: false,
input: ["text"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 202_752,
maxTokens: 16_384,
},
@@ -101,29 +101,29 @@ export const MODELSTUDIO_MODEL_CATALOG: ReadonlyArray<ModelDefinitionConfig> = [
name: "kimi-k2.5",
reasoning: false,
input: ["text", "image"],
cost: MODELSTUDIO_DEFAULT_COST,
cost: QWEN_DEFAULT_COST,
contextWindow: 262_144,
maxTokens: 32_768,
},
];
export function isNativeModelStudioBaseUrl(baseUrl: string | undefined): boolean {
export function isNativeQwenBaseUrl(baseUrl: string | undefined): boolean {
return supportsNativeStreamingUsageCompat({
providerId: "modelstudio",
providerId: "qwen",
baseUrl,
});
}
export function applyModelStudioNativeStreamingUsageCompat(
export function applyQwenNativeStreamingUsageCompat(
provider: ModelProviderConfig,
): ModelProviderConfig {
return applyProviderNativeStreamingUsageCompat({
providerId: "modelstudio",
providerId: "qwen",
providerConfig: provider,
});
}
export function buildModelStudioModelDefinition(params: {
export function buildQwenModelDefinition(params: {
id: string;
name?: string;
reasoning?: boolean;
@@ -132,19 +132,34 @@ export function buildModelStudioModelDefinition(params: {
contextWindow?: number;
maxTokens?: number;
}): ModelDefinitionConfig {
const catalog = MODELSTUDIO_MODEL_CATALOG.find((model) => model.id === params.id);
const catalog = QWEN_MODEL_CATALOG.find((model) => model.id === params.id);
return {
id: params.id,
name: params.name ?? catalog?.name ?? params.id,
reasoning: params.reasoning ?? catalog?.reasoning ?? false,
input:
(params.input as ("text" | "image")[]) ?? (catalog?.input ? [...catalog.input] : ["text"]),
cost: params.cost ?? catalog?.cost ?? MODELSTUDIO_DEFAULT_COST,
cost: params.cost ?? catalog?.cost ?? QWEN_DEFAULT_COST,
contextWindow: params.contextWindow ?? catalog?.contextWindow ?? 262_144,
maxTokens: params.maxTokens ?? catalog?.maxTokens ?? 65_536,
};
}
export function buildModelStudioDefaultModelDefinition(): ModelDefinitionConfig {
return buildModelStudioModelDefinition({ id: MODELSTUDIO_DEFAULT_MODEL_ID });
export function buildQwenDefaultModelDefinition(): ModelDefinitionConfig {
return buildQwenModelDefinition({ id: QWEN_DEFAULT_MODEL_ID });
}
// Backward-compatible aliases while `modelstudio` references are still in the wild.
export const MODELSTUDIO_BASE_URL = QWEN_BASE_URL;
export const MODELSTUDIO_GLOBAL_BASE_URL = QWEN_GLOBAL_BASE_URL;
export const MODELSTUDIO_CN_BASE_URL = QWEN_CN_BASE_URL;
export const MODELSTUDIO_STANDARD_CN_BASE_URL = QWEN_STANDARD_CN_BASE_URL;
export const MODELSTUDIO_STANDARD_GLOBAL_BASE_URL = QWEN_STANDARD_GLOBAL_BASE_URL;
export const MODELSTUDIO_DEFAULT_MODEL_ID = QWEN_DEFAULT_MODEL_ID;
export const MODELSTUDIO_DEFAULT_COST = QWEN_DEFAULT_COST;
export const MODELSTUDIO_DEFAULT_MODEL_REF = `modelstudio/${QWEN_DEFAULT_MODEL_ID}`;
export const MODELSTUDIO_MODEL_CATALOG = QWEN_MODEL_CATALOG;
export const isNativeModelStudioBaseUrl = isNativeQwenBaseUrl;
export const applyModelStudioNativeStreamingUsageCompat = applyQwenNativeStreamingUsageCompat;
export const buildModelStudioModelDefinition = buildQwenModelDefinition;
export const buildModelStudioDefaultModelDefinition = buildQwenDefaultModelDefinition;

View File

@@ -0,0 +1,81 @@
import {
createModelCatalogPresetAppliers,
type OpenClawConfig,
} from "openclaw/plugin-sdk/provider-onboard";
import {
QWEN_CN_BASE_URL,
QWEN_DEFAULT_MODEL_REF,
QWEN_GLOBAL_BASE_URL,
QWEN_STANDARD_CN_BASE_URL,
QWEN_STANDARD_GLOBAL_BASE_URL,
} from "./models.js";
import { buildQwenProvider } from "./provider-catalog.js";
export {
QWEN_CN_BASE_URL,
QWEN_DEFAULT_MODEL_REF,
QWEN_GLOBAL_BASE_URL,
QWEN_STANDARD_CN_BASE_URL,
QWEN_STANDARD_GLOBAL_BASE_URL,
};
const qwenPresetAppliers = createModelCatalogPresetAppliers<[string]>({
primaryModelRef: QWEN_DEFAULT_MODEL_REF,
resolveParams: (_cfg: OpenClawConfig, baseUrl: string) => {
const provider = buildQwenProvider();
return {
providerId: "qwen",
api: provider.api ?? "openai-completions",
baseUrl,
catalogModels: provider.models ?? [],
aliases: [
...(provider.models ?? []).flatMap((model) => [
`qwen/${model.id}`,
`modelstudio/${model.id}`,
]),
{ modelRef: QWEN_DEFAULT_MODEL_REF, alias: "Qwen" },
],
};
},
});
export function applyQwenProviderConfig(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_GLOBAL_BASE_URL);
}
export function applyQwenProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_CN_BASE_URL);
}
export function applyQwenConfig(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyConfig(cfg, QWEN_GLOBAL_BASE_URL);
}
export function applyQwenConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyConfig(cfg, QWEN_CN_BASE_URL);
}
export function applyQwenStandardProviderConfig(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_STANDARD_GLOBAL_BASE_URL);
}
export function applyQwenStandardProviderConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyProviderConfig(cfg, QWEN_STANDARD_CN_BASE_URL);
}
export function applyQwenStandardConfig(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyConfig(cfg, QWEN_STANDARD_GLOBAL_BASE_URL);
}
export function applyQwenStandardConfigCn(cfg: OpenClawConfig): OpenClawConfig {
return qwenPresetAppliers.applyConfig(cfg, QWEN_STANDARD_CN_BASE_URL);
}
export const applyModelStudioProviderConfig = applyQwenProviderConfig;
export const applyModelStudioProviderConfigCn = applyQwenProviderConfigCn;
export const applyModelStudioConfig = applyQwenConfig;
export const applyModelStudioConfigCn = applyQwenConfigCn;
export const applyModelStudioStandardProviderConfig = applyQwenStandardProviderConfig;
export const applyModelStudioStandardProviderConfigCn = applyQwenStandardProviderConfigCn;
export const applyModelStudioStandardConfig = applyQwenStandardConfig;
export const applyModelStudioStandardConfigCn = applyQwenStandardConfigCn;

View File

@@ -0,0 +1,79 @@
{
"id": "qwen",
"enabledByDefault": true,
"providers": ["qwen"],
"contracts": {
"mediaUnderstandingProviders": ["qwen"],
"videoGenerationProviders": ["qwen"]
},
"providerAuthEnvVars": {
"qwen": ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"]
},
"providerAuthChoices": [
{
"provider": "qwen",
"method": "standard-api-key-cn",
"choiceId": "qwen-standard-api-key-cn",
"deprecatedChoiceIds": ["modelstudio-standard-api-key-cn"],
"choiceLabel": "Standard API Key for China (pay-as-you-go)",
"choiceHint": "Endpoint: dashscope.aliyuncs.com",
"groupId": "qwen",
"groupLabel": "Qwen Cloud",
"groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap",
"optionKey": "modelstudioStandardApiKeyCn",
"cliFlag": "--modelstudio-standard-api-key-cn",
"cliOption": "--modelstudio-standard-api-key-cn <key>",
"cliDescription": "Qwen Cloud standard API key (China)"
},
{
"provider": "qwen",
"method": "standard-api-key",
"choiceId": "qwen-standard-api-key",
"deprecatedChoiceIds": ["modelstudio-standard-api-key"],
"choiceLabel": "Standard API Key for Global/Intl (pay-as-you-go)",
"choiceHint": "Endpoint: dashscope-intl.aliyuncs.com",
"groupId": "qwen",
"groupLabel": "Qwen Cloud",
"groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap",
"optionKey": "modelstudioStandardApiKey",
"cliFlag": "--modelstudio-standard-api-key",
"cliOption": "--modelstudio-standard-api-key <key>",
"cliDescription": "Qwen Cloud standard API key (Global/Intl)"
},
{
"provider": "qwen",
"method": "api-key-cn",
"choiceId": "qwen-api-key-cn",
"deprecatedChoiceIds": ["modelstudio-api-key-cn"],
"choiceLabel": "Coding Plan API Key for China (subscription)",
"choiceHint": "Endpoint: coding.dashscope.aliyuncs.com",
"groupId": "qwen",
"groupLabel": "Qwen Cloud",
"groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap",
"optionKey": "modelstudioApiKeyCn",
"cliFlag": "--modelstudio-api-key-cn",
"cliOption": "--modelstudio-api-key-cn <key>",
"cliDescription": "Qwen Cloud Coding Plan API key (China)"
},
{
"provider": "qwen",
"method": "api-key",
"choiceId": "qwen-api-key",
"deprecatedChoiceIds": ["modelstudio-api-key"],
"choiceLabel": "Coding Plan API Key for Global/Intl (subscription)",
"choiceHint": "Endpoint: coding-intl.dashscope.aliyuncs.com",
"groupId": "qwen",
"groupLabel": "Qwen Cloud",
"groupHint": "Standard / Coding Plan (CN / Global) + multimodal roadmap",
"optionKey": "modelstudioApiKey",
"cliFlag": "--modelstudio-api-key",
"cliOption": "--modelstudio-api-key <key>",
"cliDescription": "Qwen Cloud Coding Plan API key (Global/Intl)"
}
],
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {}
}
}

View File

@@ -1,8 +1,8 @@
{
"name": "@openclaw/modelstudio-provider",
"name": "@openclaw/qwen-provider",
"version": "2026.4.1-beta.1",
"private": true,
"description": "OpenClaw Model Studio provider plugin",
"description": "OpenClaw Qwen Cloud provider plugin",
"type": "module",
"openclaw": {
"extensions": [

View File

@@ -0,0 +1,10 @@
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
describePluginRegistrationContract({
pluginId: "qwen",
providerIds: ["qwen"],
mediaUnderstandingProviderIds: ["qwen"],
videoGenerationProviderIds: ["qwen"],
requireDescribeImages: true,
requireGenerateVideo: true,
});

View File

@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import {
applyQwenNativeStreamingUsageCompat,
buildQwenProvider,
QWEN_BASE_URL,
QWEN_DEFAULT_MODEL_ID,
} from "./api.js";
describe("qwen provider catalog", () => {
it("builds the bundled Qwen provider defaults", () => {
const provider = buildQwenProvider();
expect(provider.baseUrl).toBe(QWEN_BASE_URL);
expect(provider.api).toBe("openai-completions");
expect(provider.models?.length).toBeGreaterThan(0);
expect(provider.models?.find((model) => model.id === QWEN_DEFAULT_MODEL_ID)).toBeTruthy();
expect(provider.models?.find((model) => model.id === "qwen3.6-plus")).toBeTruthy();
});
it("opts native Qwen baseUrls into streaming usage only inside the extension", () => {
const nativeProvider = applyQwenNativeStreamingUsageCompat(buildQwenProvider());
expect(
nativeProvider.models?.every((model) => model.compat?.supportsUsageInStreaming === true),
).toBe(true);
const customProvider = applyQwenNativeStreamingUsageCompat({
...buildQwenProvider(),
baseUrl: "https://proxy.example.com/v1",
});
expect(
customProvider.models?.some((model) => model.compat?.supportsUsageInStreaming === true),
).toBe(false);
});
});

View File

@@ -0,0 +1,12 @@
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import { QWEN_BASE_URL, QWEN_MODEL_CATALOG } from "./models.js";
export function buildQwenProvider(): ModelProviderConfig {
return {
baseUrl: QWEN_BASE_URL,
api: "openai-completions",
models: QWEN_MODEL_CATALOG.map((model) => ({ ...model })),
};
}
export const buildModelStudioProvider = buildQwenProvider;

View File

@@ -0,0 +1,2 @@
export { buildQwenMediaUnderstandingProvider } from "./media-understanding-provider.js";
export { buildQwenVideoGenerationProvider } from "./video-generation-provider.js";

View File

@@ -0,0 +1,110 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildQwenVideoGenerationProvider } from "./video-generation-provider.js";
const {
resolveApiKeyForProviderMock,
postJsonRequestMock,
fetchWithTimeoutMock,
assertOkOrThrowHttpErrorMock,
resolveProviderHttpRequestConfigMock,
} = vi.hoisted(() => ({
resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "qwen-key" })),
postJsonRequestMock: vi.fn(),
fetchWithTimeoutMock: vi.fn(),
assertOkOrThrowHttpErrorMock: vi.fn(async () => {}),
resolveProviderHttpRequestConfigMock: vi.fn((params) => ({
baseUrl: params.baseUrl ?? params.defaultBaseUrl,
allowPrivateNetwork: false,
headers: new Headers(params.defaultHeaders),
dispatcherPolicy: undefined,
})),
}));
vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({
resolveApiKeyForProvider: resolveApiKeyForProviderMock,
}));
vi.mock("openclaw/plugin-sdk/provider-http", () => ({
assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock,
fetchWithTimeout: fetchWithTimeoutMock,
postJsonRequest: postJsonRequestMock,
resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock,
}));
describe("qwen video generation provider", () => {
afterEach(() => {
resolveApiKeyForProviderMock.mockClear();
postJsonRequestMock.mockReset();
fetchWithTimeoutMock.mockReset();
assertOkOrThrowHttpErrorMock.mockClear();
resolveProviderHttpRequestConfigMock.mockClear();
});
it("submits async Wan generation, polls task status, and downloads the resulting video", async () => {
postJsonRequestMock.mockResolvedValue({
response: {
json: async () => ({
request_id: "req-1",
output: {
task_id: "task-1",
},
}),
},
release: vi.fn(async () => {}),
});
fetchWithTimeoutMock
.mockResolvedValueOnce({
json: async () => ({
output: {
task_status: "SUCCEEDED",
results: [{ video_url: "https://example.com/out.mp4" }],
},
}),
headers: new Headers(),
})
.mockResolvedValueOnce({
arrayBuffer: async () => Buffer.from("mp4-bytes"),
headers: new Headers({ "content-type": "video/mp4" }),
});
const provider = buildQwenVideoGenerationProvider();
const result = await provider.generateVideo({
provider: "qwen",
model: "wan2.6-r2v-flash",
prompt: "animate this shot",
cfg: {},
inputImages: [{ url: "https://example.com/ref.png" }],
durationSeconds: 6,
audio: true,
});
expect(postJsonRequestMock).toHaveBeenCalledWith(
expect.objectContaining({
url: "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis",
body: expect.objectContaining({
model: "wan2.6-r2v-flash",
input: expect.objectContaining({
prompt: "animate this shot",
img_url: "https://example.com/ref.png",
}),
}),
}),
);
expect(fetchWithTimeoutMock).toHaveBeenNthCalledWith(
1,
"https://dashscope-intl.aliyuncs.com/api/v1/tasks/task-1",
expect.objectContaining({ method: "GET" }),
120000,
fetch,
);
expect(result.videos).toHaveLength(1);
expect(result.videos[0]?.mimeType).toBe("video/mp4");
expect(result.metadata).toEqual(
expect.objectContaining({
requestId: "req-1",
taskId: "task-1",
taskStatus: "SUCCEEDED",
}),
);
});
});

View File

@@ -0,0 +1,300 @@
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
fetchWithTimeout,
postJsonRequest,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import type {
GeneratedVideoAsset,
VideoGenerationProvider,
VideoGenerationRequest,
VideoGenerationResult,
VideoGenerationSourceAsset,
} from "openclaw/plugin-sdk/video-generation";
import { QWEN_STANDARD_CN_BASE_URL, QWEN_STANDARD_GLOBAL_BASE_URL } from "./models.js";
const DEFAULT_QWEN_VIDEO_BASE_URL = "https://dashscope-intl.aliyuncs.com";
const DEFAULT_QWEN_VIDEO_MODEL = "wan2.6-t2v";
const DEFAULT_DURATION_SECONDS = 5;
const DEFAULT_REQUEST_TIMEOUT_MS = 120_000;
const POLL_INTERVAL_MS = 2_500;
const MAX_POLL_ATTEMPTS = 120;
const RESOLUTION_TO_SIZE: Record<string, string> = {
"480P": "832*480",
"720P": "1280*720",
"1080P": "1920*1080",
};
type QwenVideoGenerationResponse = {
output?: {
task_id?: string;
task_status?: string;
submit_time?: string;
results?: Array<{
video_url?: string;
orig_prompt?: string;
actual_prompt?: string;
}>;
video_url?: string;
code?: string;
message?: string;
};
request_id?: string;
code?: string;
message?: string;
};
function resolveQwenVideoBaseUrl(req: VideoGenerationRequest): string {
const direct = req.cfg?.models?.providers?.qwen?.baseUrl?.trim();
if (!direct) {
return DEFAULT_QWEN_VIDEO_BASE_URL;
}
try {
const url = new URL(direct);
if (url.hostname === "coding-intl.dashscope.aliyuncs.com") {
return "https://dashscope-intl.aliyuncs.com";
}
if (url.hostname === "coding.dashscope.aliyuncs.com") {
return "https://dashscope.aliyuncs.com";
}
if (url.hostname === "dashscope-intl.aliyuncs.com") {
return "https://dashscope-intl.aliyuncs.com";
}
if (url.hostname === "dashscope.aliyuncs.com") {
return "https://dashscope.aliyuncs.com";
}
return url.origin;
} catch {
return DEFAULT_QWEN_VIDEO_BASE_URL;
}
}
function resolveDashscopeAigcApiBaseUrl(baseUrl: string): string {
if (baseUrl.startsWith(QWEN_STANDARD_CN_BASE_URL)) {
return "https://dashscope.aliyuncs.com";
}
if (baseUrl.startsWith(QWEN_STANDARD_GLOBAL_BASE_URL)) {
return DEFAULT_QWEN_VIDEO_BASE_URL;
}
return baseUrl.replace(/\/+$/u, "");
}
function resolveReferenceUrls(
inputImages: VideoGenerationSourceAsset[] | undefined,
inputVideos: VideoGenerationSourceAsset[] | undefined,
): string[] {
return [...(inputImages ?? []), ...(inputVideos ?? [])]
.map((asset) => asset.url?.trim())
.filter((value): value is string => Boolean(value));
}
function buildQwenVideoGenerationInput(req: VideoGenerationRequest): Record<string, unknown> {
const input: Record<string, unknown> = {
prompt: req.prompt,
};
const referenceUrls = resolveReferenceUrls(req.inputImages, req.inputVideos);
if (
referenceUrls.length === 1 &&
(req.inputImages?.length ?? 0) === 1 &&
!req.inputVideos?.length
) {
input.img_url = referenceUrls[0];
} else if (referenceUrls.length > 0) {
input.reference_urls = referenceUrls;
}
return input;
}
function buildQwenVideoGenerationParameters(
req: VideoGenerationRequest,
): Record<string, unknown> | undefined {
const parameters: Record<string, unknown> = {};
const size =
req.size?.trim() || (req.resolution ? RESOLUTION_TO_SIZE[req.resolution] : undefined);
if (size) {
parameters.size = size;
}
if (req.aspectRatio?.trim()) {
parameters.aspect_ratio = req.aspectRatio.trim();
}
if (typeof req.durationSeconds === "number" && Number.isFinite(req.durationSeconds)) {
parameters.duration = Math.max(1, Math.round(req.durationSeconds));
}
if (typeof req.audio === "boolean") {
parameters.enable_audio = req.audio;
}
if (typeof req.watermark === "boolean") {
parameters.watermark = req.watermark;
}
return Object.keys(parameters).length > 0 ? parameters : undefined;
}
function extractVideoUrls(payload: QwenVideoGenerationResponse): string[] {
const urls = [
...(payload.output?.results?.map((entry) => entry.video_url).filter(Boolean) ?? []),
payload.output?.video_url,
].filter((value): value is string => typeof value === "string" && value.trim().length > 0);
return [...new Set(urls)];
}
async function pollTaskUntilComplete(params: {
taskId: string;
headers: Headers;
timeoutMs?: number;
fetchFn: typeof fetch;
baseUrl: string;
}): Promise<QwenVideoGenerationResponse> {
for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
const response = await fetchWithTimeout(
`${params.baseUrl}/api/v1/tasks/${params.taskId}`,
{
method: "GET",
headers: params.headers,
},
params.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
params.fetchFn,
);
await assertOkOrThrowHttpError(response, "Qwen video-generation task poll failed");
const payload = (await response.json()) as QwenVideoGenerationResponse;
const status = payload.output?.task_status?.trim().toUpperCase();
if (status === "SUCCEEDED") {
return payload;
}
if (status === "FAILED" || status === "CANCELED") {
throw new Error(
payload.output?.message?.trim() ||
payload.message?.trim() ||
`Qwen video generation task ${params.taskId} ${status.toLowerCase()}`,
);
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
throw new Error(`Qwen video generation task ${params.taskId} did not finish in time`);
}
async function downloadGeneratedVideos(params: {
urls: string[];
timeoutMs?: number;
fetchFn: typeof fetch;
}): Promise<GeneratedVideoAsset[]> {
const videos: GeneratedVideoAsset[] = [];
for (const [index, url] of params.urls.entries()) {
const response = await fetchWithTimeout(
url,
{ method: "GET" },
params.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
params.fetchFn,
);
await assertOkOrThrowHttpError(response, "Qwen generated video download failed");
const arrayBuffer = await response.arrayBuffer();
videos.push({
buffer: Buffer.from(arrayBuffer),
mimeType: response.headers.get("content-type")?.trim() || "video/mp4",
fileName: `video-${index + 1}.mp4`,
metadata: { sourceUrl: url },
});
}
return videos;
}
export function buildQwenVideoGenerationProvider(): VideoGenerationProvider {
return {
id: "qwen",
label: "Qwen Cloud",
defaultModel: DEFAULT_QWEN_VIDEO_MODEL,
models: ["wan2.6-t2v", "wan2.6-i2v", "wan2.6-r2v", "wan2.6-r2v-flash", "wan2.7-r2v"],
capabilities: {
maxVideos: 1,
maxInputImages: 1,
maxInputVideos: 4,
maxDurationSeconds: 10,
supportsSize: true,
supportsAspectRatio: true,
supportsResolution: true,
supportsAudio: true,
supportsWatermark: true,
},
async generateVideo(req): Promise<VideoGenerationResult> {
const fetchFn = fetch;
const auth = await resolveApiKeyForProvider({
provider: "qwen",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("Qwen API key missing");
}
const requestBaseUrl = resolveQwenVideoBaseUrl(req);
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: requestBaseUrl,
defaultBaseUrl: DEFAULT_QWEN_VIDEO_BASE_URL,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"Content-Type": "application/json",
"X-DashScope-Async": "enable",
},
provider: "qwen",
capability: "video",
transport: "http",
});
const model = req.model?.trim() || DEFAULT_QWEN_VIDEO_MODEL;
const { response, release } = await postJsonRequest({
url: `${resolveDashscopeAigcApiBaseUrl(baseUrl)}/api/v1/services/aigc/video-generation/video-synthesis`,
headers,
body: {
model,
input: buildQwenVideoGenerationInput(req),
parameters: buildQwenVideoGenerationParameters({
...req,
durationSeconds: req.durationSeconds ?? DEFAULT_DURATION_SECONDS,
}),
},
timeoutMs: req.timeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(response, "Qwen video generation failed");
const submitted = (await response.json()) as QwenVideoGenerationResponse;
const taskId = submitted.output?.task_id?.trim();
if (!taskId) {
throw new Error("Qwen video generation response missing task_id");
}
const completed = await pollTaskUntilComplete({
taskId,
headers,
timeoutMs: req.timeoutMs,
fetchFn,
baseUrl: resolveDashscopeAigcApiBaseUrl(baseUrl),
});
const urls = extractVideoUrls(completed);
if (urls.length === 0) {
throw new Error("Qwen video generation completed without output video URLs");
}
const videos = await downloadGeneratedVideos({
urls,
timeoutMs: req.timeoutMs,
fetchFn,
});
return {
videos,
model,
metadata: {
requestId: submitted.request_id,
taskId,
taskStatus: completed.output?.task_status,
},
};
} finally {
await release();
}
},
};
}

View File

@@ -0,0 +1 @@
export * from "openclaw/plugin-sdk/video-generation-core";

View File

@@ -0,0 +1,7 @@
{
"name": "@openclaw/video-generation-core",
"version": "2026.4.1-beta.1",
"private": true,
"description": "OpenClaw video generation runtime package",
"type": "module"
}

View File

@@ -0,0 +1,6 @@
export {
generateVideo,
listRuntimeVideoGenerationProviders,
type GenerateVideoParams,
type GenerateVideoRuntimeResult,
} from "./src/runtime.js";

View File

@@ -0,0 +1,164 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../../src/config/config.js";
import type { VideoGenerationProvider } from "../api.js";
import { generateVideo, listRuntimeVideoGenerationProviders } from "./runtime.js";
const mocks = vi.hoisted(() => {
const debug = vi.fn();
return {
createSubsystemLogger: vi.fn(() => ({ debug })),
describeFailoverError: vi.fn(),
getProviderEnvVars: vi.fn<(providerId: string) => string[]>(() => []),
getVideoGenerationProvider: vi.fn<
(providerId: string, config?: OpenClawConfig) => VideoGenerationProvider | undefined
>(() => undefined),
isFailoverError: vi.fn<(err: unknown) => boolean>(() => false),
listVideoGenerationProviders: vi.fn<(config?: OpenClawConfig) => VideoGenerationProvider[]>(
() => [],
),
parseVideoGenerationModelRef: vi.fn<
(raw?: string) => { provider: string; model: string } | undefined
>((raw?: string) => {
const trimmed = raw?.trim();
if (!trimmed) {
return undefined;
}
const slash = trimmed.indexOf("/");
if (slash <= 0 || slash === trimmed.length - 1) {
return undefined;
}
return {
provider: trimmed.slice(0, slash),
model: trimmed.slice(slash + 1),
};
}),
resolveAgentModelFallbackValues: vi.fn<(value: unknown) => string[]>(() => []),
resolveAgentModelPrimaryValue: vi.fn<(value: unknown) => string | undefined>(() => undefined),
debug,
};
});
vi.mock("../api.js", () => ({
createSubsystemLogger: mocks.createSubsystemLogger,
describeFailoverError: mocks.describeFailoverError,
getProviderEnvVars: mocks.getProviderEnvVars,
getVideoGenerationProvider: mocks.getVideoGenerationProvider,
isFailoverError: mocks.isFailoverError,
listVideoGenerationProviders: mocks.listVideoGenerationProviders,
parseVideoGenerationModelRef: mocks.parseVideoGenerationModelRef,
resolveAgentModelFallbackValues: mocks.resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue: mocks.resolveAgentModelPrimaryValue,
}));
describe("video-generation runtime", () => {
beforeEach(() => {
mocks.createSubsystemLogger.mockClear();
mocks.describeFailoverError.mockReset();
mocks.getProviderEnvVars.mockReset();
mocks.getProviderEnvVars.mockReturnValue([]);
mocks.getVideoGenerationProvider.mockReset();
mocks.isFailoverError.mockReset();
mocks.isFailoverError.mockReturnValue(false);
mocks.listVideoGenerationProviders.mockReset();
mocks.listVideoGenerationProviders.mockReturnValue([]);
mocks.parseVideoGenerationModelRef.mockClear();
mocks.resolveAgentModelFallbackValues.mockReset();
mocks.resolveAgentModelFallbackValues.mockReturnValue([]);
mocks.resolveAgentModelPrimaryValue.mockReset();
mocks.resolveAgentModelPrimaryValue.mockReturnValue(undefined);
mocks.debug.mockReset();
});
it("generates videos through the active video-generation provider", async () => {
const authStore = { version: 1, profiles: {} } as const;
let seenAuthStore: unknown;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const provider: VideoGenerationProvider = {
id: "video-plugin",
capabilities: {},
async generateVideo(req: { authStore?: unknown }) {
seenAuthStore = req.authStore;
return {
videos: [
{
buffer: Buffer.from("mp4-bytes"),
mimeType: "video/mp4",
fileName: "sample.mp4",
},
],
model: "vid-v1",
};
},
};
mocks.getVideoGenerationProvider.mockReturnValue(provider);
const result = await generateVideo({
cfg: {
agents: {
defaults: {
videoGenerationModel: { primary: "video-plugin/vid-v1" },
},
},
} as OpenClawConfig,
prompt: "animate a cat",
agentDir: "/tmp/agent",
authStore,
});
expect(result.provider).toBe("video-plugin");
expect(result.model).toBe("vid-v1");
expect(result.attempts).toEqual([]);
expect(seenAuthStore).toEqual(authStore);
expect(result.videos).toEqual([
{
buffer: Buffer.from("mp4-bytes"),
mimeType: "video/mp4",
fileName: "sample.mp4",
},
]);
});
it("lists runtime video-generation providers through the owner runtime", () => {
const providers: VideoGenerationProvider[] = [
{
id: "video-plugin",
defaultModel: "vid-v1",
models: ["vid-v1"],
capabilities: {
supportsAudio: true,
},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
}),
},
];
mocks.listVideoGenerationProviders.mockReturnValue(providers);
expect(listRuntimeVideoGenerationProviders({ config: {} as OpenClawConfig })).toEqual(
providers,
);
expect(mocks.listVideoGenerationProviders).toHaveBeenCalledWith({} as OpenClawConfig);
});
it("explains native video-generation config and provider auth when no model is configured", async () => {
mocks.listVideoGenerationProviders.mockReturnValue([
{
id: "qwen",
defaultModel: "wan2.6-t2v",
capabilities: {},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
}),
},
]);
mocks.getProviderEnvVars.mockReturnValue(["QWEN_API_KEY"]);
const promise = generateVideo({ cfg: {} as OpenClawConfig, prompt: "animate a cat" });
await expect(promise).rejects.toThrow("No video-generation model configured.");
await expect(promise).rejects.toThrow(
'Set agents.defaults.videoGenerationModel.primary to a provider/model like "',
);
await expect(promise).rejects.toThrow("qwen: QWEN_API_KEY");
});
});

View File

@@ -0,0 +1,189 @@
import {
createSubsystemLogger,
describeFailoverError,
getProviderEnvVars,
getVideoGenerationProvider,
isFailoverError,
listVideoGenerationProviders,
parseVideoGenerationModelRef,
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
type AuthProfileStore,
type FallbackAttempt,
type GeneratedVideoAsset,
type OpenClawConfig,
type VideoGenerationResolution,
type VideoGenerationResult,
type VideoGenerationSourceAsset,
} from "../api.js";
const log = createSubsystemLogger("video-generation");
export type GenerateVideoParams = {
cfg: OpenClawConfig;
prompt: string;
agentDir?: string;
authStore?: AuthProfileStore;
modelOverride?: string;
size?: string;
aspectRatio?: string;
resolution?: VideoGenerationResolution;
durationSeconds?: number;
audio?: boolean;
watermark?: boolean;
inputImages?: VideoGenerationSourceAsset[];
inputVideos?: VideoGenerationSourceAsset[];
};
export type GenerateVideoRuntimeResult = {
videos: GeneratedVideoAsset[];
provider: string;
model: string;
attempts: FallbackAttempt[];
metadata?: Record<string, unknown>;
};
function resolveVideoGenerationCandidates(params: {
cfg: OpenClawConfig;
modelOverride?: string;
}): Array<{ provider: string; model: string }> {
const candidates: Array<{ provider: string; model: string }> = [];
const seen = new Set<string>();
const add = (raw: string | undefined) => {
const parsed = parseVideoGenerationModelRef(raw);
if (!parsed) {
return;
}
const key = `${parsed.provider}/${parsed.model}`;
if (seen.has(key)) {
return;
}
seen.add(key);
candidates.push(parsed);
};
add(params.modelOverride);
add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.videoGenerationModel));
for (const fallback of resolveAgentModelFallbackValues(
params.cfg.agents?.defaults?.videoGenerationModel,
)) {
add(fallback);
}
return candidates;
}
function throwVideoGenerationFailure(params: {
attempts: FallbackAttempt[];
lastError: unknown;
}): never {
if (params.attempts.length <= 1 && params.lastError) {
throw params.lastError;
}
const summary =
params.attempts.length > 0
? params.attempts
.map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`)
.join(" | ")
: "unknown";
throw new Error(`All video generation models failed (${params.attempts.length}): ${summary}`, {
cause: params.lastError instanceof Error ? params.lastError : undefined,
});
}
function buildNoVideoGenerationModelConfiguredMessage(cfg: OpenClawConfig): string {
const providers = listVideoGenerationProviders(cfg);
const sampleModel =
providers.find((provider) => provider.defaultModel) ??
({ id: "qwen", defaultModel: "wan2.6-t2v" } as const);
const authHints = providers
.flatMap((provider) => {
const envVars = getProviderEnvVars(provider.id);
if (envVars.length === 0) {
return [];
}
return [`${provider.id}: ${envVars.join(" / ")}`];
})
.slice(0, 3);
return [
`No video-generation model configured. Set agents.defaults.videoGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`,
authHints.length > 0
? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).`
: "If you want a specific provider, also configure that provider's auth/API key first.",
].join(" ");
}
export function listRuntimeVideoGenerationProviders(params?: { config?: OpenClawConfig }) {
return listVideoGenerationProviders(params?.config);
}
export async function generateVideo(
params: GenerateVideoParams,
): Promise<GenerateVideoRuntimeResult> {
const candidates = resolveVideoGenerationCandidates({
cfg: params.cfg,
modelOverride: params.modelOverride,
});
if (candidates.length === 0) {
throw new Error(buildNoVideoGenerationModelConfiguredMessage(params.cfg));
}
const attempts: FallbackAttempt[] = [];
let lastError: unknown;
for (const candidate of candidates) {
const provider = getVideoGenerationProvider(candidate.provider, params.cfg);
if (!provider) {
const error = `No video-generation provider registered for ${candidate.provider}`;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error,
});
lastError = new Error(error);
continue;
}
try {
const result: VideoGenerationResult = await provider.generateVideo({
provider: candidate.provider,
model: candidate.model,
prompt: params.prompt,
cfg: params.cfg,
agentDir: params.agentDir,
authStore: params.authStore,
size: params.size,
aspectRatio: params.aspectRatio,
resolution: params.resolution,
durationSeconds: params.durationSeconds,
audio: params.audio,
watermark: params.watermark,
inputImages: params.inputImages,
inputVideos: params.inputVideos,
});
if (!Array.isArray(result.videos) || result.videos.length === 0) {
throw new Error("Video generation provider returned no videos.");
}
return {
videos: result.videos,
provider: candidate.provider,
model: result.model ?? candidate.model,
attempts,
metadata: result.metadata,
};
} catch (err) {
lastError = err;
const described = isFailoverError(err) ? describeFailoverError(err) : undefined;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error: described?.message ?? (err instanceof Error ? err.message : String(err)),
reason: described?.reason,
status: described?.status,
code: described?.code,
});
log.debug(`video-generation candidate failed: ${candidate.provider}/${candidate.model}`);
}
}
throwVideoGenerationFailure({ attempts, lastError });
}

View File

@@ -3,6 +3,13 @@ import {
type OpenClawPluginApi,
type ProviderAuthMethodNonInteractiveContext,
} from "openclaw/plugin-sdk/plugin-entry";
import {
buildVllmProvider,
VLLM_DEFAULT_API_KEY_ENV_VAR,
VLLM_DEFAULT_BASE_URL,
VLLM_MODEL_PLACEHOLDER,
VLLM_PROVIDER_LABEL,
} from "./api.js";
const PROVIDER_ID = "vllm";
@@ -14,15 +21,7 @@ export default definePluginEntry({
id: "vllm",
name: "vLLM Provider",
description: "Bundled vLLM provider plugin",
async register(api: OpenClawPluginApi) {
const {
buildVllmProvider,
VLLM_DEFAULT_API_KEY_ENV_VAR,
VLLM_DEFAULT_BASE_URL,
VLLM_MODEL_PLACEHOLDER,
VLLM_PROVIDER_LABEL,
} = await import("./register.runtime.js");
register(api: OpenClawPluginApi) {
api.registerProvider({
id: PROVIDER_ID,
label: "vLLM",

View File

@@ -567,6 +567,14 @@
"types": "./dist/plugin-sdk/image-generation-core.d.ts",
"default": "./dist/plugin-sdk/image-generation-core.js"
},
"./plugin-sdk/video-generation": {
"types": "./dist/plugin-sdk/video-generation.d.ts",
"default": "./dist/plugin-sdk/video-generation.js"
},
"./plugin-sdk/video-generation-core": {
"types": "./dist/plugin-sdk/video-generation-core.d.ts",
"default": "./dist/plugin-sdk/video-generation-core.js"
},
"./plugin-sdk/irc": {
"types": "./dist/plugin-sdk/irc.d.ts",
"default": "./dist/plugin-sdk/irc.js"
@@ -727,6 +735,14 @@
"types": "./dist/plugin-sdk/minimax.d.ts",
"default": "./dist/plugin-sdk/minimax.js"
},
"./plugin-sdk/qwen": {
"types": "./dist/plugin-sdk/qwen.d.ts",
"default": "./dist/plugin-sdk/qwen.js"
},
"./plugin-sdk/qwen-definitions": {
"types": "./dist/plugin-sdk/qwen-definitions.d.ts",
"default": "./dist/plugin-sdk/qwen-definitions.js"
},
"./plugin-sdk/modelstudio": {
"types": "./dist/plugin-sdk/modelstudio.d.ts",
"default": "./dist/plugin-sdk/modelstudio.js"

View File

@@ -130,6 +130,8 @@
"googlechat",
"image-generation",
"image-generation-core",
"video-generation",
"video-generation-core",
"irc",
"irc-surface",
"kimi-coding",
@@ -171,6 +173,8 @@
"memory-core-host-runtime-files",
"memory-lancedb",
"minimax",
"qwen",
"qwen-definitions",
"modelstudio",
"modelstudio-definitions",
"moonshot",

View File

@@ -315,6 +315,18 @@ export const GENERATED_PLUGIN_SDK_FACADES = [
],
typeExports: ["GenerateImageParams", "GenerateImageRuntimeResult"],
},
{
subpath: "video-generation-runtime",
source: pluginSource("video-generation-core", "runtime-api.js"),
loadPolicy: "activated",
exports: [
"generateVideo",
"listRuntimeVideoGenerationProviders",
"GenerateVideoParams",
"GenerateVideoRuntimeResult",
],
typeExports: ["GenerateVideoParams", "GenerateVideoRuntimeResult"],
},
{
subpath: "kimi-coding",
source: pluginSource("kimi-coding", "api.js"),
@@ -370,13 +382,19 @@ export const GENERATED_PLUGIN_SDK_FACADES = [
source: pluginSource("memory-core", "runtime-api.js"),
loadPolicy: "activated",
exports: [
"auditShortTermPromotionArtifacts",
"BuiltinMemoryEmbeddingProviderDoctorMetadata",
"getBuiltinMemoryEmbeddingProviderDoctorMetadata",
"getMemorySearchManager",
"listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata",
"MemoryIndexManager",
"repairShortTermPromotionArtifacts",
],
typeExports: [
"BuiltinMemoryEmbeddingProviderDoctorMetadata",
"RepairShortTermPromotionArtifactsResult",
"ShortTermAuditSummary",
],
typeExports: ["BuiltinMemoryEmbeddingProviderDoctorMetadata"],
},
{
subpath: "mattermost-policy",
@@ -532,9 +550,44 @@ export const GENERATED_PLUGIN_SDK_FACADES = [
"MINIMAX_TEXT_MODEL_REFS",
],
},
{
subpath: "qwen",
source: pluginSource("qwen", "api.js"),
exports: [
"applyQwenNativeStreamingUsageCompat",
"buildQwenDefaultModelDefinition",
"buildQwenModelDefinition",
"QWEN_BASE_URL",
"QWEN_CN_BASE_URL",
"QWEN_DEFAULT_COST",
"QWEN_DEFAULT_MODEL_ID",
"QWEN_DEFAULT_MODEL_REF",
"QWEN_GLOBAL_BASE_URL",
"QWEN_STANDARD_CN_BASE_URL",
"QWEN_STANDARD_GLOBAL_BASE_URL",
"QWEN_MODEL_CATALOG",
"isNativeQwenBaseUrl",
"buildQwenProvider",
],
},
{
subpath: "qwen-definitions",
source: pluginSource("qwen", "api.js"),
exports: [
"buildQwenDefaultModelDefinition",
"buildQwenModelDefinition",
"QWEN_CN_BASE_URL",
"QWEN_DEFAULT_COST",
"QWEN_DEFAULT_MODEL_ID",
"QWEN_DEFAULT_MODEL_REF",
"QWEN_GLOBAL_BASE_URL",
"QWEN_STANDARD_CN_BASE_URL",
"QWEN_STANDARD_GLOBAL_BASE_URL",
],
},
{
subpath: "modelstudio",
source: pluginSource("modelstudio", "api.js"),
source: pluginSource("qwen", "api.js"),
exports: [
"applyModelStudioNativeStreamingUsageCompat",
"buildModelStudioDefaultModelDefinition",
@@ -554,7 +607,7 @@ export const GENERATED_PLUGIN_SDK_FACADES = [
},
{
subpath: "modelstudio-definitions",
source: pluginSource("modelstudio", "api.js"),
source: pluginSource("qwen", "api.js"),
exports: [
"buildModelStudioDefaultModelDefinition",
"buildModelStudioModelDefinition",

View File

@@ -14,8 +14,8 @@ describe("formatAuthDoctorHint", () => {
provider: "qwen-portal",
});
expect(hint).toContain("openclaw onboard --auth-choice modelstudio-api-key");
expect(hint).toContain("modelstudio-api-key-cn");
expect(hint).not.toContain("--provider modelstudio");
expect(hint).toContain("openclaw onboard --auth-choice qwen-api-key");
expect(hint).toContain("qwen-api-key-cn");
expect(hint).not.toContain("--provider qwen");
});
});

View File

@@ -13,7 +13,15 @@ import {
} from "./model-auth.js";
vi.mock("../plugins/provider-runtime.js", () => ({
buildProviderMissingAuthMessageWithPlugin: () => undefined,
buildProviderMissingAuthMessageWithPlugin: (params: {
provider: string;
context: { listProfileIds: (providerId: string) => string[] };
}) => {
if (params.provider === "openai" && params.context.listProfileIds("openai-codex").length > 0) {
return 'No API key found for provider "openai". Use openai-codex/gpt-5.4.';
}
return undefined;
},
formatProviderAuthProfileApiKeyWithPlugin: async () => undefined,
refreshProviderOAuthCredentialWithPlugin: async () => null,
resolveProviderSyntheticAuthWithPlugin: (params: {
@@ -311,13 +319,13 @@ describe("getApiKeyForModel", () => {
});
});
it("resolves Model Studio API key from env", async () => {
it("resolves Qwen API key from env", async () => {
await withEnvAsync(
{ [envVar("MODELSTUDIO", "API", "KEY")]: "modelstudio-test-key" },
async () => {
// pragma: allowlist secret
const resolved = await resolveApiKeyForProvider({
provider: "modelstudio",
provider: "qwen",
store: { version: 1, profiles: {} },
});
expect(resolved.apiKey).toBe("modelstudio-test-key");

View File

@@ -157,10 +157,10 @@ describe("normalizeModelCompat", () => {
});
});
it("keeps supportsUsageInStreaming on for native ModelStudio endpoints", () => {
it("keeps supportsUsageInStreaming on for native Qwen endpoints", () => {
const model = {
...baseModel(),
provider: "modelstudio",
provider: "qwen",
baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
};
delete (model as { compat?: unknown }).compat;

View File

@@ -419,7 +419,7 @@ describe("model-selection", () => {
"qwen-dashscope": {
models: [{ id: "qwen-max" }],
},
modelstudio: {
qwen: {
models: [{ id: "qwen-max" }],
},
},

View File

@@ -834,13 +834,13 @@ describe("openai transport stream", () => {
expect(params.messages?.[0]?.content).toBe("Stable prefix\nDynamic suffix");
});
it("uses system role and streaming usage compat for native ModelStudio completions providers", () => {
it("uses system role and streaming usage compat for native Qwen completions providers", () => {
const params = buildOpenAICompletionsParams(
{
id: "qwen3.6-plus",
name: "Qwen 3.6 Plus",
api: "openai-completions",
provider: "modelstudio",
provider: "qwen",
baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
reasoning: true,
input: ["text"],

View File

@@ -561,7 +561,7 @@ describe("provider attribution", () => {
expect(
resolveProviderRequestCapabilities({
provider: "modelstudio",
provider: "qwen",
api: "openai-completions",
baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
capability: "llm",
@@ -745,9 +745,9 @@ describe("provider attribution", () => {
},
},
{
name: "native ModelStudio completions",
name: "native Qwen completions",
input: {
provider: "modelstudio",
provider: "qwen",
api: "openai-completions",
baseUrl: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
capability: "llm" as const,

View File

@@ -295,6 +295,8 @@ function resolveKnownProviderFamily(provider: string | undefined): string {
case "moonshot":
case "kimi":
return "moonshot";
case "qwen":
case "qwencloud":
case "modelstudio":
case "dashscope":
return "modelstudio";

View File

@@ -1,5 +1,8 @@
export function normalizeProviderId(provider: string): string {
const normalized = provider.trim().toLowerCase();
if (normalized === "modelstudio" || normalized === "qwencloud") {
return "qwen";
}
if (normalized === "z.ai" || normalized === "z-ai") {
return "zai";
}

View File

@@ -537,17 +537,17 @@ vi.mock("./onboard-non-interactive/local/auth-choice.plugin-providers.js", async
}),
],
[
"modelstudio-api-key",
"qwen-api-key",
createApiKeyChoice({
providerId: "modelstudio",
label: "Model Studio",
choiceId: "modelstudio-api-key",
providerId: "qwen",
label: "Qwen Cloud",
choiceId: "qwen-api-key",
optionKey: "modelstudioApiKey",
flagName: "--modelstudio-api-key",
envVar: "MODELSTUDIO_API_KEY",
defaultModel: "modelstudio/qwen3.5-plus",
envVar: "QWEN_API_KEY",
defaultModel: "qwen/qwen3.5-plus",
applyConfig: (cfg) =>
withProviderConfig(cfg, "modelstudio", {
withProviderConfig(cfg, "qwen", {
baseUrl: "https://coding-intl.dashscope.aliyuncs.com/v1",
api: "openai-completions",
models: [buildTestProviderModel("qwen3.5-plus")],
@@ -1071,18 +1071,9 @@ describe("onboard (non-interactive): provider auth", () => {
token,
tokenProfileId: "anthropic:default",
}),
).rejects.toThrow("Process exited with code 1");
).rejects.toThrow('Auth choice "token" is no longer supported for Anthropic onboarding.');
expect(runtime.error).toHaveBeenCalledWith(
[
'Auth choice "token" is no longer supported for Anthropic onboarding.',
"Existing Anthropic token profiles still run if they are already configured.",
'Use "--auth-choice anthropic-cli" or "--auth-choice apiKey" instead.',
].join("\n"),
);
const cfg = await readJsonFile<ProviderAuthConfigSnapshot>(configPath);
expect(cfg.auth?.profiles?.["anthropic:default"]).toBeUndefined();
await expect(fs.access(configPath)).rejects.toMatchObject({ code: "ENOENT" });
expect(ensureAuthProfileStore().profiles["anthropic:default"]).toBeUndefined();
});
});
@@ -1365,21 +1356,21 @@ describe("onboard (non-interactive): provider auth", () => {
});
});
it("infers Model Studio auth choice from --modelstudio-api-key and sets default model", async () => {
it("infers Qwen auth choice from --modelstudio-api-key and sets default model", async () => {
await withOnboardEnv("openclaw-onboard-modelstudio-infer-", async (env) => {
const cfg = await runOnboardingAndReadConfig(env, {
modelstudioApiKey: "modelstudio-test-key", // pragma: allowlist secret
});
expect(cfg.auth?.profiles?.["modelstudio:default"]?.provider).toBe("modelstudio");
expect(cfg.auth?.profiles?.["modelstudio:default"]?.mode).toBe("api_key");
expect(cfg.models?.providers?.modelstudio?.baseUrl).toBe(
expect(cfg.auth?.profiles?.["qwen:default"]?.provider).toBe("qwen");
expect(cfg.auth?.profiles?.["qwen:default"]?.mode).toBe("api_key");
expect(cfg.models?.providers?.qwen?.baseUrl).toBe(
"https://coding-intl.dashscope.aliyuncs.com/v1",
);
expect(cfg.agents?.defaults?.model?.primary).toBe("modelstudio/qwen3.5-plus");
expect(cfg.agents?.defaults?.model?.primary).toBe("qwen/qwen3.5-plus");
await expectApiKeyProfile({
profileId: "modelstudio:default",
provider: "modelstudio",
profileId: "qwen:default",
provider: "qwen",
key: "modelstudio-test-key",
});
});

View File

@@ -46,6 +46,10 @@ export type BuiltInAuthChoice =
| "volcengine-api-key"
| "byteplus-api-key"
| "qianfan-api-key"
| "qwen-standard-api-key-cn"
| "qwen-standard-api-key"
| "qwen-api-key-cn"
| "qwen-api-key"
| "modelstudio-standard-api-key-cn"
| "modelstudio-standard-api-key"
| "modelstudio-api-key-cn"
@@ -77,6 +81,7 @@ export type BuiltInAuthChoiceGroupId =
| "together"
| "huggingface"
| "qianfan"
| "qwen"
| "modelstudio"
| "xai"
| "volcengine"

View File

@@ -80,6 +80,7 @@ const SHELL_ENV_EXPECTED_KEYS = [
"OPENROUTER_API_KEY",
"AI_GATEWAY_API_KEY",
"MINIMAX_API_KEY",
"QWEN_API_KEY",
"MODELSTUDIO_API_KEY",
"SYNTHETIC_API_KEY",
"KILOCODE_API_KEY",

View File

@@ -2573,6 +2573,28 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
},
],
},
videoGenerationModel: {
anyOf: [
{
type: "string",
},
{
type: "object",
properties: {
primary: {
type: "string",
},
fallbacks: {
type: "array",
items: {
type: "string",
},
},
},
additionalProperties: false,
},
],
},
pdfModel: {
anyOf: [
{
@@ -22637,6 +22659,16 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
help: "Ordered fallback image-generation models (provider/model).",
tags: ["reliability", "media"],
},
"agents.defaults.videoGenerationModel.primary": {
label: "Video Generation Model",
help: "Optional video-generation model (provider/model) used by the shared video generation capability.",
tags: ["media"],
},
"agents.defaults.videoGenerationModel.fallbacks": {
label: "Video Generation Model Fallbacks",
help: "Ordered fallback video-generation models (provider/model).",
tags: ["reliability", "media"],
},
"agents.defaults.pdfModel.primary": {
label: "PDF Model",
help: "Optional PDF model (provider/model) for the PDF analysis tool. Defaults to imageModel, then session model.",

View File

@@ -1098,6 +1098,10 @@ export const FIELD_HELP: Record<string, string> = {
"Optional image-generation model (provider/model) used by the shared image generation capability.",
"agents.defaults.imageGenerationModel.fallbacks":
"Ordered fallback image-generation models (provider/model).",
"agents.defaults.videoGenerationModel.primary":
"Optional video-generation model (provider/model) used by the shared video generation capability.",
"agents.defaults.videoGenerationModel.fallbacks":
"Ordered fallback video-generation models (provider/model).",
"agents.defaults.pdfModel.primary":
"Optional PDF model (provider/model) for the PDF analysis tool. Defaults to imageModel, then session model.",
"agents.defaults.pdfModel.fallbacks": "Ordered fallback PDF models (provider/model).",

View File

@@ -499,6 +499,8 @@ export const FIELD_LABELS: Record<string, string> = {
"agents.defaults.imageModel.fallbacks": "Image Model Fallbacks",
"agents.defaults.imageGenerationModel.primary": "Image Generation Model",
"agents.defaults.imageGenerationModel.fallbacks": "Image Generation Model Fallbacks",
"agents.defaults.videoGenerationModel.primary": "Video Generation Model",
"agents.defaults.videoGenerationModel.fallbacks": "Video Generation Model Fallbacks",
"agents.defaults.pdfModel.primary": "PDF Model",
"agents.defaults.pdfModel.fallbacks": "PDF Model Fallbacks",
"agents.defaults.pdfMaxBytesMb": "PDF Max Size (MB)",

View File

@@ -126,6 +126,8 @@ export type AgentDefaultsConfig = {
imageModel?: AgentModelConfig;
/** Optional image-generation model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */
imageGenerationModel?: AgentModelConfig;
/** Optional video-generation model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */
videoGenerationModel?: AgentModelConfig;
/** Optional PDF-capable model and fallbacks (provider/model). Accepts string or {primary,fallbacks}. */
pdfModel?: AgentModelConfig;
/** Maximum PDF file size in megabytes (default: 10). */

View File

@@ -21,6 +21,7 @@ export const AgentDefaultsSchema = z
model: AgentModelSchema.optional(),
imageModel: AgentModelSchema.optional(),
imageGenerationModel: AgentModelSchema.optional(),
videoGenerationModel: AgentModelSchema.optional(),
pdfModel: AgentModelSchema.optional(),
pdfMaxBytesMb: z.number().positive().optional(),
pdfMaxPages: z.number().int().positive().optional(),

View File

@@ -73,6 +73,7 @@ const createRegistry = (diagnostics: PluginDiagnostic[]): PluginRegistry => ({
realtimeVoiceProviders: [],
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
gatewayHandlers: {},

View File

@@ -771,7 +771,7 @@ describe("resolveSessionModelIdentityRef", () => {
"qwen-dashscope": {
models: [{ id: "qwen-max" }],
},
modelstudio: {
qwen: {
models: [{ id: "qwen-max" }],
},
},

View File

@@ -205,6 +205,7 @@ const createStubPluginRegistry = (): PluginRegistry => ({
realtimeVoiceProviders: [],
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
gatewayHandlers: {},

View File

@@ -166,6 +166,18 @@ export interface PluginSdkFacadeTypeMap {
GenerateImageRuntimeResult: import("@openclaw/image-generation-core/runtime-api.js").GenerateImageRuntimeResult;
};
};
"video-generation-runtime": {
module: typeof import("@openclaw/video-generation-core/runtime-api.js");
sourceModules: {
source1: {
module: typeof import("@openclaw/video-generation-core/runtime-api.js");
};
};
types: {
GenerateVideoParams: import("@openclaw/video-generation-core/runtime-api.js").GenerateVideoParams;
GenerateVideoRuntimeResult: import("@openclaw/video-generation-core/runtime-api.js").GenerateVideoRuntimeResult;
};
};
"kimi-coding": {
module: typeof import("@openclaw/kimi-coding/api.js");
sourceModules: {
@@ -337,20 +349,38 @@ export interface PluginSdkFacadeTypeMap {
};
types: {};
};
modelstudio: {
module: typeof import("@openclaw/modelstudio/api.js");
qwen: {
module: typeof import("@openclaw/qwen/api.js");
sourceModules: {
source1: {
module: typeof import("@openclaw/modelstudio/api.js");
module: typeof import("@openclaw/qwen/api.js");
};
};
types: {};
};
"qwen-definitions": {
module: typeof import("@openclaw/qwen/api.js");
sourceModules: {
source1: {
module: typeof import("@openclaw/qwen/api.js");
};
};
types: {};
};
modelstudio: {
module: typeof import("@openclaw/qwen/api.js");
sourceModules: {
source1: {
module: typeof import("@openclaw/qwen/api.js");
};
};
types: {};
};
"modelstudio-definitions": {
module: typeof import("@openclaw/modelstudio/api.js");
module: typeof import("@openclaw/qwen/api.js");
sourceModules: {
source1: {
module: typeof import("@openclaw/modelstudio/api.js");
module: typeof import("@openclaw/qwen/api.js");
};
};
types: {};

View File

@@ -51,7 +51,11 @@ describe("resolveAutoMediaKeyProviders", () => {
});
it("keeps the bundled video fallback order", () => {
expect(resolveAutoMediaKeyProviders({ capability: "video" })).toEqual(["google", "moonshot"]);
expect(resolveAutoMediaKeyProviders({ capability: "video" })).toEqual([
"google",
"qwen",
"moonshot",
]);
});
});

View File

@@ -11,10 +11,10 @@ describe("media-understanding provider registry", () => {
setActivePluginRegistry(createEmptyPluginRegistry());
});
it("returns no providers by default when no active registry is present", () => {
it("loads bundled providers by default when no active registry is present", () => {
const registry = buildMediaUnderstandingRegistry();
expect(getMediaUnderstandingProvider("groq", registry)).toBeUndefined();
expect(getMediaUnderstandingProvider("deepgram", registry)).toBeUndefined();
expect(getMediaUnderstandingProvider("groq", registry)?.id).toBe("groq");
expect(getMediaUnderstandingProvider("deepgram", registry)?.id).toBe("deepgram");
});
it("merges plugin-registered media providers into the active registry", async () => {

View File

@@ -78,6 +78,8 @@ export const getRichMenuIdOfUser: FacadeModule["getRichMenuIdOfUser"] = ((...arg
loadFacadeModule()["getRichMenuIdOfUser"](...args)) as FacadeModule["getRichMenuIdOfUser"];
export const getRichMenuList: FacadeModule["getRichMenuList"] = ((...args) =>
loadFacadeModule()["getRichMenuList"](...args)) as FacadeModule["getRichMenuList"];
export const hasLineDirectives: FacadeModule["hasLineDirectives"] = ((...args) =>
loadFacadeModule()["hasLineDirectives"](...args)) as FacadeModule["hasLineDirectives"];
export const isSenderAllowed: FacadeModule["isSenderAllowed"] = ((...args) =>
loadFacadeModule()["isSenderAllowed"](...args)) as FacadeModule["isSenderAllowed"];
export const linkRichMenuToUser: FacadeModule["linkRichMenuToUser"] = ((...args) =>
@@ -96,6 +98,8 @@ export const normalizeDmAllowFromWithStore: FacadeModule["normalizeDmAllowFromWi
loadFacadeModule()["normalizeDmAllowFromWithStore"](
...args,
)) as FacadeModule["normalizeDmAllowFromWithStore"];
export const parseLineDirectives: FacadeModule["parseLineDirectives"] = ((...args) =>
loadFacadeModule()["parseLineDirectives"](...args)) as FacadeModule["parseLineDirectives"];
export const postbackAction: FacadeModule["postbackAction"] = ((...args) =>
loadFacadeModule()["postbackAction"](...args)) as FacadeModule["postbackAction"];
export const probeLineBot: FacadeModule["probeLineBot"] = ((...args) =>

View File

@@ -13,17 +13,17 @@ function loadFacadeModule(): FacadeModule {
artifactBasename: "runtime-api.js",
});
}
export const getBuiltinMemoryEmbeddingProviderDoctorMetadata: FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"] =
((...args) =>
loadFacadeModule()["getBuiltinMemoryEmbeddingProviderDoctorMetadata"](
...args,
)) as FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"];
export const auditShortTermPromotionArtifacts: FacadeModule["auditShortTermPromotionArtifacts"] = ((
...args
) =>
loadFacadeModule()["auditShortTermPromotionArtifacts"](
...args,
)) as FacadeModule["auditShortTermPromotionArtifacts"];
export const getBuiltinMemoryEmbeddingProviderDoctorMetadata: FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"] =
((...args) =>
loadFacadeModule()["getBuiltinMemoryEmbeddingProviderDoctorMetadata"](
...args,
)) as FacadeModule["getBuiltinMemoryEmbeddingProviderDoctorMetadata"];
export const getMemorySearchManager: FacadeModule["getMemorySearchManager"] = ((...args) =>
loadFacadeModule()["getMemorySearchManager"](...args)) as FacadeModule["getMemorySearchManager"];
export const listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata: FacadeModule["listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata"] =

View File

@@ -9,7 +9,7 @@ import {
function loadFacadeModule(): FacadeModule {
return loadBundledPluginPublicSurfaceModuleSync<FacadeModule>({
dirName: "modelstudio",
dirName: "qwen",
artifactBasename: "api.js",
});
}

View File

@@ -10,7 +10,7 @@ import {
function loadFacadeModule(): FacadeModule {
return loadBundledPluginPublicSurfaceModuleSync<FacadeModule>({
dirName: "modelstudio",
dirName: "qwen",
artifactBasename: "api.js",
});
}

View File

@@ -0,0 +1,40 @@
// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually.
import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js";
type FacadeEntry = PluginSdkFacadeTypeMap["qwen-definitions"];
type FacadeModule = FacadeEntry["module"];
import {
createLazyFacadeObjectValue,
loadBundledPluginPublicSurfaceModuleSync,
} from "./facade-runtime.js";
function loadFacadeModule(): FacadeModule {
return loadBundledPluginPublicSurfaceModuleSync<FacadeModule>({
dirName: "qwen",
artifactBasename: "api.js",
});
}
export const buildQwenDefaultModelDefinition: FacadeModule["buildQwenDefaultModelDefinition"] = ((
...args
) =>
loadFacadeModule()["buildQwenDefaultModelDefinition"](
...args,
)) as FacadeModule["buildQwenDefaultModelDefinition"];
export const buildQwenModelDefinition: FacadeModule["buildQwenModelDefinition"] = ((...args) =>
loadFacadeModule()["buildQwenModelDefinition"](
...args,
)) as FacadeModule["buildQwenModelDefinition"];
export const QWEN_CN_BASE_URL: FacadeModule["QWEN_CN_BASE_URL"] =
loadFacadeModule()["QWEN_CN_BASE_URL"];
export const QWEN_DEFAULT_COST: FacadeModule["QWEN_DEFAULT_COST"] = createLazyFacadeObjectValue(
() => loadFacadeModule()["QWEN_DEFAULT_COST"] as object,
) as FacadeModule["QWEN_DEFAULT_COST"];
export const QWEN_DEFAULT_MODEL_ID: FacadeModule["QWEN_DEFAULT_MODEL_ID"] =
loadFacadeModule()["QWEN_DEFAULT_MODEL_ID"];
export const QWEN_DEFAULT_MODEL_REF: FacadeModule["QWEN_DEFAULT_MODEL_REF"] =
loadFacadeModule()["QWEN_DEFAULT_MODEL_REF"];
export const QWEN_GLOBAL_BASE_URL: FacadeModule["QWEN_GLOBAL_BASE_URL"] =
loadFacadeModule()["QWEN_GLOBAL_BASE_URL"];
export const QWEN_STANDARD_CN_BASE_URL: FacadeModule["QWEN_STANDARD_CN_BASE_URL"] =
loadFacadeModule()["QWEN_STANDARD_CN_BASE_URL"];
export const QWEN_STANDARD_GLOBAL_BASE_URL: FacadeModule["QWEN_STANDARD_GLOBAL_BASE_URL"] =
loadFacadeModule()["QWEN_STANDARD_GLOBAL_BASE_URL"];

54
src/plugin-sdk/qwen.ts Normal file
View File

@@ -0,0 +1,54 @@
// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually.
import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js";
type FacadeEntry = PluginSdkFacadeTypeMap["qwen"];
type FacadeModule = FacadeEntry["module"];
import {
createLazyFacadeArrayValue,
createLazyFacadeObjectValue,
loadBundledPluginPublicSurfaceModuleSync,
} from "./facade-runtime.js";
function loadFacadeModule(): FacadeModule {
return loadBundledPluginPublicSurfaceModuleSync<FacadeModule>({
dirName: "qwen",
artifactBasename: "api.js",
});
}
export const applyQwenNativeStreamingUsageCompat: FacadeModule["applyQwenNativeStreamingUsageCompat"] =
((...args) =>
loadFacadeModule()["applyQwenNativeStreamingUsageCompat"](
...args,
)) as FacadeModule["applyQwenNativeStreamingUsageCompat"];
export const buildQwenDefaultModelDefinition: FacadeModule["buildQwenDefaultModelDefinition"] = ((
...args
) =>
loadFacadeModule()["buildQwenDefaultModelDefinition"](
...args,
)) as FacadeModule["buildQwenDefaultModelDefinition"];
export const buildQwenModelDefinition: FacadeModule["buildQwenModelDefinition"] = ((...args) =>
loadFacadeModule()["buildQwenModelDefinition"](
...args,
)) as FacadeModule["buildQwenModelDefinition"];
export const QWEN_BASE_URL: FacadeModule["QWEN_BASE_URL"] = loadFacadeModule()["QWEN_BASE_URL"];
export const QWEN_CN_BASE_URL: FacadeModule["QWEN_CN_BASE_URL"] =
loadFacadeModule()["QWEN_CN_BASE_URL"];
export const QWEN_DEFAULT_COST: FacadeModule["QWEN_DEFAULT_COST"] = createLazyFacadeObjectValue(
() => loadFacadeModule()["QWEN_DEFAULT_COST"] as object,
) as FacadeModule["QWEN_DEFAULT_COST"];
export const QWEN_DEFAULT_MODEL_ID: FacadeModule["QWEN_DEFAULT_MODEL_ID"] =
loadFacadeModule()["QWEN_DEFAULT_MODEL_ID"];
export const QWEN_DEFAULT_MODEL_REF: FacadeModule["QWEN_DEFAULT_MODEL_REF"] =
loadFacadeModule()["QWEN_DEFAULT_MODEL_REF"];
export const QWEN_GLOBAL_BASE_URL: FacadeModule["QWEN_GLOBAL_BASE_URL"] =
loadFacadeModule()["QWEN_GLOBAL_BASE_URL"];
export const QWEN_STANDARD_CN_BASE_URL: FacadeModule["QWEN_STANDARD_CN_BASE_URL"] =
loadFacadeModule()["QWEN_STANDARD_CN_BASE_URL"];
export const QWEN_STANDARD_GLOBAL_BASE_URL: FacadeModule["QWEN_STANDARD_GLOBAL_BASE_URL"] =
loadFacadeModule()["QWEN_STANDARD_GLOBAL_BASE_URL"];
export const QWEN_MODEL_CATALOG: FacadeModule["QWEN_MODEL_CATALOG"] = createLazyFacadeArrayValue(
() => loadFacadeModule()["QWEN_MODEL_CATALOG"] as unknown as readonly unknown[],
) as FacadeModule["QWEN_MODEL_CATALOG"];
export const isNativeQwenBaseUrl: FacadeModule["isNativeQwenBaseUrl"] = ((...args) =>
loadFacadeModule()["isNativeQwenBaseUrl"](...args)) as FacadeModule["isNativeQwenBaseUrl"];
export const buildQwenProvider: FacadeModule["buildQwenProvider"] = ((...args) =>
loadFacadeModule()["buildQwenProvider"](...args)) as FacadeModule["buildQwenProvider"];

View File

@@ -0,0 +1,27 @@
// Shared video-generation implementation helpers for bundled and third-party plugins.
export type { AuthProfileStore } from "../agents/auth-profiles.js";
export type { FallbackAttempt } from "../agents/model-fallback.types.js";
export type { VideoGenerationProviderPlugin } from "../plugins/types.js";
export type {
GeneratedVideoAsset,
VideoGenerationProvider,
VideoGenerationRequest,
VideoGenerationResolution,
VideoGenerationResult,
VideoGenerationSourceAsset,
} from "../video-generation/types.js";
export type { OpenClawConfig } from "../config/config.js";
export { describeFailoverError, isFailoverError } from "../agents/failover-error.js";
export {
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
} from "../config/model-input.js";
export {
getVideoGenerationProvider,
listVideoGenerationProviders,
} from "../video-generation/provider-registry.js";
export { parseVideoGenerationModelRef } from "../video-generation/model-ref.js";
export { createSubsystemLogger } from "../logging/subsystem.js";
export { getProviderEnvVars } from "../secrets/provider-env-vars.js";

View File

@@ -0,0 +1,21 @@
// Generated by scripts/generate-plugin-sdk-facades.mjs. Do not edit manually.
import type { PluginSdkFacadeTypeMap } from "../generated/plugin-sdk-facade-type-map.generated.js";
type FacadeEntry = PluginSdkFacadeTypeMap["video-generation-runtime"];
type FacadeModule = FacadeEntry["module"];
import { loadActivatedBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
function loadFacadeModule(): FacadeModule {
return loadActivatedBundledPluginPublicSurfaceModuleSync<FacadeModule>({
dirName: "video-generation-core",
artifactBasename: "runtime-api.js",
});
}
export const generateVideo: FacadeModule["generateVideo"] = ((...args) =>
loadFacadeModule()["generateVideo"](...args)) as FacadeModule["generateVideo"];
export const listRuntimeVideoGenerationProviders: FacadeModule["listRuntimeVideoGenerationProviders"] =
((...args) =>
loadFacadeModule()["listRuntimeVideoGenerationProviders"](
...args,
)) as FacadeModule["listRuntimeVideoGenerationProviders"];
export type GenerateVideoParams = FacadeEntry["types"]["GenerateVideoParams"];
export type GenerateVideoRuntimeResult = FacadeEntry["types"]["GenerateVideoRuntimeResult"];

View File

@@ -0,0 +1,10 @@
// Public video-generation helpers and types for provider plugins.
export type {
GeneratedVideoAsset,
VideoGenerationProvider,
VideoGenerationRequest,
VideoGenerationResolution,
VideoGenerationResult,
VideoGenerationSourceAsset,
} from "../video-generation/types.js";

View File

@@ -32,6 +32,7 @@ export type BuildPluginApiParams = {
| "registerRealtimeVoiceProvider"
| "registerMediaUnderstandingProvider"
| "registerImageGenerationProvider"
| "registerVideoGenerationProvider"
| "registerWebFetchProvider"
| "registerWebSearchProvider"
| "registerInteractiveHandler"
@@ -65,6 +66,8 @@ const noopRegisterMediaUnderstandingProvider: OpenClawPluginApi["registerMediaUn
() => {};
const noopRegisterImageGenerationProvider: OpenClawPluginApi["registerImageGenerationProvider"] =
() => {};
const noopRegisterVideoGenerationProvider: OpenClawPluginApi["registerVideoGenerationProvider"] =
() => {};
const noopRegisterWebFetchProvider: OpenClawPluginApi["registerWebFetchProvider"] = () => {};
const noopRegisterWebSearchProvider: OpenClawPluginApi["registerWebSearchProvider"] = () => {};
const noopRegisterInteractiveHandler: OpenClawPluginApi["registerInteractiveHandler"] = () => {};
@@ -111,6 +114,8 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi
handlers.registerMediaUnderstandingProvider ?? noopRegisterMediaUnderstandingProvider,
registerImageGenerationProvider:
handlers.registerImageGenerationProvider ?? noopRegisterImageGenerationProvider,
registerVideoGenerationProvider:
handlers.registerVideoGenerationProvider ?? noopRegisterVideoGenerationProvider,
registerWebFetchProvider: handlers.registerWebFetchProvider ?? noopRegisterWebFetchProvider,
registerWebSearchProvider: handlers.registerWebSearchProvider ?? noopRegisterWebSearchProvider,
registerInteractiveHandler:

View File

@@ -9,6 +9,7 @@ export type BundledPluginContractSnapshot = {
realtimeVoiceProviderIds: string[];
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
toolNames: string[];
@@ -45,6 +46,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
realtimeVoiceProviderIds: uniqueStrings(manifest.contracts?.realtimeVoiceProviders),
mediaUnderstandingProviderIds: uniqueStrings(manifest.contracts?.mediaUnderstandingProviders),
imageGenerationProviderIds: uniqueStrings(manifest.contracts?.imageGenerationProviders),
videoGenerationProviderIds: uniqueStrings(manifest.contracts?.videoGenerationProviders),
webFetchProviderIds: uniqueStrings(manifest.contracts?.webFetchProviders),
webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders),
toolNames: uniqueStrings(manifest.contracts?.tools),
@@ -58,6 +60,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
entry.realtimeVoiceProviderIds.length > 0 ||
entry.mediaUnderstandingProviderIds.length > 0 ||
entry.imageGenerationProviderIds.length > 0 ||
entry.videoGenerationProviderIds.length > 0 ||
entry.webFetchProviderIds.length > 0 ||
entry.webSearchProviderIds.length > 0 ||
entry.toolNames.length > 0,
@@ -92,6 +95,10 @@ export const BUNDLED_IMAGE_GENERATION_PLUGIN_IDS = collectPluginIds(
(entry) => entry.imageGenerationProviderIds,
);
export const BUNDLED_VIDEO_GENERATION_PLUGIN_IDS = collectPluginIds(
(entry) => entry.videoGenerationProviderIds,
);
export const BUNDLED_WEB_FETCH_PLUGIN_IDS = collectPluginIds((entry) => entry.webFetchProviderIds);
export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [
@@ -104,6 +111,7 @@ export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [
entry.realtimeVoiceProviderIds.length > 0 ||
entry.mediaUnderstandingProviderIds.length > 0 ||
entry.imageGenerationProviderIds.length > 0 ||
entry.videoGenerationProviderIds.length > 0 ||
entry.webFetchProviderIds.length > 0 ||
entry.webSearchProviderIds.length > 0,
).map((entry) => entry.pluginId),

View File

@@ -126,6 +126,7 @@ function createCapabilityPluginRecord(params: {
realtimeVoiceProviderIds: [],
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
gatewayMethods: [],
@@ -286,6 +287,9 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
record.imageGenerationProviderIds.push(
...captured.imageGenerationProviders.map((entry) => entry.id),
);
record.videoGenerationProviderIds.push(
...captured.videoGenerationProviders.map((entry) => entry.id),
);
record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id));
record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id));
record.toolNames.push(...captured.tools.map((entry) => entry.name));
@@ -353,6 +357,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
rootDir: record.rootDir,
})),
);
registry.videoGenerationProviders.push(
...captured.videoGenerationProviders.map((provider) => ({
pluginId: record.id,
pluginName: record.name,
provider,
source: record.source,
rootDir: record.rootDir,
})),
);
registry.webFetchProviders.push(
...captured.webFetchProviders.map((provider) => ({
pluginId: record.id,

View File

@@ -23,7 +23,8 @@ export const BUNDLED_PROVIDER_AUTH_ENV_VAR_CANDIDATES = {
minimax: ["MINIMAX_API_KEY"],
"minimax-portal": ["MINIMAX_OAUTH_TOKEN", "MINIMAX_API_KEY"],
mistral: ["MISTRAL_API_KEY"],
modelstudio: ["MODELSTUDIO_API_KEY"],
qwen: ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"],
modelstudio: ["QWEN_API_KEY", "MODELSTUDIO_API_KEY", "DASHSCOPE_API_KEY"],
moonshot: ["MOONSHOT_API_KEY"],
nvidia: ["NVIDIA_API_KEY"],
ollama: ["OLLAMA_API_KEY"],

View File

@@ -265,4 +265,44 @@ describe("resolvePluginCapabilityProviders", () => {
config: expect.anything(),
});
});
it("loads bundled capability providers even without an explicit cfg", () => {
const compatConfig = {
plugins: {
enabled: true,
allow: ["google"],
entries: { google: { enabled: true } },
},
} as OpenClawConfig;
const loaded = createEmptyPluginRegistry();
loaded.mediaUnderstandingProviders.push({
pluginId: "google",
pluginName: "google",
source: "test",
provider: {
id: "google",
capabilities: ["image", "audio", "video"],
describeImage: vi.fn(),
transcribeAudio: vi.fn(),
describeVideo: vi.fn(),
autoPriority: { image: 30, audio: 40, video: 10 },
nativeDocumentInputs: ["pdf"],
},
} as never);
setBundledCapabilityFixture("mediaUnderstandingProviders");
mocks.withBundledPluginEnablementCompat.mockReturnValue(compatConfig);
mocks.withBundledPluginVitestCompat.mockReturnValue(compatConfig);
mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) =>
params === undefined ? undefined : loaded,
);
const providers = resolvePluginCapabilityProviders({ key: "mediaUnderstandingProviders" });
expectResolvedCapabilityProviderIds(providers, ["google"]);
expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({
config: undefined,
env: process.env,
});
expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ config: compatConfig });
});
});

View File

@@ -12,14 +12,16 @@ type CapabilityProviderRegistryKey =
| "realtimeTranscriptionProviders"
| "realtimeVoiceProviders"
| "mediaUnderstandingProviders"
| "imageGenerationProviders";
| "imageGenerationProviders"
| "videoGenerationProviders";
type CapabilityContractKey =
| "speechProviders"
| "realtimeTranscriptionProviders"
| "realtimeVoiceProviders"
| "mediaUnderstandingProviders"
| "imageGenerationProviders";
| "imageGenerationProviders"
| "videoGenerationProviders";
type CapabilityProviderForKey<K extends CapabilityProviderRegistryKey> =
PluginRegistry[K][number] extends { provider: infer T } ? T : never;
@@ -30,6 +32,7 @@ const CAPABILITY_CONTRACT_KEY: Record<CapabilityProviderRegistryKey, CapabilityC
realtimeVoiceProviders: "realtimeVoiceProviders",
mediaUnderstandingProviders: "mediaUnderstandingProviders",
imageGenerationProviders: "imageGenerationProviders",
videoGenerationProviders: "videoGenerationProviders",
};
function resolveBundledCapabilityCompatPluginIds(params: {
@@ -73,12 +76,8 @@ export function resolvePluginCapabilityProviders<K extends CapabilityProviderReg
if (activeProviders.length > 0) {
return activeProviders.map((entry) => entry.provider) as CapabilityProviderForKey<K>[];
}
const loadOptions =
params.cfg === undefined
? undefined
: {
config: resolveCapabilityProviderConfig({ key: params.key, cfg: params.cfg }),
};
const compatConfig = resolveCapabilityProviderConfig({ key: params.key, cfg: params.cfg });
const loadOptions = compatConfig === undefined ? undefined : { config: compatConfig };
const registry = resolveRuntimePluginRegistry(loadOptions);
return (registry?.[params.key] ?? []).map(
(entry) => entry.provider,

View File

@@ -13,6 +13,7 @@ import type {
RealtimeTranscriptionProviderPlugin,
RealtimeVoiceProviderPlugin,
SpeechProviderPlugin,
VideoGenerationProviderPlugin,
WebFetchProviderPlugin,
WebSearchProviderPlugin,
} from "./types.js";
@@ -33,6 +34,7 @@ export type CapturedPluginRegistration = {
realtimeVoiceProviders: RealtimeVoiceProviderPlugin[];
mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[];
imageGenerationProviders: ImageGenerationProviderPlugin[];
videoGenerationProviders: VideoGenerationProviderPlugin[];
webFetchProviders: WebFetchProviderPlugin[];
webSearchProviders: WebSearchProviderPlugin[];
tools: AnyAgentTool[];
@@ -50,6 +52,7 @@ export function createCapturedPluginRegistration(params?: {
const realtimeVoiceProviders: RealtimeVoiceProviderPlugin[] = [];
const mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[] = [];
const imageGenerationProviders: ImageGenerationProviderPlugin[] = [];
const videoGenerationProviders: VideoGenerationProviderPlugin[] = [];
const webFetchProviders: WebFetchProviderPlugin[] = [];
const webSearchProviders: WebSearchProviderPlugin[] = [];
const tools: AnyAgentTool[] = [];
@@ -69,6 +72,7 @@ export function createCapturedPluginRegistration(params?: {
realtimeVoiceProviders,
mediaUnderstandingProviders,
imageGenerationProviders,
videoGenerationProviders,
webFetchProviders,
webSearchProviders,
tools,
@@ -126,6 +130,9 @@ export function createCapturedPluginRegistration(params?: {
registerImageGenerationProvider(provider: ImageGenerationProviderPlugin) {
imageGenerationProviders.push(provider);
},
registerVideoGenerationProvider(provider: VideoGenerationProviderPlugin) {
videoGenerationProviders.push(provider);
},
registerWebFetchProvider(provider: WebFetchProviderPlugin) {
webFetchProviders.push(provider);
},

View File

@@ -15,6 +15,7 @@ function hasRuntimeContractSurface(plugin: PluginManifestRecord): boolean {
plugin.contracts?.speechProviders?.length ||
plugin.contracts?.mediaUnderstandingProviders?.length ||
plugin.contracts?.imageGenerationProviders?.length ||
plugin.contracts?.videoGenerationProviders?.length ||
plugin.contracts?.webFetchProviders?.length ||
plugin.contracts?.webSearchProviders?.length ||
hasKind(plugin.kind, "memory"),

View File

@@ -8,6 +8,7 @@ import {
BUNDLED_SPEECH_PLUGIN_IDS,
BUNDLED_WEB_FETCH_PLUGIN_IDS,
BUNDLED_WEB_SEARCH_PLUGIN_IDS,
BUNDLED_VIDEO_GENERATION_PLUGIN_IDS,
} from "../bundled-capability-metadata.js";
import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js";
import type {
@@ -17,6 +18,7 @@ import type {
RealtimeTranscriptionProviderPlugin,
RealtimeVoiceProviderPlugin,
SpeechProviderPlugin,
VideoGenerationProviderPlugin,
WebFetchProviderPlugin,
WebSearchProviderPlugin,
} from "../types.js";
@@ -26,6 +28,7 @@ import {
loadVitestRealtimeTranscriptionProviderContractRegistry,
loadVitestRealtimeVoiceProviderContractRegistry,
loadVitestSpeechProviderContractRegistry,
loadVitestVideoGenerationProviderContractRegistry,
} from "./speech-vitest-registry.js";
type BundledCapabilityRuntimeRegistry = ReturnType<typeof loadBundledCapabilityRuntimeRegistry>;
@@ -50,6 +53,7 @@ type RealtimeVoiceProviderContractEntry = CapabilityContractEntry<RealtimeVoiceP
type MediaUnderstandingProviderContractEntry =
CapabilityContractEntry<MediaUnderstandingProviderPlugin>;
type ImageGenerationProviderContractEntry = CapabilityContractEntry<ImageGenerationProviderPlugin>;
type VideoGenerationProviderContractEntry = CapabilityContractEntry<VideoGenerationProviderPlugin>;
type PluginRegistrationContractEntry = {
pluginId: string;
@@ -60,6 +64,7 @@ type PluginRegistrationContractEntry = {
realtimeVoiceProviderIds: string[];
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
toolNames: string[];
@@ -114,6 +119,8 @@ let mediaUnderstandingProviderContractRegistryCache:
| null = null;
let imageGenerationProviderContractRegistryCache: ImageGenerationProviderContractEntry[] | null =
null;
let videoGenerationProviderContractRegistryCache: VideoGenerationProviderContractEntry[] | null =
null;
const providerContractPluginIdsByProviderId = createProviderContractPluginIdsByProviderId();
export let providerContractLoadError: Error | undefined;
@@ -462,6 +469,21 @@ function loadImageGenerationProviderContractRegistry(): ImageGenerationProviderC
return imageGenerationProviderContractRegistryCache;
}
function loadVideoGenerationProviderContractRegistry(): VideoGenerationProviderContractEntry[] {
if (!videoGenerationProviderContractRegistryCache) {
videoGenerationProviderContractRegistryCache = process.env.VITEST
? loadVitestVideoGenerationProviderContractRegistry()
: loadBundledCapabilityRuntimeRegistry({
pluginIds: BUNDLED_VIDEO_GENERATION_PLUGIN_IDS,
pluginSdkResolution: "dist",
}).videoGenerationProviders.map((entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
}));
}
return videoGenerationProviderContractRegistryCache;
}
function createLazyArrayView<T>(load: () => T[]): T[] {
return new Proxy([] as T[], {
get(_target, prop) {
@@ -576,6 +598,9 @@ export const mediaUnderstandingProviderContractRegistry: MediaUnderstandingProvi
export const imageGenerationProviderContractRegistry: ImageGenerationProviderContractEntry[] =
createLazyArrayView(loadImageGenerationProviderContractRegistry);
export const videoGenerationProviderContractRegistry: VideoGenerationProviderContractEntry[] =
createLazyArrayView(loadVideoGenerationProviderContractRegistry);
function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEntry[] {
return BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.map((entry) => ({
pluginId: entry.pluginId,
@@ -586,6 +611,7 @@ function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEnt
realtimeVoiceProviderIds: uniqueStrings(entry.realtimeVoiceProviderIds),
mediaUnderstandingProviderIds: uniqueStrings(entry.mediaUnderstandingProviderIds),
imageGenerationProviderIds: uniqueStrings(entry.imageGenerationProviderIds),
videoGenerationProviderIds: uniqueStrings(entry.videoGenerationProviderIds),
webFetchProviderIds: uniqueStrings(entry.webFetchProviderIds),
webSearchProviderIds: uniqueStrings(entry.webSearchProviderIds),
toolNames: uniqueStrings(entry.toolNames),

View File

@@ -8,6 +8,7 @@ import {
BUNDLED_REALTIME_TRANSCRIPTION_PLUGIN_IDS,
BUNDLED_REALTIME_VOICE_PLUGIN_IDS,
BUNDLED_SPEECH_PLUGIN_IDS,
BUNDLED_VIDEO_GENERATION_PLUGIN_IDS,
} from "../bundled-capability-metadata.js";
import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js";
import { loadPluginManifestRegistry } from "../manifest-registry.js";
@@ -18,6 +19,7 @@ import type {
RealtimeTranscriptionProviderPlugin,
RealtimeVoiceProviderPlugin,
SpeechProviderPlugin,
VideoGenerationProviderPlugin,
} from "../types.js";
export type SpeechProviderContractEntry = {
@@ -45,6 +47,11 @@ export type ImageGenerationProviderContractEntry = {
provider: ImageGenerationProviderPlugin;
};
export type VideoGenerationProviderContractEntry = {
pluginId: string;
provider: VideoGenerationProviderPlugin;
};
function buildVitestCapabilityAliasMap(modulePath: string): Record<string, string> {
const { ["openclaw/plugin-sdk"]: _ignoredRootAlias, ...scopedAliasMap } =
buildPluginLoaderAliasMap(modulePath, process.argv[1], import.meta.url, "dist");
@@ -338,3 +345,48 @@ export function loadVitestImageGenerationProviderContractRegistry(): ImageGenera
);
return registrations;
}
export function loadVitestVideoGenerationProviderContractRegistry(): VideoGenerationProviderContractEntry[] {
const registrations: VideoGenerationProviderContractEntry[] = [];
const { manifests, unresolvedPluginIds } = resolveTestApiModuleRecords(
BUNDLED_VIDEO_GENERATION_PLUGIN_IDS,
);
for (const plugin of manifests) {
if (!plugin.rootDir) {
continue;
}
const testApiPath = path.join(plugin.rootDir, "test-api.ts");
if (!fs.existsSync(testApiPath)) {
continue;
}
const builder = resolveNamedBuilder<VideoGenerationProviderPlugin>(
createVitestCapabilityLoader(testApiPath)(testApiPath),
/^build.+VideoGenerationProvider$/u,
);
if (!builder) {
continue;
}
registrations.push({
pluginId: plugin.id,
provider: builder(),
});
unresolvedPluginIds.delete(plugin.id);
}
if (unresolvedPluginIds.size === 0) {
return registrations;
}
const runtimeRegistry = loadBundledCapabilityRuntimeRegistry({
pluginIds: [...unresolvedPluginIds],
pluginSdkResolution: "dist",
});
registrations.push(
...runtimeRegistry.videoGenerationProviders.map((entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
})),
);
return registrations;
}

View File

@@ -38,6 +38,7 @@ export function createMockPluginRegistry(
speechProviders: [],
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
webSearchProviders: [],
httpRoutes: [],
gatewayHandlers: {},

View File

@@ -594,6 +594,7 @@ function createPluginRecord(params: {
realtimeVoiceProviderIds: [],
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
gatewayMethods: [],

View File

@@ -76,6 +76,7 @@ export type PluginManifestContracts = {
realtimeVoiceProviders?: string[];
mediaUnderstandingProviders?: string[];
imageGenerationProviders?: string[];
videoGenerationProviders?: string[];
webFetchProviders?: string[];
webSearchProviders?: string[];
tools?: string[];
@@ -155,6 +156,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u
const realtimeVoiceProviders = normalizeStringList(value.realtimeVoiceProviders);
const mediaUnderstandingProviders = normalizeStringList(value.mediaUnderstandingProviders);
const imageGenerationProviders = normalizeStringList(value.imageGenerationProviders);
const videoGenerationProviders = normalizeStringList(value.videoGenerationProviders);
const webFetchProviders = normalizeStringList(value.webFetchProviders);
const webSearchProviders = normalizeStringList(value.webSearchProviders);
const tools = normalizeStringList(value.tools);
@@ -164,6 +166,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u
...(realtimeVoiceProviders.length > 0 ? { realtimeVoiceProviders } : {}),
...(mediaUnderstandingProviders.length > 0 ? { mediaUnderstandingProviders } : {}),
...(imageGenerationProviders.length > 0 ? { imageGenerationProviders } : {}),
...(videoGenerationProviders.length > 0 ? { videoGenerationProviders } : {}),
...(webFetchProviders.length > 0 ? { webFetchProviders } : {}),
...(webSearchProviders.length > 0 ? { webSearchProviders } : {}),
...(tools.length > 0 ? { tools } : {}),

View File

@@ -85,6 +85,7 @@ const {
setTogetherApiKey,
setHuggingfaceApiKey,
setQianfanApiKey,
setQwenApiKey,
setModelStudioApiKey,
setXaiApiKey,
setMistralApiKey,
@@ -110,7 +111,8 @@ const {
setTogetherApiKey: { provider: "together" },
setHuggingfaceApiKey: { provider: "huggingface" },
setQianfanApiKey: { provider: "qianfan" },
setModelStudioApiKey: { provider: "modelstudio" },
setQwenApiKey: { provider: "qwen" },
setModelStudioApiKey: { provider: "qwen" },
setXaiApiKey: { provider: "xai" },
setMistralApiKey: { provider: "mistral" },
setKilocodeApiKey: { provider: "kilocode" },
@@ -134,6 +136,7 @@ export {
setTogetherApiKey,
setHuggingfaceApiKey,
setQianfanApiKey,
setQwenApiKey,
setModelStudioApiKey,
setXaiApiKey,
setMistralApiKey,

View File

@@ -16,16 +16,19 @@ const applyPluginAutoEnableMock = vi.fn<ApplyPluginAutoEnable>();
let resolveOwningPluginIdsForProvider: typeof import("./providers.js").resolveOwningPluginIdsForProvider;
let resolveOwningPluginIdsForModelRef: typeof import("./providers.js").resolveOwningPluginIdsForModelRef;
let resolveEnabledProviderPluginIds: typeof import("./providers.js").resolveEnabledProviderPluginIds;
let resolvePluginProviders: typeof import("./providers.runtime.js").resolvePluginProviders;
function createManifestProviderPlugin(params: {
id: string;
providerIds: string[];
origin?: "bundled" | "workspace";
enabledByDefault?: boolean;
modelSupport?: { modelPrefixes?: string[]; modelPatterns?: string[] };
}): PluginManifestRecord {
return {
id: params.id,
enabledByDefault: params.enabledByDefault,
channels: [],
cliBackends: [],
providers: params.providerIds,
@@ -230,8 +233,11 @@ describe("resolvePluginProviders", () => {
loadPluginManifestRegistry: (...args: Parameters<LoadPluginManifestRegistry>) =>
loadPluginManifestRegistryMock(...args),
}));
({ resolveOwningPluginIdsForProvider, resolveOwningPluginIdsForModelRef } =
await import("./providers.js"));
({
resolveOwningPluginIdsForProvider,
resolveOwningPluginIdsForModelRef,
resolveEnabledProviderPluginIds,
} = await import("./providers.js"));
({ resolvePluginProviders } = await import("./providers.runtime.js"));
});
@@ -255,10 +261,22 @@ describe("resolvePluginProviders", () => {
}),
);
setManifestPlugins([
createManifestProviderPlugin({ id: "google", providerIds: ["google"] }),
createManifestProviderPlugin({
id: "google",
providerIds: ["google"],
enabledByDefault: true,
}),
createManifestProviderPlugin({ id: "browser", providerIds: [] }),
createManifestProviderPlugin({ id: "kilocode", providerIds: ["kilocode"] }),
createManifestProviderPlugin({ id: "moonshot", providerIds: ["moonshot"] }),
createManifestProviderPlugin({
id: "kilocode",
providerIds: ["kilocode"],
enabledByDefault: true,
}),
createManifestProviderPlugin({
id: "moonshot",
providerIds: ["moonshot"],
enabledByDefault: true,
}),
createManifestProviderPlugin({ id: "google-gemini-cli-auth", providerIds: [] }),
createManifestProviderPlugin({
id: "workspace-provider",
@@ -292,6 +310,14 @@ describe("resolvePluginProviders", () => {
);
});
it("keeps bundled provider plugins enabled when they default on outside Vitest compat", () => {
expect(resolveEnabledProviderPluginIds({ config: {}, env: {} as NodeJS.ProcessEnv })).toEqual([
"google",
"kilocode",
"moonshot",
]);
});
it.each([
{
name: "can augment restrictive allowlists for bundled provider compatibility",

View File

@@ -62,6 +62,7 @@ export function resolveEnabledProviderPluginIds(params: {
origin: plugin.origin,
config: normalizedConfig,
rootConfig: params.config,
enabledByDefault: plugin.enabledByDefault,
}).activated,
)
.map((plugin) => plugin.id)

View File

@@ -15,6 +15,7 @@ export function createEmptyPluginRegistry(): PluginRegistry {
realtimeVoiceProviders: [],
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
gatewayHandlers: {},

View File

@@ -69,6 +69,7 @@ import type {
PluginHookHandlerMap,
PluginHookRegistration as TypedPluginHookRegistration,
SpeechProviderPlugin,
VideoGenerationProviderPlugin,
WebFetchProviderPlugin,
WebSearchProviderPlugin,
} from "./types.js";
@@ -153,6 +154,8 @@ export type PluginMediaUnderstandingProviderRegistration =
PluginOwnedProviderRegistration<MediaUnderstandingProviderPlugin>;
export type PluginImageGenerationProviderRegistration =
PluginOwnedProviderRegistration<ImageGenerationProviderPlugin>;
export type PluginVideoGenerationProviderRegistration =
PluginOwnedProviderRegistration<VideoGenerationProviderPlugin>;
export type PluginWebFetchProviderRegistration =
PluginOwnedProviderRegistration<WebFetchProviderPlugin>;
export type PluginWebSearchProviderRegistration =
@@ -224,6 +227,7 @@ export type PluginRecord = {
realtimeVoiceProviderIds: string[];
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
gatewayMethods: string[];
@@ -252,6 +256,7 @@ export type PluginRegistry = {
realtimeVoiceProviders: PluginRealtimeVoiceProviderRegistration[];
mediaUnderstandingProviders: PluginMediaUnderstandingProviderRegistration[];
imageGenerationProviders: PluginImageGenerationProviderRegistration[];
videoGenerationProviders: PluginVideoGenerationProviderRegistration[];
webFetchProviders: PluginWebFetchProviderRegistration[];
webSearchProviders: PluginWebSearchProviderRegistration[];
gatewayHandlers: GatewayRequestHandlers;
@@ -772,6 +777,19 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
});
};
const registerVideoGenerationProvider = (
record: PluginRecord,
provider: VideoGenerationProviderPlugin,
) => {
registerUniqueProviderLike({
record,
provider,
kindLabel: "video-generation provider",
registrations: registry.videoGenerationProviders,
ownedIds: record.videoGenerationProviderIds,
});
};
const registerWebFetchProvider = (record: PluginRecord, provider: WebFetchProviderPlugin) => {
registerUniqueProviderLike({
record,
@@ -1064,6 +1082,8 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
registerMediaUnderstandingProvider(record, provider),
registerImageGenerationProvider: (provider) =>
registerImageGenerationProvider(record, provider),
registerVideoGenerationProvider: (provider) =>
registerVideoGenerationProvider(record, provider),
registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider),
registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider),
registerGatewayMethod: (method, handler, opts) =>
@@ -1253,6 +1273,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
registerRealtimeVoiceProvider,
registerMediaUnderstandingProvider,
registerImageGenerationProvider,
registerVideoGenerationProvider,
registerWebSearchProvider,
registerGatewayMethod,
registerCli,

View File

@@ -203,6 +203,7 @@ describe("setActivePluginRegistry", () => {
realtimeVoiceProviderIds: [],
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
gatewayMethods: [],
@@ -231,6 +232,7 @@ describe("setActivePluginRegistry", () => {
realtimeVoiceProviderIds: [],
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
gatewayMethods: [],

View File

@@ -58,6 +58,12 @@ type RuntimeImageGenerationModule = Pick<
>;
let cachedRuntimeImageGenerationModule: RuntimeImageGenerationModule | null = null;
type RuntimeVideoGenerationModule = Pick<
typeof import("../../plugin-sdk/video-generation-runtime.js"),
"generateVideo" | "listRuntimeVideoGenerationProviders"
>;
let cachedRuntimeVideoGenerationModule: RuntimeVideoGenerationModule | null = null;
function loadRuntimeImageGenerationModule(): RuntimeImageGenerationModule {
cachedRuntimeImageGenerationModule ??=
loadBundledPluginPublicSurfaceModuleSync<RuntimeImageGenerationModule>({
@@ -75,6 +81,23 @@ function createRuntimeImageGeneration(): PluginRuntime["imageGeneration"] {
};
}
function loadRuntimeVideoGenerationModule(): RuntimeVideoGenerationModule {
cachedRuntimeVideoGenerationModule ??=
loadBundledPluginPublicSurfaceModuleSync<RuntimeVideoGenerationModule>({
dirName: "video-generation-core",
artifactBasename: "runtime-api.js",
});
return cachedRuntimeVideoGenerationModule;
}
function createRuntimeVideoGeneration(): PluginRuntime["videoGeneration"] {
return {
generate: (params) => loadRuntimeVideoGenerationModule().generateVideo(params),
listProviders: (params) =>
loadRuntimeVideoGenerationModule().listRuntimeVideoGenerationProviders(params),
};
}
function createRuntimeModelAuth(): PluginRuntime["modelAuth"] {
const getApiKeyForModel = createLazyRuntimeMethod(
loadModelAuthRuntime,
@@ -213,10 +236,13 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}):
taskFlow,
} satisfies Omit<
PluginRuntime,
"tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration"
"tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration"
> &
Partial<
Pick<PluginRuntime, "tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration">
Pick<
PluginRuntime,
"tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration"
>
>;
defineCachedValue(runtime, "tts", createRuntimeTts);
@@ -226,6 +252,7 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}):
}));
defineCachedValue(runtime, "modelAuth", createRuntimeModelAuth);
defineCachedValue(runtime, "imageGeneration", createRuntimeImageGeneration);
defineCachedValue(runtime, "videoGeneration", createRuntimeVideoGeneration);
return runtime as PluginRuntime;
}

View File

@@ -82,6 +82,10 @@ export type PluginRuntimeCore = {
generate: typeof import("../../plugin-sdk/image-generation-runtime.js").generateImage;
listProviders: typeof import("../../plugin-sdk/image-generation-runtime.js").listRuntimeImageGenerationProviders;
};
videoGeneration: {
generate: typeof import("../../plugin-sdk/video-generation-runtime.js").generateVideo;
listProviders: typeof import("../../plugin-sdk/video-generation-runtime.js").listRuntimeVideoGenerationProviders;
};
webSearch: {
listProviders: typeof import("../../web-search/runtime.js").listWebSearchProviders;
search: typeof import("../../web-search/runtime.js").runWebSearch;

View File

@@ -55,6 +55,7 @@ export function createPluginRecord(
realtimeVoiceProviderIds: [],
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
gatewayMethods: [],
@@ -119,6 +120,7 @@ export function createPluginLoadResult(
speechProviders: [],
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
tools: [],

View File

@@ -69,6 +69,7 @@ import type {
SpeechVoiceOption,
} from "../tts/provider-types.js";
import type { DeliveryContext } from "../utils/delivery-context.js";
import type { VideoGenerationProvider } from "../video-generation/types.js";
import type { WizardPrompter } from "../wizard/prompts.js";
import type { SecretInputMode } from "./provider-auth-types.js";
import type { createVpsAwareOAuthHandlers } from "./provider-oauth-flow.js";
@@ -1698,6 +1699,7 @@ export type PluginRealtimeVoiceProviderEntry = RealtimeVoiceProviderPlugin & {
export type MediaUnderstandingProviderPlugin = MediaUnderstandingProvider;
export type ImageGenerationProviderPlugin = ImageGenerationProvider;
export type VideoGenerationProviderPlugin = VideoGenerationProvider;
export type OpenClawPluginGatewayMethod = {
method: string;
@@ -2035,6 +2037,8 @@ export type OpenClawPluginApi = {
registerMediaUnderstandingProvider: (provider: MediaUnderstandingProviderPlugin) => void;
/** Register an image generation provider (image generation capability). */
registerImageGenerationProvider: (provider: ImageGenerationProviderPlugin) => void;
/** Register a video generation provider (video generation capability). */
registerVideoGenerationProvider: (provider: VideoGenerationProviderPlugin) => void;
/** Register a web fetch provider (web fetch capability). */
registerWebFetchProvider: (provider: WebFetchProviderPlugin) => void;
/** Register a web search provider (web search capability). */

View File

@@ -0,0 +1,16 @@
export function parseVideoGenerationModelRef(
raw: string | undefined,
): { provider: string; model: string } | null {
const trimmed = raw?.trim();
if (!trimmed) {
return null;
}
const slashIndex = trimmed.indexOf("/");
if (slashIndex <= 0 || slashIndex === trimmed.length - 1) {
return null;
}
return {
provider: trimmed.slice(0, slashIndex).trim(),
model: trimmed.slice(slashIndex + 1).trim(),
};
}

View File

@@ -0,0 +1,93 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createEmptyPluginRegistry } from "../plugins/registry.js";
const { resolveRuntimePluginRegistryMock } = vi.hoisted(() => ({
resolveRuntimePluginRegistryMock: vi.fn<
(params?: unknown) => ReturnType<typeof createEmptyPluginRegistry> | undefined
>(() => undefined),
}));
vi.mock("../plugins/loader.js", () => ({
resolveRuntimePluginRegistry: resolveRuntimePluginRegistryMock,
}));
let getVideoGenerationProvider: typeof import("./provider-registry.js").getVideoGenerationProvider;
let listVideoGenerationProviders: typeof import("./provider-registry.js").listVideoGenerationProviders;
describe("video-generation provider registry", () => {
beforeAll(async () => {
({ getVideoGenerationProvider, listVideoGenerationProviders } =
await import("./provider-registry.js"));
});
beforeEach(() => {
resolveRuntimePluginRegistryMock.mockReset();
resolveRuntimePluginRegistryMock.mockReturnValue(undefined);
});
it("does not load plugins when listing without config", () => {
expect(listVideoGenerationProviders()).toEqual([]);
expect(resolveRuntimePluginRegistryMock).toHaveBeenCalledWith();
});
it("uses active plugin providers without loading from disk", () => {
const registry = createEmptyPluginRegistry();
registry.videoGenerationProviders.push({
pluginId: "custom-video",
pluginName: "Custom Video",
source: "test",
provider: {
id: "custom-video",
label: "Custom Video",
capabilities: {},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }],
}),
},
});
resolveRuntimePluginRegistryMock.mockReturnValue(registry);
const provider = getVideoGenerationProvider("custom-video");
expect(provider?.id).toBe("custom-video");
expect(resolveRuntimePluginRegistryMock).toHaveBeenCalledWith();
});
it("ignores prototype-like provider ids and aliases", () => {
const registry = createEmptyPluginRegistry();
registry.videoGenerationProviders.push(
{
pluginId: "blocked-video",
pluginName: "Blocked Video",
source: "test",
provider: {
id: "__proto__",
aliases: ["constructor", "prototype"],
capabilities: {},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }],
}),
},
},
{
pluginId: "safe-video",
pluginName: "Safe Video",
source: "test",
provider: {
id: "safe-video",
aliases: ["safe-alias", "constructor"],
capabilities: {},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("video"), mimeType: "video/mp4" }],
}),
},
},
);
resolveRuntimePluginRegistryMock.mockReturnValue(registry);
expect(listVideoGenerationProviders().map((provider) => provider.id)).toEqual(["safe-video"]);
expect(getVideoGenerationProvider("__proto__")).toBeUndefined();
expect(getVideoGenerationProvider("constructor")).toBeUndefined();
expect(getVideoGenerationProvider("safe-alias")?.id).toBe("safe-video");
});
});

View File

@@ -0,0 +1,77 @@
import { normalizeProviderId } from "../agents/model-selection.js";
import type { OpenClawConfig } from "../config/config.js";
import { isBlockedObjectKey } from "../infra/prototype-keys.js";
import { resolvePluginCapabilityProviders } from "../plugins/capability-provider-runtime.js";
import type { VideoGenerationProviderPlugin } from "../plugins/types.js";
const BUILTIN_VIDEO_GENERATION_PROVIDERS: readonly VideoGenerationProviderPlugin[] = [];
const UNSAFE_PROVIDER_IDS = new Set(["__proto__", "constructor", "prototype"]);
function normalizeVideoGenerationProviderId(id: string | undefined): string | undefined {
const normalized = normalizeProviderId(id ?? "");
if (!normalized || isBlockedObjectKey(normalized)) {
return undefined;
}
return normalized;
}
function isSafeVideoGenerationProviderId(id: string | undefined): id is string {
return Boolean(id && !UNSAFE_PROVIDER_IDS.has(id));
}
function resolvePluginVideoGenerationProviders(
cfg?: OpenClawConfig,
): VideoGenerationProviderPlugin[] {
return resolvePluginCapabilityProviders({
key: "videoGenerationProviders",
cfg,
});
}
function buildProviderMaps(cfg?: OpenClawConfig): {
canonical: Map<string, VideoGenerationProviderPlugin>;
aliases: Map<string, VideoGenerationProviderPlugin>;
} {
const canonical = new Map<string, VideoGenerationProviderPlugin>();
const aliases = new Map<string, VideoGenerationProviderPlugin>();
const register = (provider: VideoGenerationProviderPlugin) => {
const id = normalizeVideoGenerationProviderId(provider.id);
if (!isSafeVideoGenerationProviderId(id)) {
return;
}
canonical.set(id, provider);
aliases.set(id, provider);
for (const alias of provider.aliases ?? []) {
const normalizedAlias = normalizeVideoGenerationProviderId(alias);
if (isSafeVideoGenerationProviderId(normalizedAlias)) {
aliases.set(normalizedAlias, provider);
}
}
};
for (const provider of BUILTIN_VIDEO_GENERATION_PROVIDERS) {
register(provider);
}
for (const provider of resolvePluginVideoGenerationProviders(cfg)) {
register(provider);
}
return { canonical, aliases };
}
export function listVideoGenerationProviders(
cfg?: OpenClawConfig,
): VideoGenerationProviderPlugin[] {
return [...buildProviderMaps(cfg).canonical.values()];
}
export function getVideoGenerationProvider(
providerId: string | undefined,
cfg?: OpenClawConfig,
): VideoGenerationProviderPlugin | undefined {
const normalized = normalizeVideoGenerationProviderId(providerId);
if (!normalized) {
return undefined;
}
return buildProviderMaps(cfg).aliases.get(normalized);
}

View File

@@ -0,0 +1,72 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import {
generateVideo,
listRuntimeVideoGenerationProviders,
type GenerateVideoRuntimeResult,
} from "../plugin-sdk/video-generation-runtime.js";
import type { VideoGenerationProvider } from "../video-generation/types.js";
const mocks = vi.hoisted(() => ({
generateVideo: vi.fn<typeof generateVideo>(),
listRuntimeVideoGenerationProviders: vi.fn<typeof listRuntimeVideoGenerationProviders>(),
}));
vi.mock("../plugin-sdk/video-generation-runtime.js", () => ({
generateVideo: mocks.generateVideo,
listRuntimeVideoGenerationProviders: mocks.listRuntimeVideoGenerationProviders,
}));
describe("video-generation runtime facade", () => {
afterEach(() => {
mocks.generateVideo.mockReset();
mocks.listRuntimeVideoGenerationProviders.mockReset();
});
it("delegates video generation to the plugin-sdk runtime", async () => {
const result: GenerateVideoRuntimeResult = {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4", fileName: "sample.mp4" }],
provider: "video-plugin",
model: "vid-v1",
attempts: [],
};
mocks.generateVideo.mockResolvedValue(result);
const params = {
cfg: {
agents: {
defaults: {
videoGenerationModel: { primary: "video-plugin/vid-v1" },
},
},
} as OpenClawConfig,
prompt: "animate a cat",
agentDir: "/tmp/agent",
authStore: { version: 1, profiles: {} },
};
await expect(generateVideo(params)).resolves.toBe(result);
expect(mocks.generateVideo).toHaveBeenCalledWith(params);
});
it("delegates provider listing to the plugin-sdk runtime", () => {
const providers: VideoGenerationProvider[] = [
{
id: "video-plugin",
defaultModel: "vid-v1",
models: ["vid-v1", "vid-v2"],
capabilities: {
maxDurationSeconds: 10,
supportsAudio: true,
},
generateVideo: async () => ({
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
}),
},
];
mocks.listRuntimeVideoGenerationProviders.mockReturnValue(providers);
const params = { config: {} as OpenClawConfig };
expect(listRuntimeVideoGenerationProviders(params)).toBe(providers);
expect(mocks.listRuntimeVideoGenerationProviders).toHaveBeenCalledWith(params);
});
});

View File

@@ -0,0 +1,6 @@
export {
generateVideo,
listRuntimeVideoGenerationProviders,
type GenerateVideoParams,
type GenerateVideoRuntimeResult,
} from "../plugin-sdk/video-generation-runtime.js";

View File

@@ -0,0 +1,65 @@
import type { AuthProfileStore } from "../agents/auth-profiles.js";
import type { OpenClawConfig } from "../config/config.js";
export type GeneratedVideoAsset = {
buffer: Buffer;
mimeType: string;
fileName?: string;
metadata?: Record<string, unknown>;
};
export type VideoGenerationResolution = "480P" | "720P" | "1080P";
export type VideoGenerationSourceAsset = {
url?: string;
buffer?: Buffer;
mimeType?: string;
fileName?: string;
metadata?: Record<string, unknown>;
};
export type VideoGenerationRequest = {
provider: string;
model: string;
prompt: string;
cfg: OpenClawConfig;
agentDir?: string;
authStore?: AuthProfileStore;
timeoutMs?: number;
size?: string;
aspectRatio?: string;
resolution?: VideoGenerationResolution;
durationSeconds?: number;
audio?: boolean;
watermark?: boolean;
inputImages?: VideoGenerationSourceAsset[];
inputVideos?: VideoGenerationSourceAsset[];
};
export type VideoGenerationResult = {
videos: GeneratedVideoAsset[];
model?: string;
metadata?: Record<string, unknown>;
};
export type VideoGenerationProviderCapabilities = {
maxVideos?: number;
maxInputImages?: number;
maxInputVideos?: number;
maxDurationSeconds?: number;
supportsSize?: boolean;
supportsAspectRatio?: boolean;
supportsResolution?: boolean;
supportsAudio?: boolean;
supportsWatermark?: boolean;
};
export type VideoGenerationProvider = {
id: string;
aliases?: string[];
label?: string;
defaultModel?: string;
models?: string[];
capabilities: VideoGenerationProviderCapabilities;
generateVideo: (req: VideoGenerationRequest) => Promise<VideoGenerationResult>;
};

View File

@@ -24,6 +24,7 @@ export function createTestPluginApi(api: TestPluginApiInput): OpenClawPluginApi
registerRealtimeVoiceProvider() {},
registerMediaUnderstandingProvider() {},
registerImageGenerationProvider() {},
registerVideoGenerationProvider() {},
registerWebFetchProvider() {},
registerWebSearchProvider() {},
registerInteractiveHandler() {},

View File

@@ -4,6 +4,7 @@ import {
mediaUnderstandingProviderContractRegistry,
pluginRegistrationContractRegistry,
speechProviderContractRegistry,
videoGenerationProviderContractRegistry,
} from "../../../src/plugins/contracts/registry.js";
import { loadPluginManifestRegistry } from "../../../src/plugins/manifest-registry.js";
@@ -17,11 +18,13 @@ type PluginRegistrationContractParams = {
realtimeVoiceProviderIds?: string[];
mediaUnderstandingProviderIds?: string[];
imageGenerationProviderIds?: string[];
videoGenerationProviderIds?: string[];
cliBackendIds?: string[];
toolNames?: string[];
requireSpeechVoices?: boolean;
requireDescribeImages?: boolean;
requireGenerateImage?: boolean;
requireGenerateVideo?: boolean;
manifestAuthChoice?: {
pluginId: string;
choiceId: string;
@@ -91,6 +94,23 @@ function findImageGenerationProvider(pluginId: string) {
return entry.provider;
}
function findVideoGenerationProviderIds(pluginId: string) {
return videoGenerationProviderContractRegistry
.filter((entry) => entry.pluginId === pluginId)
.map((entry) => entry.provider.id)
.toSorted((left, right) => left.localeCompare(right));
}
function findVideoGenerationProvider(pluginId: string) {
const entry = videoGenerationProviderContractRegistry.find(
(candidate) => candidate.pluginId === pluginId,
);
if (!entry) {
throw new Error(`video-generation provider contract missing for ${pluginId}`);
}
return entry.provider;
}
export function describePluginRegistrationContract(params: PluginRegistrationContractParams) {
describe(`${params.pluginId} plugin registration contract`, () => {
if (params.providerIds) {
@@ -162,6 +182,17 @@ export function describePluginRegistrationContract(params: PluginRegistrationCon
});
}
if (params.videoGenerationProviderIds) {
it("keeps bundled video-generation ownership explicit", () => {
expect(findRegistration(params.pluginId).videoGenerationProviderIds).toEqual(
params.videoGenerationProviderIds,
);
expect(findVideoGenerationProviderIds(params.pluginId)).toEqual(
params.videoGenerationProviderIds,
);
});
}
if (params.cliBackendIds) {
it("keeps bundled CLI backend ownership explicit", () => {
expect(findRegistration(params.pluginId).cliBackendIds).toEqual(params.cliBackendIds);
@@ -196,6 +227,14 @@ export function describePluginRegistrationContract(params: PluginRegistrationCon
});
}
if (params.requireGenerateVideo) {
it("keeps bundled video-generation support explicit", () => {
expect(findVideoGenerationProvider(params.pluginId).generateVideo).toEqual(
expect.any(Function),
);
});
}
const manifestAuthChoice = params.manifestAuthChoice;
if (manifestAuthChoice) {
it("keeps onboarding auth grouping explicit", () => {

Some files were not shown because too many files have changed in this diff Show More