feat: add vydra media provider

This commit is contained in:
Peter Steinberger
2026-04-06 02:15:51 +01:00
parent 7d2dc7a9fb
commit 9b2b22f350
21 changed files with 1358 additions and 11 deletions

View File

@@ -22,6 +22,7 @@ import openaiPlugin from "./openai/index.js";
import qwenPlugin from "./qwen/index.js";
import runwayPlugin from "./runway/index.js";
import togetherPlugin from "./together/index.js";
import vydraPlugin from "./vydra/index.js";
import xaiPlugin from "./xai/index.js";
const LIVE = isLiveTestEnabled();
@@ -65,6 +66,7 @@ const CASES: LiveProviderCase[] = [
pluginName: "Together Provider",
providerId: "together",
},
{ plugin: vydraPlugin, pluginId: "vydra", pluginName: "Vydra Provider", providerId: "vydra" },
{ plugin: xaiPlugin, pluginId: "xai", pluginName: "xAI Plugin", providerId: "xai" },
]
.filter((entry) => (providerFilter ? providerFilter.has(entry.providerId) : true))

View File

@@ -0,0 +1,128 @@
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildVydraImageGenerationProvider } from "./image-generation-provider.js";
describe("vydra image-generation provider", () => {
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
it("posts to the www api and downloads the generated image", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "vydra-test-key",
source: "env",
mode: "api-key",
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(
JSON.stringify({
jobId: "job-123",
status: "completed",
imageUrl: "https://cdn.vydra.ai/generated/test.png",
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
),
)
.mockResolvedValueOnce(
new Response(Buffer.from("png-data"), {
status: 200,
headers: { "Content-Type": "image/png" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const provider = buildVydraImageGenerationProvider();
const result = await provider.generateImage({
provider: "vydra",
model: "grok-imagine",
prompt: "draw a cat",
cfg: {},
});
expect(fetchMock).toHaveBeenNthCalledWith(
1,
"https://www.vydra.ai/api/v1/models/grok-imagine",
expect.objectContaining({
method: "POST",
body: JSON.stringify({
prompt: "draw a cat",
model: "text-to-image",
}),
}),
);
const [, init] = fetchMock.mock.calls[0] as [string, RequestInit];
const headers = new Headers(init.headers);
expect(headers.get("authorization")).toBe("Bearer vydra-test-key");
expect(result).toEqual({
images: [
{
buffer: Buffer.from("png-data"),
mimeType: "image/png",
fileName: "image-1.png",
},
],
model: "grok-imagine",
metadata: {
jobId: "job-123",
imageUrl: "https://cdn.vydra.ai/generated/test.png",
status: "completed",
},
});
});
it("polls jobs when the create response is not completed yet", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "vydra-test-key",
source: "env",
mode: "api-key",
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ jobId: "job-456", status: "queued" }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
)
.mockResolvedValueOnce(
new Response(
JSON.stringify({
jobId: "job-456",
status: "completed",
resultUrls: ["https://cdn.vydra.ai/generated/polled.png"],
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
),
)
.mockResolvedValueOnce(
new Response(Buffer.from("png-data"), {
status: 200,
headers: { "Content-Type": "image/png" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const provider = buildVydraImageGenerationProvider();
await provider.generateImage({
provider: "vydra",
model: "grok-imagine",
prompt: "draw a cat",
cfg: {},
});
expect(fetchMock).toHaveBeenNthCalledWith(
2,
"https://www.vydra.ai/api/v1/jobs/job-456",
expect.objectContaining({ method: "GET" }),
);
});
});

View File

@@ -0,0 +1,152 @@
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
postJsonRequest,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import {
DEFAULT_VYDRA_BASE_URL,
DEFAULT_VYDRA_IMAGE_MODEL,
downloadVydraAsset,
extractVydraResultUrls,
resolveVydraBaseUrlFromConfig,
resolveVydraErrorMessage,
resolveVydraResponseJobId,
resolveVydraResponseStatus,
waitForVydraJob,
} from "./shared.js";
export function buildVydraImageGenerationProvider(): ImageGenerationProvider {
return {
id: "vydra",
label: "Vydra",
defaultModel: DEFAULT_VYDRA_IMAGE_MODEL,
models: [DEFAULT_VYDRA_IMAGE_MODEL],
isConfigured: ({ agentDir }) =>
isProviderApiKeyConfigured({
provider: "vydra",
agentDir,
}),
capabilities: {
generate: {
maxCount: 1,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
},
edit: {
enabled: false,
maxCount: 1,
maxInputImages: 0,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
},
},
async generateImage(req) {
if ((req.inputImages?.length ?? 0) > 0) {
throw new Error(
"Vydra image generation currently supports text-to-image only in the bundled plugin.",
);
}
if ((req.count ?? 1) > 1) {
throw new Error("Vydra image generation supports at most one image per request.");
}
const auth = await resolveApiKeyForProvider({
provider: "vydra",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("Vydra API key missing");
}
const fetchFn = fetch;
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: resolveVydraBaseUrlFromConfig(req.cfg),
defaultBaseUrl: DEFAULT_VYDRA_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"Content-Type": "application/json",
},
provider: "vydra",
capability: "image",
transport: "http",
});
const model = req.model?.trim() || DEFAULT_VYDRA_IMAGE_MODEL;
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${model}`,
headers,
body: {
prompt: req.prompt,
model: "text-to-image",
},
timeoutMs: req.timeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(response, "Vydra image generation failed");
const submitted = await response.json();
const completedPayload =
resolveVydraResponseStatus(submitted) === "completed" ||
extractVydraResultUrls(submitted, "image").length > 0
? submitted
: await (() => {
const jobId = resolveVydraResponseJobId(submitted);
if (!jobId) {
throw new Error(
resolveVydraErrorMessage(submitted) ??
"Vydra image generation response missing job id",
);
}
return waitForVydraJob({
baseUrl,
jobId,
headers,
timeoutMs: req.timeoutMs,
fetchFn,
kind: "image",
});
})();
const imageUrl = extractVydraResultUrls(completedPayload, "image")[0];
if (!imageUrl) {
throw new Error("Vydra image generation completed without an image URL");
}
const image = await downloadVydraAsset({
url: imageUrl,
kind: "image",
timeoutMs: req.timeoutMs,
fetchFn,
});
return {
images: [
{
buffer: image.buffer,
mimeType: image.mimeType,
fileName: image.fileName,
},
],
model,
metadata: {
jobId:
resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted),
imageUrl,
status: resolveVydraResponseStatus(completedPayload) ?? "completed",
},
};
} finally {
await release();
}
},
};
}

49
extensions/vydra/index.ts Normal file
View File

@@ -0,0 +1,49 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
import { buildVydraImageGenerationProvider } from "./image-generation-provider.js";
import { applyVydraConfig, VYDRA_DEFAULT_IMAGE_MODEL_REF } from "./onboard.js";
import { buildVydraSpeechProvider } from "./speech-provider.js";
import { buildVydraVideoGenerationProvider } from "./video-generation-provider.js";
const PROVIDER_ID = "vydra";
export default definePluginEntry({
id: PROVIDER_ID,
name: "Vydra Provider",
description: "Bundled Vydra image, video, and speech provider",
register(api) {
api.registerProvider({
id: PROVIDER_ID,
label: "Vydra",
docsPath: "/providers/vydra",
envVars: ["VYDRA_API_KEY"],
auth: [
createProviderApiKeyAuthMethod({
providerId: PROVIDER_ID,
methodId: "api-key",
label: "Vydra API key",
hint: "Image, video, and speech API key",
optionKey: "vydraApiKey",
flagName: "--vydra-api-key",
envVar: "VYDRA_API_KEY",
promptMessage: "Enter Vydra API key",
defaultModel: VYDRA_DEFAULT_IMAGE_MODEL_REF,
expectedProviders: [PROVIDER_ID],
applyConfig: (cfg) => applyVydraConfig(cfg),
wizard: {
choiceId: "vydra-api-key",
choiceLabel: "Vydra API key",
choiceHint: "Image, video, and speech API key",
groupId: "vydra",
groupLabel: "Vydra",
groupHint: "Image, video, and speech",
onboardingScopes: ["image-generation"],
},
}),
],
});
api.registerSpeechProvider(buildVydraSpeechProvider());
api.registerImageGenerationProvider(buildVydraImageGenerationProvider());
api.registerVideoGenerationProvider(buildVydraVideoGenerationProvider());
},
});

View File

@@ -0,0 +1,21 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
export const VYDRA_DEFAULT_IMAGE_MODEL_REF = "vydra/grok-imagine";
export function applyVydraConfig(cfg: OpenClawConfig): OpenClawConfig {
if (cfg.agents?.defaults?.imageGenerationModel) {
return cfg;
}
return {
...cfg,
agents: {
...cfg.agents,
defaults: {
...cfg.agents?.defaults,
imageGenerationModel: {
primary: VYDRA_DEFAULT_IMAGE_MODEL_REF,
},
},
},
};
}

View File

@@ -0,0 +1,34 @@
{
"id": "vydra",
"enabledByDefault": true,
"providers": ["vydra"],
"providerAuthEnvVars": {
"vydra": ["VYDRA_API_KEY"]
},
"providerAuthChoices": [
{
"provider": "vydra",
"method": "api-key",
"choiceId": "vydra-api-key",
"choiceLabel": "Vydra API key",
"groupId": "vydra",
"groupLabel": "Vydra",
"groupHint": "Image, video, and speech",
"onboardingScopes": ["image-generation"],
"optionKey": "vydraApiKey",
"cliFlag": "--vydra-api-key",
"cliOption": "--vydra-api-key <key>",
"cliDescription": "Vydra API key"
}
],
"contracts": {
"speechProviders": ["vydra"],
"imageGenerationProviders": ["vydra"],
"videoGenerationProviders": ["vydra"]
},
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {}
}
}

View File

@@ -0,0 +1,12 @@
{
"name": "@openclaw/vydra-provider",
"version": "2026.4.6",
"private": true,
"description": "OpenClaw Vydra media provider plugin",
"type": "module",
"openclaw": {
"extensions": [
"./index.ts"
]
}
}

View File

@@ -0,0 +1,20 @@
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
describePluginRegistrationContract({
pluginId: "vydra",
providerIds: ["vydra"],
speechProviderIds: ["vydra"],
imageGenerationProviderIds: ["vydra"],
videoGenerationProviderIds: ["vydra"],
requireSpeechVoices: true,
requireGenerateImage: true,
requireGenerateVideo: true,
manifestAuthChoice: {
pluginId: "vydra",
choiceId: "vydra-api-key",
choiceLabel: "Vydra API key",
groupId: "vydra",
groupLabel: "Vydra",
groupHint: "Image, video, and speech",
},
});

218
extensions/vydra/shared.ts Normal file
View File

@@ -0,0 +1,218 @@
import { assertOkOrThrowHttpError, fetchWithTimeout } from "openclaw/plugin-sdk/provider-http";
export const DEFAULT_VYDRA_BASE_URL = "https://www.vydra.ai/api/v1";
export const DEFAULT_VYDRA_IMAGE_MODEL = "grok-imagine";
export const DEFAULT_VYDRA_VIDEO_MODEL = "veo3";
export const DEFAULT_VYDRA_SPEECH_MODEL = "elevenlabs/tts";
export const DEFAULT_VYDRA_VOICE_ID = "21m00Tcm4TlvDq8ikWAM";
export const DEFAULT_HTTP_TIMEOUT_MS = 120_000;
const POLL_INTERVAL_MS = 2_500;
const MAX_POLL_ATTEMPTS = 120;
type VydraMediaKind = "audio" | "image" | "video";
type VydraJobPayload = {
id?: string;
jobId?: string;
status?: string;
message?: string;
error?: string | { message?: string; detail?: string } | null;
};
function asObject(value: unknown): Record<string, unknown> | undefined {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: undefined;
}
function addUrlValue(value: unknown, urls: Set<string>): void {
if (typeof value === "string") {
const trimmed = value.trim();
if (/^https?:\/\//iu.test(trimmed)) {
urls.add(trimmed);
}
return;
}
if (Array.isArray(value)) {
for (const entry of value) {
addUrlValue(entry, urls);
}
}
}
export function trimToUndefined(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value.trim() : undefined;
}
export function normalizeVydraBaseUrl(value: string | undefined): string {
const fallback = DEFAULT_VYDRA_BASE_URL;
const trimmed = trimToUndefined(value);
if (!trimmed) {
return fallback;
}
try {
const url = new URL(trimmed);
if (url.hostname === "vydra.ai") {
url.hostname = "www.vydra.ai";
}
const pathname = url.pathname.replace(/\/+$/u, "");
if (!pathname) {
url.pathname = "/api/v1";
} else {
url.pathname = pathname;
}
return url.toString().replace(/\/$/u, "");
} catch {
return fallback;
}
}
export function resolveVydraBaseUrlFromConfig(cfg: unknown): string {
const models = asObject(asObject(cfg)?.models);
const providers = asObject(models?.providers);
const vydra = asObject(providers?.vydra);
return normalizeVydraBaseUrl(trimToUndefined(vydra?.baseUrl));
}
export function resolveVydraResponseJobId(payload: unknown): string | undefined {
const object = asObject(payload) as VydraJobPayload | undefined;
return trimToUndefined(object?.jobId) ?? trimToUndefined(object?.id);
}
export function resolveVydraResponseStatus(payload: unknown): string | undefined {
return trimToUndefined(asObject(payload)?.status)?.toLowerCase();
}
export function resolveVydraErrorMessage(payload: unknown): string | undefined {
const object = asObject(payload) as VydraJobPayload | undefined;
const error = object?.error;
if (typeof error === "string" && error.trim()) {
return error.trim();
}
const errorObject = asObject(error);
return (
trimToUndefined(errorObject?.message) ??
trimToUndefined(errorObject?.detail) ??
trimToUndefined(object?.message)
);
}
export function extractVydraResultUrls(payload: unknown, kind: VydraMediaKind): string[] {
const urls = new Set<string>();
const preferredKeys =
kind === "audio"
? ["audioUrl", "audioUrls"]
: kind === "image"
? ["imageUrl", "imageUrls"]
: ["videoUrl", "videoUrls"];
const sharedKeys = ["resultUrl", "resultUrls", "outputUrl", "outputUrls", "url", "urls"];
const recurseKeys = ["output", "outputs", "result", "results", "data", "asset", "assets"];
const visit = (value: unknown, depth = 0) => {
if (depth > 5) {
return;
}
if (Array.isArray(value)) {
for (const entry of value) {
visit(entry, depth + 1);
}
return;
}
const object = asObject(value);
if (!object) {
return;
}
for (const key of [...preferredKeys, ...sharedKeys]) {
addUrlValue(object[key], urls);
}
for (const key of recurseKeys) {
if (key in object) {
visit(object[key], depth + 1);
}
}
};
visit(payload);
return [...urls];
}
function inferExtension(kind: VydraMediaKind, mimeType: string): string {
const normalized = mimeType.toLowerCase();
if (normalized.includes("jpeg")) {
return "jpg";
}
if (normalized.includes("webp")) {
return "webp";
}
if (normalized.includes("wav")) {
return "wav";
}
if (normalized.includes("mpeg") || normalized.includes("mp3")) {
return "mp3";
}
if (normalized.includes("webm")) {
return "webm";
}
if (normalized.includes("quicktime")) {
return "mov";
}
return kind === "image" ? "png" : kind === "audio" ? "mp3" : "mp4";
}
export async function downloadVydraAsset(params: {
url: string;
kind: VydraMediaKind;
timeoutMs?: number;
fetchFn: typeof fetch;
}): Promise<{ buffer: Buffer; mimeType: string; fileName: string }> {
const response = await fetchWithTimeout(
params.url,
{ method: "GET" },
params.timeoutMs ?? DEFAULT_HTTP_TIMEOUT_MS,
params.fetchFn,
);
await assertOkOrThrowHttpError(response, `Vydra ${params.kind} download failed`);
const mimeType =
response.headers.get("content-type")?.trim() ||
(params.kind === "image" ? "image/png" : params.kind === "audio" ? "audio/mpeg" : "video/mp4");
const arrayBuffer = await response.arrayBuffer();
const extension = inferExtension(params.kind, mimeType);
const fileStem = params.kind === "image" ? "image" : params.kind === "audio" ? "audio" : "video";
return {
buffer: Buffer.from(arrayBuffer),
mimeType,
fileName: `${fileStem}-1.${extension}`,
};
}
export async function waitForVydraJob(params: {
baseUrl: string;
jobId: string;
headers: Headers;
timeoutMs?: number;
fetchFn: typeof fetch;
kind: VydraMediaKind;
}): Promise<unknown> {
for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
const response = await fetchWithTimeout(
`${params.baseUrl}/jobs/${params.jobId}`,
{
method: "GET",
headers: params.headers,
},
params.timeoutMs ?? DEFAULT_HTTP_TIMEOUT_MS,
params.fetchFn,
);
await assertOkOrThrowHttpError(response, "Vydra job status request failed");
const payload = await response.json();
const status = resolveVydraResponseStatus(payload);
if (status === "completed" || extractVydraResultUrls(payload, params.kind).length > 0) {
return payload;
}
if (status === "failed" || status === "error" || status === "cancelled") {
throw new Error(resolveVydraErrorMessage(payload) ?? `Vydra job ${params.jobId} failed`);
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
throw new Error(`Vydra job ${params.jobId} did not finish in time`);
}

View File

@@ -0,0 +1,71 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildVydraSpeechProvider } from "./speech-provider.js";
describe("vydra speech provider", () => {
const provider = buildVydraSpeechProvider();
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
it("exposes the default voice and model", async () => {
expect(provider.models).toEqual(["elevenlabs/tts"]);
const voices = await provider.listVoices?.({});
expect(voices).toEqual([
{
id: "21m00Tcm4TlvDq8ikWAM",
name: "Rachel",
},
]);
});
it("posts to the tts endpoint and downloads the audio", async () => {
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(
JSON.stringify({
audioUrl: "https://cdn.vydra.ai/generated/test.mp3",
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
),
)
.mockResolvedValueOnce(
new Response(Buffer.from("mp3-data"), {
status: 200,
headers: { "Content-Type": "audio/mpeg" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const result = await provider.synthesize({
text: "OpenClaw test",
cfg: {} as never,
providerConfig: { apiKey: "vydra-test-key" },
target: "audio-file",
timeoutMs: 30_000,
});
expect(fetchMock).toHaveBeenNthCalledWith(
1,
"https://www.vydra.ai/api/v1/models/elevenlabs/tts",
expect.objectContaining({
method: "POST",
body: JSON.stringify({
text: "OpenClaw test",
voice_id: "21m00Tcm4TlvDq8ikWAM",
}),
}),
);
const [, init] = fetchMock.mock.calls[0] as [string, RequestInit];
const headers = new Headers(init.headers);
expect(headers.get("authorization")).toBe("Bearer vydra-test-key");
expect(result.outputFormat).toBe("mp3");
expect(result.fileExtension).toBe(".mp3");
expect(result.audioBuffer).toEqual(Buffer.from("mp3-data"));
});
});

View File

@@ -0,0 +1,157 @@
import {
assertOkOrThrowHttpError,
postJsonRequest,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
import type {
SpeechProviderConfig,
SpeechProviderOverrides,
SpeechProviderPlugin,
} from "openclaw/plugin-sdk/speech-core";
import {
DEFAULT_VYDRA_BASE_URL,
DEFAULT_VYDRA_SPEECH_MODEL,
DEFAULT_VYDRA_VOICE_ID,
downloadVydraAsset,
extractVydraResultUrls,
normalizeVydraBaseUrl,
trimToUndefined,
} from "./shared.js";
type VydraSpeechConfig = {
apiKey?: string;
baseUrl: string;
model: string;
voiceId: string;
};
const VYDRA_SPEECH_VOICES = [
{
id: DEFAULT_VYDRA_VOICE_ID,
name: "Rachel",
},
] as const;
function asObject(value: unknown): Record<string, unknown> | undefined {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: undefined;
}
function normalizeVydraSpeechConfig(rawConfig: Record<string, unknown>): VydraSpeechConfig {
const providers = asObject(rawConfig.providers);
const raw = asObject(providers?.vydra) ?? asObject(rawConfig.vydra);
return {
apiKey: normalizeResolvedSecretInputString({
value: raw?.apiKey,
path: "messages.tts.providers.vydra.apiKey",
}),
baseUrl: normalizeVydraBaseUrl(
trimToUndefined(raw?.baseUrl) ?? trimToUndefined(process.env.VYDRA_BASE_URL),
),
model:
trimToUndefined(raw?.model) ??
trimToUndefined(process.env.VYDRA_TTS_MODEL) ??
DEFAULT_VYDRA_SPEECH_MODEL,
voiceId:
trimToUndefined(raw?.voiceId) ??
trimToUndefined(process.env.VYDRA_TTS_VOICE_ID) ??
DEFAULT_VYDRA_VOICE_ID,
};
}
function readVydraSpeechConfig(config: SpeechProviderConfig): VydraSpeechConfig {
const normalized = normalizeVydraSpeechConfig({});
return {
apiKey: trimToUndefined(config.apiKey) ?? normalized.apiKey,
baseUrl: normalizeVydraBaseUrl(trimToUndefined(config.baseUrl) ?? normalized.baseUrl),
model: trimToUndefined(config.model) ?? normalized.model,
voiceId: trimToUndefined(config.voiceId) ?? normalized.voiceId,
};
}
function readVydraOverrides(overrides: SpeechProviderOverrides | undefined): {
model?: string;
voiceId?: string;
} {
if (!overrides) {
return {};
}
return {
model: trimToUndefined(overrides.model),
voiceId: trimToUndefined(overrides.voiceId),
};
}
export function buildVydraSpeechProvider(): SpeechProviderPlugin {
return {
id: "vydra",
label: "Vydra",
models: [DEFAULT_VYDRA_SPEECH_MODEL],
voices: VYDRA_SPEECH_VOICES.map((voice) => voice.id),
resolveConfig: ({ rawConfig }) => normalizeVydraSpeechConfig(rawConfig),
listVoices: async () => VYDRA_SPEECH_VOICES.map((voice) => ({ ...voice })),
isConfigured: ({ providerConfig }) =>
Boolean(readVydraSpeechConfig(providerConfig).apiKey || process.env.VYDRA_API_KEY),
synthesize: async (req) => {
const config = readVydraSpeechConfig(req.providerConfig);
const overrides = readVydraOverrides(req.providerOverrides);
const apiKey = config.apiKey || process.env.VYDRA_API_KEY;
if (!apiKey) {
throw new Error("Vydra API key missing");
}
const fetchFn = fetch;
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: config.baseUrl,
defaultBaseUrl: DEFAULT_VYDRA_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
provider: "vydra",
capability: "audio",
transport: "http",
});
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${overrides.model ?? config.model}`,
headers,
body: {
text: req.text,
voice_id: overrides.voiceId ?? config.voiceId,
},
timeoutMs: req.timeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(response, "Vydra speech synthesis failed");
const payload = await response.json();
const audioUrl = extractVydraResultUrls(payload, "audio")[0];
if (!audioUrl) {
throw new Error("Vydra speech synthesis response missing audio URL");
}
const audio = await downloadVydraAsset({
url: audioUrl,
kind: "audio",
timeoutMs: req.timeoutMs,
fetchFn,
});
return {
audioBuffer: audio.buffer,
outputFormat: audio.mimeType.includes("wav") ? "wav" : "mp3",
fileExtension: audio.fileName.endsWith(".wav") ? ".wav" : ".mp3",
voiceCompatible: false,
};
} finally {
await release();
}
},
};
}

View File

@@ -0,0 +1,94 @@
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildVydraVideoGenerationProvider } from "./video-generation-provider.js";
describe("vydra video-generation provider", () => {
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
it("submits veo3 jobs and downloads the completed video", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "vydra-test-key",
source: "env",
mode: "api-key",
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ jobId: "job-123", status: "processing" }), {
status: 200,
headers: { "Content-Type": "application/json" },
}),
)
.mockResolvedValueOnce(
new Response(
JSON.stringify({
jobId: "job-123",
status: "completed",
videoUrl: "https://cdn.vydra.ai/generated/test.mp4",
}),
{
status: 200,
headers: { "Content-Type": "application/json" },
},
),
)
.mockResolvedValueOnce(
new Response(Buffer.from("mp4-data"), {
status: 200,
headers: { "Content-Type": "video/mp4" },
}),
);
vi.stubGlobal("fetch", fetchMock);
const provider = buildVydraVideoGenerationProvider();
const result = await provider.generateVideo({
provider: "vydra",
model: "veo3",
prompt: "tiny city at sunrise",
cfg: {},
});
expect(fetchMock).toHaveBeenNthCalledWith(
1,
"https://www.vydra.ai/api/v1/models/veo3",
expect.objectContaining({
method: "POST",
body: JSON.stringify({ prompt: "tiny city at sunrise" }),
}),
);
expect(fetchMock).toHaveBeenNthCalledWith(
2,
"https://www.vydra.ai/api/v1/jobs/job-123",
expect.objectContaining({ method: "GET" }),
);
expect(result.videos[0]?.mimeType).toBe("video/mp4");
expect(result.metadata).toEqual({
jobId: "job-123",
videoUrl: "https://cdn.vydra.ai/generated/test.mp4",
status: "completed",
});
});
it("requires a remote image url for kling", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "vydra-test-key",
source: "env",
mode: "api-key",
});
vi.stubGlobal("fetch", vi.fn());
const provider = buildVydraVideoGenerationProvider();
await expect(
provider.generateVideo({
provider: "vydra",
model: "kling",
prompt: "animate this image",
cfg: {},
inputImages: [{ buffer: Buffer.from("png"), mimeType: "image/png" }],
}),
).rejects.toThrow("Vydra kling currently requires a remote image URL reference.");
});
});

View File

@@ -0,0 +1,165 @@
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
postJsonRequest,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation";
import {
DEFAULT_VYDRA_BASE_URL,
DEFAULT_VYDRA_VIDEO_MODEL,
downloadVydraAsset,
extractVydraResultUrls,
resolveVydraBaseUrlFromConfig,
resolveVydraErrorMessage,
resolveVydraResponseJobId,
resolveVydraResponseStatus,
waitForVydraJob,
} from "./shared.js";
const VYDRA_KLING_MODEL = "kling";
function resolveVydraVideoRequestBody(
req: Parameters<VideoGenerationProvider["generateVideo"]>[0],
) {
const model = req.model?.trim() || DEFAULT_VYDRA_VIDEO_MODEL;
if (model === VYDRA_KLING_MODEL) {
const input = req.inputImages?.[0];
const imageUrl = input?.url?.trim();
if (!imageUrl) {
throw new Error("Vydra kling currently requires a remote image URL reference.");
}
return {
model,
body: {
prompt: req.prompt,
image_url: imageUrl,
},
};
}
if ((req.inputImages?.length ?? 0) > 0) {
throw new Error(
`Vydra ${model} does not support image reference inputs in the bundled plugin.`,
);
}
return {
model,
body: {
prompt: req.prompt,
},
};
}
export function buildVydraVideoGenerationProvider(): VideoGenerationProvider {
return {
id: "vydra",
label: "Vydra",
defaultModel: DEFAULT_VYDRA_VIDEO_MODEL,
models: [DEFAULT_VYDRA_VIDEO_MODEL, VYDRA_KLING_MODEL],
isConfigured: ({ agentDir }) =>
isProviderApiKeyConfigured({
provider: "vydra",
agentDir,
}),
capabilities: {
maxVideos: 1,
maxInputImages: 1,
maxInputVideos: 0,
},
async generateVideo(req) {
if ((req.inputVideos?.length ?? 0) > 0) {
throw new Error("Vydra video generation does not support video reference inputs.");
}
const auth = await resolveApiKeyForProvider({
provider: "vydra",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("Vydra API key missing");
}
const fetchFn = fetch;
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: resolveVydraBaseUrlFromConfig(req.cfg),
defaultBaseUrl: DEFAULT_VYDRA_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"Content-Type": "application/json",
},
provider: "vydra",
capability: "video",
transport: "http",
});
const { model, body } = resolveVydraVideoRequestBody(req);
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${model}`,
headers,
body,
timeoutMs: req.timeoutMs,
fetchFn,
allowPrivateNetwork,
dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(response, "Vydra video generation failed");
const submitted = await response.json();
const completedPayload =
resolveVydraResponseStatus(submitted) === "completed" ||
extractVydraResultUrls(submitted, "video").length > 0
? submitted
: await (() => {
const jobId = resolveVydraResponseJobId(submitted);
if (!jobId) {
throw new Error(
resolveVydraErrorMessage(submitted) ??
"Vydra video generation response missing job id",
);
}
return waitForVydraJob({
baseUrl,
jobId,
headers,
timeoutMs: req.timeoutMs,
fetchFn,
kind: "video",
});
})();
const videoUrl = extractVydraResultUrls(completedPayload, "video")[0];
if (!videoUrl) {
throw new Error("Vydra video generation completed without a video URL");
}
const video = await downloadVydraAsset({
url: videoUrl,
kind: "video",
timeoutMs: req.timeoutMs,
fetchFn,
});
return {
videos: [
{
buffer: video.buffer,
mimeType: video.mimeType,
fileName: video.fileName,
},
],
model,
metadata: {
jobId:
resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted),
videoUrl,
status: resolveVydraResponseStatus(completedPayload) ?? "completed",
},
};
} finally {
await release();
}
},
};
}

View File

@@ -0,0 +1,81 @@
import { describe, expect, it } from "vitest";
import { isLiveTestEnabled } from "../../src/agents/live-test-helpers.js";
import {
registerProviderPlugin,
requireRegisteredProvider,
} from "../../test/helpers/plugins/provider-registration.js";
import plugin from "./index.js";
const LIVE = isLiveTestEnabled();
const VYDRA_API_KEY = process.env.VYDRA_API_KEY?.trim() ?? "";
const ENABLE_VYDRA_VIDEO_LIVE = process.env.OPENCLAW_LIVE_VYDRA_VIDEO === "1";
const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_VYDRA_IMAGE_MODEL?.trim() || "grok-imagine";
const LIVE_VIDEO_MODEL = process.env.OPENCLAW_LIVE_VYDRA_VIDEO_MODEL?.trim() || "veo3";
const registerVydraPlugin = () =>
registerProviderPlugin({
plugin,
id: "vydra",
name: "Vydra Provider",
});
describe.skipIf(!LIVE || !VYDRA_API_KEY)("vydra live", () => {
it("generates an image through the registered provider", async () => {
const { imageProviders } = await registerVydraPlugin();
const provider = requireRegisteredProvider(imageProviders, "vydra");
const result = await provider.generateImage({
provider: "vydra",
model: LIVE_IMAGE_MODEL,
prompt: "Create a minimal flat orange square centered on a white background.",
cfg: { plugins: { enabled: true } } as never,
agentDir: "/tmp/openclaw-live-vydra-image",
});
expect(result.images.length).toBeGreaterThan(0);
expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true);
expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(512);
}, 60_000);
it("synthesizes speech through the registered provider", async () => {
const { speechProviders } = await registerVydraPlugin();
const provider = requireRegisteredProvider(speechProviders, "vydra");
const voices = await provider.listVoices?.({});
expect(voices).toEqual(
expect.arrayContaining([expect.objectContaining({ id: "21m00Tcm4TlvDq8ikWAM" })]),
);
const result = await provider.synthesize({
text: "OpenClaw integration test OK.",
cfg: { plugins: { enabled: true } } as never,
providerConfig: { apiKey: VYDRA_API_KEY },
target: "audio-file",
timeoutMs: 45_000,
});
expect(result.outputFormat).toBe("mp3");
expect(result.audioBuffer.byteLength).toBeGreaterThan(512);
}, 60_000);
it.skipIf(!ENABLE_VYDRA_VIDEO_LIVE)(
"generates a short video through the registered provider",
async () => {
const { videoProviders } = await registerVydraPlugin();
const provider = requireRegisteredProvider(videoProviders, "vydra");
const result = await provider.generateVideo({
provider: "vydra",
model: LIVE_VIDEO_MODEL,
prompt:
"A tiny paper diorama city at sunrise with slow cinematic camera motion and no text.",
cfg: { plugins: { enabled: true } } as never,
agentDir: "/tmp/openclaw-live-vydra-video",
});
expect(result.videos.length).toBeGreaterThan(0);
expect(result.videos[0]?.mimeType.startsWith("video/")).toBe(true);
expect(result.videos[0]?.buffer.byteLength).toBeGreaterThan(1024);
},
8 * 60_000,
);
});