vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 07:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

test: speed media runtime specs

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-28 04:53:21 +01:00
parent 6f38425e5c
commit fed337b164
2 changed files with 389 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

343
src/image-generation/runtime.test.ts
View File

@@ -1,24 +1,48 @@
import { beforeEach, describe, expect, it } from "vitest";
import {
getMediaGenerationRuntimeMocks,
resetImageGenerationRuntimeMocks,
} from "../../test/helpers/media-generation/runtime-module-mocks.js";
import type { OpenClawConfig } from "../config/config.js";
import { generateImage, listRuntimeImageGenerationProviders } from "./runtime.js";
import {
generateImage,
listRuntimeImageGenerationProviders,
type GenerateImageParams,
type ImageGenerationRuntimeDeps,
} from "./runtime.js";
import type { ImageGenerationProvider } from "./types.js";
const mocks = getMediaGenerationRuntimeMocks();
let providers: ImageGenerationProvider[] = [];
let listedConfigs: Array<OpenClawConfig | undefined> = [];
let providerEnvVars: Record<string, string[]> = {};
let warnings: string[] = [];
const runtimeDeps: ImageGenerationRuntimeDeps = {
getProvider: (providerId) => providers.find((provider) => provider.id === providerId),
listProviders: (config) => {
listedConfigs.push(config);
return providers;
},
getProviderEnvVars: (providerId) => providerEnvVars[providerId] ?? [],
log: {
warn: (message) => {
warnings.push(message);
},
},
};
function runGenerateImage(params: GenerateImageParams) {
return generateImage(params, runtimeDeps);
}
describe("image-generation runtime", () => {
beforeEach(() => {
resetImageGenerationRuntimeMocks();
providers = [];
listedConfigs = [];
providerEnvVars = {};
warnings = [];
});
it("generates images through the active image-generation provider", async () => {
const authStore = { version: 1, profiles: {} } as const;
let seenAuthStore: unknown;
let seenTimeoutMs: number | undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("image-plugin/img-v1");
const provider: ImageGenerationProvider = {
id: "image-plugin",
capabilities: {
@@ -40,9 +64,9 @@ describe("image-generation runtime", () => {
};
},
};
mocks.getImageGenerationProvider.mockReturnValue(provider);
providers = [provider];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -73,7 +97,6 @@ describe("image-generation runtime", () => {
it("uses configured image-generation timeout when the call omits timeoutMs", async () => {
let seenTimeoutMs: number | undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("image-plugin/img-v1");
const provider: ImageGenerationProvider = {
id: "image-plugin",
capabilities: {
@@ -94,9 +117,9 @@ describe("image-generation runtime", () => {
};
},
};
mocks.getImageGenerationProvider.mockReturnValue(provider);
providers = [provider];
await generateImage({
await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -114,41 +137,7 @@ describe("image-generation runtime", () => {
});
it("auto-detects and falls through to another configured image-generation provider by default", async () => {
mocks.getImageGenerationProvider.mockImplementation((providerId: string) => {
if (providerId === "openai") {
return {
id: "openai",
defaultModel: "gpt-image-1",
capabilities: {
generate: {},
edit: { enabled: true },
},
isConfigured: () => true,
async generateImage() {
throw new Error("OpenAI API key missing");
},
};
}
if (providerId === "google") {
return {
id: "google",
defaultModel: "gemini-3.1-flash-image-preview",
capabilities: {
generate: {},
edit: { enabled: true },
},
isConfigured: () => true,
async generateImage() {
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
model: "gemini-3.1-flash-image-preview",
};
},
};
}
return undefined;
});
mocks.listImageGenerationProviders.mockReturnValue([
providers = [
{
id: "openai",
defaultModel: "gpt-image-1",
@@ -157,7 +146,9 @@ describe("image-generation runtime", () => {
edit: { enabled: true },
},
isConfigured: () => true,
generateImage: async () => ({ images: [] }),
async generateImage() {
throw new Error("OpenAI API key missing");
},
},
{
id: "google",
@@ -167,11 +158,16 @@ describe("image-generation runtime", () => {
edit: { enabled: true },
},
isConfigured: () => true,
generateImage: async () => ({ images: [] }),
async generateImage() {
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
model: "gemini-3.1-flash-image-preview",
};
},
},
]);
];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {} as OpenClawConfig,
prompt: "draw a cat",
});
@@ -185,7 +181,7 @@ describe("image-generation runtime", () => {
error: "OpenAI API key missing",
},
]);
expect(mocks.warn).toHaveBeenCalledWith(
expect(warnings).toContain(
"image-generation candidate failed: openai/gpt-image-1: OpenAI API key missing",
);
});
@@ -198,38 +194,39 @@ describe("image-generation runtime", () => {
resolution?: string;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/gpt-image-1");
mocks.getImageGenerationProvider.mockReturnValue({
id: "openai",
capabilities: {
generate: {
supportsSize: true,
supportsAspectRatio: false,
supportsResolution: false,
providers = [
{
id: "openai",
capabilities: {
generate: {
supportsSize: true,
supportsAspectRatio: false,
supportsResolution: false,
},
edit: {
enabled: true,
supportsSize: true,
supportsAspectRatio: false,
supportsResolution: false,
},
geometry: {
sizes: ["1024x1024", "1024x1536", "1536x1024"],
},
},
edit: {
enabled: true,
supportsSize: true,
supportsAspectRatio: false,
supportsResolution: false,
},
geometry: {
sizes: ["1024x1024", "1024x1536", "1536x1024"],
async generateImage(req) {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
};
},
},
async generateImage(req) {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
};
},
});
];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -263,37 +260,38 @@ describe("image-generation runtime", () => {
providerOptions?: unknown;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/gpt-image-2");
mocks.getImageGenerationProvider.mockReturnValue({
id: "openai",
capabilities: {
generate: {
supportsSize: true,
providers = [
{
id: "openai",
capabilities: {
generate: {
supportsSize: true,
},
edit: {
enabled: true,
supportsSize: true,
},
output: {
qualities: ["low", "medium", "high", "auto"],
formats: ["png", "jpeg", "webp"],
backgrounds: ["transparent", "opaque", "auto"],
},
},
edit: {
enabled: true,
supportsSize: true,
},
output: {
qualities: ["low", "medium", "high", "auto"],
formats: ["png", "jpeg", "webp"],
backgrounds: ["transparent", "opaque", "auto"],
async generateImage(req) {
seenRequest = {
quality: req.quality,
outputFormat: req.outputFormat,
background: req.background,
providerOptions: req.providerOptions,
};
return {
images: [{ buffer: Buffer.from("jpeg-bytes"), mimeType: "image/jpeg" }],
};
},
},
async generateImage(req) {
seenRequest = {
quality: req.quality,
outputFormat: req.outputFormat,
background: req.background,
providerOptions: req.providerOptions,
};
return {
images: [{ buffer: Buffer.from("jpeg-bytes"), mimeType: "image/jpeg" }],
};
},
});
];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -339,28 +337,29 @@ describe("image-generation runtime", () => {
background?: string;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("vydra/grok-imagine");
mocks.getImageGenerationProvider.mockReturnValue({
id: "vydra",
capabilities: {
generate: {},
edit: {
enabled: false,
providers = [
{
id: "vydra",
capabilities: {
generate: {},
edit: {
enabled: false,
},
},
async generateImage(req) {
seenRequest = {
quality: req.quality,
outputFormat: req.outputFormat,
background: req.background,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
};
},
},
async generateImage(req) {
seenRequest = {
quality: req.quality,
outputFormat: req.outputFormat,
background: req.background,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
};
},
});
];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -394,39 +393,40 @@ describe("image-generation runtime", () => {
resolution?: string;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("minimax/image-01");
mocks.getImageGenerationProvider.mockReturnValue({
id: "minimax",
capabilities: {
generate: {
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
providers = [
{
id: "minimax",
capabilities: {
generate: {
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
},
edit: {
enabled: true,
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
},
geometry: {
aspectRatios: ["1:1", "16:9"],
},
},
edit: {
enabled: true,
supportsSize: false,
supportsAspectRatio: true,
supportsResolution: false,
},
geometry: {
aspectRatios: ["1:1", "16:9"],
async generateImage(req) {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
model: "image-01",
};
},
},
async generateImage(req) {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
model: "image-01",
};
},
});
];
const result = await generateImage({
const result = await runGenerateImage({
cfg: {
agents: {
defaults: {
@@ -458,7 +458,7 @@ describe("image-generation runtime", () => {
});
it("lists runtime image-generation providers through the provider registry", () => {
const providers: ImageGenerationProvider[] = [
const registryProviders: ImageGenerationProvider[] = [
{
id: "image-plugin",
defaultModel: "img-v1",
@@ -480,16 +480,16 @@ describe("image-generation runtime", () => {
}),
},
];
mocks.listImageGenerationProviders.mockReturnValue(providers);
providers = registryProviders;
expect(listRuntimeImageGenerationProviders({ config: {} as OpenClawConfig })).toEqual(
providers,
);
expect(mocks.listImageGenerationProviders).toHaveBeenCalledWith({} as OpenClawConfig);
expect(
listRuntimeImageGenerationProviders({ config: {} as OpenClawConfig }, runtimeDeps),
).toEqual(registryProviders);
expect(listedConfigs).toEqual([{} as OpenClawConfig]);
});
it("builds a generic config hint without hardcoded provider ids", async () => {
mocks.listImageGenerationProviders.mockReturnValue([
providers = [
{
id: "vision-one",
defaultModel: "paint-v1",
@@ -514,19 +514,14 @@ describe("image-generation runtime", () => {
images: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
}),
},
]);
mocks.getProviderEnvVars.mockImplementation((providerId: string) => {
if (providerId === "vision-one") {
return ["VISION_ONE_API_KEY"];
}
if (providerId === "vision-two") {
return ["VISION_TWO_API_KEY"];
}
return [];
});
];
providerEnvVars = {
"vision-one": ["VISION_ONE_API_KEY"],
"vision-two": ["VISION_TWO_API_KEY"],
};
await expect(
generateImage({ cfg: {} as OpenClawConfig, prompt: "draw a cat" }),
runGenerateImage({ cfg: {} as OpenClawConfig, prompt: "draw a cat" }),
).rejects.toThrow(
'No image-generation model configured. Set agents.defaults.imageGenerationModel.primary to a provider/model like "vision-one/paint-v1". If you want a specific provider, also configure that provider\'s auth/API key first (vision-one: VISION_ONE_API_KEY; vision-two: VISION_TWO_API_KEY).',
);

517
src/video-generation/runtime.test.ts
View File

@@ -1,13 +1,33 @@
import { beforeEach, describe, expect, it } from "vitest";
import {
getMediaGenerationRuntimeMocks,
resetVideoGenerationRuntimeMocks,
} from "../../test/helpers/media-generation/runtime-module-mocks.js";
import type { OpenClawConfig } from "../config/types.js";
import { generateVideo, listRuntimeVideoGenerationProviders } from "./runtime.js";
import {
generateVideo,
listRuntimeVideoGenerationProviders,
type GenerateVideoParams,
type VideoGenerationRuntimeDeps,
} from "./runtime.js";
import type { VideoGenerationProvider, VideoGenerationProviderOptionType } from "./types.js";
const mocks = getMediaGenerationRuntimeMocks();
let providers: VideoGenerationProvider[] = [];
let listedConfigs: Array<OpenClawConfig | undefined> = [];
let providerEnvVars: Record<string, string[]> = {};
const runtimeDeps: VideoGenerationRuntimeDeps = {
getProvider: (providerId) => providers.find((provider) => provider.id === providerId),
listProviders: (config) => {
listedConfigs.push(config);
return providers;
},
getProviderEnvVars: (providerId) => providerEnvVars[providerId] ?? [],
log: {
debug: () => {},
warn: () => {},
},
};
function runGenerateVideo(params: GenerateVideoParams) {
return generateVideo(params, runtimeDeps);
}
function createProviderOptionsCaptureProvider(
capabilities: VideoGenerationProvider["capabilities"],
@@ -28,14 +48,15 @@ function createProviderOptionsCaptureProvider(
describe("video-generation runtime", () => {
beforeEach(() => {
resetVideoGenerationRuntimeMocks();
providers = [];
listedConfigs = [];
providerEnvVars = {};
});
it("generates videos through the active video-generation provider", async () => {
const authStore = { version: 1, profiles: {} } as const;
let seenAuthStore: unknown;
let seenTimeoutMs: number | undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const provider: VideoGenerationProvider = {
id: "video-plugin",
capabilities: {},
@@ -54,9 +75,9 @@ describe("video-generation runtime", () => {
};
},
};
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -86,52 +107,31 @@ describe("video-generation runtime", () => {
});
it("auto-detects and falls through to another configured video-generation provider by default", async () => {
mocks.getVideoGenerationProvider.mockImplementation((providerId: string) => {
if (providerId === "openai") {
return {
id: "openai",
defaultModel: "sora-2",
capabilities: {},
isConfigured: () => true,
async generateVideo() {
throw new Error("Your request was blocked by our moderation system.");
},
};
}
if (providerId === "runway") {
return {
id: "runway",
defaultModel: "gen4.5",
capabilities: {},
isConfigured: () => true,
async generateVideo() {
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
};
}
return undefined;
});
mocks.listVideoGenerationProviders.mockReturnValue([
providers = [
{
id: "openai",
defaultModel: "sora-2",
capabilities: {},
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo() {
throw new Error("Your request was blocked by our moderation system.");
},
},
{
id: "runway",
defaultModel: "gen4.5",
capabilities: {},
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo() {
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
},
]);
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {} as OpenClawConfig,
prompt: "animate a cat",
});
@@ -148,7 +148,6 @@ describe("video-generation runtime", () => {
});
it("forwards providerOptions to providers that declare the matching schema", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const { provider, getSeenProviderOptions } = createProviderOptionsCaptureProvider({
providerOptions: {
seed: "number",
@@ -156,9 +155,9 @@ describe("video-generation runtime", () => {
camera_fixed: "boolean",
},
});
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
await generateVideo({
await runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "video-plugin/vid-v1" } } },
} as OpenClawConfig,
@@ -172,11 +171,10 @@ describe("video-generation runtime", () => {
it("passes providerOptions through to providers that do not declare any schema", async () => {
// Undeclared schema = backward-compatible pass-through: the provider receives the
// options and can handle or ignore them. No skip occurs.
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const { provider, getSeenProviderOptions } = createProviderOptionsCaptureProvider({});
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
await generateVideo({
await runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "video-plugin/vid-v1" } } },
} as OpenClawConfig,
@@ -189,7 +187,6 @@ describe("video-generation runtime", () => {
it("skips candidates that explicitly declare an empty providerOptions schema", async () => {
// Explicitly declared empty schema ({}) = provider has opted in and supports no options.
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const provider: VideoGenerationProvider = {
id: "video-plugin",
capabilities: {
@@ -201,10 +198,10 @@ describe("video-generation runtime", () => {
throw new Error("should not be called");
},
};
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "video-plugin/vid-v1" } } },
} as OpenClawConfig,
@@ -215,7 +212,6 @@ describe("video-generation runtime", () => {
});
it("skips candidates that declare a providerOptions schema missing the requested key", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const provider: VideoGenerationProvider = {
id: "video-plugin",
capabilities: {
@@ -225,10 +221,10 @@ describe("video-generation runtime", () => {
throw new Error("should not be called");
},
};
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "video-plugin/vid-v1" } } },
} as OpenClawConfig,
@@ -239,7 +235,6 @@ describe("video-generation runtime", () => {
});
it("skips candidates when providerOptions values do not match the declared type", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
const provider: VideoGenerationProvider = {
id: "video-plugin",
capabilities: {
@@ -249,10 +244,10 @@ describe("video-generation runtime", () => {
throw new Error("should not be called");
},
};
mocks.getVideoGenerationProvider.mockReturnValue(provider);
providers = [provider];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "video-plugin/vid-v1" } } },
} as OpenClawConfig,
@@ -265,57 +260,32 @@ describe("video-generation runtime", () => {
it("falls over from a provider with explicitly empty providerOptions schema to one that has it", async () => {
// Explicitly empty schema ({}) causes a skip; undeclared schema passes through.
// Here "openai" declares {} to signal it has been audited and truly accepts no options.
mocks.getVideoGenerationProvider.mockImplementation((providerId: string) => {
if (providerId === "openai") {
return {
id: "openai",
defaultModel: "sora-2",
capabilities: {
providerOptions: {} as Record<string, VideoGenerationProviderOptionType>,
}, // explicitly empty: accepts no options
isConfigured: () => true,
async generateVideo() {
throw new Error("should not be called");
},
};
}
if (providerId === "byteplus") {
return {
id: "byteplus",
defaultModel: "seedance-1-0-pro-250528",
capabilities: {
providerOptions: { seed: "number" },
},
isConfigured: () => true,
async generateVideo(req) {
expect(req.providerOptions).toEqual({ seed: 42 });
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "seedance-1-0-pro-250528",
};
},
};
}
return undefined;
});
mocks.listVideoGenerationProviders.mockReturnValue([
providers = [
{
id: "openai",
defaultModel: "sora-2",
capabilities: { providerOptions: {} as Record<string, VideoGenerationProviderOptionType> },
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo() {
throw new Error("should not be called");
},
},
{
id: "byteplus",
defaultModel: "seedance-1-0-pro-250528",
capabilities: { providerOptions: { seed: "number" } },
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo(req) {
expect(req.providerOptions).toEqual({ seed: 42 });
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "seedance-1-0-pro-250528",
};
},
},
]);
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {} as OpenClawConfig,
prompt: "animate a cat",
providerOptions: { seed: 42 },
@@ -328,57 +298,34 @@ describe("video-generation runtime", () => {
});
it("skips providers that cannot satisfy reference audio inputs and falls back", async () => {
mocks.getVideoGenerationProvider.mockImplementation((providerId: string) => {
if (providerId === "openai") {
return {
id: "openai",
defaultModel: "sora-2",
capabilities: {},
isConfigured: () => true,
async generateVideo() {
throw new Error("should not be called");
},
};
}
if (providerId === "byteplus") {
return {
id: "byteplus",
defaultModel: "seedance-1-0-pro-250528",
capabilities: {
maxInputAudios: 1,
},
isConfigured: () => true,
async generateVideo(req) {
expect(req.inputAudios).toEqual([
{ url: "https://example.com/reference-audio.mp3", role: "reference_audio" },
]);
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "seedance-1-0-pro-250528",
};
},
};
}
return undefined;
});
mocks.listVideoGenerationProviders.mockReturnValue([
providers = [
{
id: "openai",
defaultModel: "sora-2",
capabilities: {},
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo() {
throw new Error("should not be called");
},
},
{
id: "byteplus",
defaultModel: "seedance-1-0-pro-250528",
capabilities: { maxInputAudios: 1 },
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo(req) {
expect(req.inputAudios).toEqual([
{ url: "https://example.com/reference-audio.mp3", role: "reference_audio" },
]);
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "seedance-1-0-pro-250528",
};
},
},
]);
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -402,31 +349,30 @@ describe("video-generation runtime", () => {
inputVideos?: unknown;
inputAudios?: unknown;
} = {};
mocks.resolveAgentModelPrimaryValue.mockReturnValue(
"fal/bytedance/seedance-2.0/fast/reference-to-video",
);
mocks.getVideoGenerationProvider.mockReturnValue({
id: "fal",
capabilities: {
videoToVideo: {
enabled: true,
maxInputImages: 9,
maxInputVideos: 3,
maxInputAudios: 3,
providers = [
{
id: "fal",
capabilities: {
videoToVideo: {
enabled: true,
maxInputImages: 9,
maxInputVideos: 3,
maxInputAudios: 3,
},
},
async generateVideo(req) {
seenRequest.inputImages = req.inputImages;
seenRequest.inputVideos = req.inputVideos;
seenRequest.inputAudios = req.inputAudios;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "bytedance/seedance-2.0/fast/reference-to-video",
};
},
},
async generateVideo(req) {
seenRequest.inputImages = req.inputImages;
seenRequest.inputVideos = req.inputVideos;
seenRequest.inputAudios = req.inputAudios;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "bytedance/seedance-2.0/fast/reference-to-video",
};
},
});
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -452,17 +398,18 @@ describe("video-generation runtime", () => {
});
it("fails when every candidate is skipped for unsupported reference audio inputs", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/sora-2");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "openai",
capabilities: {},
async generateVideo() {
throw new Error("should not be called");
providers = [
{
id: "openai",
capabilities: {},
async generateVideo() {
throw new Error("should not be called");
},
},
});
];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "openai/sora-2" } } },
} as OpenClawConfig,
@@ -474,61 +421,32 @@ describe("video-generation runtime", () => {
it("skips providers whose hard duration cap is below the request and falls back", async () => {
let seenDurationSeconds: number | undefined;
mocks.getVideoGenerationProvider.mockImplementation((providerId: string) => {
if (providerId === "openai") {
return {
id: "openai",
defaultModel: "sora-2",
capabilities: {
generate: {
maxDurationSeconds: 4,
},
},
isConfigured: () => true,
async generateVideo() {
throw new Error("should not be called");
},
};
}
if (providerId === "runway") {
return {
id: "runway",
defaultModel: "gen4.5",
capabilities: {
generate: {
maxDurationSeconds: 8,
},
},
isConfigured: () => true,
async generateVideo(req) {
seenDurationSeconds = req.durationSeconds;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
};
}
return undefined;
});
mocks.listVideoGenerationProviders.mockReturnValue([
providers = [
{
id: "openai",
defaultModel: "sora-2",
capabilities: { generate: { maxDurationSeconds: 4 } },
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo() {
throw new Error("should not be called");
},
},
{
id: "runway",
defaultModel: "gen4.5",
capabilities: { generate: { maxDurationSeconds: 8 } },
isConfigured: () => true,
generateVideo: async () => ({ videos: [] }),
async generateVideo(req) {
seenDurationSeconds = req.durationSeconds;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
},
]);
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -548,21 +466,22 @@ describe("video-generation runtime", () => {
});
it("fails when every candidate is skipped for exceeding hard duration caps", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/sora-2");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "openai",
capabilities: {
generate: {
maxDurationSeconds: 4,
providers = [
{
id: "openai",
capabilities: {
generate: {
maxDurationSeconds: 4,
},
},
async generateVideo() {
throw new Error("should not be called");
},
},
async generateVideo() {
throw new Error("should not be called");
},
});
];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: { defaults: { videoGenerationModel: { primary: "openai/sora-2" } } },
} as OpenClawConfig,
@@ -573,17 +492,18 @@ describe("video-generation runtime", () => {
});
it("rejects provider results that contain undeliverable assets", async () => {
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "video-plugin",
capabilities: {},
generateVideo: async () => ({
videos: [{ mimeType: "video/mp4" }],
}),
});
providers = [
{
id: "video-plugin",
capabilities: {},
generateVideo: async () => ({
videos: [{ mimeType: "video/mp4" }],
}),
},
];
await expect(
generateVideo({
runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -597,7 +517,7 @@ describe("video-generation runtime", () => {
});
it("lists runtime video-generation providers through the provider registry", () => {
const providers: VideoGenerationProvider[] = [
const registryProviders: VideoGenerationProvider[] = [
{
id: "video-plugin",
defaultModel: "vid-v1",
@@ -612,34 +532,35 @@ describe("video-generation runtime", () => {
}),
},
];
mocks.listVideoGenerationProviders.mockReturnValue(providers);
providers = registryProviders;
expect(listRuntimeVideoGenerationProviders({ config: {} as OpenClawConfig })).toEqual(
providers,
);
expect(mocks.listVideoGenerationProviders).toHaveBeenCalledWith({} as OpenClawConfig);
expect(
listRuntimeVideoGenerationProviders({ config: {} as OpenClawConfig }, runtimeDeps),
).toEqual(registryProviders);
expect(listedConfigs).toEqual([{} as OpenClawConfig]);
});
it("normalizes requested durations to supported provider values", async () => {
let seenDurationSeconds: number | undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("video-plugin/vid-v1");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "video-plugin",
capabilities: {
generate: {
supportedDurationSeconds: [4, 6, 8],
providers = [
{
id: "video-plugin",
capabilities: {
generate: {
supportedDurationSeconds: [4, 6, 8],
},
},
generateVideo: async (req) => {
seenDurationSeconds = req.durationSeconds;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "vid-v1",
};
},
},
generateVideo: async (req) => {
seenDurationSeconds = req.durationSeconds;
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "vid-v1",
};
},
});
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -677,30 +598,31 @@ describe("video-generation runtime", () => {
watermark?: boolean;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/sora-2");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "openai",
capabilities: {
generate: {
supportsSize: true,
providers = [
{
id: "openai",
capabilities: {
generate: {
supportsSize: true,
},
},
generateVideo: async (req) => {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
audio: req.audio,
watermark: req.watermark,
};
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "sora-2",
};
},
},
generateVideo: async (req) => {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
audio: req.audio,
watermark: req.watermark,
};
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "sora-2",
};
},
});
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -739,35 +661,36 @@ describe("video-generation runtime", () => {
resolution?: string;
}
| undefined;
mocks.resolveAgentModelPrimaryValue.mockReturnValue("runway/gen4.5");
mocks.getVideoGenerationProvider.mockReturnValue({
id: "runway",
capabilities: {
generate: {
supportsSize: true,
supportsAspectRatio: false,
providers = [
{
id: "runway",
capabilities: {
generate: {
supportsSize: true,
supportsAspectRatio: false,
},
imageToVideo: {
enabled: true,
maxInputImages: 1,
supportsSize: false,
supportsAspectRatio: true,
},
},
imageToVideo: {
enabled: true,
maxInputImages: 1,
supportsSize: false,
supportsAspectRatio: true,
generateVideo: async (req) => {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
},
generateVideo: async (req) => {
seenRequest = {
size: req.size,
aspectRatio: req.aspectRatio,
resolution: req.resolution,
};
return {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
model: "gen4.5",
};
},
});
];
const result = await generateVideo({
const result = await runGenerateVideo({
cfg: {
agents: {
defaults: {
@@ -800,7 +723,7 @@ describe("video-generation runtime", () => {
});
it("builds a generic config hint without hardcoded provider ids", async () => {
mocks.listVideoGenerationProviders.mockReturnValue([
providers = [
{
id: "motion-one",
defaultModel: "animate-v1",
@@ -809,11 +732,11 @@ describe("video-generation runtime", () => {
videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
}),
},
]);
mocks.getProviderEnvVars.mockReturnValue(["MOTION_ONE_API_KEY"]);
];
providerEnvVars = { "motion-one": ["MOTION_ONE_API_KEY"] };
await expect(
generateVideo({ cfg: {} as OpenClawConfig, prompt: "animate a cat" }),
runGenerateVideo({ cfg: {} as OpenClawConfig, prompt: "animate a cat" }),
).rejects.toThrow(
'No video-generation model configured. Set agents.defaults.videoGenerationModel.primary to a provider/model like "motion-one/animate-v1". If you want a specific provider, also configure that provider\'s auth/API key first (motion-one: MOTION_ONE_API_KEY).',
);