mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-16 11:41:08 +00:00
365 lines
11 KiB
TypeScript
365 lines
11 KiB
TypeScript
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
|
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
|
import {
|
|
assertOkOrThrowHttpError,
|
|
fetchWithTimeout,
|
|
postJsonRequest,
|
|
resolveProviderHttpRequestConfig,
|
|
} from "openclaw/plugin-sdk/provider-http";
|
|
import type {
|
|
GeneratedVideoAsset,
|
|
VideoGenerationProvider,
|
|
VideoGenerationRequest,
|
|
VideoGenerationResult,
|
|
VideoGenerationSourceAsset,
|
|
} from "openclaw/plugin-sdk/video-generation";
|
|
|
|
const DEFAULT_RUNWAY_BASE_URL = "https://api.dev.runwayml.com";
|
|
const DEFAULT_RUNWAY_MODEL = "gen4.5";
|
|
const RUNWAY_API_VERSION = "2024-11-06";
|
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
const POLL_INTERVAL_MS = 5_000;
|
|
const MAX_POLL_ATTEMPTS = 120;
|
|
const MAX_DURATION_SECONDS = 10;
|
|
|
|
type RunwayTaskStatus = "PENDING" | "RUNNING" | "THROTTLED" | "SUCCEEDED" | "FAILED" | "CANCELLED";
|
|
|
|
type RunwayTaskCreateResponse = {
|
|
id?: string;
|
|
};
|
|
|
|
type RunwayTaskDetailResponse = {
|
|
id?: string;
|
|
status?: RunwayTaskStatus;
|
|
output?: string[];
|
|
failure?: string | { message?: string } | null;
|
|
};
|
|
|
|
const TEXT_ONLY_MODELS = new Set(["gen4.5", "veo3.1", "veo3.1_fast", "veo3"]);
|
|
const IMAGE_MODELS = new Set([
|
|
"gen4.5",
|
|
"gen4_turbo",
|
|
"gen3a_turbo",
|
|
"veo3.1",
|
|
"veo3.1_fast",
|
|
"veo3",
|
|
]);
|
|
const VIDEO_MODELS = new Set(["gen4_aleph"]);
|
|
const RUNWAY_TEXT_ASPECT_RATIOS = ["16:9", "9:16"] as const;
|
|
const RUNWAY_EDIT_ASPECT_RATIOS = ["1:1", "16:9", "9:16", "3:4", "4:3", "21:9"] as const;
|
|
|
|
function resolveRunwayBaseUrl(req: VideoGenerationRequest): string {
|
|
return req.cfg?.models?.providers?.runway?.baseUrl?.trim() || DEFAULT_RUNWAY_BASE_URL;
|
|
}
|
|
|
|
function toDataUrl(buffer: Buffer, mimeType: string): string {
|
|
return `data:${mimeType};base64,${buffer.toString("base64")}`;
|
|
}
|
|
|
|
function resolveSourceUri(
|
|
asset: VideoGenerationSourceAsset | undefined,
|
|
fallbackMimeType: string,
|
|
): string | undefined {
|
|
if (!asset) {
|
|
return undefined;
|
|
}
|
|
const url = asset.url?.trim();
|
|
if (url) {
|
|
return url;
|
|
}
|
|
if (!asset.buffer) {
|
|
return undefined;
|
|
}
|
|
return toDataUrl(asset.buffer, asset.mimeType?.trim() || fallbackMimeType);
|
|
}
|
|
|
|
function resolveDurationSeconds(value: number | undefined): number {
|
|
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
return 5;
|
|
}
|
|
return Math.max(2, Math.min(MAX_DURATION_SECONDS, Math.round(value)));
|
|
}
|
|
|
|
function resolveRunwayRatio(req: VideoGenerationRequest): string {
|
|
const hasImageInput = (req.inputImages?.length ?? 0) > 0;
|
|
const requested =
|
|
req.size?.trim() ||
|
|
(() => {
|
|
switch (req.aspectRatio?.trim()) {
|
|
case "9:16":
|
|
return "720:1280";
|
|
case "16:9":
|
|
return "1280:720";
|
|
case "1:1":
|
|
return "960:960";
|
|
case "3:4":
|
|
return "832:1104";
|
|
case "4:3":
|
|
return "1104:832";
|
|
case "21:9":
|
|
return "1584:672";
|
|
default:
|
|
return undefined;
|
|
}
|
|
})();
|
|
if (requested) {
|
|
if (!hasImageInput && requested !== "1280:720" && requested !== "720:1280") {
|
|
throw new Error("Runway text-to-video currently supports only 16:9 or 9:16 output ratios.");
|
|
}
|
|
return requested;
|
|
}
|
|
return "1280:720";
|
|
}
|
|
|
|
function resolveEndpoint(
|
|
req: VideoGenerationRequest,
|
|
): "/v1/text_to_video" | "/v1/image_to_video" | "/v1/video_to_video" {
|
|
const imageCount = req.inputImages?.length ?? 0;
|
|
const videoCount = req.inputVideos?.length ?? 0;
|
|
if (imageCount > 0 && videoCount > 0) {
|
|
throw new Error("Runway video generation does not support image and video inputs together.");
|
|
}
|
|
if (imageCount > 1 || videoCount > 1) {
|
|
throw new Error("Runway video generation supports at most one input image or one input video.");
|
|
}
|
|
if (videoCount > 0) {
|
|
return "/v1/video_to_video";
|
|
}
|
|
if (imageCount > 0) {
|
|
return "/v1/image_to_video";
|
|
}
|
|
return "/v1/text_to_video";
|
|
}
|
|
|
|
function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
|
|
const endpoint = resolveEndpoint(req);
|
|
const duration = resolveDurationSeconds(req.durationSeconds);
|
|
const ratio = resolveRunwayRatio(req);
|
|
const model = req.model?.trim() || DEFAULT_RUNWAY_MODEL;
|
|
if (endpoint === "/v1/text_to_video") {
|
|
if (!TEXT_ONLY_MODELS.has(model)) {
|
|
throw new Error(
|
|
`Runway text-to-video does not support model ${model}. Use one of: ${[...TEXT_ONLY_MODELS].join(", ")}.`,
|
|
);
|
|
}
|
|
return {
|
|
model,
|
|
promptText: req.prompt,
|
|
ratio,
|
|
duration,
|
|
};
|
|
}
|
|
|
|
if (endpoint === "/v1/image_to_video") {
|
|
if (!IMAGE_MODELS.has(model)) {
|
|
throw new Error(
|
|
`Runway image-to-video does not support model ${model}. Use one of: ${[...IMAGE_MODELS].join(", ")}.`,
|
|
);
|
|
}
|
|
const promptImage = resolveSourceUri(req.inputImages?.[0], "image/png");
|
|
if (!promptImage) {
|
|
throw new Error("Runway image-to-video input is missing image data.");
|
|
}
|
|
return {
|
|
model,
|
|
promptText: req.prompt,
|
|
promptImage,
|
|
ratio,
|
|
duration,
|
|
};
|
|
}
|
|
|
|
if (!VIDEO_MODELS.has(model)) {
|
|
throw new Error("Runway video-to-video currently requires model gen4_aleph.");
|
|
}
|
|
const videoUri = resolveSourceUri(req.inputVideos?.[0], "video/mp4");
|
|
if (!videoUri) {
|
|
throw new Error("Runway video-to-video input is missing video data.");
|
|
}
|
|
return {
|
|
model,
|
|
promptText: req.prompt,
|
|
videoUri,
|
|
ratio,
|
|
};
|
|
}
|
|
|
|
async function pollRunwayTask(params: {
|
|
taskId: string;
|
|
headers: Headers;
|
|
timeoutMs?: number;
|
|
baseUrl: string;
|
|
fetchFn: typeof fetch;
|
|
}): Promise<RunwayTaskDetailResponse> {
|
|
for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
|
|
const response = await fetchWithTimeout(
|
|
`${params.baseUrl}/v1/tasks/${params.taskId}`,
|
|
{
|
|
method: "GET",
|
|
headers: params.headers,
|
|
},
|
|
params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
params.fetchFn,
|
|
);
|
|
await assertOkOrThrowHttpError(response, "Runway video status request failed");
|
|
const payload = (await response.json()) as RunwayTaskDetailResponse;
|
|
switch (payload.status) {
|
|
case "SUCCEEDED":
|
|
return payload;
|
|
case "FAILED":
|
|
case "CANCELLED":
|
|
throw new Error(
|
|
(typeof payload.failure === "string"
|
|
? payload.failure
|
|
: payload.failure?.message
|
|
)?.trim() || `Runway video generation ${payload.status.toLowerCase()}`,
|
|
);
|
|
case "PENDING":
|
|
case "RUNNING":
|
|
case "THROTTLED":
|
|
default:
|
|
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
|
break;
|
|
}
|
|
}
|
|
throw new Error(`Runway video generation task ${params.taskId} did not finish in time`);
|
|
}
|
|
|
|
async function downloadRunwayVideos(params: {
|
|
urls: string[];
|
|
timeoutMs?: number;
|
|
fetchFn: typeof fetch;
|
|
}): Promise<GeneratedVideoAsset[]> {
|
|
const videos: GeneratedVideoAsset[] = [];
|
|
for (const [index, url] of params.urls.entries()) {
|
|
const response = await fetchWithTimeout(
|
|
url,
|
|
{ method: "GET" },
|
|
params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
params.fetchFn,
|
|
);
|
|
await assertOkOrThrowHttpError(response, "Runway generated video download failed");
|
|
const mimeType = response.headers.get("content-type")?.trim() || "video/mp4";
|
|
const arrayBuffer = await response.arrayBuffer();
|
|
videos.push({
|
|
buffer: Buffer.from(arrayBuffer),
|
|
mimeType,
|
|
fileName: `video-${index + 1}.${mimeType.includes("webm") ? "webm" : "mp4"}`,
|
|
metadata: { sourceUrl: url },
|
|
});
|
|
}
|
|
return videos;
|
|
}
|
|
|
|
export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider {
|
|
return {
|
|
id: "runway",
|
|
label: "Runway",
|
|
defaultModel: DEFAULT_RUNWAY_MODEL,
|
|
models: ["gen4.5", "gen4_turbo", "gen4_aleph", "gen3a_turbo", "veo3.1", "veo3.1_fast", "veo3"],
|
|
isConfigured: ({ agentDir }) =>
|
|
isProviderApiKeyConfigured({
|
|
provider: "runway",
|
|
agentDir,
|
|
}),
|
|
capabilities: {
|
|
generate: {
|
|
maxVideos: 1,
|
|
maxDurationSeconds: MAX_DURATION_SECONDS,
|
|
aspectRatios: RUNWAY_TEXT_ASPECT_RATIOS,
|
|
supportsAspectRatio: true,
|
|
},
|
|
imageToVideo: {
|
|
enabled: true,
|
|
maxVideos: 1,
|
|
maxInputImages: 1,
|
|
maxDurationSeconds: MAX_DURATION_SECONDS,
|
|
aspectRatios: RUNWAY_EDIT_ASPECT_RATIOS,
|
|
supportsAspectRatio: true,
|
|
},
|
|
videoToVideo: {
|
|
enabled: true,
|
|
maxVideos: 1,
|
|
maxInputVideos: 1,
|
|
aspectRatios: RUNWAY_EDIT_ASPECT_RATIOS,
|
|
supportsAspectRatio: true,
|
|
},
|
|
},
|
|
async generateVideo(req): Promise<VideoGenerationResult> {
|
|
const auth = await resolveApiKeyForProvider({
|
|
provider: "runway",
|
|
cfg: req.cfg,
|
|
agentDir: req.agentDir,
|
|
store: req.authStore,
|
|
});
|
|
if (!auth.apiKey) {
|
|
throw new Error("Runway API key missing");
|
|
}
|
|
|
|
const fetchFn = fetch;
|
|
const requestBody = buildCreateBody(req);
|
|
const endpoint = resolveEndpoint(req);
|
|
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
|
resolveProviderHttpRequestConfig({
|
|
baseUrl: resolveRunwayBaseUrl(req),
|
|
defaultBaseUrl: DEFAULT_RUNWAY_BASE_URL,
|
|
defaultHeaders: {
|
|
Authorization: `Bearer ${auth.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
"X-Runway-Version": RUNWAY_API_VERSION,
|
|
},
|
|
provider: "runway",
|
|
capability: "video",
|
|
transport: "http",
|
|
});
|
|
const { response, release } = await postJsonRequest({
|
|
url: `${baseUrl}${endpoint}`,
|
|
headers,
|
|
body: requestBody,
|
|
timeoutMs: req.timeoutMs,
|
|
fetchFn,
|
|
allowPrivateNetwork,
|
|
dispatcherPolicy,
|
|
});
|
|
try {
|
|
await assertOkOrThrowHttpError(response, "Runway video generation failed");
|
|
const submitted = (await response.json()) as RunwayTaskCreateResponse;
|
|
const taskId = submitted.id?.trim();
|
|
if (!taskId) {
|
|
throw new Error("Runway video generation response missing task id");
|
|
}
|
|
const completed = await pollRunwayTask({
|
|
taskId,
|
|
headers,
|
|
timeoutMs: req.timeoutMs,
|
|
baseUrl,
|
|
fetchFn,
|
|
});
|
|
const outputUrls = completed.output?.filter(
|
|
(value) => typeof value === "string" && value.trim(),
|
|
);
|
|
if (!outputUrls?.length) {
|
|
throw new Error("Runway video generation completed without output URLs");
|
|
}
|
|
const videos = await downloadRunwayVideos({
|
|
urls: outputUrls,
|
|
timeoutMs: req.timeoutMs,
|
|
fetchFn,
|
|
});
|
|
return {
|
|
videos,
|
|
model: req.model?.trim() || DEFAULT_RUNWAY_MODEL,
|
|
metadata: {
|
|
taskId,
|
|
status: completed.status,
|
|
endpoint,
|
|
outputUrls,
|
|
},
|
|
};
|
|
} finally {
|
|
await release();
|
|
}
|
|
},
|
|
};
|
|
}
|