import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, fetchWithTimeout, postJsonRequest, resolveProviderHttpRequestConfig, } from "openclaw/plugin-sdk/provider-http"; import type { GeneratedVideoAsset, VideoGenerationProvider, VideoGenerationRequest, VideoGenerationResult, VideoGenerationSourceAsset, } from "openclaw/plugin-sdk/video-generation"; const DEFAULT_RUNWAY_BASE_URL = "https://api.dev.runwayml.com"; const DEFAULT_RUNWAY_MODEL = "gen4.5"; const RUNWAY_API_VERSION = "2024-11-06"; const DEFAULT_TIMEOUT_MS = 120_000; const POLL_INTERVAL_MS = 5_000; const MAX_POLL_ATTEMPTS = 120; const MAX_DURATION_SECONDS = 10; type RunwayTaskStatus = "PENDING" | "RUNNING" | "THROTTLED" | "SUCCEEDED" | "FAILED" | "CANCELLED"; type RunwayTaskCreateResponse = { id?: string; }; type RunwayTaskDetailResponse = { id?: string; status?: RunwayTaskStatus; output?: string[]; failure?: string | { message?: string } | null; }; const TEXT_ONLY_MODELS = new Set(["gen4.5", "veo3.1", "veo3.1_fast", "veo3"]); const IMAGE_MODELS = new Set([ "gen4.5", "gen4_turbo", "gen3a_turbo", "veo3.1", "veo3.1_fast", "veo3", ]); const VIDEO_MODELS = new Set(["gen4_aleph"]); const RUNWAY_TEXT_ASPECT_RATIOS = ["16:9", "9:16"] as const; const RUNWAY_EDIT_ASPECT_RATIOS = ["1:1", "16:9", "9:16", "3:4", "4:3", "21:9"] as const; function resolveRunwayBaseUrl(req: VideoGenerationRequest): string { return req.cfg?.models?.providers?.runway?.baseUrl?.trim() || DEFAULT_RUNWAY_BASE_URL; } function toDataUrl(buffer: Buffer, mimeType: string): string { return `data:${mimeType};base64,${buffer.toString("base64")}`; } function resolveSourceUri( asset: VideoGenerationSourceAsset | undefined, fallbackMimeType: string, ): string | undefined { if (!asset) { return undefined; } const url = asset.url?.trim(); if (url) { return url; } if (!asset.buffer) { return undefined; } return toDataUrl(asset.buffer, asset.mimeType?.trim() || fallbackMimeType); } function resolveDurationSeconds(value: number | undefined): number { if (typeof value !== "number" || !Number.isFinite(value)) { return 5; } return Math.max(2, Math.min(MAX_DURATION_SECONDS, Math.round(value))); } function resolveRunwayRatio(req: VideoGenerationRequest): string { const hasImageInput = (req.inputImages?.length ?? 0) > 0; const requested = req.size?.trim() || (() => { switch (req.aspectRatio?.trim()) { case "9:16": return "720:1280"; case "16:9": return "1280:720"; case "1:1": return "960:960"; case "3:4": return "832:1104"; case "4:3": return "1104:832"; case "21:9": return "1584:672"; default: return undefined; } })(); if (requested) { if (!hasImageInput && requested !== "1280:720" && requested !== "720:1280") { throw new Error("Runway text-to-video currently supports only 16:9 or 9:16 output ratios."); } return requested; } return "1280:720"; } function resolveEndpoint( req: VideoGenerationRequest, ): "/v1/text_to_video" | "/v1/image_to_video" | "/v1/video_to_video" { const imageCount = req.inputImages?.length ?? 0; const videoCount = req.inputVideos?.length ?? 0; if (imageCount > 0 && videoCount > 0) { throw new Error("Runway video generation does not support image and video inputs together."); } if (imageCount > 1 || videoCount > 1) { throw new Error("Runway video generation supports at most one input image or one input video."); } if (videoCount > 0) { return "/v1/video_to_video"; } if (imageCount > 0) { return "/v1/image_to_video"; } return "/v1/text_to_video"; } function buildCreateBody(req: VideoGenerationRequest): Record { const endpoint = resolveEndpoint(req); const duration = resolveDurationSeconds(req.durationSeconds); const ratio = resolveRunwayRatio(req); const model = req.model?.trim() || DEFAULT_RUNWAY_MODEL; if (endpoint === "/v1/text_to_video") { if (!TEXT_ONLY_MODELS.has(model)) { throw new Error( `Runway text-to-video does not support model ${model}. Use one of: ${[...TEXT_ONLY_MODELS].join(", ")}.`, ); } return { model, promptText: req.prompt, ratio, duration, }; } if (endpoint === "/v1/image_to_video") { if (!IMAGE_MODELS.has(model)) { throw new Error( `Runway image-to-video does not support model ${model}. Use one of: ${[...IMAGE_MODELS].join(", ")}.`, ); } const promptImage = resolveSourceUri(req.inputImages?.[0], "image/png"); if (!promptImage) { throw new Error("Runway image-to-video input is missing image data."); } return { model, promptText: req.prompt, promptImage, ratio, duration, }; } if (!VIDEO_MODELS.has(model)) { throw new Error("Runway video-to-video currently requires model gen4_aleph."); } const videoUri = resolveSourceUri(req.inputVideos?.[0], "video/mp4"); if (!videoUri) { throw new Error("Runway video-to-video input is missing video data."); } return { model, promptText: req.prompt, videoUri, ratio, }; } async function pollRunwayTask(params: { taskId: string; headers: Headers; timeoutMs?: number; baseUrl: string; fetchFn: typeof fetch; }): Promise { for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/v1/tasks/${params.taskId}`, { method: "GET", headers: params.headers, }, params.timeoutMs ?? DEFAULT_TIMEOUT_MS, params.fetchFn, ); await assertOkOrThrowHttpError(response, "Runway video status request failed"); const payload = (await response.json()) as RunwayTaskDetailResponse; switch (payload.status) { case "SUCCEEDED": return payload; case "FAILED": case "CANCELLED": throw new Error( (typeof payload.failure === "string" ? payload.failure : payload.failure?.message )?.trim() || `Runway video generation ${payload.status.toLowerCase()}`, ); case "PENDING": case "RUNNING": case "THROTTLED": default: await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); break; } } throw new Error(`Runway video generation task ${params.taskId} did not finish in time`); } async function downloadRunwayVideos(params: { urls: string[]; timeoutMs?: number; fetchFn: typeof fetch; }): Promise { const videos: GeneratedVideoAsset[] = []; for (const [index, url] of params.urls.entries()) { const response = await fetchWithTimeout( url, { method: "GET" }, params.timeoutMs ?? DEFAULT_TIMEOUT_MS, params.fetchFn, ); await assertOkOrThrowHttpError(response, "Runway generated video download failed"); const mimeType = response.headers.get("content-type")?.trim() || "video/mp4"; const arrayBuffer = await response.arrayBuffer(); videos.push({ buffer: Buffer.from(arrayBuffer), mimeType, fileName: `video-${index + 1}.${mimeType.includes("webm") ? "webm" : "mp4"}`, metadata: { sourceUrl: url }, }); } return videos; } export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { return { id: "runway", label: "Runway", defaultModel: DEFAULT_RUNWAY_MODEL, models: ["gen4.5", "gen4_turbo", "gen4_aleph", "gen3a_turbo", "veo3.1", "veo3.1_fast", "veo3"], isConfigured: ({ agentDir }) => isProviderApiKeyConfigured({ provider: "runway", agentDir, }), capabilities: { generate: { maxVideos: 1, maxDurationSeconds: MAX_DURATION_SECONDS, aspectRatios: RUNWAY_TEXT_ASPECT_RATIOS, supportsAspectRatio: true, }, imageToVideo: { enabled: true, maxVideos: 1, maxInputImages: 1, maxDurationSeconds: MAX_DURATION_SECONDS, aspectRatios: RUNWAY_EDIT_ASPECT_RATIOS, supportsAspectRatio: true, }, videoToVideo: { enabled: true, maxVideos: 1, maxInputVideos: 1, aspectRatios: RUNWAY_EDIT_ASPECT_RATIOS, supportsAspectRatio: true, }, }, async generateVideo(req): Promise { const auth = await resolveApiKeyForProvider({ provider: "runway", cfg: req.cfg, agentDir: req.agentDir, store: req.authStore, }); if (!auth.apiKey) { throw new Error("Runway API key missing"); } const fetchFn = fetch; const requestBody = buildCreateBody(req); const endpoint = resolveEndpoint(req); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveRunwayBaseUrl(req), defaultBaseUrl: DEFAULT_RUNWAY_BASE_URL, defaultHeaders: { Authorization: `Bearer ${auth.apiKey}`, "Content-Type": "application/json", "X-Runway-Version": RUNWAY_API_VERSION, }, provider: "runway", capability: "video", transport: "http", }); const { response, release } = await postJsonRequest({ url: `${baseUrl}${endpoint}`, headers, body: requestBody, timeoutMs: req.timeoutMs, fetchFn, allowPrivateNetwork, dispatcherPolicy, }); try { await assertOkOrThrowHttpError(response, "Runway video generation failed"); const submitted = (await response.json()) as RunwayTaskCreateResponse; const taskId = submitted.id?.trim(); if (!taskId) { throw new Error("Runway video generation response missing task id"); } const completed = await pollRunwayTask({ taskId, headers, timeoutMs: req.timeoutMs, baseUrl, fetchFn, }); const outputUrls = completed.output?.filter( (value) => typeof value === "string" && value.trim(), ); if (!outputUrls?.length) { throw new Error("Runway video generation completed without output URLs"); } const videos = await downloadRunwayVideos({ urls: outputUrls, timeoutMs: req.timeoutMs, fetchFn, }); return { videos, model: req.model?.trim() || DEFAULT_RUNWAY_MODEL, metadata: { taskId, status: completed.status, endpoint, outputUrls, }, }; } finally { await release(); } }, }; }