openclaw/extensions/openrouter/video-generation-provider.ts

// Openrouter provider module implements model/runtime integration.
import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation";
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
  assertOkOrThrowHttpError,
  createProviderOperationDeadline,
  postJsonRequest,
  readProviderJsonResponse,
  resolveProviderHttpRequestConfig,
  resolveProviderOperationTimeoutMs,
  sanitizeConfiguredModelProviderRequest,
  waitProviderOperationPollInterval,
} from "openclaw/plugin-sdk/provider-http";
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import type {
  GeneratedVideoAsset,
  VideoGenerationProvider,
  VideoGenerationRequest,
  VideoGenerationSourceAsset,
} from "openclaw/plugin-sdk/video-generation";
import { OPENROUTER_BASE_URL } from "./provider-catalog.js";
import {
  fetchOpenRouterVideoGet,
  resolveOpenRouterVideoUrl,
  type OpenRouterVideoDispatcherPolicy,
} from "./video-http.js";
import { resolveOpenRouterVideoModelCapabilities } from "./video-model-catalog.js";

export { listOpenRouterVideoModelCatalog } from "./video-model-catalog.js";

const DEFAULT_MODEL = "google/veo-3.1-fast";
const DEFAULT_TIMEOUT_MS = 600_000;
const DEFAULT_HTTP_TIMEOUT_MS = 60_000;
const POLL_INTERVAL_MS = 5_000;
const MAX_POLL_ATTEMPTS = 120;
const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024;
const SUPPORTED_ASPECT_RATIOS = ["16:9", "9:16"] as const;
const OPENROUTER_VIDEO_MALFORMED_RESPONSE = "OpenRouter video generation response malformed";
const SUPPORTED_DURATION_SECONDS = [4, 6, 8] as const;
// Runtime sets this after normalizing against live model capabilities.
const SUPPORTED_DURATIONS_HINT = Symbol.for("openclaw.videoGeneration.supportedDurations");
const SUPPORTED_RESOLUTIONS = ["720P", "1080P"] as const;

type OpenRouterVideoResponse = {
  id?: string;
  generation_id?: string | null;
  polling_url?: string;
  status?: string;
  unsigned_urls?: string[];
  error?: string | null;
  model?: string | null;
  usage?: {
    cost?: number | null;
    is_byok?: boolean;
  };
};

type OpenRouterImagePart = {
  type: "image_url";
  image_url: { url: string };
};

type OpenRouterFrameImagePart = OpenRouterImagePart & {
  frame_type: "first_frame" | "last_frame";
};

async function readOpenRouterVideoJson(response: Response): Promise<Record<string, unknown>> {
  let payload: unknown;
  try {
    payload = await readProviderJsonResponse<unknown>(response, "OpenRouter video generation");
  } catch (error) {
    if (error instanceof Error && error.message.endsWith(": malformed JSON response")) {
      throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE, { cause: error });
    }
    throw error;
  }
  if (!isRecord(payload)) {
    throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
  }
  return payload;
}

function readOpenRouterVideoResponse(payload: Record<string, unknown>): OpenRouterVideoResponse {
  const unsignedUrls = payload.unsigned_urls;
  if (unsignedUrls !== undefined && unsignedUrls !== null && !Array.isArray(unsignedUrls)) {
    throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
  }
  const usage = payload.usage;
  if (usage !== undefined && usage !== null && !isRecord(usage)) {
    throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
  }
  return {
    id: normalizeOptionalString(payload.id),
    generation_id: normalizeOptionalString(payload.generation_id) ?? null,
    polling_url: normalizeOptionalString(payload.polling_url),
    status: normalizeOptionalString(payload.status),
    unsigned_urls: Array.isArray(unsignedUrls)
      ? unsignedUrls.map((url) => {
          const normalized = normalizeOptionalString(url);
          if (!normalized) {
            throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
          }
          return normalized;
        })
      : undefined,
    error: normalizeOptionalString(payload.error) ?? null,
    model: normalizeOptionalString(payload.model) ?? null,
    usage: isRecord(usage)
      ? {
          cost: typeof usage.cost === "number" ? usage.cost : null,
          is_byok: typeof usage.is_byok === "boolean" ? usage.is_byok : undefined,
        }
      : undefined,
  };
}

function toDataUrl(asset: VideoGenerationSourceAsset): string {
  if (asset.buffer) {
    return toImageDataUrl({ ...asset, buffer: asset.buffer, defaultMimeType: "image/png" });
  }
  const url = normalizeOptionalString(asset.url);
  if (url) {
    return url;
  }
  throw new Error(
    "OpenRouter video generation requires image references to include a URL or buffer.",
  );
}

function toImagePart(asset: VideoGenerationSourceAsset): OpenRouterImagePart {
  return {
    type: "image_url",
    image_url: { url: toDataUrl(asset) },
  };
}

function buildImageInputs(inputImages: VideoGenerationSourceAsset[] | undefined): {
  frameImages: OpenRouterFrameImagePart[];
  inputReferences: OpenRouterImagePart[];
} {
  const frameImages: OpenRouterFrameImagePart[] = [];
  const inputReferences: OpenRouterImagePart[] = [];
  let hasFirstFrame = false;
  let hasLastFrame = false;

  for (const image of inputImages ?? []) {
    const role = normalizeOptionalString(image.role);
    if (role === "reference_image") {
      inputReferences.push(toImagePart(image));
      continue;
    }

    const frameType =
      role === "last_frame"
        ? "last_frame"
        : role === "first_frame"
          ? "first_frame"
          : hasFirstFrame
            ? "last_frame"
            : "first_frame";

    if (frameType === "first_frame" && !hasFirstFrame) {
      frameImages.push({ ...toImagePart(image), frame_type: "first_frame" });
      hasFirstFrame = true;
      continue;
    }
    if (frameType === "last_frame" && !hasLastFrame) {
      frameImages.push({ ...toImagePart(image), frame_type: "last_frame" });
      hasLastFrame = true;
      continue;
    }
    inputReferences.push(toImagePart(image));
  }

  return { frameImages, inputReferences };
}

function resolveDurationSeconds(
  durationSeconds: number | undefined,
  supportedDurations: readonly number[] = SUPPORTED_DURATION_SECONDS,
): number | undefined {
  if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
    return undefined;
  }
  const effectiveDurations =
    supportedDurations.length > 0 ? supportedDurations : SUPPORTED_DURATION_SECONDS;
  const rounded = Math.max(1, Math.round(durationSeconds));
  if (durationSeconds === rounded && effectiveDurations.includes(rounded)) {
    return rounded;
  }
  return effectiveDurations.reduce((best, current) => {
    const currentDistance = Math.abs(current - rounded);
    const bestDistance = Math.abs(best - rounded);
    if (currentDistance < bestDistance) {
      return current;
    }
    if (currentDistance === bestDistance && current > best) {
      return current;
    }
    return best;
  });
}

function resolveResolution(resolution: VideoGenerationRequest["resolution"]): string | undefined {
  const normalized = normalizeOptionalString(resolution);
  return normalized ? normalized.toLowerCase() : undefined;
}

function resolveSeed(seed: unknown): number | undefined {
  if (seed === undefined) {
    return undefined;
  }
  if (typeof seed !== "number") {
    return undefined;
  }
  if (!Number.isSafeInteger(seed)) {
    throw new Error("OpenRouter video seed must be an integer");
  }
  return seed;
}

function buildRequestBody(req: VideoGenerationRequest, model: string): Record<string, unknown> {
  const { frameImages, inputReferences } = buildImageInputs(req.inputImages);
  const supportedDurations =
    (req as VideoGenerationRequest & { [SUPPORTED_DURATIONS_HINT]?: readonly number[] })[
      SUPPORTED_DURATIONS_HINT
    ] ?? SUPPORTED_DURATION_SECONDS;
  const body: Record<string, unknown> = {
    model,
    prompt: req.prompt,
  };

  const duration = resolveDurationSeconds(req.durationSeconds, supportedDurations);
  if (duration != null) {
    body.duration = duration;
  }
  const resolution = resolveResolution(req.resolution);
  if (resolution) {
    body.resolution = resolution;
  }
  const aspectRatio = normalizeOptionalString(req.aspectRatio);
  if (aspectRatio) {
    body.aspect_ratio = aspectRatio;
  }
  const size = normalizeOptionalString(req.size);
  if (size) {
    body.size = size;
  }
  if (typeof req.audio === "boolean") {
    body.generate_audio = req.audio;
  }
  if (frameImages.length > 0) {
    body.frame_images = frameImages;
  }
  if (inputReferences.length > 0) {
    body.input_references = inputReferences;
  }

  const seed = resolveSeed(req.providerOptions?.seed);
  if (seed !== undefined) {
    body.seed = seed;
  }
  const callbackUrl =
    typeof req.providerOptions?.callback_url === "string"
      ? normalizeOptionalString(req.providerOptions.callback_url)
      : undefined;
  if (callbackUrl) {
    body.callback_url = callbackUrl;
  }

  return body;
}

function isTerminalFailure(status: string | undefined): boolean {
  return status === "failed" || status === "cancelled" || status === "expired";
}

async function fetchOpenRouterJson(params: {
  url: string;
  baseUrl: string;
  headers: Headers;
  timeoutMs: number;
  allowPrivateNetwork: boolean;
  dispatcherPolicy: OpenRouterVideoDispatcherPolicy;
  errorContext: string;
  auditContext: string;
}): Promise<OpenRouterVideoResponse> {
  const { response, release } = await fetchOpenRouterVideoGet(params);
  try {
    await assertOkOrThrowHttpError(response, params.errorContext);
    return readOpenRouterVideoResponse(await readOpenRouterVideoJson(response));
  } finally {
    await release();
  }
}

async function pollOpenRouterVideo(params: {
  pollingUrl: string;
  baseUrl: string;
  headers: Headers;
  timeoutMs: number;
  allowPrivateNetwork: boolean;
  dispatcherPolicy: OpenRouterVideoDispatcherPolicy;
}): Promise<OpenRouterVideoResponse> {
  const deadline = createProviderOperationDeadline({
    timeoutMs: params.timeoutMs,
    label: "OpenRouter video generation",
  });

  for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
    const payload = await fetchOpenRouterJson({
      url: params.pollingUrl,
      baseUrl: params.baseUrl,
      headers: params.headers,
      timeoutMs: resolveProviderOperationTimeoutMs({
        deadline,
        defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
      }),
      allowPrivateNetwork: params.allowPrivateNetwork,
      dispatcherPolicy: params.dispatcherPolicy,
      errorContext: "OpenRouter video status request failed",
      auditContext: "openrouter-video-status",
    });
    const status = normalizeOptionalString(payload.status);
    if (
      !status ||
      (!["queued", "pending", "processing", "running", "completed"].includes(status) &&
        !isTerminalFailure(status))
    ) {
      throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
    }
    if (status === "completed") {
      return payload;
    }
    if (isTerminalFailure(status)) {
      throw new Error(
        normalizeOptionalString(payload.error) ?? `OpenRouter video generation ${status}`,
      );
    }
    await waitProviderOperationPollInterval({
      deadline,
      pollIntervalMs: POLL_INTERVAL_MS,
    });
  }

  throw new Error("OpenRouter video generation did not finish in time");
}

function resolveOpenRouterContentUrl(params: { baseUrl: string; jobId: string }): string {
  return resolveOpenRouterVideoUrl(
    `videos/${encodeURIComponent(params.jobId)}/content?index=0`,
    params.baseUrl,
  );
}

function resolveDeliverableOpenRouterVideoUrl(value: string | undefined): string | undefined {
  const normalized = normalizeOptionalString(value);
  if (!normalized) {
    return undefined;
  }
  try {
    const url = new URL(normalized);
    return url.protocol === "https:" || url.protocol === "http:" ? normalized : undefined;
  } catch {
    return undefined;
  }
}

function resolveGeneratedVideoMaxBytes(req: VideoGenerationRequest): number {
  const configured = req.cfg.agents?.defaults?.mediaMaxMb;
  if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
    return Math.floor(configured * 1024 * 1024);
  }
  return DEFAULT_GENERATED_VIDEO_MAX_BYTES;
}

async function downloadOpenRouterVideo(params: {
  url: string;
  deliveryUrl?: string;
  baseUrl: string;
  headers: Headers;
  timeoutMs: number;
  allowPrivateNetwork: boolean;
  dispatcherPolicy: OpenRouterVideoDispatcherPolicy;
  maxBytes: number;
}): Promise<GeneratedVideoAsset> {
  const { response, release } = await fetchOpenRouterVideoGet({
    ...params,
    auditContext: "openrouter-video-download",
  });
  try {
    await assertOkOrThrowHttpError(response, "OpenRouter generated video download failed");
    const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "video/mp4";
    const fileName = `video-1.${extensionForMime(mimeType)?.slice(1) ?? "mp4"}`;
    let exceededMaxBytes = false;
    let buffer: Buffer;
    try {
      buffer = await readResponseWithLimit(response, params.maxBytes, {
        onOverflow: ({ maxBytes }) => {
          exceededMaxBytes = true;
          return new Error(`OpenRouter generated video download exceeds ${maxBytes} bytes`);
        },
      });
    } catch (error) {
      if (exceededMaxBytes && params.deliveryUrl) {
        return {
          url: params.deliveryUrl,
          mimeType,
          fileName,
        };
      }
      throw error;
    }
    return {
      buffer,
      mimeType,
      fileName,
    };
  } finally {
    await release();
  }
}

export function buildOpenRouterVideoGenerationProvider(): VideoGenerationProvider {
  return {
    id: "openrouter",
    label: "OpenRouter",
    defaultModel: DEFAULT_MODEL,
    models: [DEFAULT_MODEL],
    isConfigured: ({ agentDir }) =>
      isProviderApiKeyConfigured({ provider: "openrouter", agentDir }),
    resolveModelCapabilities: resolveOpenRouterVideoModelCapabilities,
    capabilities: {
      providerOptions: {
        callback_url: "string",
        seed: "number",
      },
      generate: {
        maxVideos: 1,
        supportedDurationSeconds: [...SUPPORTED_DURATION_SECONDS],
        supportsAspectRatio: true,
        supportsResolution: true,
        supportsSize: true,
        supportsAudio: true,
        aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
        resolutions: [...SUPPORTED_RESOLUTIONS],
      },
      imageToVideo: {
        enabled: true,
        maxVideos: 1,
        maxInputImages: 4,
        supportedDurationSeconds: [...SUPPORTED_DURATION_SECONDS],
        supportsAspectRatio: true,
        supportsResolution: true,
        supportsSize: true,
        supportsAudio: true,
        aspectRatios: [...SUPPORTED_ASPECT_RATIOS],
        resolutions: [...SUPPORTED_RESOLUTIONS],
      },
      videoToVideo: {
        enabled: false,
      },
    },
    async generateVideo(req) {
      if ((req.inputVideos?.length ?? 0) > 0) {
        throw new Error("OpenRouter video generation does not support video reference inputs.");
      }

      const auth = await resolveApiKeyForProvider({
        provider: "openrouter",
        cfg: req.cfg,
        agentDir: req.agentDir,
        store: req.authStore,
      });
      if (!auth.apiKey) {
        throw new Error("OpenRouter API key missing");
      }

      const model = normalizeOptionalString(req.model) ?? DEFAULT_MODEL;
      const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
        resolveProviderHttpRequestConfig({
          baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl,
          defaultBaseUrl: OPENROUTER_BASE_URL,
          allowPrivateNetwork: false,
          defaultHeaders: {
            Authorization: `Bearer ${auth.apiKey}`,
            "Content-Type": "application/json",
            "HTTP-Referer": "https://openclaw.ai",
            "X-OpenRouter-Title": "OpenClaw",
          },
          request: sanitizeConfiguredModelProviderRequest(
            req.cfg?.models?.providers?.openrouter?.request,
          ),
          provider: "openrouter",
          capability: "video",
          transport: "http",
        });
      const deadline = createProviderOperationDeadline({
        timeoutMs: req.timeoutMs,
        label: "OpenRouter video generation",
      });
      const { response, release } = await postJsonRequest({
        url: `${baseUrl}/videos`,
        headers,
        body: buildRequestBody(req, model),
        timeoutMs: resolveProviderOperationTimeoutMs({
          deadline,
          defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
        }),
        fetchFn: fetch,
        allowPrivateNetwork,
        dispatcherPolicy,
        auditContext: "openrouter-video-submit",
      });

      try {
        await assertOkOrThrowHttpError(response, "OpenRouter video generation failed");
        const submitted = readOpenRouterVideoResponse(await readOpenRouterVideoJson(response));
        const jobId = normalizeOptionalString(submitted.id);
        const pollingUrl = normalizeOptionalString(submitted.polling_url);
        if (!jobId || !pollingUrl) {
          throw new Error("OpenRouter video generation response missing job details");
        }
        const submittedStatus = normalizeOptionalString(submitted.status);
        if (
          submittedStatus &&
          !["queued", "pending", "processing", "running", "completed"].includes(submittedStatus) &&
          !isTerminalFailure(submittedStatus)
        ) {
          throw new Error(OPENROUTER_VIDEO_MALFORMED_RESPONSE);
        }
        if (isTerminalFailure(submittedStatus)) {
          throw new Error(
            normalizeOptionalString(submitted.error) ??
              `OpenRouter video generation ${submittedStatus}`,
          );
        }
        const completed =
          submittedStatus === "completed"
            ? submitted
            : await pollOpenRouterVideo({
                pollingUrl,
                baseUrl,
                headers,
                timeoutMs: resolveProviderOperationTimeoutMs({
                  deadline,
                  defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
                }),
                allowPrivateNetwork,
                dispatcherPolicy,
              });
        const completedJobId = normalizeOptionalString(completed.id) ?? jobId;
        const unsignedUrl = completed.unsigned_urls?.find((url) => normalizeOptionalString(url));
        const videoUrl =
          unsignedUrl ?? resolveOpenRouterContentUrl({ baseUrl, jobId: completedJobId });
        const video = await downloadOpenRouterVideo({
          url: videoUrl,
          deliveryUrl: resolveDeliverableOpenRouterVideoUrl(unsignedUrl),
          baseUrl,
          headers,
          timeoutMs: resolveProviderOperationTimeoutMs({
            deadline,
            defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
          }),
          allowPrivateNetwork,
          dispatcherPolicy,
          maxBytes: resolveGeneratedVideoMaxBytes(req),
        });

        return {
          videos: [video],
          model: normalizeOptionalString(completed.model) ?? model,
          metadata: {
            jobId,
            status: completed.status,
            ...(normalizeOptionalString(completed.generation_id)
              ? { generationId: normalizeOptionalString(completed.generation_id) }
              : {}),
            ...(completed.usage ? { usage: completed.usage } : {}),
          },
        };
      } finally {
        await release();
      }
    },
  };
}