From a88c6f0fe7137047b6ea86309dadade48d2496e7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 14 Apr 2026 14:59:01 +0100 Subject: [PATCH] fix: bound live video generation smoke --- .../openclaw-release-maintainer/SKILL.md | 9 + CHANGELOG.md | 2 + .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/help/testing.md | 8 +- docs/tools/video-generation.md | 19 +- .../byteplus/video-generation-provider.ts | 30 +- .../google/video-generation-provider.ts | 17 +- .../minimax/provider-http.test-helpers.ts | 13 + .../minimax/video-generation-provider.ts | 35 +- .../openai/video-generation-provider.ts | 40 +- .../runway/video-generation-provider.ts | 35 +- .../together/video-generation-provider.ts | 30 +- .../video-generation-providers.live.test.ts | 471 ++++++++++++------ extensions/vydra/shared.ts | 11 +- extensions/vydra/video-generation-provider.ts | 26 +- extensions/xai/video-generation-provider.ts | 30 +- scripts/test-live-media.ts | 21 +- src/media-understanding/shared.test.ts | 69 +++ src/media-understanding/shared.ts | 56 +++ src/plugin-sdk/provider-http.ts | 4 + src/scripts/test-live-media.test.ts | 5 +- src/video-generation/dashscope-compatible.ts | 30 +- .../media-generation/provider-http-mocks.ts | 13 + 23 files changed, 749 insertions(+), 229 deletions(-) diff --git a/.agents/skills/openclaw-release-maintainer/SKILL.md b/.agents/skills/openclaw-release-maintainer/SKILL.md index 0d7772fd567..3b9e087668a 100644 --- a/.agents/skills/openclaw-release-maintainer/SKILL.md +++ b/.agents/skills/openclaw-release-maintainer/SKILL.md @@ -90,6 +90,15 @@ node --import tsx scripts/openclaw-npm-postpublish-verify.ts now fails the candidate update tarball when npm reports an oversized `unpackedSize`, so release-time e2e cannot miss pack bloat that would risk low-memory install/startup failures. +- Use `pnpm test:live:media video` for bounded video-provider smoke when video + generation is in release scope. The default video smoke skips `fal`, runs one + text-to-video attempt per provider with a one-second lobster prompt, and caps + each provider operation with `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS` + (`180000` by default). +- Run `pnpm test:live:media video --video-providers fal` only when FAL-specific + proof is required. Its queue latency can dominate release time. +- Set `OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES=1` only when intentionally + validating the slower image-to-video and video-to-video transform lanes. ## Check all relevant release builds diff --git a/CHANGELOG.md b/CHANGELOG.md index a5799630313..07aea968bc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Docs: https://docs.openclaw.ai ### Fixes +- Video generation/live tests: bound provider polling for live video smoke, default to the fast non-FAL text-to-video path, and use a one-second lobster prompt so release validation no longer waits indefinitely on slow provider queues. + ## 2026.4.14 ### Changes diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 365a8adcd0a..dc50c76ede1 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -7b121e2b694f80433fa91ce9037527ca58be546a7f18798470a4ade66593e5e1 plugin-sdk-api-baseline.json -7b802cc04f0eac0b498b50711e39a7afe93bbb6b682a2013d2c303583fb73f40 plugin-sdk-api-baseline.jsonl +6eaa32f7af20ef87952510a09f400f3c48acf25db408f067eea4ea9b86d70514 plugin-sdk-api-baseline.json +d4276644b0bd99bd37a54d1498ec3487375d20fde6906fb0bfa146d91f352d1d plugin-sdk-api-baseline.jsonl diff --git a/docs/help/testing.md b/docs/help/testing.md index 417d7a7de18..26924635889 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -781,11 +781,13 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Harness: `pnpm test:live:media video` - Scope: - Exercises the shared bundled video-generation provider path + - Defaults to the release-safe smoke path: non-FAL providers, one text-to-video request per provider, one-second lobster prompt, and a per-provider operation cap from `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS` (`180000` by default) + - Skips FAL by default because provider-side queue latency can dominate release time; pass `--video-providers fal` or `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="fal"` to run it explicitly - Loads provider env vars from your login shell (`~/.profile`) before probing - Uses live/env API keys ahead of stored auth profiles by default, so stale test keys in `auth-profiles.json` do not mask real shell credentials - Skips providers with no usable auth/profile/model - - Runs both declared runtime modes when available: - - `generate` with prompt-only input + - Runs only `generate` by default + - Set `OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES=1` to also run declared transform modes when available: - `imageToVideo` when the provider declares `capabilities.imageToVideo.enabled` and the selected provider/model accepts buffer-backed local image input in the shared sweep - `videoToVideo` when the provider declares `capabilities.videoToVideo.enabled` and the selected provider/model accepts buffer-backed local video input in the shared sweep - Current declared-but-skipped `imageToVideo` providers in the shared sweep: @@ -802,6 +804,8 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Optional narrowing: - `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="google,openai,runway"` - `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,openai/sora-2,runway/gen4_aleph"` + - `OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS=""` to include every provider in the default sweep, including FAL + - `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS=60000` to reduce each provider operation cap for an aggressive smoke run - Optional auth behavior: - `OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS=1` to force profile-store auth and ignore env-only overrides diff --git a/docs/tools/video-generation.md b/docs/tools/video-generation.md index 7de3bc29082..eb46780b608 100644 --- a/docs/tools/video-generation.md +++ b/docs/tools/video-generation.md @@ -316,10 +316,23 @@ pnpm test:live:media video ``` This live file loads missing provider env vars from `~/.profile`, prefers -live/env API keys ahead of stored auth profiles by default, and runs the -declared modes it can exercise safely with local media: +live/env API keys ahead of stored auth profiles by default, and runs a +release-safe smoke by default: + +- `generate` for every non-FAL provider in the sweep +- one-second lobster prompt +- per-provider operation cap from `OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS` + (`180000` by default) + +FAL is opt-in because provider-side queue latency can dominate release time: + +```bash +pnpm test:live:media video --video-providers fal +``` + +Set `OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES=1` to also run declared transform +modes the shared sweep can exercise safely with local media: -- `generate` for every provider in the sweep - `imageToVideo` when `capabilities.imageToVideo.enabled` - `videoToVideo` when `capabilities.videoToVideo.enabled` and the provider/model accepts buffer-backed local video input in the shared sweep diff --git a/extensions/byteplus/video-generation-provider.ts b/extensions/byteplus/video-generation-provider.ts index 10e1de91432..2e817588615 100644 --- a/extensions/byteplus/video-generation-provider.ts +++ b/extensions/byteplus/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -73,6 +76,10 @@ async function pollBytePlusTask(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `BytePlus video generation task ${params.taskId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/contents/generations/tasks/${params.taskId}`, @@ -80,7 +87,7 @@ async function pollBytePlusTask(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "BytePlus video status request failed"); @@ -96,7 +103,7 @@ async function pollBytePlusTask(params: { case "queued": case "running": default: - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); break; } } @@ -183,6 +190,10 @@ export function buildBytePlusVideoGenerationProvider(): VideoGenerationProvider } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "BytePlus video generation", + }); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveBytePlusVideoBaseUrl(req), @@ -261,7 +272,10 @@ export function buildBytePlusVideoGenerationProvider(): VideoGenerationProvider url: `${baseUrl}/contents/generations/tasks`, headers, body, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -276,7 +290,10 @@ export function buildBytePlusVideoGenerationProvider(): VideoGenerationProvider const completed = await pollBytePlusTask({ taskId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); @@ -286,7 +303,10 @@ export function buildBytePlusVideoGenerationProvider(): VideoGenerationProvider } const video = await downloadBytePlusVideo({ url: videoUrl, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, }); return { diff --git a/extensions/google/video-generation-provider.ts b/extensions/google/video-generation-provider.ts index b0b321dcc3b..627f8ebf51e 100644 --- a/extensions/google/video-generation-provider.ts +++ b/extensions/google/video-generation-provider.ts @@ -3,6 +3,11 @@ import path from "node:path"; import { GoogleGenAI } from "@google/genai"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; +import { + createProviderOperationDeadline, + resolveProviderOperationTimeoutMs, + waitProviderOperationPollInterval, +} from "openclaw/plugin-sdk/provider-http"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -225,11 +230,18 @@ export function buildGoogleVideoGenerationProvider(): VideoGenerationProvider { const configuredBaseUrl = resolveConfiguredGoogleVideoBaseUrl(req); const durationSeconds = resolveDurationSeconds(req.durationSeconds); + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "Google video generation", + }); const client = new GoogleGenAI({ apiKey: auth.apiKey, httpOptions: { ...(configuredBaseUrl ? { baseUrl: configuredBaseUrl } : {}), - timeout: req.timeoutMs ?? DEFAULT_TIMEOUT_MS, + timeout: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), }, }); let operation = await client.models.generateVideos({ @@ -253,7 +265,8 @@ export function buildGoogleVideoGenerationProvider(): VideoGenerationProvider { if (attempt >= MAX_POLL_ATTEMPTS) { throw new Error("Google video generation did not finish in time"); } - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }); operation = await client.operations.getVideosOperation({ operation }); } if (operation.error) { diff --git a/extensions/minimax/provider-http.test-helpers.ts b/extensions/minimax/provider-http.test-helpers.ts index b56c984ac81..25854d20150 100644 --- a/extensions/minimax/provider-http.test-helpers.ts +++ b/extensions/minimax/provider-http.test-helpers.ts @@ -24,9 +24,22 @@ vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({ vi.mock("openclaw/plugin-sdk/provider-http", () => ({ assertOkOrThrowHttpError: minimaxProviderHttpMocks.assertOkOrThrowHttpErrorMock, + createProviderOperationDeadline: ({ + label, + timeoutMs, + }: { + label: string; + timeoutMs?: number; + }) => ({ + label, + timeoutMs, + }), fetchWithTimeout: minimaxProviderHttpMocks.fetchWithTimeoutMock, postJsonRequest: minimaxProviderHttpMocks.postJsonRequestMock, + resolveProviderOperationTimeoutMs: ({ defaultTimeoutMs }: { defaultTimeoutMs: number }) => + defaultTimeoutMs, resolveProviderHttpRequestConfig: minimaxProviderHttpMocks.resolveProviderHttpRequestConfigMock, + waitProviderOperationPollInterval: async () => {}, })); export function getMinimaxProviderHttpMocks() { diff --git a/extensions/minimax/video-generation-provider.ts b/extensions/minimax/video-generation-provider.ts index cb5dd389b96..b994863053a 100644 --- a/extensions/minimax/video-generation-provider.ts +++ b/extensions/minimax/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -115,6 +118,10 @@ async function pollMinimaxVideo(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `MiniMax video generation task ${params.taskId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const url = new URL(`${params.baseUrl}/v1/query/video_generation`); url.searchParams.set("task_id", params.taskId); @@ -124,7 +131,7 @@ async function pollMinimaxVideo(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "MiniMax video status request failed"); @@ -141,7 +148,7 @@ async function pollMinimaxVideo(params: { case "Preparing": case "Processing": default: - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); break; } } @@ -269,6 +276,10 @@ export function buildMinimaxVideoGenerationProvider(): VideoGenerationProvider { } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "MiniMax video generation", + }); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveMinimaxVideoBaseUrl(req.cfg), @@ -305,7 +316,10 @@ export function buildMinimaxVideoGenerationProvider(): VideoGenerationProvider { url: `${baseUrl}/v1/video_generation`, headers, body, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -321,7 +335,10 @@ export function buildMinimaxVideoGenerationProvider(): VideoGenerationProvider { const completed = await pollMinimaxVideo({ taskId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); @@ -330,14 +347,20 @@ export function buildMinimaxVideoGenerationProvider(): VideoGenerationProvider { const video = videoUrl ? await downloadVideoFromUrl({ url: videoUrl, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, }) : fileId ? await downloadVideoFromFileId({ fileId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }) diff --git a/extensions/openai/video-generation-provider.ts b/extensions/openai/video-generation-provider.ts index 0aef298e4cc..ecc289f4959 100644 --- a/extensions/openai/video-generation-provider.ts +++ b/extensions/openai/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -121,6 +124,10 @@ async function pollOpenAIVideo(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `OpenAI video generation task ${params.videoId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/videos/${params.videoId}`, @@ -128,7 +135,7 @@ async function pollOpenAIVideo(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "OpenAI video status request failed"); @@ -141,7 +148,7 @@ async function pollOpenAIVideo(params: { normalizeOptionalString(payload.error?.message) || "OpenAI video generation failed", ); } - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); } throw new Error(`OpenAI video generation task ${params.videoId} did not finish in time`); } @@ -227,6 +234,10 @@ export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider { } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "OpenAI video generation", + }); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg), @@ -270,7 +281,10 @@ export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider { ), }, }, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -296,7 +310,10 @@ export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider { headers: multipartHeaders, body: form, }, - req.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, ).then((response) => ({ response, @@ -315,7 +332,10 @@ export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider { ...(seconds ? { seconds } : {}), ...(size ? { size } : {}), }, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -333,14 +353,20 @@ export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider { const completed = await pollOpenAIVideo({ videoId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); const video = await downloadOpenAIVideo({ videoId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); diff --git a/extensions/runway/video-generation-provider.ts b/extensions/runway/video-generation-provider.ts index 09f9edff403..aa186db11d7 100644 --- a/extensions/runway/video-generation-provider.ts +++ b/extensions/runway/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeLowercaseStringOrEmpty, @@ -15,7 +18,6 @@ import type { VideoGenerationProvider, VideoGenerationRequest, VideoGenerationResult, - VideoGenerationSourceAsset, } from "openclaw/plugin-sdk/video-generation"; const DEFAULT_RUNWAY_BASE_URL = "https://api.dev.runwayml.com"; @@ -39,6 +41,8 @@ type RunwayTaskDetailResponse = { failure?: string | { message?: string } | null; }; +type RunwaySourceAsset = Pick; + const TEXT_ONLY_MODELS = new Set(["gen4.5", "veo3.1", "veo3.1_fast", "veo3"]); const IMAGE_MODELS = new Set([ "gen4.5", @@ -63,7 +67,7 @@ function toDataUrl(buffer: Buffer, mimeType: string): string { } function resolveSourceUri( - asset: VideoGenerationSourceAsset | undefined, + asset: RunwaySourceAsset | undefined, fallbackMimeType: string, ): string | undefined { if (!asset) { @@ -197,6 +201,10 @@ async function pollRunwayTask(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `Runway video generation task ${params.taskId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/v1/tasks/${params.taskId}`, @@ -204,7 +212,7 @@ async function pollRunwayTask(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "Runway video status request failed"); @@ -223,7 +231,7 @@ async function pollRunwayTask(params: { case "RUNNING": case "THROTTLED": default: - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); break; } } @@ -302,6 +310,10 @@ export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "Runway video generation", + }); const requestBody = buildCreateBody(req); const endpoint = resolveEndpoint(req); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = @@ -321,7 +333,10 @@ export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { url: `${baseUrl}${endpoint}`, headers, body: requestBody, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -336,7 +351,10 @@ export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { const completed = await pollRunwayTask({ taskId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); @@ -348,7 +366,10 @@ export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { } const videos = await downloadRunwayVideos({ urls: outputUrls, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, }); return { diff --git a/extensions/together/video-generation-provider.ts b/extensions/together/video-generation-provider.ts index 5114465d83e..9fbeb4a143a 100644 --- a/extensions/together/video-generation-provider.ts +++ b/extensions/together/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -71,6 +74,10 @@ async function pollTogetherVideo(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `Together video generation task ${params.videoId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/videos/${params.videoId}`, @@ -78,7 +85,7 @@ async function pollTogetherVideo(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "Together video status request failed"); @@ -91,7 +98,7 @@ async function pollTogetherVideo(params: { normalizeOptionalString(payload.error?.message) ?? "Together video generation failed", ); } - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); } throw new Error(`Together video generation task ${params.videoId} did not finish in time`); } @@ -165,6 +172,10 @@ export function buildTogetherVideoGenerationProvider(): VideoGenerationProvider } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "Together video generation", + }); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveTogetherVideoBaseUrl(req), @@ -209,7 +220,10 @@ export function buildTogetherVideoGenerationProvider(): VideoGenerationProvider url: `${baseUrl}/videos`, headers, body, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -224,7 +238,10 @@ export function buildTogetherVideoGenerationProvider(): VideoGenerationProvider const completed = await pollTogetherVideo({ videoId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); @@ -234,7 +251,10 @@ export function buildTogetherVideoGenerationProvider(): VideoGenerationProvider } const video = await downloadTogetherVideo({ url: videoUrl, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, }); return { diff --git a/extensions/video-generation-providers.live.test.ts b/extensions/video-generation-providers.live.test.ts index 4697de38b27..432aab647ab 100644 --- a/extensions/video-generation-providers.live.test.ts +++ b/extensions/video-generation-providers.live.test.ts @@ -16,6 +16,7 @@ import { isTruthyEnvValue } from "../src/infra/env.js"; import { getShellEnvAppliedKeys, loadShellEnvFallback } from "../src/infra/shell-env.js"; import { encodePngRgba, fillPixel } from "../src/media/png-encode.js"; import { getProviderEnvVars } from "../src/secrets/provider-env-vars.js"; +import { normalizeVideoGenerationDuration } from "../src/video-generation/duration-support.js"; import { canRunBufferBackedImageToVideoLiveLane, canRunBufferBackedVideoToVideoLiveLane, @@ -28,6 +29,13 @@ import { resolveLiveVideoResolution, } from "../src/video-generation/live-test-helpers.js"; import { parseVideoGenerationModelRef } from "../src/video-generation/model-ref.js"; +import type { + GeneratedVideoAsset, + VideoGenerationMode, + VideoGenerationModeCapabilities, + VideoGenerationProvider, + VideoGenerationRequest, +} from "../src/video-generation/types.js"; import { registerProviderPlugin, requireRegisteredProvider, @@ -49,7 +57,22 @@ const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled() || isTruthyEnvValue(process.env.OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS); const describeLive = LIVE ? describe : describe.skip; const providerFilter = parseCsvFilter(process.env.OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS); +const defaultSkippedProviders = providerFilter + ? null + : parseCsvFilter(process.env.OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS ?? "fal"); const envModelMap = parseProviderModelMap(process.env.OPENCLAW_LIVE_VIDEO_GENERATION_MODELS); +const RUN_FULL_VIDEO_MODES = isTruthyEnvValue( + process.env.OPENCLAW_LIVE_VIDEO_GENERATION_FULL_MODES, +); +const LIVE_VIDEO_REQUESTED_DURATION_SECONDS = 1; +const LIVE_VIDEO_OPERATION_TIMEOUT_MS = readPositiveIntegerEnv( + process.env.OPENCLAW_LIVE_VIDEO_GENERATION_TIMEOUT_MS, + 180_000, +); +const LIVE_VIDEO_TEST_TIMEOUT_MS = + (RUN_FULL_VIDEO_MODES ? 3 : 1) * LIVE_VIDEO_OPERATION_TIMEOUT_MS + 30_000; +const LIVE_VIDEO_SMOKE_PROMPT = + "A one-second low-motion video of a lobster walking across wet sand, no text."; type LiveProviderCase = { plugin: Parameters[0]["plugin"]; @@ -58,6 +81,14 @@ type LiveProviderCase = { providerId: string; }; +type BufferedGeneratedVideo = Required> & + Pick; + +type LiveVideoAttemptStatus = + | { status: "success"; video: BufferedGeneratedVideo } + | { status: "skip" } + | { status: "failure" }; + const CASES: LiveProviderCase[] = [ { plugin: alibabaPlugin, @@ -92,8 +123,16 @@ const CASES: LiveProviderCase[] = [ { plugin: xaiPlugin, pluginId: "xai", pluginName: "xAI Plugin", providerId: "xai" }, ] .filter((entry) => (providerFilter ? providerFilter.has(entry.providerId) : true)) + .filter((entry) => + defaultSkippedProviders ? !defaultSkippedProviders.has(entry.providerId) : true, + ) .toSorted((left, right) => left.providerId.localeCompare(right.providerId)); +function readPositiveIntegerEnv(raw: string | undefined, fallback: number): number { + const value = Number.parseInt(raw?.trim() ?? "", 10); + return Number.isFinite(value) && value > 0 ? value : fallback; +} + function withPluginsEnabled(cfg: OpenClawConfig): OpenClawConfig { return { ...cfg, @@ -157,6 +196,34 @@ function maybeLoadShellEnvForVideoProviders(providerIds: string[]): void { }); } +function expectBufferedVideo( + video: { buffer?: Buffer; mimeType: string; fileName?: string } | undefined, +): BufferedGeneratedVideo { + expect(video).toBeDefined(); + expect(video?.mimeType.startsWith("video/")).toBe(true); + if (!video?.buffer) { + throw new Error("expected generated video buffer"); + } + const { buffer, mimeType, fileName } = video; + expect(buffer.byteLength).toBeGreaterThan(1024); + return { buffer, mimeType, fileName }; +} + +function buildLiveCapabilityOverrides(params: { + caps: VideoGenerationModeCapabilities | undefined; + liveResolution: VideoGenerationRequest["resolution"]; + liveSize: string | undefined; +}): Pick { + const { caps, liveResolution, liveSize } = params; + return { + ...(caps?.supportsSize && liveSize ? { size: liveSize } : {}), + ...(caps?.supportsAspectRatio ? { aspectRatio: "16:9" } : {}), + ...(caps?.supportsResolution ? { resolution: liveResolution } : {}), + ...(caps?.supportsAudio ? { audio: false } : {}), + ...(caps?.supportsWatermark ? { watermark: false } : {}), + }; +} + function resolveLiveVideoSkipReason(message: string): string | null { if (isAuthErrorMessage(message)) { return "auth drift"; @@ -177,20 +244,96 @@ function resolveLiveVideoSkipReason(message: string): string | null { if (isOverloadedErrorMessage(message) || isServerErrorMessage(message)) { return "provider outage"; } + if (/access denied|not authorized|not enabled|permission denied/i.test(message)) { + return "provider/model drift"; + } return null; } -function expectBufferedVideo( - video: { buffer?: Buffer; mimeType: string; fileName?: string } | undefined, -): { buffer: Buffer; mimeType: string; fileName?: string } { - expect(video).toBeDefined(); - expect(video?.mimeType.startsWith("video/")).toBe(true); - if (!video?.buffer) { - throw new Error("expected generated video buffer"); +async function runLiveVideoAttempt(params: { + authLabel: string; + attempted: string[]; + failures: string[]; + logPrefix: string; + mode: VideoGenerationMode; + provider: VideoGenerationProvider; + providerId: string; + providerModel: string; + request: VideoGenerationRequest; + skipped: string[]; +}): Promise { + const startedAt = Date.now(); + console.error(`${params.logPrefix} mode=${params.mode} start auth=${params.authLabel}`); + try { + const result = await params.provider.generateVideo(params.request); + expect(result.videos.length).toBeGreaterThan(0); + const video = expectBufferedVideo(result.videos[0]); + params.attempted.push( + `${params.providerId}:${params.mode}:${params.providerModel} (${params.authLabel})`, + ); + console.error( + `${params.logPrefix} mode=${params.mode} done ms=${Date.now() - startedAt} videos=${result.videos.length}`, + ); + return { status: "success", video }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const skipReason = resolveLiveVideoSkipReason(message); + if (skipReason) { + params.skipped.push( + `${params.providerId}:${params.mode} (${params.authLabel}): ${skipReason}`, + ); + console.warn( + `${params.logPrefix} mode=${params.mode} skip reason=${skipReason} error=${message}`, + ); + return { status: "skip" }; + } + params.failures.push(`${params.providerId}:${params.mode} (${params.authLabel}): ${message}`); + console.error(`${params.logPrefix} mode=${params.mode} failed error=${message}`); + return { status: "failure" }; } - const { buffer, mimeType, fileName } = video; - expect(buffer.byteLength).toBeGreaterThan(1024); - return { buffer, mimeType, fileName }; +} + +function logLiveVideoSummary(params: { + attempted: string[]; + failures: string[]; + providerId: string; + skipped: string[]; +}): void { + console.log( + `[live:video-generation] provider=${params.providerId} attempted=${params.attempted.join(", ") || "none"} skipped=${params.skipped.join(", ") || "none"} failures=${params.failures.join(" | ") || "none"} shellEnv=${getShellEnvAppliedKeys().join(", ") || "none"}`, + ); +} + +function expectLiveVideoCasePassed(params: { + attempted: string[]; + failures: string[]; + providerId: string; + skipped: string[]; +}): void { + logLiveVideoSummary(params); + if (params.attempted.length === 0) { + expect(params.failures).toEqual([]); + console.warn("[live:video-generation] no live video attempt completed; skipping assertions"); + return; + } + expect(params.failures).toEqual([]); +} + +function resolveLiveSmokeDurationSeconds(params: { + provider: Parameters[0]["provider"]; + model: string; + inputImageCount?: number; + inputVideoCount?: number; +}): number { + return ( + normalizeVideoGenerationDuration({ + provider: params.provider, + model: params.model, + durationSeconds: LIVE_VIDEO_REQUESTED_DURATION_SECONDS, + inputImageCount: params.inputImageCount ?? 0, + inputVideoCount: params.inputVideoCount ?? 0, + }) ?? LIVE_VIDEO_REQUESTED_DURATION_SECONDS + ); } async function runLiveVideoProviderCase(testCase: LiveProviderCase): Promise { @@ -200,6 +343,7 @@ async function runLiveVideoProviderCase(testCase: LiveProviderCase): Promise { + if (CASES.length === 0) { + it("skips when no video generation providers are selected", () => { + expect(CASES).toHaveLength(0); + }); + } + for (const testCase of CASES) { // One provider per test keeps cumulative suite runtime from tripping a single timeout cap. it( @@ -427,7 +566,7 @@ describeLive("video generation provider live", () => { async () => { await runLiveVideoProviderCase(testCase); }, - 15 * 60_000, + LIVE_VIDEO_TEST_TIMEOUT_MS, ); } }); diff --git a/extensions/vydra/shared.ts b/extensions/vydra/shared.ts index 2c0b88f26d1..f1f526b660f 100644 --- a/extensions/vydra/shared.ts +++ b/extensions/vydra/shared.ts @@ -2,8 +2,11 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeLowercaseStringOrEmpty, @@ -247,6 +250,10 @@ export async function waitForVydraJob(params: { fetchFn: typeof fetch; kind: VydraMediaKind; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `Vydra job ${params.jobId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/jobs/${params.jobId}`, @@ -254,7 +261,7 @@ export async function waitForVydraJob(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_HTTP_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "Vydra job status request failed"); @@ -266,7 +273,7 @@ export async function waitForVydraJob(params: { if (status === "failed" || status === "error" || status === "cancelled") { throw new Error(resolveVydraErrorMessage(payload) ?? `Vydra job ${params.jobId} failed`); } - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); } throw new Error(`Vydra job ${params.jobId} did not finish in time`); } diff --git a/extensions/vydra/video-generation-provider.ts b/extensions/vydra/video-generation-provider.ts index b03b282cf2e..89dbd7ef5d4 100644 --- a/extensions/vydra/video-generation-provider.ts +++ b/extensions/vydra/video-generation-provider.ts @@ -1,5 +1,10 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { assertOkOrThrowHttpError, postJsonRequest } from "openclaw/plugin-sdk/provider-http"; +import { + assertOkOrThrowHttpError, + createProviderOperationDeadline, + postJsonRequest, + resolveProviderOperationTimeoutMs, +} from "openclaw/plugin-sdk/provider-http"; import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation"; import { DEFAULT_VYDRA_VIDEO_MODEL, @@ -82,12 +87,19 @@ export function buildVydraVideoGenerationProvider(): VideoGenerationProvider { authStore: req.authStore, capability: "video", }); + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "Vydra video generation", + }); const { model, body } = resolveVydraVideoRequestBody(req); const { response, release } = await postJsonRequest({ url: `${baseUrl}/models/${model}`, headers, body, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: 120_000, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -100,7 +112,10 @@ export function buildVydraVideoGenerationProvider(): VideoGenerationProvider { submitted, baseUrl, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: 120_000, + }), fetchFn, kind: "video", missingJobIdMessage: "Vydra video generation response missing job id", @@ -112,7 +127,10 @@ export function buildVydraVideoGenerationProvider(): VideoGenerationProvider { const video = await downloadVydraAsset({ url: videoUrl, kind: "video", - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: 120_000, + }), fetchFn, }); return { diff --git a/extensions/xai/video-generation-provider.ts b/extensions/xai/video-generation-provider.ts index be9a3929f5b..bb007f98f0b 100644 --- a/extensions/xai/video-generation-provider.ts +++ b/extensions/xai/video-generation-provider.ts @@ -2,9 +2,12 @@ import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime"; import type { @@ -202,6 +205,10 @@ async function pollXaiVideo(params: { baseUrl: string; fetchFn: typeof fetch; }): Promise { + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `xAI video generation request ${params.requestId}`, + }); for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/videos/${params.requestId}`, @@ -209,7 +216,7 @@ async function pollXaiVideo(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }), params.fetchFn, ); await assertOkOrThrowHttpError(response, "xAI video status request failed"); @@ -226,7 +233,7 @@ async function pollXaiVideo(params: { case "queued": case "processing": default: - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS }); break; } } @@ -305,6 +312,10 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { } const fetchFn = fetch; + const deadline = createProviderOperationDeadline({ + timeoutMs: req.timeoutMs, + label: "xAI video generation", + }); const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: resolveXaiVideoBaseUrl(req), @@ -322,7 +333,10 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { url: `${baseUrl}${resolveCreateEndpoint(req)}`, headers, body: buildCreateBody(req), - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, allowPrivateNetwork, dispatcherPolicy, @@ -340,7 +354,10 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { const completed = await pollXaiVideo({ requestId, headers, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), baseUrl, fetchFn, }); @@ -350,7 +367,10 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { } const video = await downloadXaiVideo({ url: videoUrl, - timeoutMs: req.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_TIMEOUT_MS, + }), fetchFn, }); return { diff --git a/scripts/test-live-media.ts b/scripts/test-live-media.ts index b7602cd35be..04fcb529f48 100644 --- a/scripts/test-live-media.ts +++ b/scripts/test-live-media.ts @@ -25,6 +25,7 @@ export type MediaSuiteConfig = { testFile: string; providerEnvVar: string; providers: string[]; + defaultProviders?: string[]; }; export const MEDIA_SUITES: Record = { @@ -57,6 +58,18 @@ export const MEDIA_SUITES: Record = { "vydra", "xai", ], + defaultProviders: [ + "alibaba", + "byteplus", + "google", + "minimax", + "openai", + "qwen", + "runway", + "together", + "vydra", + "xai", + ], }, }; @@ -213,9 +226,10 @@ function selectProviders(params: { requireAuth: boolean; }): string[] { const explicit = params.suiteProviders ?? params.globalProviders; - let providers = params.suite.providers.filter((provider) => - explicit ? explicit.has(provider) : true, - ); + const candidates = explicit + ? params.suite.providers + : (params.suite.defaultProviders ?? params.suite.providers); + let providers = candidates.filter((provider) => (explicit ? explicit.has(provider) : true)); if (!params.requireAuth) { return providers; } @@ -275,6 +289,7 @@ Defaults: - runs image + music + video - auto-loads missing provider env vars from ~/.profile - narrows each suite to providers that currently have usable auth + - skips the slow fal video smoke by default; pass --video-providers fal to run it - forwards extra args to scripts/test-live.mjs Flags: diff --git a/src/media-understanding/shared.test.ts b/src/media-understanding/shared.test.ts index 56b7c3e3f4e..a065b3f546c 100644 --- a/src/media-understanding/shared.test.ts +++ b/src/media-understanding/shared.test.ts @@ -30,11 +30,14 @@ vi.mock("../infra/net/proxy-env.js", async () => { }); import { + createProviderOperationDeadline, fetchWithTimeoutGuarded, postJsonRequest, postTranscriptionRequest, readErrorResponse, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, + waitProviderOperationPollInterval, } from "./shared.js"; beforeEach(() => { @@ -44,6 +47,72 @@ beforeEach(() => { afterEach(() => { vi.clearAllMocks(); + vi.useRealTimers(); +}); + +describe("provider operation deadlines", () => { + it("keeps default per-call timeouts when no operation timeout is configured", () => { + const deadline = createProviderOperationDeadline({ + label: "video generation", + }); + + expect(resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: 60_000 })).toBe(60_000); + }); + + it("clamps per-call timeouts to the remaining operation deadline", () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createProviderOperationDeadline({ + label: "video generation", + timeoutMs: 5_000, + }); + + vi.setSystemTime(4_250); + + expect(resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: 60_000 })).toBe(1_750); + }); + + it("throws once the operation deadline has expired", () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createProviderOperationDeadline({ + label: "video generation", + timeoutMs: 2_000, + }); + + vi.setSystemTime(3_001); + + expect(() => resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: 60_000 })).toThrow( + "video generation timed out after 2000ms", + ); + }); + + it("clamps poll waits to the remaining operation deadline", async () => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + + const deadline = createProviderOperationDeadline({ + label: "video generation", + timeoutMs: 1_000, + }); + const wait = waitProviderOperationPollInterval({ + deadline, + pollIntervalMs: 10_000, + }); + + await vi.advanceTimersByTimeAsync(999); + let settled = false; + void wait.then(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + await expect(wait).resolves.toBeUndefined(); + }); }); describe("resolveProviderHttpRequestConfig", () => { diff --git a/src/media-understanding/shared.ts b/src/media-understanding/shared.ts index 90df939f8f9..f0c6f2c50b6 100644 --- a/src/media-understanding/shared.ts +++ b/src/media-understanding/shared.ts @@ -21,6 +21,62 @@ const MAX_ERROR_RESPONSE_BYTES = 4096; const DEFAULT_GUARDED_HTTP_TIMEOUT_MS = 60_000; const MAX_AUDIT_CONTEXT_CHARS = 80; +export type ProviderOperationDeadline = { + deadlineAtMs?: number; + label: string; + timeoutMs?: number; +}; + +export function createProviderOperationDeadline(params: { + timeoutMs?: number; + label: string; +}): ProviderOperationDeadline { + if ( + typeof params.timeoutMs !== "number" || + !Number.isFinite(params.timeoutMs) || + params.timeoutMs <= 0 + ) { + return { label: params.label }; + } + const timeoutMs = Math.floor(params.timeoutMs); + return { + deadlineAtMs: Date.now() + timeoutMs, + label: params.label, + timeoutMs, + }; +} + +export function resolveProviderOperationTimeoutMs(params: { + deadline: ProviderOperationDeadline; + defaultTimeoutMs: number; +}): number { + const deadlineAtMs = params.deadline.deadlineAtMs; + if (typeof deadlineAtMs !== "number") { + return params.defaultTimeoutMs; + } + const remainingMs = deadlineAtMs - Date.now(); + if (remainingMs <= 0) { + throw new Error(`${params.deadline.label} timed out after ${params.deadline.timeoutMs}ms`); + } + return Math.max(1, Math.min(params.defaultTimeoutMs, remainingMs)); +} + +export async function waitProviderOperationPollInterval(params: { + deadline: ProviderOperationDeadline; + pollIntervalMs: number; +}): Promise { + const deadlineAtMs = params.deadline.deadlineAtMs; + if (typeof deadlineAtMs !== "number") { + await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs)); + return; + } + const remainingMs = deadlineAtMs - Date.now(); + if (remainingMs <= 0) { + throw new Error(`${params.deadline.label} timed out after ${params.deadline.timeoutMs}ms`); + } + await new Promise((resolve) => setTimeout(resolve, Math.min(params.pollIntervalMs, remainingMs))); +} + function resolveGuardedHttpTimeoutMs(timeoutMs: number | undefined): number { if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || timeoutMs <= 0) { return DEFAULT_GUARDED_HTTP_TIMEOUT_MS; diff --git a/src/plugin-sdk/provider-http.ts b/src/plugin-sdk/provider-http.ts index 15bb882fcf8..71844827a4e 100644 --- a/src/plugin-sdk/provider-http.ts +++ b/src/plugin-sdk/provider-http.ts @@ -3,14 +3,18 @@ export { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, fetchWithTimeoutGuarded, normalizeBaseUrl, postJsonRequest, postTranscriptionRequest, + resolveProviderOperationTimeoutMs, resolveProviderHttpRequestConfig, requireTranscriptionText, + waitProviderOperationPollInterval, } from "../media-understanding/shared.js"; +export type { ProviderOperationDeadline } from "../media-understanding/shared.js"; export type { ProviderAttributionPolicy, ProviderRequestCapabilities, diff --git a/src/scripts/test-live-media.test.ts b/src/scripts/test-live-media.test.ts index b53b4dcf78c..4adbab23c62 100644 --- a/src/scripts/test-live-media.test.ts +++ b/src/scripts/test-live-media.test.ts @@ -43,7 +43,6 @@ describe("test-live-media", () => { "minimax", ]); expect(plan.find((entry) => entry.suite.id === "video")?.providers).toEqual([ - "fal", "google", "minimax", "openai", @@ -54,12 +53,12 @@ describe("test-live-media", () => { it("supports suite-specific provider filters without auth narrowing", async () => { const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); const plan = buildRunPlan( - parseArgs(["video", "--video-providers", "openai,runway", "--all-providers"]), + parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), ); expect(plan).toHaveLength(1); expect(plan[0]?.suite.id).toBe("video"); - expect(plan[0]?.providers).toEqual(["openai", "runway"]); + expect(plan[0]?.providers).toEqual(["fal", "openai", "runway"]); }); it("forwards quiet flags separately from passthrough args", async () => { diff --git a/src/video-generation/dashscope-compatible.ts b/src/video-generation/dashscope-compatible.ts index ef1fd24ea74..ea847fe4f04 100644 --- a/src/video-generation/dashscope-compatible.ts +++ b/src/video-generation/dashscope-compatible.ts @@ -1,7 +1,10 @@ import { assertOkOrThrowHttpError, + createProviderOperationDeadline, fetchWithTimeout, postJsonRequest, + resolveProviderOperationTimeoutMs, + waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; import type { @@ -165,6 +168,11 @@ export async function pollDashscopeVideoTaskUntilComplete(params: { baseUrl: string; defaultTimeoutMs?: number; }): Promise { + const defaultTimeoutMs = params.defaultTimeoutMs ?? DEFAULT_VIDEO_GENERATION_TIMEOUT_MS; + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `${params.providerLabel} video generation task ${params.taskId}`, + }); for (let attempt = 0; attempt < DEFAULT_VIDEO_GENERATION_MAX_POLL_ATTEMPTS; attempt += 1) { const response = await fetchWithTimeout( `${params.baseUrl}/api/v1/tasks/${params.taskId}`, @@ -172,7 +180,7 @@ export async function pollDashscopeVideoTaskUntilComplete(params: { method: "GET", headers: params.headers, }, - params.timeoutMs ?? params.defaultTimeoutMs ?? DEFAULT_VIDEO_GENERATION_TIMEOUT_MS, + resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs }), params.fetchFn, ); await assertOkOrThrowHttpError( @@ -191,7 +199,10 @@ export async function pollDashscopeVideoTaskUntilComplete(params: { `${params.providerLabel} video generation task ${params.taskId} ${normalizeLowercaseStringOrEmpty(status)}`, ); } - await new Promise((resolve) => setTimeout(resolve, DEFAULT_VIDEO_GENERATION_POLL_INTERVAL_MS)); + await waitProviderOperationPollInterval({ + deadline, + pollIntervalMs: DEFAULT_VIDEO_GENERATION_POLL_INTERVAL_MS, + }); } throw new Error( `${params.providerLabel} video generation task ${params.taskId} did not finish in time`, @@ -211,6 +222,11 @@ export async function runDashscopeVideoGenerationTask(params: { dispatcherPolicy?: Parameters[0]["dispatcherPolicy"]; defaultTimeoutMs?: number; }): Promise { + const defaultTimeoutMs = params.defaultTimeoutMs ?? DEFAULT_VIDEO_GENERATION_TIMEOUT_MS; + const deadline = createProviderOperationDeadline({ + timeoutMs: params.timeoutMs, + label: `${params.providerLabel} video generation`, + }); const { response, release } = await postJsonRequest({ url: params.url, headers: params.headers, @@ -228,7 +244,7 @@ export async function runDashscopeVideoGenerationTask(params: { DEFAULT_VIDEO_RESOLUTION_TO_SIZE, ), }, - timeoutMs: params.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs }), fetchFn: params.fetchFn, allowPrivateNetwork: params.allowPrivateNetwork, dispatcherPolicy: params.dispatcherPolicy, @@ -245,10 +261,10 @@ export async function runDashscopeVideoGenerationTask(params: { providerLabel: params.providerLabel, taskId, headers: params.headers, - timeoutMs: params.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs }), fetchFn: params.fetchFn, baseUrl: params.baseUrl, - defaultTimeoutMs: params.defaultTimeoutMs ?? DEFAULT_VIDEO_GENERATION_TIMEOUT_MS, + defaultTimeoutMs, }); const urls = extractDashscopeVideoUrls(completed); if (urls.length === 0) { @@ -259,9 +275,9 @@ export async function runDashscopeVideoGenerationTask(params: { const videos = await downloadDashscopeGeneratedVideos({ providerLabel: params.providerLabel, urls, - timeoutMs: params.timeoutMs, + timeoutMs: resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs }), fetchFn: params.fetchFn, - defaultTimeoutMs: params.defaultTimeoutMs ?? DEFAULT_VIDEO_GENERATION_TIMEOUT_MS, + defaultTimeoutMs, }); return { videos, diff --git a/test/helpers/media-generation/provider-http-mocks.ts b/test/helpers/media-generation/provider-http-mocks.ts index 337469973ac..3ab7fb4d923 100644 --- a/test/helpers/media-generation/provider-http-mocks.ts +++ b/test/helpers/media-generation/provider-http-mocks.ts @@ -24,9 +24,22 @@ vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({ vi.mock("openclaw/plugin-sdk/provider-http", () => ({ assertOkOrThrowHttpError: providerHttpMocks.assertOkOrThrowHttpErrorMock, + createProviderOperationDeadline: ({ + label, + timeoutMs, + }: { + label: string; + timeoutMs?: number; + }) => ({ + label, + timeoutMs, + }), fetchWithTimeout: providerHttpMocks.fetchWithTimeoutMock, postJsonRequest: providerHttpMocks.postJsonRequestMock, + resolveProviderOperationTimeoutMs: ({ defaultTimeoutMs }: { defaultTimeoutMs: number }) => + defaultTimeoutMs, resolveProviderHttpRequestConfig: providerHttpMocks.resolveProviderHttpRequestConfigMock, + waitProviderOperationPollInterval: async () => {}, })); export function getProviderHttpMocks() {