From 75dfd3dc0972cd5eb4b869aae6158be427cdfd07 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 04:49:42 +0100 Subject: [PATCH] feat(xai): support Grok Imagine Video 1.5 (#103316) * feat(media): add per-model generation catalog metadata * feat(xai): support Grok Imagine Video 1.5 * test(plugin-sdk): update public surface budget --- docs/help/testing-live.md | 3 + docs/plugins/sdk-provider-plugins.md | 23 +++ docs/providers/xai.md | 30 ++-- docs/tools/video-generation.md | 17 +- .../xai/video-generation-provider.test.ts | 145 +++++++++++++++++- extensions/xai/video-generation-provider.ts | 97 +++++++++++- extensions/xai/xai.live.test.ts | 72 +++++++++ .../media-generation-core/src/catalog.test.ts | 43 ++++++ packages/media-generation-core/src/catalog.ts | 15 +- scripts/plugin-sdk-surface-report.mjs | 2 +- .../media-generate-tool-actions-shared.ts | 28 +++- .../tools/video-generate-tool.actions.ts | 52 +++++-- src/agents/tools/video-generate-tool.test.ts | 67 ++++++++ src/plugin-sdk/video-generation-core.ts | 1 + src/plugin-sdk/video-generation.ts | 10 ++ src/video-generation/types.ts | 7 + 16 files changed, 569 insertions(+), 43 deletions(-) diff --git a/docs/help/testing-live.md b/docs/help/testing-live.md index 36d7a69a35a1..fd1d70b9f8b1 100644 --- a/docs/help/testing-live.md +++ b/docs/help/testing-live.md @@ -582,6 +582,9 @@ request. Plugin dependencies are expected to be present before runtime load. - Provider-specific Vydra coverage: - `OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_VYDRA_VIDEO=1 pnpm test:live -- extensions/vydra/vydra.live.test.ts` - That file runs `veo3` text-to-video plus a `kling` image-to-video lane that uses a remote image URL fixture by default (`OPENCLAW_LIVE_VYDRA_KLING_IMAGE_URL` to override). + - Provider-specific xAI Video 1.5 coverage: + - `OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_XAI_VIDEO_15=1 pnpm test:live -- extensions/xai/xai.live.test.ts -t "Grok Imagine Video 1.5"` + - The case generates a local PNG first frame, requests a one-second 1080P image-to-video clip, polls to completion, and verifies the downloaded video buffer. - Current `videoToVideo` live coverage: - `runway` only when the selected model resolves to `gen4_aleph` - Current declared-but-skipped `videoToVideo` providers in the shared sweep: diff --git a/docs/plugins/sdk-provider-plugins.md b/docs/plugins/sdk-provider-plugins.md index 99a784bdf300..e856b1a53e45 100644 --- a/docs/plugins/sdk-provider-plugins.md +++ b/docs/plugins/sdk-provider-plugins.md @@ -934,6 +934,7 @@ catalog, API-key auth, and dynamic model resolution. id: "acme-ai", label: "Acme Video", defaultTimeoutMs: 600_000, + models: ["acme-video", "acme-image-video"], capabilities: { generate: { maxVideos: 1, maxDurationSeconds: 10, supportsResolution: true }, imageToVideo: { @@ -945,6 +946,21 @@ catalog, API-key auth, and dynamic model resolution. }, videoToVideo: { enabled: false }, }, + catalogByModel: { + "acme-image-video": { + modes: ["imageToVideo"], + capabilities: { + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 1, + resolutions: ["480P", "720P", "1080P"], + supportsResolution: true, + }, + videoToVideo: { enabled: false }, + }, + }, + }, generateVideo: async (req) => ({ videos: [] }), }); ``` @@ -952,6 +968,13 @@ catalog, API-key auth, and dynamic model resolution. `capabilities` is required on both provider types; `edit` and the video transform blocks (`imageToVideo`, `videoToVideo`) always need an explicit `enabled` flag. + + Use `catalogByModel` when a listed model's static modes or capabilities + differ from the provider defaults. This metadata keeps + `video_generate action=list` and model catalogs accurate without + invoking provider code. Request-time capability lookup and enforcement + still belong in `resolveModelCapabilities` and `generateVideo`; reuse + the same capability constant for both paths when possible. ```typescript diff --git a/docs/providers/xai.md b/docs/providers/xai.md index e3305eda23f4..628efd8a846d 100644 --- a/docs/providers/xai.md +++ b/docs/providers/xai.md @@ -134,7 +134,7 @@ below or under known limits. | Server-side X search | `x_search` tool | Yes | | Server-side code execution | `code_execution` tool | Yes | | Images | `image_generate` | Yes | -| Videos | `video_generate` | Classic model; Video 1.5 is not exposed yet | +| Videos | `video_generate` | Classic full workflow; Video 1.5 image-to-video | | Batch text-to-speech | `messages.tts.provider: "xai"` / `tts` | Yes | | Streaming TTS | - | Not implemented by the xAI provider yet | | Batch speech-to-text | `tools.media.audio` media understanding | Yes | @@ -210,13 +210,17 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 The bundled `xai` plugin registers video generation through the shared `video_generate` tool. - - Default video model: `xai/grok-imagine-video` - - Modes: text-to-video, image-to-video, reference-image generation, remote - video edit, and remote video extension - - Aspect ratios: `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3` - - Resolutions: `480P`, `720P` + - Default model: `xai/grok-imagine-video` + - Additional model: `xai/grok-imagine-video-1.5` + - Classic modes: text-to-video, image-to-video, reference-image generation, + remote video edit, and remote video extension + - Video 1.5 mode: image-to-video only, with exactly one first-frame image + - Aspect ratios: `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `3:2`, `2:3`; + Video 1.5 inherits the source image ratio when omitted + - Resolutions: classic `480P`/`720P`; Video 1.5 supports `480P`, `720P`, + and `1080P`, and defaults to `480P` - Duration: 1-15 seconds for generation/image-to-video, 1-10 seconds when - using `reference_image` roles, 2-10 seconds for extension + using classic `reference_image` roles, 2-10 seconds for classic extension - Reference-image generation: set `imageRoles` to `reference_image` for every supplied image; xAI accepts up to 7 such images - Default operation timeout: 600 seconds unless `video_generate.timeoutMs` @@ -228,6 +232,10 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 OpenClaw encodes those as data URLs for xAI. + Video 1.5 also recognizes xAI's `grok-imagine-video-1.5-preview` and + `grok-imagine-video-1.5-2026-05-30` identifiers. OpenClaw forwards the + selected identifier unchanged, but applies the same image-only validation. + To use xAI as the default video provider: ```json5 @@ -502,9 +510,6 @@ stale context metadata on active 4.20 rows. It does not pin active 4.20 - xAI Realtime voice is not registered as an OpenClaw provider yet. It needs a different bidirectional voice session contract than batch STT or streaming transcription. - - `grok-imagine-video-1.5` is not exposed yet. Unlike the classic video - model, it is image-to-video only and needs model-specific mode and 1080p - validation in the shared provider contract. - xAI image `quality`, image `mask`, and extra native-only aspect ratios are not exposed until the shared `image_generate` tool has corresponding cross-provider controls. @@ -546,6 +551,7 @@ The xAI media paths are covered by unit tests and opt-in live suites. Export ```bash pnpm test extensions/xai OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 pnpm test:live -- extensions/xai/xai.live.test.ts +OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_XAI_VIDEO_15=1 pnpm test:live -- extensions/xai/xai.live.test.ts -t "Grok Imagine Video 1.5" OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 pnpm test:live -- extensions/xai/x-search.live.test.ts OPENCLAW_LIVE_GATEWAY_MODELS="xai/grok-4.5,xai/grok-build-0.1,xai/grok-4.3,xai/grok-4.20-0309-reasoning,xai/grok-4.20-0309-non-reasoning" OPENCLAW_LIVE_GATEWAY_MAX_MODELS=0 OPENCLAW_LIVE_GATEWAY_SMOKE=0 pnpm test:live -- src/gateway/gateway-models.profiles.live.test.ts OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_TEST_QUIET=1 OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS=xai pnpm test:live -- test/image-generation.runtime.live.test.ts @@ -555,7 +561,9 @@ The provider-specific live file synthesizes normal TTS, telephony-friendly PCM TTS, transcribes audio through xAI batch STT, streams the same PCM through xAI realtime STT, generates text-to-image output, and edits a reference image. The shared image live file verifies the same xAI provider through OpenClaw's -runtime selection, fallback, normalization, and media attachment path. +runtime selection, fallback, normalization, and media attachment path. The +opt-in Video 1.5 case submits one generated first-frame image at 1080P and +verifies the completed video download. ## Related diff --git a/docs/tools/video-generation.md b/docs/tools/video-generation.md index d5d0c5832d16..6fe10a249ba4 100644 --- a/docs/tools/video-generation.md +++ b/docs/tools/video-generation.md @@ -119,7 +119,7 @@ openclaw tasks cancel | Runway | `gen4.5` | ✓ | 1 image | 1 video | `RUNWAYML_API_SECRET` | | Together | `Wan-AI/Wan2.2-T2V-A14B` | ✓ | `Wan-AI/Wan2.2-I2V-A14B` only | - | `TOGETHER_API_KEY` | | Vydra | `veo3` | ✓ | 1 image (`kling`) | - | `VYDRA_API_KEY` | -| xAI | `grok-imagine-video` | ✓ | 1 first-frame image or up to 7 `reference_image`s | 1 video | `XAI_API_KEY` | +| xAI | `grok-imagine-video` | ✓ | Classic: 1 first frame or 7 references; 1.5: 1 frame | Classic: 1 video | `XAI_API_KEY` | Some providers accept additional or alternate API key env vars. See individual [provider pages](#related) for details. @@ -147,7 +147,7 @@ the shared live sweep: | Runway | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` | | Together | ✓ | ✓ | - | `generate`, `imageToVideo` | | Vydra | ✓ | ✓ | - | `generate`; shared `imageToVideo` skipped because bundled `veo3` is text-only and bundled `kling` requires a remote image URL | -| xAI | ✓ | ✓ | ✓ | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider currently needs a remote MP4 URL | +| xAI | ✓ | ✓ | ✓ | Classic supports all modes; Video 1.5 is image-to-video only; remote MP4 input keeps `videoToVideo` out of the shared sweep | ## Tool parameters @@ -423,9 +423,16 @@ only the explicit `model`, `primary`, and `fallbacks` entries. a remote image URL. - Supports text-to-video, single first-frame image-to-video, up to 7 - `reference_image` inputs through xAI `reference_images`, and remote - video edit/extend flows. + The default `grok-imagine-video` model supports text-to-video, single + first-frame image-to-video, up to 7 `reference_image` inputs through xAI + `reference_images`, and remote video edit/extend flows. + + `grok-imagine-video-1.5` is image-to-video only: provide exactly one image. + It supports 1-15 seconds and `480P`, `720P`, or `1080P`, defaulting to + `480P`; omit `aspectRatio` to inherit the source image ratio. The preview + and dated 1.5 identifiers receive the same validation and are forwarded + unchanged. + diff --git a/extensions/xai/video-generation-provider.test.ts b/extensions/xai/video-generation-provider.test.ts index 80d6bcc234e4..b07b016964ec 100644 --- a/extensions/xai/video-generation-provider.test.ts +++ b/extensions/xai/video-generation-provider.test.ts @@ -4,10 +4,15 @@ import { installProviderHttpMockCleanup, } from "openclaw/plugin-sdk/provider-http-test-mocks"; import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts"; +import type { VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -const { postJsonRequestMock, fetchWithTimeoutMock, readProviderJsonResponseMock } = - getProviderHttpMocks(); +const { + postJsonRequestMock, + fetchWithTimeoutMock, + readProviderJsonResponseMock, + resolveApiKeyForProviderMock, +} = getProviderHttpMocks(); let buildXaiVideoGenerationProvider: typeof import("./video-generation-provider.js").buildXaiVideoGenerationProvider; @@ -161,6 +166,142 @@ describe("xai video generation provider", () => { expectExplicitVideoGenerationCapabilities(buildXaiVideoGenerationProvider()); }); + it("advertises canonical 1.5 and resolves capabilities for all API aliases", async () => { + const provider = buildXaiVideoGenerationProvider(); + + expect(provider.defaultModel).toBe("grok-imagine-video"); + expect(provider.models).toEqual(["grok-imagine-video", "grok-imagine-video-1.5"]); + expect(provider.catalogByModel?.["grok-imagine-video-1.5"]).toMatchObject({ + modes: ["imageToVideo"], + capabilities: { + imageToVideo: { + enabled: true, + maxInputImages: 1, + resolutions: ["480P", "720P", "1080P"], + }, + videoToVideo: { enabled: false }, + }, + }); + + for (const model of [ + "grok-imagine-video-1.5", + "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5-2026-05-30", + ]) { + const capabilities = await provider.resolveModelCapabilities?.({ + provider: "xai", + model, + cfg: {}, + }); + expect(capabilities?.imageToVideo).toMatchObject({ + enabled: true, + maxInputImages: 1, + maxDurationSeconds: 15, + resolutions: ["480P", "720P", "1080P"], + }); + expect(capabilities?.videoToVideo?.enabled).toBe(false); + } + }); + + it("uses the 1.5 default while preserving aliases, 1080p, and source aspect ratio", async () => { + const models = [ + "grok-imagine-video-1.5", + "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5-2026-05-30", + ]; + const provider = buildXaiVideoGenerationProvider(); + + for (const [index, model] of models.entries()) { + const requestId = `req_15_${index}`; + postJsonRequestMock.mockResolvedValueOnce({ + response: { json: async () => ({ request_id: requestId }) }, + release: vi.fn(async () => {}), + }); + fetchWithTimeoutMock + .mockResolvedValueOnce({ + json: async () => ({ + request_id: requestId, + status: "done", + video: { url: `https://cdn.x.ai/${requestId}.mp4` }, + }), + }) + .mockResolvedValueOnce({ + headers: new Headers({ "content-type": "video/mp4" }), + arrayBuffer: async () => Buffer.from("video-bytes"), + }); + + const result = await provider.generateVideo({ + provider: "xai", + model, + prompt: "Animate this still image", + cfg: {}, + durationSeconds: 20, + resolution: index === 0 ? undefined : "1080P", + inputImages: [ + { + url: "https://example.com/first-frame.png", + ...(index === 0 ? {} : { role: "first_frame" as const }), + }, + ], + }); + + const body = requirePostJsonCall(index).body ?? {}; + expect(body.model).toBe(model); + expect(body.image).toEqual({ url: "https://example.com/first-frame.png" }); + expect(body.duration).toBe(15); + expect(body.resolution).toBe(index === 0 ? "480p" : "1080p"); + expect(body).not.toHaveProperty("aspect_ratio"); + expect(result.model).toBe(model); + } + }); + + it("rejects unsupported 1.5 modes before submitting a request", async () => { + const provider = buildXaiVideoGenerationProvider(); + const cases: Array< + Pick & { error: string } + > = [ + { + model: "grok-imagine-video-1.5", + inputImages: undefined, + error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.", + }, + { + model: "grok-imagine-video-1.5-preview", + inputImages: [{ url: "https://example.com/reference.png", role: "reference_image" }], + error: "xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image.", + }, + { + model: "grok-imagine-video-1.5-2026-05-30", + inputImages: [ + { url: "https://example.com/first.png" }, + { url: "https://example.com/second.png", role: "first_frame" }, + ], + error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.", + }, + { + model: "grok-imagine-video-1.5", + inputImages: undefined, + inputVideos: [{ url: "https://example.com/input.mp4" }], + error: "xAI grok-imagine-video-1.5 does not support video inputs.", + }, + ]; + + for (const testCase of cases) { + await expect( + provider.generateVideo({ + provider: "xai", + model: testCase.model, + prompt: "Unsupported 1.5 request", + cfg: {}, + inputImages: testCase.inputImages, + inputVideos: testCase.inputVideos, + }), + ).rejects.toThrow(testCase.error); + } + expect(resolveApiKeyForProviderMock).not.toHaveBeenCalled(); + expect(postJsonRequestMock).not.toHaveBeenCalled(); + }); + it("creates, polls, and downloads a generated video", async () => { postJsonRequestMock.mockResolvedValue({ response: { diff --git a/extensions/xai/video-generation-provider.ts b/extensions/xai/video-generation-provider.ts index 790c0f46e7b6..fa64954b5286 100644 --- a/extensions/xai/video-generation-provider.ts +++ b/extensions/xai/video-generation-provider.ts @@ -21,15 +21,37 @@ import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-co import type { GeneratedVideoAsset, VideoGenerationProvider, + VideoGenerationProviderCapabilities, VideoGenerationRequest, } from "openclaw/plugin-sdk/video-generation"; const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1"; const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video"; +const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5"; +const XAI_VIDEO_15_MODEL_IDS = new Set([ + XAI_VIDEO_15_MODEL, + "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5-2026-05-30", +]); const DEFAULT_TIMEOUT_MS = 600_000; const POLL_INTERVAL_MS = 5_000; const MAX_POLL_ATTEMPTS = 120; const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]); +const XAI_VIDEO_15_CAPABILITIES = { + imageToVideo: { + enabled: true, + maxVideos: 1, + maxInputImages: 1, + maxDurationSeconds: 15, + aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS], + resolutions: ["480P", "720P", "1080P"], + supportsAspectRatio: true, + supportsResolution: true, + }, + videoToVideo: { + enabled: false, + }, +} satisfies VideoGenerationProviderCapabilities; const XAI_VIDEO_MALFORMED_RESPONSE = "xAI video generation response malformed"; // xAI documents these as the only meaningful values; everything else (queued, // processing, submitted, pending, in_progress, ...) means "keep polling". @@ -37,6 +59,7 @@ const XAI_VIDEO_TERMINAL_FAILURE_STATUSES = new Set(["failed", "error", "expired const XAI_VIDEO_DEFAULT_DURATION_SECONDS = 8; const XAI_VIDEO_DEFAULT_ASPECT_RATIO = "16:9"; const XAI_VIDEO_DEFAULT_RESOLUTION = "720p"; +const XAI_VIDEO_15_DEFAULT_RESOLUTION = "480p"; const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024; type XaiVideoCreateResponse = { @@ -155,6 +178,32 @@ function isReferenceImage(input: VideoGenerationSourceInput): boolean { return normalizeOptionalString(input.role)?.toLowerCase() === "reference_image"; } +function isXaiVideo15Model(model: string | undefined): boolean { + const normalized = normalizeOptionalString(model); + return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false; +} + +function isFirstFrameImage(input: VideoGenerationSourceInput): boolean { + const role = normalizeOptionalString(input.role)?.toLowerCase(); + return role === undefined || role === "first_frame"; +} + +function validateXaiVideo15Request(req: VideoGenerationRequest): void { + if (!isXaiVideo15Model(req.model)) { + return; + } + if ((req.inputVideos?.length ?? 0) > 0) { + throw new Error("xAI grok-imagine-video-1.5 does not support video inputs."); + } + const inputImages = req.inputImages ?? []; + if (inputImages.length !== 1) { + throw new Error("xAI grok-imagine-video-1.5 requires exactly one first-frame image."); + } + if (!isFirstFrameImage(inputImages[0])) { + throw new Error("xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image."); + } +} + function resolveInputVideoUrl(input: VideoGenerationSourceInput | undefined): string | undefined { if (!input) { return undefined; @@ -189,7 +238,10 @@ function resolveAspectRatio(value: string | undefined): string | undefined { return trimmed; } -function resolveResolution(value: string | undefined): "480p" | "720p" | undefined { +function resolveResolution( + value: string | undefined, + options?: { allow1080p?: boolean }, +): "480p" | "720p" | "1080p" | undefined { if (typeof value !== "string") { return undefined; } @@ -197,9 +249,12 @@ function resolveResolution(value: string | undefined): "480p" | "720p" | undefin if (normalized === "480p") { return "480p"; } - if (normalized === "720p" || normalized === "1080p") { + if (normalized === "720p") { return "720p"; } + if (normalized === "1080p") { + return options?.allow1080p ? "1080p" : "720p"; + } return undefined; } @@ -223,6 +278,7 @@ function resolveXaiVideoMode( } function buildCreateBody(req: VideoGenerationRequest): Record { + validateXaiVideo15Request(req); const inputImages = req.inputImages ?? []; const hasReferenceImages = inputImages.some(isReferenceImage); if (hasReferenceImages && !inputImages.every(isReferenceImage)) { @@ -245,11 +301,14 @@ function buildCreateBody(req: VideoGenerationRequest): Record { const mode = resolveXaiVideoMode(req); const body: Record = { + // Aliases are API-owned routing choices. Preserve the selected identifier + // instead of silently pinning it to the canonical 1.5 model. model: normalizeOptionalString(req.model) ?? DEFAULT_XAI_VIDEO_MODEL, prompt: req.prompt, }; if (mode === "generate") { + const isVideo15 = isXaiVideo15Model(req.model); const imageUrl = resolveImageUrl(req.inputImages?.[0]); if (imageUrl) { body.image = { url: imageUrl }; @@ -260,8 +319,16 @@ function buildCreateBody(req: VideoGenerationRequest): Record { min: 1, max: 15, }) ?? XAI_VIDEO_DEFAULT_DURATION_SECONDS; - body.aspect_ratio = resolveAspectRatio(req.aspectRatio) ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO; - body.resolution = resolveResolution(req.resolution) ?? XAI_VIDEO_DEFAULT_RESOLUTION; + const aspectRatio = resolveAspectRatio(req.aspectRatio); + // 1.5 inherits the source image ratio when callers do not choose one. + if (aspectRatio || !isVideo15) { + body.aspect_ratio = aspectRatio ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO; + } + const defaultResolution = isVideo15 + ? XAI_VIDEO_15_DEFAULT_RESOLUTION + : XAI_VIDEO_DEFAULT_RESOLUTION; + body.resolution = + resolveResolution(req.resolution, { allow1080p: isVideo15 }) ?? defaultResolution; return body; } @@ -380,7 +447,13 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { label: "xAI", defaultModel: DEFAULT_XAI_VIDEO_MODEL, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, - models: [DEFAULT_XAI_VIDEO_MODEL], + models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL], + catalogByModel: { + [XAI_VIDEO_15_MODEL]: { + capabilities: XAI_VIDEO_15_CAPABILITIES, + modes: ["imageToVideo"], + }, + }, isConfigured: ({ agentDir }) => isProviderApiKeyConfigured({ provider: "xai", @@ -414,7 +487,17 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { supportsResolution: true, }, }, + resolveModelCapabilities: ({ model }): VideoGenerationProviderCapabilities | undefined => { + if (!isXaiVideo15Model(model)) { + return undefined; + } + return XAI_VIDEO_15_CAPABILITIES; + }, async generateVideo(req) { + // Validate provider/model mode constraints before auth or HTTP setup so + // unsupported 1.5 requests cannot be submitted and billed accidentally. + const createBody = buildCreateBody(req); + const createEndpoint = resolveCreateEndpoint(req); const auth = await resolveApiKeyForProvider({ provider: "xai", cfg: req.cfg, @@ -448,9 +531,9 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider { const submitHeaders = new Headers(headers); submitHeaders.set("x-idempotency-key", crypto.randomUUID()); const { response, release } = await postJsonRequest({ - url: `${baseUrl}${resolveCreateEndpoint(req)}`, + url: `${baseUrl}${createEndpoint}`, headers: submitHeaders, - body: buildCreateBody(req), + body: createBody, timeoutMs: resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS, diff --git a/extensions/xai/xai.live.test.ts b/extensions/xai/xai.live.test.ts index a342551f4508..5a6944c9052e 100644 --- a/extensions/xai/xai.live.test.ts +++ b/extensions/xai/xai.live.test.ts @@ -21,6 +21,7 @@ import { XAI_DEFAULT_STT_MODEL } from "./stt.js"; const XAI_API_KEY = process.env.XAI_API_KEY ?? ""; const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_XAI_IMAGE_MODEL?.trim() || "grok-imagine-image"; +const ENABLE_VIDEO_15_LIVE = process.env.OPENCLAW_LIVE_XAI_VIDEO_15 === "1"; const liveEnabled = XAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1"; const describeLive = liveEnabled ? describe : describe.skip; const EMPTY_AUTH_STORE = { version: 1, profiles: {} } as const; @@ -63,6 +64,27 @@ function createReferencePng(): Buffer { return encodePngRgba(buf, width, height); } +function createVideoReferencePng(): Buffer { + const width = 384; + const height = 384; + const buf = Buffer.alloc(width * height * 4, 255); + + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const blue = Math.round(160 + (80 * y) / height); + fillPixel(buf, x, y, width, 32, 96, blue, 255); + } + } + + for (let y = 112; y < 272; y += 1) { + for (let x = 112; x < 272; x += 1) { + fillPixel(buf, x, y, width, 255, 153, 51, 255); + } + } + + return encodePngRgba(buf, width, height); +} + async function createTempAgentDir(): Promise { return await fs.mkdtemp(path.join(os.tmpdir(), "xai-plugin-live-")); } @@ -357,4 +379,54 @@ describeLive("xai plugin live", () => { } }); }, 300_000); + + it.skipIf(!ENABLE_VIDEO_15_LIVE)( + "generates a Grok Imagine Video 1.5 clip from one image", + async () => { + await runXaiLiveCase("video-1.5", async () => { + const { videoProviders } = await registerXaiPlugin(); + const videoProvider = requireRegisteredProvider(videoProviders, "xai"); + const cfg = createLiveConfig(); + const agentDir = await createTempAgentDir(); + + try { + const generated = await videoProvider.generateVideo({ + provider: "xai", + model: "grok-imagine-video-1.5", + prompt: + "Animate the orange square with a subtle slow rotation. Keep the framing fixed.", + cfg, + agentDir, + authStore: EMPTY_AUTH_STORE, + timeoutMs: 10 * 60_000, + durationSeconds: 1, + resolution: "1080P", + inputImages: [ + { + buffer: createVideoReferencePng(), + mimeType: "image/png", + fileName: "video-reference.png", + }, + ], + }); + + expect(generated.model).toBe("grok-imagine-video-1.5"); + expect(generated.videos).toHaveLength(1); + const video = generated.videos[0]; + if (!video?.buffer) { + throw new Error("xAI Video 1.5 did not return a buffered video"); + } + expect(video.mimeType.startsWith("video/")).toBe(true); + expect(video.buffer.byteLength).toBeGreaterThan(1_000); + const outputPath = process.env.OPENCLAW_LIVE_XAI_VIDEO_15_OUTPUT?.trim(); + if (outputPath) { + await fs.writeFile(outputPath, video.buffer); + } + } finally { + await fs.rm(agentDir, { recursive: true, force: true }); + } + }); + }, + 12 * 60_000, + ); }); diff --git a/packages/media-generation-core/src/catalog.test.ts b/packages/media-generation-core/src/catalog.test.ts index f42b58a2f55c..d0cf91496d6f 100644 --- a/packages/media-generation-core/src/catalog.test.ts +++ b/packages/media-generation-core/src/catalog.test.ts @@ -56,6 +56,49 @@ describe("media-generation catalog", () => { ).toEqual(["video-default", "video-pro"]); }); + it("uses per-model capabilities and modes when provided", () => { + type VideoCapabilities = { + generate?: { maxVideos: number }; + imageToVideo?: { enabled: boolean; maxInputImages: number }; + }; + const providerCapabilities: VideoCapabilities = { + generate: { maxVideos: 1 }, + }; + const alternateCapabilities: VideoCapabilities = { + imageToVideo: { enabled: true, maxInputImages: 1 }, + }; + + const rows = synthesizeMediaGenerationCatalogEntries({ + kind: "video_generation", + provider: { + id: "example", + defaultModel: "default-video", + models: ["default-video", "image-video"], + capabilities: providerCapabilities, + catalogByModel: { + "image-video": { + capabilities: alternateCapabilities, + modes: ["imageToVideo"], + }, + }, + }, + modes: ["generate"], + }); + + expect(rows).toEqual([ + expect.objectContaining({ + model: "default-video", + capabilities: providerCapabilities, + modes: ["generate"], + }), + expect.objectContaining({ + model: "image-video", + capabilities: alternateCapabilities, + modes: ["imageToVideo"], + }), + ]); + }); + it("marks a trimmed default model as the catalog default", () => { expect( synthesizeMediaGenerationCatalogEntries({ diff --git a/packages/media-generation-core/src/catalog.ts b/packages/media-generation-core/src/catalog.ts index d87c8e946b28..a6a82cd77325 100644 --- a/packages/media-generation-core/src/catalog.ts +++ b/packages/media-generation-core/src/catalog.ts @@ -30,6 +30,12 @@ export type MediaGenerationCatalogEntry = { warnings?: readonly string[]; }; +/** Static catalog metadata that overrides provider defaults for one model. */ +export type MediaGenerationCatalogModelEntry = { + capabilities?: TCapabilities; + modes?: readonly string[]; +}; + /** Provider metadata used to synthesize static media generation catalog entries. */ export type MediaGenerationCatalogProvider = { id: string; @@ -38,6 +44,7 @@ export type MediaGenerationCatalogProvider = { defaultModel?: string; models?: readonly string[]; capabilities: TCapabilities; + catalogByModel?: Readonly>>; }; /** Return unique configured models with default model first when present. */ @@ -53,12 +60,13 @@ export function synthesizeMediaGenerationCatalogEntries(params: { }): Array> { const defaultModel = uniqueTrimmedStrings([params.provider.defaultModel])[0]; return uniqueModels(params.provider).map((model) => { + const modelCatalogEntry = params.provider.catalogByModel?.[model]; const entry: MediaGenerationCatalogEntry = { kind: params.kind, provider: params.provider.id, model, source: "static", - capabilities: params.provider.capabilities, + capabilities: modelCatalogEntry?.capabilities ?? params.provider.capabilities, }; if (params.provider.label) { entry.label = params.provider.label; @@ -66,8 +74,9 @@ export function synthesizeMediaGenerationCatalogEntries(params: { if (model === defaultModel) { entry.default = true; } - if (params.modes) { - entry.modes = params.modes; + const modes = modelCatalogEntry?.modes ?? params.modes; + if (modes) { + entry.modes = modes; } return entry; }); diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index e928eb952f02..3486463bf1e2 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -195,7 +195,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { ), publicExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", - 10478, + 10480, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/agents/tools/media-generate-tool-actions-shared.ts b/src/agents/tools/media-generate-tool-actions-shared.ts index 429a9ad5cc4c..75172a31a2e2 100644 --- a/src/agents/tools/media-generate-tool-actions-shared.ts +++ b/src/agents/tools/media-generate-tool-actions-shared.ts @@ -26,6 +26,7 @@ type MediaGenerateProvider = { defaultModel?: string; models?: readonly string[]; capabilities: unknown; + catalogByModel?: Readonly>; isConfigured?: (ctx: { cfg?: OpenClawConfig; agentDir?: string }) => boolean; }; @@ -41,6 +42,11 @@ type MediaGenerateListProviderDetails = catalog: ReturnType>; }; +type MediaGenerateCapabilitySummaryOptions = { + modes?: readonly string[]; + includeModes?: boolean; +}; + /** Common tool result shape for media generation list/status actions. */ export type { MediaGenerateActionResult }; @@ -56,7 +62,10 @@ export function createMediaGenerateProviderListActionResult< agentDir?: string; authStore?: AuthProfileStore; listModes: (provider: TProvider) => string[]; - summarizeCapabilities: (provider: TProvider) => string; + summarizeCapabilities: ( + provider: TProvider, + options?: MediaGenerateCapabilitySummaryOptions, + ) => string; formatAuthHint?: (provider: { id: string; authEnvVars: readonly string[] }) => string | undefined; }): MediaGenerateActionResult { if (params.providers.length === 0) { @@ -104,6 +113,22 @@ export function createMediaGenerateProviderListActionResult< const authHint = params.formatAuthHint?.({ id: details.id, authEnvVars: authHints }) ?? (authHints.length > 0 ? `set ${authHints.join(" / ")} to use ${details.id}/*` : undefined); + const modelCapabilityLines = details.catalog.flatMap((entry) => { + if (!provider.catalogByModel?.[entry.model]) { + return []; + } + const modelProvider = { + ...provider, + capabilities: entry.capabilities ?? provider.capabilities, + } as TProvider; + const modelCapabilities = params.summarizeCapabilities(modelProvider, { + modes: entry.modes, + includeModes: false, + }); + const modelModes = entry.modes?.length ? `modes=${entry.modes.join("/")}` : undefined; + const modelSummary = [modelModes, modelCapabilities || undefined].filter(Boolean).join(", "); + return [` model ${entry.model}: ${modelSummary || "no capabilities declared"}`]; + }); return [ `${details.id}${details.defaultModel ? ` (default ${details.defaultModel})` : ""}`, ` models: ${modelLine}`, @@ -111,6 +136,7 @@ export function createMediaGenerateProviderListActionResult< ...(authHint ? [` auth: ${authHint}`] : []), " source: static", ...(capabilities ? [` capabilities: ${capabilities}`] : []), + ...modelCapabilityLines, ]; }); diff --git a/src/agents/tools/video-generate-tool.actions.ts b/src/agents/tools/video-generate-tool.actions.ts index 765768b24e69..d449a4cb5405 100644 --- a/src/agents/tools/video-generate-tool.actions.ts +++ b/src/agents/tools/video-generate-tool.actions.ts @@ -24,11 +24,26 @@ type VideoGenerateActionResult = MediaGenerateActionResult; function summarizeVideoGenerationCapabilities( provider: ReturnType[number], + options?: { modes?: readonly string[]; includeModes?: boolean }, ): string { - const supportedModes = listSupportedVideoGenerationModes(provider); + const supportedModes = options?.modes ?? listSupportedVideoGenerationModes(provider); const generate = provider.capabilities.generate; const imageToVideo = provider.capabilities.imageToVideo; const videoToVideo = provider.capabilities.videoToVideo; + const activeModeCapabilities = [ + supportedModes.includes("generate") ? generate : undefined, + supportedModes.includes("imageToVideo") && imageToVideo?.enabled ? imageToVideo : undefined, + supportedModes.includes("videoToVideo") && videoToVideo?.enabled ? videoToVideo : undefined, + ].filter((capabilities) => capabilities !== undefined); + const maxDurationSeconds = activeModeCapabilities + .map((capabilities) => capabilities.maxDurationSeconds) + .find((value) => typeof value === "number"); + const supportedDurationSeconds = activeModeCapabilities + .map((capabilities) => capabilities.supportedDurationSeconds) + .find((value) => value && value.length > 0); + const supportedDurationSecondsByModel = activeModeCapabilities + .map((capabilities) => capabilities.supportedDurationSecondsByModel) + .find((value) => value && Object.keys(value).length > 0); // providerOptions may be declared at the mode level (generate) or at the flat // provider-capabilities level. The runtime checks both; surface the union so // the agent sees a single merged view of which opaque keys each provider @@ -52,28 +67,39 @@ function summarizeVideoGenerationCapabilities( videoToVideo?.maxInputAudios ?? provider.capabilities.maxInputAudios; const capabilities = [ - supportedModes.length > 0 ? `modes=${supportedModes.join("/")}` : null, + options?.includeModes !== false && supportedModes.length > 0 + ? `modes=${supportedModes.join("/")}` + : null, generate?.maxVideos ? `maxVideos=${generate.maxVideos}` : null, imageToVideo?.maxInputImages ? `maxInputImages=${imageToVideo.maxInputImages}` : null, videoToVideo?.maxInputVideos ? `maxInputVideos=${videoToVideo.maxInputVideos}` : null, typeof maxInputAudios === "number" && maxInputAudios > 0 ? `maxInputAudios=${maxInputAudios}` : null, - generate?.maxDurationSeconds ? `maxDurationSeconds=${generate.maxDurationSeconds}` : null, - generate?.supportedDurationSeconds?.length - ? `supportedDurationSeconds=${generate.supportedDurationSeconds.join("/")}` + maxDurationSeconds ? `maxDurationSeconds=${maxDurationSeconds}` : null, + supportedDurationSeconds + ? `supportedDurationSeconds=${supportedDurationSeconds.join("/")}` : null, - generate?.supportedDurationSecondsByModel && - Object.keys(generate.supportedDurationSecondsByModel).length > 0 - ? `supportedDurationSecondsByModel=${Object.entries(generate.supportedDurationSecondsByModel) + supportedDurationSecondsByModel + ? `supportedDurationSecondsByModel=${Object.entries(supportedDurationSecondsByModel) .map(([modelId, durations]) => `${modelId}:${durations.join("/")}`) .join("; ")}` : null, - generate?.supportsResolution ? "resolution" : null, - generate?.supportsAspectRatio ? "aspectRatio" : null, - generate?.supportsSize ? "size" : null, - generate?.supportsAudio ? "audio" : null, - generate?.supportsWatermark ? "watermark" : null, + activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsResolution) + ? "resolution" + : null, + activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsAspectRatio) + ? "aspectRatio" + : null, + activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsSize) + ? "size" + : null, + activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsAudio) + ? "audio" + : null, + activeModeCapabilities.some((modeCapabilities) => modeCapabilities.supportsWatermark) + ? "watermark" + : null, Object.keys(declaredProviderOptions).length > 0 ? `providerOptions={${Object.entries(declaredProviderOptions) .map(([key, type]) => `${key}:${type}`) diff --git a/src/agents/tools/video-generate-tool.test.ts b/src/agents/tools/video-generate-tool.test.ts index d66888ffc013..4442e1a79b56 100644 --- a/src/agents/tools/video-generate-tool.test.ts +++ b/src/agents/tools/video-generate-tool.test.ts @@ -1295,6 +1295,73 @@ describe("createVideoGenerateTool", () => { expect(providers[0]?.modes).toEqual(["generate", "imageToVideo"]); }); + it("lists model-specific catalog capabilities and modes", async () => { + const imageToVideoCapabilities = { + imageToVideo: { + enabled: true, + maxInputImages: 1, + maxDurationSeconds: 15, + resolutions: ["480P", "720P", "1080P"] as const, + aspectRatios: ["16:9", "9:16"] as const, + supportsResolution: true, + supportsAspectRatio: true, + }, + }; + vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([ + { + id: "video-plugin", + defaultModel: "text-video", + models: ["text-video", "image-video"], + capabilities: { + generate: { + maxDurationSeconds: 10, + }, + }, + catalogByModel: { + "image-video": { + capabilities: imageToVideoCapabilities, + modes: ["imageToVideo"], + }, + }, + generateVideo: vi.fn(async () => { + throw new Error("not used"); + }), + }, + ]); + + const tool = createVideoGenerateTool({ + config: asConfig({ + agents: { + defaults: { + videoGenerationModel: { primary: "video-plugin/text-video" }, + }, + }, + }), + }); + if (!tool) { + throw new Error("expected video_generate tool"); + } + + const result = await tool.execute("call-1", { action: "list" }); + const text = (result.content?.[0] as { text: string } | undefined)?.text ?? ""; + expect(text).toContain( + "model image-video: modes=imageToVideo, maxInputImages=1, maxDurationSeconds=15, resolution, aspectRatio", + ); + const providers = resultDetails(result).providers as Array<{ + catalog?: Array<{ + model?: string; + capabilities?: unknown; + modes?: string[]; + }>; + }>; + const catalogEntry = providers[0]?.catalog?.find((entry) => entry.model === "image-video"); + expect(catalogEntry).toMatchObject({ + model: "image-video", + capabilities: imageToVideoCapabilities, + modes: ["imageToVideo"], + }); + }); + it("rejects image-to-video when the provider disables that mode", async () => { vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([ { diff --git a/src/plugin-sdk/video-generation-core.ts b/src/plugin-sdk/video-generation-core.ts index 63385ee3dbfe..c5577d08612c 100644 --- a/src/plugin-sdk/video-generation-core.ts +++ b/src/plugin-sdk/video-generation-core.ts @@ -5,6 +5,7 @@ export type { FallbackAttempt } from "../agents/model-fallback.types.js"; export type { VideoGenerationProviderPlugin } from "../plugins/types.js"; export type { GeneratedVideoAsset, + VideoGenerationCatalogModelEntry, VideoGenerationIgnoredOverride, VideoGenerationMode, VideoGenerationModeCapabilities, diff --git a/src/plugin-sdk/video-generation.ts b/src/plugin-sdk/video-generation.ts index 024683e630f0..fa0e9ca0d6ed 100644 --- a/src/plugin-sdk/video-generation.ts +++ b/src/plugin-sdk/video-generation.ts @@ -8,6 +8,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { GeneratedVideoAsset as CoreGeneratedVideoAsset, VideoGenerationAssetRole as CoreVideoGenerationAssetRole, + VideoGenerationCatalogModelEntry as CoreVideoGenerationCatalogModelEntry, VideoGenerationMode as CoreVideoGenerationMode, VideoGenerationModeCapabilities as CoreVideoGenerationModeCapabilities, VideoGenerationModelCapabilitiesContext as CoreVideoGenerationModelCapabilitiesContext, @@ -171,6 +172,12 @@ export type VideoGenerationProviderCapabilities = VideoGenerationModeCapabilitie videoToVideo?: VideoGenerationTransformCapabilities; }; +/** Static catalog metadata that overrides provider defaults for one video model. */ +export type VideoGenerationCatalogModelEntry = { + capabilities?: VideoGenerationProviderCapabilities; + modes?: readonly VideoGenerationMode[]; +}; + /** Video generation provider contract implemented by provider plugins. */ export type VideoGenerationProvider = { id: string; @@ -181,6 +188,7 @@ export type VideoGenerationProvider = { defaultTimeoutMs?: number; models?: string[]; capabilities: VideoGenerationProviderCapabilities; + catalogByModel?: Readonly>; isConfigured?: (ctx: VideoGenerationProviderConfiguredContext) => boolean; resolveModelCapabilities?: ( ctx: VideoGenerationModelCapabilitiesContext, @@ -201,6 +209,8 @@ const videoGenerationSdkCompat: [ AssertAssignable, AssertAssignable, AssertAssignable, + AssertAssignable, + AssertAssignable, AssertAssignable, AssertAssignable, AssertAssignable, diff --git a/src/video-generation/types.ts b/src/video-generation/types.ts index ce9b5e790529..828260b22199 100644 --- a/src/video-generation/types.ts +++ b/src/video-generation/types.ts @@ -156,6 +156,12 @@ export type VideoGenerationProviderCapabilities = VideoGenerationModeCapabilitie videoToVideo?: VideoGenerationTransformCapabilities; }; +/** Static catalog metadata that overrides provider defaults for one video model. */ +export type VideoGenerationCatalogModelEntry = { + capabilities?: VideoGenerationProviderCapabilities; + modes?: readonly VideoGenerationMode[]; +}; + export type VideoGenerationNormalization = { size?: MediaNormalizationEntry; aspectRatio?: MediaNormalizationEntry; @@ -172,6 +178,7 @@ export type VideoGenerationProvider = { defaultTimeoutMs?: number; models?: string[]; capabilities: VideoGenerationProviderCapabilities; + catalogByModel?: Readonly>; isConfigured?: (ctx: VideoGenerationProviderConfiguredContext) => boolean; resolveModelCapabilities?: ( ctx: VideoGenerationModelCapabilitiesContext,