feat(xai): support Grok Imagine Video 1.5 (#103316)

* feat(media): add per-model generation catalog metadata * feat(xai): support Grok Imagine Video 1.5 * test(plugin-sdk): update public surface budget
2026-07-19 01:21:42 +00:00 · 2026-07-10 04:49:42 +01:00
parent 071e30ecd5
commit 75dfd3dc09
16 changed files with 569 additions and 43 deletions
--- a/extensions/xai/video-generation-provider.test.ts
+++ b/extensions/xai/video-generation-provider.test.ts
@@ -4,10 +4,15 @@ import {
  installProviderHttpMockCleanup,
 } from "openclaw/plugin-sdk/provider-http-test-mocks";
 import { expectExplicitVideoGenerationCapabilities } from "openclaw/plugin-sdk/provider-test-contracts";
+import type { VideoGenerationRequest } from "openclaw/plugin-sdk/video-generation";
 import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";

-const { postJsonRequestMock, fetchWithTimeoutMock, readProviderJsonResponseMock } =
-  getProviderHttpMocks();
+const {
+  postJsonRequestMock,
+  fetchWithTimeoutMock,
+  readProviderJsonResponseMock,
+  resolveApiKeyForProviderMock,
+} = getProviderHttpMocks();

 let buildXaiVideoGenerationProvider: typeof import("./video-generation-provider.js").buildXaiVideoGenerationProvider;

@@ -161,6 +166,142 @@ describe("xai video generation provider", () => {
    expectExplicitVideoGenerationCapabilities(buildXaiVideoGenerationProvider());
  });

+  it("advertises canonical 1.5 and resolves capabilities for all API aliases", async () => {
+    const provider = buildXaiVideoGenerationProvider();
+
+    expect(provider.defaultModel).toBe("grok-imagine-video");
+    expect(provider.models).toEqual(["grok-imagine-video", "grok-imagine-video-1.5"]);
+    expect(provider.catalogByModel?.["grok-imagine-video-1.5"]).toMatchObject({
+      modes: ["imageToVideo"],
+      capabilities: {
+        imageToVideo: {
+          enabled: true,
+          maxInputImages: 1,
+          resolutions: ["480P", "720P", "1080P"],
+        },
+        videoToVideo: { enabled: false },
+      },
+    });
+
+    for (const model of [
+      "grok-imagine-video-1.5",
+      "grok-imagine-video-1.5-preview",
+      "grok-imagine-video-1.5-2026-05-30",
+    ]) {
+      const capabilities = await provider.resolveModelCapabilities?.({
+        provider: "xai",
+        model,
+        cfg: {},
+      });
+      expect(capabilities?.imageToVideo).toMatchObject({
+        enabled: true,
+        maxInputImages: 1,
+        maxDurationSeconds: 15,
+        resolutions: ["480P", "720P", "1080P"],
+      });
+      expect(capabilities?.videoToVideo?.enabled).toBe(false);
+    }
+  });
+
+  it("uses the 1.5 default while preserving aliases, 1080p, and source aspect ratio", async () => {
+    const models = [
+      "grok-imagine-video-1.5",
+      "grok-imagine-video-1.5-preview",
+      "grok-imagine-video-1.5-2026-05-30",
+    ];
+    const provider = buildXaiVideoGenerationProvider();
+
+    for (const [index, model] of models.entries()) {
+      const requestId = `req_15_${index}`;
+      postJsonRequestMock.mockResolvedValueOnce({
+        response: { json: async () => ({ request_id: requestId }) },
+        release: vi.fn(async () => {}),
+      });
+      fetchWithTimeoutMock
+        .mockResolvedValueOnce({
+          json: async () => ({
+            request_id: requestId,
+            status: "done",
+            video: { url: `https://cdn.x.ai/${requestId}.mp4` },
+          }),
+        })
+        .mockResolvedValueOnce({
+          headers: new Headers({ "content-type": "video/mp4" }),
+          arrayBuffer: async () => Buffer.from("video-bytes"),
+        });
+
+      const result = await provider.generateVideo({
+        provider: "xai",
+        model,
+        prompt: "Animate this still image",
+        cfg: {},
+        durationSeconds: 20,
+        resolution: index === 0 ? undefined : "1080P",
+        inputImages: [
+          {
+            url: "https://example.com/first-frame.png",
+            ...(index === 0 ? {} : { role: "first_frame" as const }),
+          },
+        ],
+      });
+
+      const body = requirePostJsonCall(index).body ?? {};
+      expect(body.model).toBe(model);
+      expect(body.image).toEqual({ url: "https://example.com/first-frame.png" });
+      expect(body.duration).toBe(15);
+      expect(body.resolution).toBe(index === 0 ? "480p" : "1080p");
+      expect(body).not.toHaveProperty("aspect_ratio");
+      expect(result.model).toBe(model);
+    }
+  });
+
+  it("rejects unsupported 1.5 modes before submitting a request", async () => {
+    const provider = buildXaiVideoGenerationProvider();
+    const cases: Array<
+      Pick<VideoGenerationRequest, "model" | "inputImages" | "inputVideos"> & { error: string }
+    > = [
+      {
+        model: "grok-imagine-video-1.5",
+        inputImages: undefined,
+        error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.",
+      },
+      {
+        model: "grok-imagine-video-1.5-preview",
+        inputImages: [{ url: "https://example.com/reference.png", role: "reference_image" }],
+        error: "xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image.",
+      },
+      {
+        model: "grok-imagine-video-1.5-2026-05-30",
+        inputImages: [
+          { url: "https://example.com/first.png" },
+          { url: "https://example.com/second.png", role: "first_frame" },
+        ],
+        error: "xAI grok-imagine-video-1.5 requires exactly one first-frame image.",
+      },
+      {
+        model: "grok-imagine-video-1.5",
+        inputImages: undefined,
+        inputVideos: [{ url: "https://example.com/input.mp4" }],
+        error: "xAI grok-imagine-video-1.5 does not support video inputs.",
+      },
+    ];
+
+    for (const testCase of cases) {
+      await expect(
+        provider.generateVideo({
+          provider: "xai",
+          model: testCase.model,
+          prompt: "Unsupported 1.5 request",
+          cfg: {},
+          inputImages: testCase.inputImages,
+          inputVideos: testCase.inputVideos,
+        }),
+      ).rejects.toThrow(testCase.error);
+    }
+    expect(resolveApiKeyForProviderMock).not.toHaveBeenCalled();
+    expect(postJsonRequestMock).not.toHaveBeenCalled();
+  });
+
  it("creates, polls, and downloads a generated video", async () => {
    postJsonRequestMock.mockResolvedValue({
      response: {
--- a/extensions/xai/video-generation-provider.ts
+++ b/extensions/xai/video-generation-provider.ts
@@ -21,15 +21,37 @@ import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-co
 import type {
  GeneratedVideoAsset,
  VideoGenerationProvider,
+  VideoGenerationProviderCapabilities,
  VideoGenerationRequest,
 } from "openclaw/plugin-sdk/video-generation";

 const DEFAULT_XAI_VIDEO_BASE_URL = "https://api.x.ai/v1";
 const DEFAULT_XAI_VIDEO_MODEL = "grok-imagine-video";
+const XAI_VIDEO_15_MODEL = "grok-imagine-video-1.5";
+const XAI_VIDEO_15_MODEL_IDS = new Set([
+  XAI_VIDEO_15_MODEL,
+  "grok-imagine-video-1.5-preview",
+  "grok-imagine-video-1.5-2026-05-30",
+]);
 const DEFAULT_TIMEOUT_MS = 600_000;
 const POLL_INTERVAL_MS = 5_000;
 const MAX_POLL_ATTEMPTS = 120;
 const XAI_VIDEO_ASPECT_RATIOS = new Set(["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]);
+const XAI_VIDEO_15_CAPABILITIES = {
+  imageToVideo: {
+    enabled: true,
+    maxVideos: 1,
+    maxInputImages: 1,
+    maxDurationSeconds: 15,
+    aspectRatios: [...XAI_VIDEO_ASPECT_RATIOS],
+    resolutions: ["480P", "720P", "1080P"],
+    supportsAspectRatio: true,
+    supportsResolution: true,
+  },
+  videoToVideo: {
+    enabled: false,
+  },
+} satisfies VideoGenerationProviderCapabilities;
 const XAI_VIDEO_MALFORMED_RESPONSE = "xAI video generation response malformed";
 // xAI documents these as the only meaningful values; everything else (queued,
 // processing, submitted, pending, in_progress, ...) means "keep polling".
@@ -37,6 +59,7 @@ const XAI_VIDEO_TERMINAL_FAILURE_STATUSES = new Set(["failed", "error", "expired
 const XAI_VIDEO_DEFAULT_DURATION_SECONDS = 8;
 const XAI_VIDEO_DEFAULT_ASPECT_RATIO = "16:9";
 const XAI_VIDEO_DEFAULT_RESOLUTION = "720p";
+const XAI_VIDEO_15_DEFAULT_RESOLUTION = "480p";
 const DEFAULT_GENERATED_VIDEO_MAX_BYTES = 16 * 1024 * 1024;

 type XaiVideoCreateResponse = {
@@ -155,6 +178,32 @@ function isReferenceImage(input: VideoGenerationSourceInput): boolean {
  return normalizeOptionalString(input.role)?.toLowerCase() === "reference_image";
 }

+function isXaiVideo15Model(model: string | undefined): boolean {
+  const normalized = normalizeOptionalString(model);
+  return normalized ? XAI_VIDEO_15_MODEL_IDS.has(normalized) : false;
+}
+
+function isFirstFrameImage(input: VideoGenerationSourceInput): boolean {
+  const role = normalizeOptionalString(input.role)?.toLowerCase();
+  return role === undefined || role === "first_frame";
+}
+
+function validateXaiVideo15Request(req: VideoGenerationRequest): void {
+  if (!isXaiVideo15Model(req.model)) {
+    return;
+  }
+  if ((req.inputVideos?.length ?? 0) > 0) {
+    throw new Error("xAI grok-imagine-video-1.5 does not support video inputs.");
+  }
+  const inputImages = req.inputImages ?? [];
+  if (inputImages.length !== 1) {
+    throw new Error("xAI grok-imagine-video-1.5 requires exactly one first-frame image.");
+  }
+  if (!isFirstFrameImage(inputImages[0])) {
+    throw new Error("xAI grok-imagine-video-1.5 supports only an ordinary or first_frame image.");
+  }
+}
+
 function resolveInputVideoUrl(input: VideoGenerationSourceInput | undefined): string | undefined {
  if (!input) {
    return undefined;
@@ -189,7 +238,10 @@ function resolveAspectRatio(value: string | undefined): string | undefined {
  return trimmed;
 }

-function resolveResolution(value: string | undefined): "480p" | "720p" | undefined {
+function resolveResolution(
+  value: string | undefined,
+  options?: { allow1080p?: boolean },
+): "480p" | "720p" | "1080p" | undefined {
  if (typeof value !== "string") {
    return undefined;
  }
@@ -197,9 +249,12 @@ function resolveResolution(value: string | undefined): "480p" | "720p" | undefin
  if (normalized === "480p") {
    return "480p";
  }
-  if (normalized === "720p" || normalized === "1080p") {
+  if (normalized === "720p") {
    return "720p";
  }
+  if (normalized === "1080p") {
+    return options?.allow1080p ? "1080p" : "720p";
+  }
  return undefined;
 }

@@ -223,6 +278,7 @@ function resolveXaiVideoMode(
 }

 function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
+  validateXaiVideo15Request(req);
  const inputImages = req.inputImages ?? [];
  const hasReferenceImages = inputImages.some(isReferenceImage);
  if (hasReferenceImages && !inputImages.every(isReferenceImage)) {
@@ -245,11 +301,14 @@ function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {

  const mode = resolveXaiVideoMode(req);
  const body: Record<string, unknown> = {
+    // Aliases are API-owned routing choices. Preserve the selected identifier
+    // instead of silently pinning it to the canonical 1.5 model.
    model: normalizeOptionalString(req.model) ?? DEFAULT_XAI_VIDEO_MODEL,
    prompt: req.prompt,
  };

  if (mode === "generate") {
+    const isVideo15 = isXaiVideo15Model(req.model);
    const imageUrl = resolveImageUrl(req.inputImages?.[0]);
    if (imageUrl) {
      body.image = { url: imageUrl };
@@ -260,8 +319,16 @@ function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
        min: 1,
        max: 15,
      }) ?? XAI_VIDEO_DEFAULT_DURATION_SECONDS;
-    body.aspect_ratio = resolveAspectRatio(req.aspectRatio) ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO;
-    body.resolution = resolveResolution(req.resolution) ?? XAI_VIDEO_DEFAULT_RESOLUTION;
+    const aspectRatio = resolveAspectRatio(req.aspectRatio);
+    // 1.5 inherits the source image ratio when callers do not choose one.
+    if (aspectRatio || !isVideo15) {
+      body.aspect_ratio = aspectRatio ?? XAI_VIDEO_DEFAULT_ASPECT_RATIO;
+    }
+    const defaultResolution = isVideo15
+      ? XAI_VIDEO_15_DEFAULT_RESOLUTION
+      : XAI_VIDEO_DEFAULT_RESOLUTION;
+    body.resolution =
+      resolveResolution(req.resolution, { allow1080p: isVideo15 }) ?? defaultResolution;
    return body;
  }

@@ -380,7 +447,13 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
    label: "xAI",
    defaultModel: DEFAULT_XAI_VIDEO_MODEL,
    defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
-    models: [DEFAULT_XAI_VIDEO_MODEL],
+    models: [DEFAULT_XAI_VIDEO_MODEL, XAI_VIDEO_15_MODEL],
+    catalogByModel: {
+      [XAI_VIDEO_15_MODEL]: {
+        capabilities: XAI_VIDEO_15_CAPABILITIES,
+        modes: ["imageToVideo"],
+      },
+    },
    isConfigured: ({ agentDir }) =>
      isProviderApiKeyConfigured({
        provider: "xai",
@@ -414,7 +487,17 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
        supportsResolution: true,
      },
    },
+    resolveModelCapabilities: ({ model }): VideoGenerationProviderCapabilities | undefined => {
+      if (!isXaiVideo15Model(model)) {
+        return undefined;
+      }
+      return XAI_VIDEO_15_CAPABILITIES;
+    },
    async generateVideo(req) {
+      // Validate provider/model mode constraints before auth or HTTP setup so
+      // unsupported 1.5 requests cannot be submitted and billed accidentally.
+      const createBody = buildCreateBody(req);
+      const createEndpoint = resolveCreateEndpoint(req);
      const auth = await resolveApiKeyForProvider({
        provider: "xai",
        cfg: req.cfg,
@@ -448,9 +531,9 @@ export function buildXaiVideoGenerationProvider(): VideoGenerationProvider {
      const submitHeaders = new Headers(headers);
      submitHeaders.set("x-idempotency-key", crypto.randomUUID());
      const { response, release } = await postJsonRequest({
-        url: `${baseUrl}${resolveCreateEndpoint(req)}`,
+        url: `${baseUrl}${createEndpoint}`,
        headers: submitHeaders,
-        body: buildCreateBody(req),
+        body: createBody,
        timeoutMs: resolveProviderOperationTimeoutMs({
          deadline,
          defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
--- a/extensions/xai/xai.live.test.ts
+++ b/extensions/xai/xai.live.test.ts
@@ -21,6 +21,7 @@ import { XAI_DEFAULT_STT_MODEL } from "./stt.js";

 const XAI_API_KEY = process.env.XAI_API_KEY ?? "";
 const LIVE_IMAGE_MODEL = process.env.OPENCLAW_LIVE_XAI_IMAGE_MODEL?.trim() || "grok-imagine-image";
+const ENABLE_VIDEO_15_LIVE = process.env.OPENCLAW_LIVE_XAI_VIDEO_15 === "1";
 const liveEnabled = XAI_API_KEY.trim().length > 0 && process.env.OPENCLAW_LIVE_TEST === "1";
 const describeLive = liveEnabled ? describe : describe.skip;
 const EMPTY_AUTH_STORE = { version: 1, profiles: {} } as const;
@@ -63,6 +64,27 @@ function createReferencePng(): Buffer {
  return encodePngRgba(buf, width, height);
 }

+function createVideoReferencePng(): Buffer {
+  const width = 384;
+  const height = 384;
+  const buf = Buffer.alloc(width * height * 4, 255);
+
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      const blue = Math.round(160 + (80 * y) / height);
+      fillPixel(buf, x, y, width, 32, 96, blue, 255);
+    }
+  }
+
+  for (let y = 112; y < 272; y += 1) {
+    for (let x = 112; x < 272; x += 1) {
+      fillPixel(buf, x, y, width, 255, 153, 51, 255);
+    }
+  }
+
+  return encodePngRgba(buf, width, height);
+}
+
 async function createTempAgentDir(): Promise<string> {
  return await fs.mkdtemp(path.join(os.tmpdir(), "xai-plugin-live-"));
 }
@@ -357,4 +379,54 @@ describeLive("xai plugin live", () => {
      }
    });
  }, 300_000);
+
+  it.skipIf(!ENABLE_VIDEO_15_LIVE)(
+    "generates a Grok Imagine Video 1.5 clip from one image",
+    async () => {
+      await runXaiLiveCase("video-1.5", async () => {
+        const { videoProviders } = await registerXaiPlugin();
+        const videoProvider = requireRegisteredProvider(videoProviders, "xai");
+        const cfg = createLiveConfig();
+        const agentDir = await createTempAgentDir();
+
+        try {
+          const generated = await videoProvider.generateVideo({
+            provider: "xai",
+            model: "grok-imagine-video-1.5",
+            prompt:
+              "Animate the orange square with a subtle slow rotation. Keep the framing fixed.",
+            cfg,
+            agentDir,
+            authStore: EMPTY_AUTH_STORE,
+            timeoutMs: 10 * 60_000,
+            durationSeconds: 1,
+            resolution: "1080P",
+            inputImages: [
+              {
+                buffer: createVideoReferencePng(),
+                mimeType: "image/png",
+                fileName: "video-reference.png",
+              },
+            ],
+          });
+
+          expect(generated.model).toBe("grok-imagine-video-1.5");
+          expect(generated.videos).toHaveLength(1);
+          const video = generated.videos[0];
+          if (!video?.buffer) {
+            throw new Error("xAI Video 1.5 did not return a buffered video");
+          }
+          expect(video.mimeType.startsWith("video/")).toBe(true);
+          expect(video.buffer.byteLength).toBeGreaterThan(1_000);
+          const outputPath = process.env.OPENCLAW_LIVE_XAI_VIDEO_15_OUTPUT?.trim();
+          if (outputPath) {
+            await fs.writeFile(outputPath, video.buffer);
+          }
+        } finally {
+          await fs.rm(agentDir, { recursive: true, force: true });
+        }
+      });
+    },
+    12 * 60_000,
+  );
 });