feat: add music generation tooling

extensions/minimax/index.ts

@@ -23,6 +23,7 @@ import {
   minimaxMediaUnderstandingProvider,
   minimaxPortalMediaUnderstandingProvider,
 } from "./media-understanding-provider.js";
+import { buildMinimaxMusicGenerationProvider } from "./music-generation-provider.js";
 import type { MiniMaxRegion } from "./oauth.js";
 import { applyMinimaxApiConfig, applyMinimaxApiConfigCn } from "./onboard.js";
 import { buildMinimaxPortalProvider, buildMinimaxProvider } from "./provider-catalog.js";
@@ -314,6 +315,7 @@ export default definePluginEntry({
     });
     api.registerImageGenerationProvider(buildMinimaxImageGenerationProvider());
     api.registerImageGenerationProvider(buildMinimaxPortalImageGenerationProvider());
+    api.registerMusicGenerationProvider(buildMinimaxMusicGenerationProvider());
     api.registerVideoGenerationProvider(buildMinimaxVideoGenerationProvider());
     api.registerSpeechProvider(buildMinimaxSpeechProvider());
     api.registerWebSearchProvider(createMiniMaxWebSearchProvider());

extensions/minimax/music-generation-provider.test.ts

@@ -0,0 +1,104 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { buildMinimaxMusicGenerationProvider } from "./music-generation-provider.js";
+
+const {
+  resolveApiKeyForProviderMock,
+  postJsonRequestMock,
+  fetchWithTimeoutMock,
+  assertOkOrThrowHttpErrorMock,
+  resolveProviderHttpRequestConfigMock,
+} = vi.hoisted(() => ({
+  resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "minimax-key" })),
+  postJsonRequestMock: vi.fn(),
+  fetchWithTimeoutMock: vi.fn(),
+  assertOkOrThrowHttpErrorMock: vi.fn(async () => {}),
+  resolveProviderHttpRequestConfigMock: vi.fn((params) => ({
+    baseUrl: params.baseUrl ?? params.defaultBaseUrl,
+    allowPrivateNetwork: false,
+    headers: new Headers(params.defaultHeaders),
+    dispatcherPolicy: undefined,
+  })),
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({
+  resolveApiKeyForProvider: resolveApiKeyForProviderMock,
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-http", () => ({
+  assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock,
+  fetchWithTimeout: fetchWithTimeoutMock,
+  postJsonRequest: postJsonRequestMock,
+  resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock,
+}));
+
+describe("minimax music generation provider", () => {
+  afterEach(() => {
+    resolveApiKeyForProviderMock.mockClear();
+    postJsonRequestMock.mockReset();
+    fetchWithTimeoutMock.mockReset();
+    assertOkOrThrowHttpErrorMock.mockClear();
+    resolveProviderHttpRequestConfigMock.mockClear();
+  });
+
+  it("creates music and downloads the generated track", async () => {
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({
+          task_id: "task-123",
+          audio_url: "https://example.com/out.mp3",
+          lyrics: "our city wakes",
+          base_resp: { status_code: 0 },
+        }),
+      },
+      release: vi.fn(async () => {}),
+    });
+    fetchWithTimeoutMock.mockResolvedValue({
+      headers: new Headers({ "content-type": "audio/mpeg" }),
+      arrayBuffer: async () => Buffer.from("mp3-bytes"),
+    });
+
+    const provider = buildMinimaxMusicGenerationProvider();
+    const result = await provider.generateMusic({
+      provider: "minimax",
+      model: "music-2.5+",
+      prompt: "upbeat dance-pop with female vocals",
+      cfg: {},
+      lyrics: "our city wakes",
+      durationSeconds: 45,
+    });
+
+    expect(postJsonRequestMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: "https://api.minimax.io/v1/music_generation",
+        body: expect.objectContaining({
+          model: "music-2.5+",
+          lyrics: "our city wakes",
+          output_format: "url",
+        }),
+      }),
+    );
+    expect(result.tracks).toHaveLength(1);
+    expect(result.lyrics).toEqual(["our city wakes"]);
+    expect(result.metadata).toEqual(
+      expect.objectContaining({
+        taskId: "task-123",
+        audioUrl: "https://example.com/out.mp3",
+      }),
+    );
+  });
+
+  it("rejects instrumental requests that also include lyrics", async () => {
+    const provider = buildMinimaxMusicGenerationProvider();
+
+    await expect(
+      provider.generateMusic({
+        provider: "minimax",
+        model: "music-2.5+",
+        prompt: "driving techno",
+        cfg: {},
+        instrumental: true,
+        lyrics: "do not sing this",
+      }),
+    ).rejects.toThrow("cannot use lyrics when instrumental=true");
+  });
+});

extensions/minimax/music-generation-provider.ts

@@ -0,0 +1,232 @@
+import { extensionForMime } from "openclaw/plugin-sdk/msteams";
+import type {
+  GeneratedMusicAsset,
+  MusicGenerationProvider,
+  MusicGenerationRequest,
+} from "openclaw/plugin-sdk/music-generation";
+import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
+import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
+import {
+  assertOkOrThrowHttpError,
+  fetchWithTimeout,
+  postJsonRequest,
+  resolveProviderHttpRequestConfig,
+} from "openclaw/plugin-sdk/provider-http";
+
+const DEFAULT_MINIMAX_MUSIC_BASE_URL = "https://api.minimax.io";
+const DEFAULT_MINIMAX_MUSIC_MODEL = "music-2.5+";
+const DEFAULT_TIMEOUT_MS = 120_000;
+
+type MinimaxBaseResp = {
+  status_code?: number;
+  status_msg?: string;
+};
+
+type MinimaxMusicCreateResponse = {
+  task_id?: string;
+  audio?: string;
+  audio_url?: string;
+  lyrics?: string;
+  data?: {
+    audio?: string;
+    audio_url?: string;
+    lyrics?: string;
+  };
+  base_resp?: MinimaxBaseResp;
+};
+
+function resolveMinimaxMusicBaseUrl(
+  cfg: Parameters<typeof resolveApiKeyForProvider>[0]["cfg"],
+): string {
+  const direct = cfg?.models?.providers?.minimax?.baseUrl?.trim();
+  if (!direct) {
+    return DEFAULT_MINIMAX_MUSIC_BASE_URL;
+  }
+  try {
+    return new URL(direct).origin;
+  } catch {
+    return DEFAULT_MINIMAX_MUSIC_BASE_URL;
+  }
+}
+
+function assertMinimaxBaseResp(baseResp: MinimaxBaseResp | undefined, context: string): void {
+  if (!baseResp || typeof baseResp.status_code !== "number" || baseResp.status_code === 0) {
+    return;
+  }
+  throw new Error(
+    `${context} (${baseResp.status_code}): ${baseResp.status_msg ?? "unknown error"}`,
+  );
+}
+
+function decodePossibleBinary(data: string): Buffer {
+  const trimmed = data.trim();
+  if (/^[0-9a-f]+$/iu.test(trimmed) && trimmed.length % 2 === 0) {
+    return Buffer.from(trimmed, "hex");
+  }
+  return Buffer.from(trimmed, "base64");
+}
+
+function decodePossibleText(data: string): string {
+  const trimmed = data.trim();
+  if (!trimmed) {
+    return "";
+  }
+  if (/^[0-9a-f]+$/iu.test(trimmed) && trimmed.length % 2 === 0) {
+    return Buffer.from(trimmed, "hex").toString("utf8").trim();
+  }
+  return trimmed;
+}
+
+async function downloadTrackFromUrl(params: {
+  url: string;
+  timeoutMs?: number;
+  fetchFn: typeof fetch;
+}): Promise<GeneratedMusicAsset> {
+  const response = await fetchWithTimeout(
+    params.url,
+    { method: "GET" },
+    params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    params.fetchFn,
+  );
+  await assertOkOrThrowHttpError(response, "MiniMax generated music download failed");
+  const mimeType = response.headers.get("content-type")?.trim() || "audio/mpeg";
+  const ext = extensionForMime(mimeType)?.replace(/^\./u, "") || "mp3";
+  return {
+    buffer: Buffer.from(await response.arrayBuffer()),
+    mimeType,
+    fileName: `track-1.${ext}`,
+  };
+}
+
+function buildPrompt(req: MusicGenerationRequest): string {
+  const parts = [req.prompt.trim()];
+  if (typeof req.durationSeconds === "number" && Number.isFinite(req.durationSeconds)) {
+    parts.push(`Target duration: about ${Math.max(1, Math.round(req.durationSeconds))} seconds.`);
+  }
+  return parts.join("\n\n");
+}
+
+export function buildMinimaxMusicGenerationProvider(): MusicGenerationProvider {
+  return {
+    id: "minimax",
+    label: "MiniMax",
+    defaultModel: DEFAULT_MINIMAX_MUSIC_MODEL,
+    models: [DEFAULT_MINIMAX_MUSIC_MODEL, "music-2.5", "music-2.0"],
+    isConfigured: ({ agentDir }) =>
+      isProviderApiKeyConfigured({
+        provider: "minimax",
+        agentDir,
+      }),
+    capabilities: {
+      maxTracks: 1,
+      supportsLyrics: true,
+      supportsInstrumental: true,
+      supportsDuration: true,
+      supportsFormat: true,
+      supportedFormats: ["mp3"],
+    },
+    async generateMusic(req) {
+      if ((req.inputImages?.length ?? 0) > 0) {
+        throw new Error("MiniMax music generation does not support image reference inputs.");
+      }
+      if (req.instrumental === true && req.lyrics?.trim()) {
+        throw new Error("MiniMax music generation cannot use lyrics when instrumental=true.");
+      }
+      if (req.format && req.format !== "mp3") {
+        throw new Error("MiniMax music generation currently supports mp3 output only.");
+      }
+
+      const auth = await resolveApiKeyForProvider({
+        provider: "minimax",
+        cfg: req.cfg,
+        agentDir: req.agentDir,
+        store: req.authStore,
+      });
+      if (!auth.apiKey) {
+        throw new Error("MiniMax API key missing");
+      }
+
+      const fetchFn = fetch;
+      const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
+        resolveProviderHttpRequestConfig({
+          baseUrl: resolveMinimaxMusicBaseUrl(req.cfg),
+          defaultBaseUrl: DEFAULT_MINIMAX_MUSIC_BASE_URL,
+          allowPrivateNetwork: false,
+          defaultHeaders: {
+            Authorization: `Bearer ${auth.apiKey}`,
+          },
+        });
+
+      const model = req.model?.trim() || DEFAULT_MINIMAX_MUSIC_MODEL;
+      const body = {
+        model,
+        prompt: buildPrompt(req),
+        ...(req.instrumental === true ? { is_instrumental: true } : {}),
+        ...(req.lyrics?.trim()
+          ? { lyrics: req.lyrics.trim() }
+          : req.instrumental === true
+            ? {}
+            : { lyrics_optimizer: true }),
+        output_format: "url",
+        audio_setting: {
+          format: "mp3",
+        },
+      };
+
+      const { response: res, release } = await postJsonRequest({
+        url: `${baseUrl}/v1/music_generation`,
+        headers,
+        body,
+        timeoutMs: req.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+        fetchFn,
+        pinDns: false,
+        allowPrivateNetwork,
+        dispatcherPolicy,
+      });
+
+      try {
+        await assertOkOrThrowHttpError(res, "MiniMax music generation failed");
+        const payload = (await res.json()) as MinimaxMusicCreateResponse;
+        assertMinimaxBaseResp(payload.base_resp, "MiniMax music generation failed");
+
+        const audioUrl = payload.audio_url?.trim() || payload.data?.audio_url?.trim();
+        const inlineAudio = payload.audio?.trim() || payload.data?.audio?.trim();
+        const lyrics = decodePossibleText(payload.lyrics ?? payload.data?.lyrics ?? "");
+
+        const track = audioUrl
+          ? await downloadTrackFromUrl({
+              url: audioUrl,
+              timeoutMs: req.timeoutMs,
+              fetchFn,
+            })
+          : inlineAudio
+            ? {
+                buffer: decodePossibleBinary(inlineAudio),
+                mimeType: "audio/mpeg",
+                fileName: "track-1.mp3",
+              }
+            : null;
+        if (!track) {
+          throw new Error("MiniMax music generation response missing audio output");
+        }
+
+        return {
+          tracks: [track],
+          ...(lyrics ? { lyrics: [lyrics] } : {}),
+          model,
+          metadata: {
+            ...(payload.task_id?.trim() ? { taskId: payload.task_id.trim() } : {}),
+            ...(audioUrl ? { audioUrl } : {}),
+            instrumental: req.instrumental === true,
+            ...(req.lyrics?.trim() ? { requestedLyrics: true } : {}),
+            ...(typeof req.durationSeconds === "number"
+              ? { requestedDurationSeconds: req.durationSeconds }
+              : {}),
+          },
+        };
+      } finally {
+        await release();
+      }
+    },
+  };
+}

extensions/minimax/openclaw.plugin.json

@@ -64,6 +64,7 @@
     "speechProviders": ["minimax"],
     "mediaUnderstandingProviders": ["minimax", "minimax-portal"],
     "imageGenerationProviders": ["minimax", "minimax-portal"],
+    "musicGenerationProviders": ["minimax"],
     "videoGenerationProviders": ["minimax"],
     "webSearchProviders": ["minimax"]
   },