vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-14 10:41:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add music generation tooling

Browse Source
This commit is contained in:
Peter Steinberger
2026-04-06 01:43:08 +01:00
parent 3de91d9e01
commit dc0ee2e178
79 changed files with 3538 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
src/plugins/.DS_Store vendored Normal file
View File

Binary file not shown.

5
src/plugins/api-builder.ts
View File

@@ -37,6 +37,7 @@ export type BuildPluginApiParams = {
| "registerMediaUnderstandingProvider"
| "registerImageGenerationProvider"
| "registerVideoGenerationProvider"
| "registerMusicGenerationProvider"
| "registerWebFetchProvider"
| "registerWebSearchProvider"
| "registerInteractiveHandler"
@@ -77,6 +78,8 @@ const noopRegisterImageGenerationProvider: OpenClawPluginApi["registerImageGener
() => {};
const noopRegisterVideoGenerationProvider: OpenClawPluginApi["registerVideoGenerationProvider"] =
() => {};
const noopRegisterMusicGenerationProvider: OpenClawPluginApi["registerMusicGenerationProvider"] =
() => {};
const noopRegisterWebFetchProvider: OpenClawPluginApi["registerWebFetchProvider"] = () => {};
const noopRegisterWebSearchProvider: OpenClawPluginApi["registerWebSearchProvider"] = () => {};
const noopRegisterInteractiveHandler: OpenClawPluginApi["registerInteractiveHandler"] = () => {};
@@ -130,6 +133,8 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi
handlers.registerImageGenerationProvider ?? noopRegisterImageGenerationProvider,
registerVideoGenerationProvider:
handlers.registerVideoGenerationProvider ?? noopRegisterVideoGenerationProvider,
registerMusicGenerationProvider:
handlers.registerMusicGenerationProvider ?? noopRegisterMusicGenerationProvider,
registerWebFetchProvider: handlers.registerWebFetchProvider ?? noopRegisterWebFetchProvider,
registerWebSearchProvider: handlers.registerWebSearchProvider ?? noopRegisterWebSearchProvider,
registerInteractiveHandler:

2
src/plugins/bundled-capability-metadata.test.ts
View File

@@ -36,6 +36,7 @@ describe("bundled capability metadata", () => {
),
imageGenerationProviderIds: uniqueStrings(manifest.contracts?.imageGenerationProviders),
videoGenerationProviderIds: uniqueStrings(manifest.contracts?.videoGenerationProviders),
musicGenerationProviderIds: uniqueStrings(manifest.contracts?.musicGenerationProviders),
webFetchProviderIds: uniqueStrings(manifest.contracts?.webFetchProviders),
webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders),
toolNames: uniqueStrings(manifest.contracts?.tools),
@@ -49,6 +50,7 @@ describe("bundled capability metadata", () => {
entry.mediaUnderstandingProviderIds.length > 0 ||
entry.imageGenerationProviderIds.length > 0 ||
entry.videoGenerationProviderIds.length > 0 ||
entry.musicGenerationProviderIds.length > 0 ||
entry.webFetchProviderIds.length > 0 ||
entry.webSearchProviderIds.length > 0 ||
entry.toolNames.length > 0,

13
src/plugins/bundled-capability-runtime.ts
View File

@@ -126,6 +126,7 @@ function createCapabilityPluginRecord(params: {
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
musicGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
memoryEmbeddingProviderIds: [],
@@ -289,6 +290,9 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
record.videoGenerationProviderIds.push(
...captured.videoGenerationProviders.map((entry) => entry.id),
);
record.musicGenerationProviderIds.push(
...captured.musicGenerationProviders.map((entry) => entry.id),
);
record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id));
record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id));
record.memoryEmbeddingProviderIds.push(
@@ -359,6 +363,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
rootDir: record.rootDir,
})),
);
registry.musicGenerationProviders.push(
...captured.musicGenerationProviders.map((provider) => ({
pluginId: record.id,
pluginName: record.name,
provider,
source: record.source,
rootDir: record.rootDir,
})),
);
registry.webFetchProviders.push(
...captured.webFetchProviders.map((provider) => ({
pluginId: record.id,

7
src/plugins/capability-provider-runtime.ts
View File

@@ -14,7 +14,8 @@ type CapabilityProviderRegistryKey =
| "realtimeVoiceProviders"
| "mediaUnderstandingProviders"
| "imageGenerationProviders"
| "videoGenerationProviders";
| "videoGenerationProviders"
| "musicGenerationProviders";
type CapabilityContractKey =
| "memoryEmbeddingProviders"
@@ -23,7 +24,8 @@ type CapabilityContractKey =
| "realtimeVoiceProviders"
| "mediaUnderstandingProviders"
| "imageGenerationProviders"
| "videoGenerationProviders";
| "videoGenerationProviders"
| "musicGenerationProviders";
type CapabilityProviderForKey<K extends CapabilityProviderRegistryKey> =
PluginRegistry[K][number] extends { provider: infer T } ? T : never;
@@ -36,6 +38,7 @@ const CAPABILITY_CONTRACT_KEY: Record<CapabilityProviderRegistryKey, CapabilityC
mediaUnderstandingProviders: "mediaUnderstandingProviders",
imageGenerationProviders: "imageGenerationProviders",
videoGenerationProviders: "videoGenerationProviders",
musicGenerationProviders: "musicGenerationProviders",
};
function resolveBundledCapabilityCompatPluginIds(params: {

7
src/plugins/captured-registration.ts
View File

@@ -6,6 +6,7 @@ import type {
AnyAgentTool,
ImageGenerationProviderPlugin,
MediaUnderstandingProviderPlugin,
MusicGenerationProviderPlugin,
OpenClawPluginApi,
OpenClawPluginCliCommandDescriptor,
OpenClawPluginCliRegistrar,
@@ -34,6 +35,7 @@ export type CapturedPluginRegistration = {
mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[];
imageGenerationProviders: ImageGenerationProviderPlugin[];
videoGenerationProviders: VideoGenerationProviderPlugin[];
musicGenerationProviders: MusicGenerationProviderPlugin[];
webFetchProviders: WebFetchProviderPlugin[];
webSearchProviders: WebSearchProviderPlugin[];
memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[];
@@ -52,6 +54,7 @@ export function createCapturedPluginRegistration(params?: {
const mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[] = [];
const imageGenerationProviders: ImageGenerationProviderPlugin[] = [];
const videoGenerationProviders: VideoGenerationProviderPlugin[] = [];
const musicGenerationProviders: MusicGenerationProviderPlugin[] = [];
const webFetchProviders: WebFetchProviderPlugin[] = [];
const webSearchProviders: WebSearchProviderPlugin[] = [];
const memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[] = [];
@@ -72,6 +75,7 @@ export function createCapturedPluginRegistration(params?: {
mediaUnderstandingProviders,
imageGenerationProviders,
videoGenerationProviders,
musicGenerationProviders,
webFetchProviders,
webSearchProviders,
memoryEmbeddingProviders,
@@ -130,6 +134,9 @@ export function createCapturedPluginRegistration(params?: {
registerVideoGenerationProvider(provider: VideoGenerationProviderPlugin) {
videoGenerationProviders.push(provider);
},
registerMusicGenerationProvider(provider: MusicGenerationProviderPlugin) {
musicGenerationProviders.push(provider);
},
registerWebFetchProvider(provider: WebFetchProviderPlugin) {
webFetchProviders.push(provider);
},

1
src/plugins/channel-plugin-ids.ts
View File

@@ -15,6 +15,7 @@ function hasRuntimeContractSurface(plugin: PluginManifestRecord): boolean {
plugin.contracts?.mediaUnderstandingProviders?.length ||
plugin.contracts?.imageGenerationProviders?.length ||
plugin.contracts?.videoGenerationProviders?.length ||
plugin.contracts?.musicGenerationProviders?.length ||
plugin.contracts?.webFetchProviders?.length ||
plugin.contracts?.webSearchProviders?.length ||
plugin.contracts?.memoryEmbeddingProviders?.length ||

3
src/plugins/contracts/inventory/bundled-capability-metadata.ts
View File

@@ -12,6 +12,7 @@ export type BundledPluginContractSnapshot = {
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
musicGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
toolNames: string[];
@@ -48,6 +49,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
mediaUnderstandingProviderIds: uniqueStrings(manifest.contracts?.mediaUnderstandingProviders),
imageGenerationProviderIds: uniqueStrings(manifest.contracts?.imageGenerationProviders),
videoGenerationProviderIds: uniqueStrings(manifest.contracts?.videoGenerationProviders),
musicGenerationProviderIds: uniqueStrings(manifest.contracts?.musicGenerationProviders),
webFetchProviderIds: uniqueStrings(manifest.contracts?.webFetchProviders),
webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders),
toolNames: uniqueStrings(manifest.contracts?.tools),
@@ -61,6 +63,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
entry.mediaUnderstandingProviderIds.length > 0 ||
entry.imageGenerationProviderIds.length > 0 ||
entry.videoGenerationProviderIds.length > 0 ||
entry.musicGenerationProviderIds.length > 0 ||
entry.webFetchProviderIds.length > 0 ||
entry.webSearchProviderIds.length > 0 ||
entry.toolNames.length > 0,

28
src/plugins/contracts/registry.ts
View File

@@ -6,6 +6,7 @@ import {
import type {
ImageGenerationProviderPlugin,
MediaUnderstandingProviderPlugin,
MusicGenerationProviderPlugin,
ProviderPlugin,
RealtimeTranscriptionProviderPlugin,
RealtimeVoiceProviderPlugin,
@@ -17,6 +18,7 @@ import type {
import {
loadVitestImageGenerationProviderContractRegistry,
loadVitestMediaUnderstandingProviderContractRegistry,
loadVitestMusicGenerationProviderContractRegistry,
loadVitestRealtimeTranscriptionProviderContractRegistry,
loadVitestRealtimeVoiceProviderContractRegistry,
loadVitestSpeechProviderContractRegistry,
@@ -44,6 +46,7 @@ type MediaUnderstandingProviderContractEntry =
CapabilityContractEntry<MediaUnderstandingProviderPlugin>;
type ImageGenerationProviderContractEntry = CapabilityContractEntry<ImageGenerationProviderPlugin>;
type VideoGenerationProviderContractEntry = CapabilityContractEntry<VideoGenerationProviderPlugin>;
type MusicGenerationProviderContractEntry = CapabilityContractEntry<MusicGenerationProviderPlugin>;
type PluginRegistrationContractEntry = {
pluginId: string;
@@ -54,6 +57,7 @@ type PluginRegistrationContractEntry = {
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
musicGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
toolNames: string[];
@@ -66,6 +70,7 @@ type ManifestContractKey =
| "mediaUnderstandingProviders"
| "imageGenerationProviders"
| "videoGenerationProviders"
| "musicGenerationProviders"
| "webFetchProviders"
| "webSearchProviders"
| "tools";
@@ -97,6 +102,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] {
(plugin.contracts?.mediaUnderstandingProviders?.length ?? 0) > 0 ||
(plugin.contracts?.imageGenerationProviders?.length ?? 0) > 0 ||
(plugin.contracts?.videoGenerationProviders?.length ?? 0) > 0 ||
(plugin.contracts?.musicGenerationProviders?.length ?? 0) > 0 ||
(plugin.contracts?.webFetchProviders?.length ?? 0) > 0 ||
(plugin.contracts?.webSearchProviders?.length ?? 0) > 0 ||
(plugin.contracts?.tools?.length ?? 0) > 0),
@@ -114,6 +120,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] {
),
imageGenerationProviderIds: uniqueStrings(plugin.contracts?.imageGenerationProviders ?? []),
videoGenerationProviderIds: uniqueStrings(plugin.contracts?.videoGenerationProviders ?? []),
musicGenerationProviderIds: uniqueStrings(plugin.contracts?.musicGenerationProviders ?? []),
webFetchProviderIds: uniqueStrings(plugin.contracts?.webFetchProviders ?? []),
webSearchProviderIds: uniqueStrings(plugin.contracts?.webSearchProviders ?? []),
toolNames: uniqueStrings(plugin.contracts?.tools ?? []),
@@ -166,6 +173,8 @@ function resolveBundledManifestPluginIdsForContract(contract: ManifestContractKe
return entry.imageGenerationProviderIds.length > 0;
case "videoGenerationProviders":
return entry.videoGenerationProviderIds.length > 0;
case "musicGenerationProviders":
return entry.musicGenerationProviderIds.length > 0;
case "webFetchProviders":
return entry.webFetchProviderIds.length > 0;
case "webSearchProviders":
@@ -202,6 +211,8 @@ let imageGenerationProviderContractRegistryCache: ImageGenerationProviderContrac
null;
let videoGenerationProviderContractRegistryCache: VideoGenerationProviderContractEntry[] | null =
null;
let musicGenerationProviderContractRegistryCache: MusicGenerationProviderContractEntry[] | null =
null;
export let providerContractLoadError: Error | undefined;
@@ -564,6 +575,21 @@ function loadVideoGenerationProviderContractRegistry(): VideoGenerationProviderC
return videoGenerationProviderContractRegistryCache;
}
function loadMusicGenerationProviderContractRegistry(): MusicGenerationProviderContractEntry[] {
if (!musicGenerationProviderContractRegistryCache) {
musicGenerationProviderContractRegistryCache = process.env.VITEST
? loadVitestMusicGenerationProviderContractRegistry()
: loadBundledCapabilityRuntimeRegistry({
pluginIds: resolveBundledManifestPluginIdsForContract("musicGenerationProviders"),
pluginSdkResolution: "dist",
}).musicGenerationProviders.map((entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
}));
}
return musicGenerationProviderContractRegistryCache;
}
function createLazyArrayView<T>(load: () => T[]): T[] {
return new Proxy([] as T[], {
get(_target, prop) {
@@ -667,6 +693,8 @@ export const imageGenerationProviderContractRegistry: ImageGenerationProviderCon
createLazyArrayView(loadImageGenerationProviderContractRegistry);
export const videoGenerationProviderContractRegistry: VideoGenerationProviderContractEntry[] =
createLazyArrayView(loadVideoGenerationProviderContractRegistry);
export const musicGenerationProviderContractRegistry: MusicGenerationProviderContractEntry[] =
createLazyArrayView(loadMusicGenerationProviderContractRegistry);
function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEntry[] {
return resolveBundledManifestContracts();

20
src/plugins/contracts/speech-vitest-registry.ts
View File

@@ -3,6 +3,7 @@ import { resolveManifestContractPluginIds } from "../manifest-registry.js";
import type {
ImageGenerationProviderPlugin,
MediaUnderstandingProviderPlugin,
MusicGenerationProviderPlugin,
RealtimeTranscriptionProviderPlugin,
RealtimeVoiceProviderPlugin,
SpeechProviderPlugin,
@@ -39,13 +40,19 @@ export type VideoGenerationProviderContractEntry = {
provider: VideoGenerationProviderPlugin;
};
export type MusicGenerationProviderContractEntry = {
pluginId: string;
provider: MusicGenerationProviderPlugin;
};
type ManifestContractKey =
| "imageGenerationProviders"
| "speechProviders"
| "mediaUnderstandingProviders"
| "realtimeVoiceProviders"
| "realtimeTranscriptionProviders"
| "videoGenerationProviders";
| "videoGenerationProviders"
| "musicGenerationProviders";
function loadVitestCapabilityContractEntries<T>(params: {
contract: ManifestContractKey;
@@ -134,3 +141,14 @@ export function loadVitestVideoGenerationProviderContractRegistry(): VideoGenera
})),
});
}
export function loadVitestMusicGenerationProviderContractRegistry(): MusicGenerationProviderContractEntry[] {
return loadVitestCapabilityContractEntries({
contract: "musicGenerationProviders",
pickEntries: (registry) =>
registry.musicGenerationProviders.map((entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
})),
});
}

1
src/plugins/hooks.test-helpers.ts
View File

@@ -39,6 +39,7 @@ export function createMockPluginRegistry(
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
musicGenerationProviders: [],
webSearchProviders: [],
httpRoutes: [],
gatewayHandlers: {},

1
src/plugins/loader.ts
View File

@@ -596,6 +596,7 @@ function createPluginRecord(params: {
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
musicGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
memoryEmbeddingProviderIds: [],

1
src/plugins/manifest-registry.ts
View File

@@ -36,6 +36,7 @@ type PluginManifestContractListKey =
| "realtimeTranscriptionProviders"
| "imageGenerationProviders"
| "videoGenerationProviders"
| "musicGenerationProviders"
| "memoryEmbeddingProviders"
| "webFetchProviders"
| "webSearchProviders";

3
src/plugins/manifest.ts
View File

@@ -76,6 +76,7 @@ export type PluginManifestContracts = {
mediaUnderstandingProviders?: string[];
imageGenerationProviders?: string[];
videoGenerationProviders?: string[];
musicGenerationProviders?: string[];
webFetchProviders?: string[];
webSearchProviders?: string[];
tools?: string[];
@@ -157,6 +158,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u
const mediaUnderstandingProviders = normalizeStringList(value.mediaUnderstandingProviders);
const imageGenerationProviders = normalizeStringList(value.imageGenerationProviders);
const videoGenerationProviders = normalizeStringList(value.videoGenerationProviders);
const musicGenerationProviders = normalizeStringList(value.musicGenerationProviders);
const webFetchProviders = normalizeStringList(value.webFetchProviders);
const webSearchProviders = normalizeStringList(value.webSearchProviders);
const tools = normalizeStringList(value.tools);
@@ -168,6 +170,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u
...(mediaUnderstandingProviders.length > 0 ? { mediaUnderstandingProviders } : {}),
...(imageGenerationProviders.length > 0 ? { imageGenerationProviders } : {}),
...(videoGenerationProviders.length > 0 ? { videoGenerationProviders } : {}),
...(musicGenerationProviders.length > 0 ? { musicGenerationProviders } : {}),
...(webFetchProviders.length > 0 ? { webFetchProviders } : {}),
...(webSearchProviders.length > 0 ? { webSearchProviders } : {}),
...(tools.length > 0 ? { tools } : {}),

1
src/plugins/registry-empty.ts
View File

@@ -15,6 +15,7 @@ export function createEmptyPluginRegistry(): PluginRegistry {
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
musicGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
memoryEmbeddingProviders: [],

21
src/plugins/registry.ts
View File

@@ -45,6 +45,7 @@ import {
} from "./types.js";
import type {
ImageGenerationProviderPlugin,
MusicGenerationProviderPlugin,
RealtimeTranscriptionProviderPlugin,
OpenClawPluginApi,
OpenClawPluginChannelRegistration,
@@ -157,6 +158,8 @@ export type PluginImageGenerationProviderRegistration =
PluginOwnedProviderRegistration<ImageGenerationProviderPlugin>;
export type PluginVideoGenerationProviderRegistration =
PluginOwnedProviderRegistration<VideoGenerationProviderPlugin>;
export type PluginMusicGenerationProviderRegistration =
PluginOwnedProviderRegistration<MusicGenerationProviderPlugin>;
export type PluginWebFetchProviderRegistration =
PluginOwnedProviderRegistration<WebFetchProviderPlugin>;
export type PluginWebSearchProviderRegistration =
@@ -254,6 +257,7 @@ export type PluginRecord = {
mediaUnderstandingProviderIds: string[];
imageGenerationProviderIds: string[];
videoGenerationProviderIds: string[];
musicGenerationProviderIds: string[];
webFetchProviderIds: string[];
webSearchProviderIds: string[];
memoryEmbeddingProviderIds: string[];
@@ -284,6 +288,7 @@ export type PluginRegistry = {
mediaUnderstandingProviders: PluginMediaUnderstandingProviderRegistration[];
imageGenerationProviders: PluginImageGenerationProviderRegistration[];
videoGenerationProviders: PluginVideoGenerationProviderRegistration[];
musicGenerationProviders: PluginMusicGenerationProviderRegistration[];
webFetchProviders: PluginWebFetchProviderRegistration[];
webSearchProviders: PluginWebSearchProviderRegistration[];
memoryEmbeddingProviders: PluginMemoryEmbeddingProviderRegistration[];
@@ -787,6 +792,19 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
});
};
const registerMusicGenerationProvider = (
record: PluginRecord,
provider: MusicGenerationProviderPlugin,
) => {
registerUniqueProviderLike({
record,
provider,
kindLabel: "music-generation provider",
registrations: registry.musicGenerationProviders,
ownedIds: record.musicGenerationProviderIds,
});
};
const registerWebFetchProvider = (record: PluginRecord, provider: WebFetchProviderPlugin) => {
registerUniqueProviderLike({
record,
@@ -1179,6 +1197,8 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
registerImageGenerationProvider(record, provider),
registerVideoGenerationProvider: (provider) =>
registerVideoGenerationProvider(record, provider),
registerMusicGenerationProvider: (provider) =>
registerMusicGenerationProvider(record, provider),
registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider),
registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider),
registerGatewayMethod: (method, handler, opts) =>
@@ -1381,6 +1401,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
registerMediaUnderstandingProvider,
registerImageGenerationProvider,
registerVideoGenerationProvider,
registerMusicGenerationProvider,
registerWebSearchProvider,
registerGatewayMethod,
registerCli,

2
src/plugins/runtime.test.ts
View File

@@ -203,6 +203,7 @@ describe("setActivePluginRegistry", () => {
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
musicGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
memoryEmbeddingProviderIds: [],
@@ -232,6 +233,7 @@ describe("setActivePluginRegistry", () => {
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
musicGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
memoryEmbeddingProviderIds: [],

28
src/plugins/runtime/index.ts
View File

@@ -3,6 +3,10 @@ import {
generateImage as generateRuntimeImage,
listRuntimeImageGenerationProviders,
} from "../../image-generation/runtime.js";
import {
generateMusic as generateRuntimeMusic,
listRuntimeMusicGenerationProviders,
} from "../../music-generation/runtime.js";
import { resolveGlobalSingleton } from "../../shared/global-singleton.js";
import {
createLazyRuntimeMethod,
@@ -73,6 +77,13 @@ function createRuntimeVideoGeneration(): PluginRuntime["videoGeneration"] {
};
}
function createRuntimeMusicGeneration(): PluginRuntime["musicGeneration"] {
return {
generate: (params) => generateRuntimeMusic(params),
listProviders: (params) => listRuntimeMusicGenerationProviders(params),
};
}
function createRuntimeModelAuth(): PluginRuntime["modelAuth"] {
const getApiKeyForModel = createLazyRuntimeMethod(
loadModelAuthRuntime,
@@ -211,12 +222,24 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}):
taskFlow,
} satisfies Omit<
PluginRuntime,
"tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration"
| "tts"
| "mediaUnderstanding"
| "stt"
| "modelAuth"
| "imageGeneration"
| "videoGeneration"
| "musicGeneration"
> &
Partial<
Pick<
PluginRuntime,
"tts" | "mediaUnderstanding" | "stt" | "modelAuth" | "imageGeneration" | "videoGeneration"
| "tts"
| "mediaUnderstanding"
| "stt"
| "modelAuth"
| "imageGeneration"
| "videoGeneration"
| "musicGeneration"
>
>;
@@ -228,6 +251,7 @@ export function createPluginRuntime(_options: CreatePluginRuntimeOptions = {}):
defineCachedValue(runtime, "modelAuth", createRuntimeModelAuth);
defineCachedValue(runtime, "imageGeneration", createRuntimeImageGeneration);
defineCachedValue(runtime, "videoGeneration", createRuntimeVideoGeneration);
defineCachedValue(runtime, "musicGeneration", createRuntimeMusicGeneration);
return runtime as PluginRuntime;
}

4
src/plugins/runtime/types-core.ts
View File

@@ -86,6 +86,10 @@ export type PluginRuntimeCore = {
generate: typeof import("../../video-generation/runtime.js").generateVideo;
listProviders: typeof import("../../video-generation/runtime.js").listRuntimeVideoGenerationProviders;
};
musicGeneration: {
generate: typeof import("../../music-generation/runtime.js").generateMusic;
listProviders: typeof import("../../music-generation/runtime.js").listRuntimeMusicGenerationProviders;
};
webSearch: {
listProviders: typeof import("../../web-search/runtime.js").listWebSearchProviders;
search: typeof import("../../web-search/runtime.js").runWebSearch;

2
src/plugins/status.test-helpers.ts
View File

@@ -55,6 +55,7 @@ export function createPluginRecord(
mediaUnderstandingProviderIds: [],
imageGenerationProviderIds: [],
videoGenerationProviderIds: [],
musicGenerationProviderIds: [],
webFetchProviderIds: [],
webSearchProviderIds: [],
memoryEmbeddingProviderIds: [],
@@ -121,6 +122,7 @@ export function createPluginLoadResult(
mediaUnderstandingProviders: [],
imageGenerationProviders: [],
videoGenerationProviders: [],
musicGenerationProviders: [],
webFetchProviders: [],
webSearchProviders: [],
memoryEmbeddingProviders: [],

4
src/plugins/types.ts
View File

@@ -32,6 +32,7 @@ import type { HookEntry } from "../hooks/types.js";
import type { ImageGenerationProvider } from "../image-generation/types.js";
import type { ProviderUsageSnapshot } from "../infra/provider-usage.types.js";
import type { MediaUnderstandingProvider } from "../media-understanding/types.js";
import type { MusicGenerationProvider } from "../music-generation/types.js";
import type {
RealtimeTranscriptionProviderConfig,
RealtimeTranscriptionProviderConfiguredContext,
@@ -1724,6 +1725,7 @@ export type PluginRealtimeVoiceProviderEntry = RealtimeVoiceProviderPlugin & {
export type MediaUnderstandingProviderPlugin = MediaUnderstandingProvider;
export type ImageGenerationProviderPlugin = ImageGenerationProvider;
export type VideoGenerationProviderPlugin = VideoGenerationProvider;
export type MusicGenerationProviderPlugin = MusicGenerationProvider;
export type OpenClawPluginGatewayMethod = {
method: string;
@@ -2090,6 +2092,8 @@ export type OpenClawPluginApi = {
registerImageGenerationProvider: (provider: ImageGenerationProviderPlugin) => void;
/** Register a video generation provider (video generation capability). */
registerVideoGenerationProvider: (provider: VideoGenerationProviderPlugin) => void;
/** Register a music generation provider (music generation capability). */
registerMusicGenerationProvider: (provider: MusicGenerationProviderPlugin) => void;
/** Register a web fetch provider (web fetch capability). */
registerWebFetchProvider: (provider: WebFetchProviderPlugin) => void;
/** Register a web search provider (web search capability). */