vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-06-22 11:48:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): require live media providers

Browse Source
This commit is contained in:
Vincent Koc
2026-06-07 00:25:05 +02:00
parent 84bcae95a0
commit d5ef040e65
3 changed files with 151 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
scripts/test-live-media.ts
View File

@@ -4,10 +4,7 @@
import type { ChildProcess } from "node:child_process";
import { createRequire } from "node:module";
import { pathToFileURL } from "node:url";
import { collectProviderApiKeys } from "../src/agents/live-auth-keys.js";
import { formatErrorMessage } from "../src/infra/errors.ts";
import { loadShellEnvFallback } from "../src/infra/shell-env.js";
import { getProviderEnvVars } from "../src/secrets/provider-env-vars.js";
type SpawnPnpmRunner = (params: {
pnpmArgs: string[];
stdio: "inherit";
@@ -81,6 +78,7 @@ export const MEDIA_SUITES: Record<MediaSuiteId, MediaSuiteConfig> = {
const DEFAULT_SUITES: MediaSuiteId[] = ["image", "music", "video"];
export type CliOptions = {
allowEmpty: boolean;
suites: MediaSuiteId[];
globalProviders: Set<string> | null;
suiteProviders: Partial<Record<MediaSuiteId, Set<string>>>;
@@ -96,6 +94,22 @@ export type SuiteRunPlan = {
skippedReason?: string;
};
export type BuildRunPlanDeps = {
collectProviderApiKeysImpl?: (provider: string) => Promise<unknown[]> | unknown[];
getProviderEnvVarsImpl?: (provider: string) => Promise<string[]> | string[];
loadShellEnvFallbackImpl?: (params: {
enabled: true;
env: NodeJS.ProcessEnv;
expectedKeys: string[];
logger: { warn: (message: string) => void };
}) => Promise<void> | void;
};
export type RunCliDeps = {
buildRunPlanImpl?: (options: CliOptions) => Promise<SuiteRunPlan[]> | SuiteRunPlan[];
runSuiteImpl?: typeof runSuite;
};
function formatProviderList(providers: Iterable<string>): string {
return [...providers].toSorted().join(", ");
}
@@ -108,6 +122,26 @@ function spawnLivePnpm(params: { pnpmArgs: string[]; env: NodeJS.ProcessEnv }):
});
}
async function collectProviderApiKeysForLiveMedia(provider: string): Promise<unknown[]> {
const { collectProviderApiKeys } = await import("../src/agents/live-auth-keys.js");
return collectProviderApiKeys(provider);
}
async function getProviderEnvVarsForLiveMedia(provider: string): Promise<string[]> {
const { getProviderEnvVars } = await import("../src/secrets/provider-env-vars.js");
return getProviderEnvVars(provider);
}
async function loadShellEnvFallbackForLiveMedia(params: {
enabled: true;
env: NodeJS.ProcessEnv;
expectedKeys: string[];
logger: { warn: (message: string) => void };
}): Promise<void> {
const { loadShellEnvFallback } = await import("../src/infra/shell-env.js");
loadShellEnvFallback(params);
}
function parseCsv(raw: string | undefined): Set<string> | null {
const trimmed = raw?.trim();
if (!trimmed) {
@@ -139,6 +173,7 @@ export function parseArgs(argv: string[]): CliOptions {
let globalProviders: Set<string> | null = null;
let requireAuth = true;
let help = false;
let allowEmpty = false;
const readValue = (index: number): string => {
const value = optionArgs[index + 1]?.trim();
@@ -184,6 +219,10 @@ export function parseArgs(argv: string[]): CliOptions {
requireAuth = true;
continue;
}
if (arg === "--allow-empty") {
allowEmpty = true;
continue;
}
if (arg === "--all-providers" || arg === "--no-auth-filter") {
requireAuth = false;
continue;
@@ -212,6 +251,7 @@ export function parseArgs(argv: string[]): CliOptions {
}
const options = {
allowEmpty,
suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(),
globalProviders,
suiteProviders,
@@ -276,12 +316,13 @@ export function findSkippedExplicitProviderSelections(
);
}
function selectProviders(params: {
async function selectProviders(params: {
collectProviderApiKeysImpl?: BuildRunPlanDeps["collectProviderApiKeysImpl"];
suite: MediaSuiteConfig;
globalProviders: Set<string> | null;
suiteProviders: Set<string> | undefined;
requireAuth: boolean;
}): string[] {
}): Promise<string[]> {
const explicit = params.suiteProviders ?? params.globalProviders;
const candidates = explicit
? params.suite.providers
@@ -290,20 +331,36 @@ function selectProviders(params: {
if (!params.requireAuth) {
return providers;
}
providers = providers.filter((provider) => collectProviderApiKeys(provider).length > 0);
const providerAuth = await Promise.all(
providers.map(async (provider) => ({
provider,
hasAuth:
(await (params.collectProviderApiKeysImpl ?? collectProviderApiKeysForLiveMedia)(provider))
.length > 0,
})),
);
providers = providerAuth.filter((entry) => entry.hasAuth).map((entry) => entry.provider);
return providers;
}
export function buildRunPlan(options: CliOptions): SuiteRunPlan[] {
export async function buildRunPlan(
options: CliOptions,
deps: BuildRunPlanDeps = {},
): Promise<SuiteRunPlan[]> {
const getProviderEnvVarsImpl = deps.getProviderEnvVarsImpl ?? getProviderEnvVarsForLiveMedia;
const expectedKeys = [
...new Set(
options.suites.flatMap((suiteId) =>
MEDIA_SUITES[suiteId].providers.flatMap((provider) => getProviderEnvVars(provider)),
),
(
await Promise.all(
options.suites.flatMap((suiteId) =>
MEDIA_SUITES[suiteId].providers.map((provider) => getProviderEnvVarsImpl(provider)),
),
)
).flat(),
),
];
if (expectedKeys.length) {
loadShellEnvFallback({
await (deps.loadShellEnvFallbackImpl ?? loadShellEnvFallbackForLiveMedia)({
enabled: true,
env: process.env,
expectedKeys,
@@ -311,26 +368,29 @@ export function buildRunPlan(options: CliOptions): SuiteRunPlan[] {
});
}
return options.suites.map((suiteId) => {
const suite = MEDIA_SUITES[suiteId];
const providers = selectProviders({
suite,
globalProviders: options.globalProviders,
suiteProviders: options.suiteProviders[suiteId],
requireAuth: options.requireAuth,
});
return {
suite,
providers,
...(providers.length === 0
? {
skippedReason: options.requireAuth
? "no providers with usable auth"
: "no providers selected",
}
: {}),
};
});
return await Promise.all(
options.suites.map(async (suiteId) => {
const suite = MEDIA_SUITES[suiteId];
const providers = await selectProviders({
collectProviderApiKeysImpl: deps.collectProviderApiKeysImpl,
suite,
globalProviders: options.globalProviders,
suiteProviders: options.suiteProviders[suiteId],
requireAuth: options.requireAuth,
});
return {
suite,
providers,
...(providers.length === 0
? {
skippedReason: options.requireAuth
? "no providers with usable auth"
: "no providers selected",
}
: {}),
};
}),
);
}
function printHelp(): void {
@@ -355,6 +415,7 @@ Flags:
--music-providers <csv> music-suite provider filter
--video-providers <csv> video-suite provider filter
--all-providers do not auto-filter by available auth
--allow-empty exit 0 when auth filtering leaves no runnable providers
--quiet | --no-quiet passed through to test:live
`);
}
@@ -401,13 +462,13 @@ async function runSuite(params: {
});
}
export async function runCli(argv: string[]): Promise<number> {
export async function runCli(argv: string[], deps: RunCliDeps = {}): Promise<number> {
const options = parseArgs(argv);
if (options.help) {
printHelp();
return 0;
}
const plan = buildRunPlan(options);
const plan = await (deps.buildRunPlanImpl ?? buildRunPlan)(options);
const runnable = plan.filter((entry) => entry.providers.length > 0);
const skipped = plan.filter((entry) => entry.providers.length === 0);
@@ -425,15 +486,19 @@ export async function runCli(argv: string[]): Promise<number> {
}
if (runnable.length === 0) {
console.log("[live:media] nothing to run");
if (hasExplicitProviderSelection(options)) {
console.error("[live:media] no runnable providers matched the explicit provider selection");
return 1;
if (options.allowEmpty) {
return 0;
}
return 0;
console.error(
hasExplicitProviderSelection(options)
? "[live:media] no runnable providers matched the explicit provider selection"
: "[live:media] no runnable providers matched available auth; pass --allow-empty to accept an empty live-media run",
);
return 1;
}
for (const entry of runnable) {
const exitCode = await runSuite({
const exitCode = await (deps.runSuiteImpl ?? runSuite)({
plan: entry,
quietArgs: options.quietArgs,
passthroughArgs: options.passthroughArgs,

23
src/scripts/test-live-media.test.ts
View File

@@ -6,16 +6,8 @@ const collectProviderApiKeysMock = vi.fn((provider: string) =>
process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [],
);
vi.mock("../../src/infra/shell-env.js", () => ({
loadShellEnvFallback: loadShellEnvFallbackMock,
}));
vi.mock("../../src/agents/live-auth-keys.js", () => ({
collectProviderApiKeys: collectProviderApiKeysMock,
}));
function requirePlanEntry(
plan: ReturnType<typeof import("../../scripts/test-live-media.ts").buildRunPlan>,
plan: Awaited<ReturnType<typeof import("../../scripts/test-live-media.ts").buildRunPlan>>,
suiteId: string,
) {
const entry = plan.find((candidate) => candidate.suite.id === suiteId);
@@ -40,7 +32,11 @@ describe("test-live-media", () => {
vi.stubEnv("TEST_AUTH_VYDRA", "1");
const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts");
const plan = buildRunPlan(parseArgs([]));
const plan = await buildRunPlan(parseArgs([]), {
collectProviderApiKeysImpl: collectProviderApiKeysMock,
getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`],
loadShellEnvFallbackImpl: loadShellEnvFallbackMock,
});
expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]);
expect(requirePlanEntry(plan, "image").providers).toEqual([
@@ -61,8 +57,13 @@ describe("test-live-media", () => {
it("supports suite-specific provider filters without auth narrowing", async () => {
const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts");
const plan = buildRunPlan(
const plan = await buildRunPlan(
parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]),
{
collectProviderApiKeysImpl: collectProviderApiKeysMock,
getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`],
loadShellEnvFallbackImpl: loadShellEnvFallbackMock,
},
);
expect(plan).toHaveLength(1);

36
test/scripts/test-live-media.test.ts
View File

@@ -4,6 +4,7 @@ import {
MEDIA_SUITES,
findSkippedExplicitProviderSelections,
parseArgs,
runCli,
} from "../../scripts/test-live-media.ts";
describe("scripts/test-live-media", () => {
@@ -44,6 +45,13 @@ describe("scripts/test-live-media", () => {
});
});
it("parses the explicit empty-run escape hatch", () => {
expect(parseArgs(["--allow-empty"])).toMatchObject({
allowEmpty: true,
requireAuth: true,
});
});
it("fails explicit suite selections that auth filtering would skip", () => {
const options = parseArgs([
"image",
@@ -79,4 +87,32 @@ describe("scripts/test-live-media", () => {
expect(skipped).toEqual([]);
});
it("fails default live media runs when auth filtering leaves no providers", async () => {
await expect(
runCli(["image"], {
buildRunPlanImpl: () => [
{
providers: [],
skippedReason: "no providers with usable auth",
suite: MEDIA_SUITES.image,
},
],
}),
).resolves.toBe(1);
});
it("allows empty live media runs only with an explicit escape hatch", async () => {
await expect(
runCli(["image", "--allow-empty"], {
buildRunPlanImpl: () => [
{
providers: [],
skippedReason: "no providers with usable auth",
suite: MEDIA_SUITES.image,
},
],
}),
).resolves.toBe(0);
});
});