diff --git a/scripts/test-live-media.ts b/scripts/test-live-media.ts index 2b1c362d606..8b60eda6b59 100644 --- a/scripts/test-live-media.ts +++ b/scripts/test-live-media.ts @@ -4,10 +4,7 @@ import type { ChildProcess } from "node:child_process"; import { createRequire } from "node:module"; import { pathToFileURL } from "node:url"; -import { collectProviderApiKeys } from "../src/agents/live-auth-keys.js"; import { formatErrorMessage } from "../src/infra/errors.ts"; -import { loadShellEnvFallback } from "../src/infra/shell-env.js"; -import { getProviderEnvVars } from "../src/secrets/provider-env-vars.js"; type SpawnPnpmRunner = (params: { pnpmArgs: string[]; stdio: "inherit"; @@ -81,6 +78,7 @@ export const MEDIA_SUITES: Record = { const DEFAULT_SUITES: MediaSuiteId[] = ["image", "music", "video"]; export type CliOptions = { + allowEmpty: boolean; suites: MediaSuiteId[]; globalProviders: Set | null; suiteProviders: Partial>>; @@ -96,6 +94,22 @@ export type SuiteRunPlan = { skippedReason?: string; }; +export type BuildRunPlanDeps = { + collectProviderApiKeysImpl?: (provider: string) => Promise | unknown[]; + getProviderEnvVarsImpl?: (provider: string) => Promise | string[]; + loadShellEnvFallbackImpl?: (params: { + enabled: true; + env: NodeJS.ProcessEnv; + expectedKeys: string[]; + logger: { warn: (message: string) => void }; + }) => Promise | void; +}; + +export type RunCliDeps = { + buildRunPlanImpl?: (options: CliOptions) => Promise | SuiteRunPlan[]; + runSuiteImpl?: typeof runSuite; +}; + function formatProviderList(providers: Iterable): string { return [...providers].toSorted().join(", "); } @@ -108,6 +122,26 @@ function spawnLivePnpm(params: { pnpmArgs: string[]; env: NodeJS.ProcessEnv }): }); } +async function collectProviderApiKeysForLiveMedia(provider: string): Promise { + const { collectProviderApiKeys } = await import("../src/agents/live-auth-keys.js"); + return collectProviderApiKeys(provider); +} + +async function getProviderEnvVarsForLiveMedia(provider: string): Promise { + const { getProviderEnvVars } = await import("../src/secrets/provider-env-vars.js"); + return getProviderEnvVars(provider); +} + +async function loadShellEnvFallbackForLiveMedia(params: { + enabled: true; + env: NodeJS.ProcessEnv; + expectedKeys: string[]; + logger: { warn: (message: string) => void }; +}): Promise { + const { loadShellEnvFallback } = await import("../src/infra/shell-env.js"); + loadShellEnvFallback(params); +} + function parseCsv(raw: string | undefined): Set | null { const trimmed = raw?.trim(); if (!trimmed) { @@ -139,6 +173,7 @@ export function parseArgs(argv: string[]): CliOptions { let globalProviders: Set | null = null; let requireAuth = true; let help = false; + let allowEmpty = false; const readValue = (index: number): string => { const value = optionArgs[index + 1]?.trim(); @@ -184,6 +219,10 @@ export function parseArgs(argv: string[]): CliOptions { requireAuth = true; continue; } + if (arg === "--allow-empty") { + allowEmpty = true; + continue; + } if (arg === "--all-providers" || arg === "--no-auth-filter") { requireAuth = false; continue; @@ -212,6 +251,7 @@ export function parseArgs(argv: string[]): CliOptions { } const options = { + allowEmpty, suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(), globalProviders, suiteProviders, @@ -276,12 +316,13 @@ export function findSkippedExplicitProviderSelections( ); } -function selectProviders(params: { +async function selectProviders(params: { + collectProviderApiKeysImpl?: BuildRunPlanDeps["collectProviderApiKeysImpl"]; suite: MediaSuiteConfig; globalProviders: Set | null; suiteProviders: Set | undefined; requireAuth: boolean; -}): string[] { +}): Promise { const explicit = params.suiteProviders ?? params.globalProviders; const candidates = explicit ? params.suite.providers @@ -290,20 +331,36 @@ function selectProviders(params: { if (!params.requireAuth) { return providers; } - providers = providers.filter((provider) => collectProviderApiKeys(provider).length > 0); + const providerAuth = await Promise.all( + providers.map(async (provider) => ({ + provider, + hasAuth: + (await (params.collectProviderApiKeysImpl ?? collectProviderApiKeysForLiveMedia)(provider)) + .length > 0, + })), + ); + providers = providerAuth.filter((entry) => entry.hasAuth).map((entry) => entry.provider); return providers; } -export function buildRunPlan(options: CliOptions): SuiteRunPlan[] { +export async function buildRunPlan( + options: CliOptions, + deps: BuildRunPlanDeps = {}, +): Promise { + const getProviderEnvVarsImpl = deps.getProviderEnvVarsImpl ?? getProviderEnvVarsForLiveMedia; const expectedKeys = [ ...new Set( - options.suites.flatMap((suiteId) => - MEDIA_SUITES[suiteId].providers.flatMap((provider) => getProviderEnvVars(provider)), - ), + ( + await Promise.all( + options.suites.flatMap((suiteId) => + MEDIA_SUITES[suiteId].providers.map((provider) => getProviderEnvVarsImpl(provider)), + ), + ) + ).flat(), ), ]; if (expectedKeys.length) { - loadShellEnvFallback({ + await (deps.loadShellEnvFallbackImpl ?? loadShellEnvFallbackForLiveMedia)({ enabled: true, env: process.env, expectedKeys, @@ -311,26 +368,29 @@ export function buildRunPlan(options: CliOptions): SuiteRunPlan[] { }); } - return options.suites.map((suiteId) => { - const suite = MEDIA_SUITES[suiteId]; - const providers = selectProviders({ - suite, - globalProviders: options.globalProviders, - suiteProviders: options.suiteProviders[suiteId], - requireAuth: options.requireAuth, - }); - return { - suite, - providers, - ...(providers.length === 0 - ? { - skippedReason: options.requireAuth - ? "no providers with usable auth" - : "no providers selected", - } - : {}), - }; - }); + return await Promise.all( + options.suites.map(async (suiteId) => { + const suite = MEDIA_SUITES[suiteId]; + const providers = await selectProviders({ + collectProviderApiKeysImpl: deps.collectProviderApiKeysImpl, + suite, + globalProviders: options.globalProviders, + suiteProviders: options.suiteProviders[suiteId], + requireAuth: options.requireAuth, + }); + return { + suite, + providers, + ...(providers.length === 0 + ? { + skippedReason: options.requireAuth + ? "no providers with usable auth" + : "no providers selected", + } + : {}), + }; + }), + ); } function printHelp(): void { @@ -355,6 +415,7 @@ Flags: --music-providers music-suite provider filter --video-providers video-suite provider filter --all-providers do not auto-filter by available auth + --allow-empty exit 0 when auth filtering leaves no runnable providers --quiet | --no-quiet passed through to test:live `); } @@ -401,13 +462,13 @@ async function runSuite(params: { }); } -export async function runCli(argv: string[]): Promise { +export async function runCli(argv: string[], deps: RunCliDeps = {}): Promise { const options = parseArgs(argv); if (options.help) { printHelp(); return 0; } - const plan = buildRunPlan(options); + const plan = await (deps.buildRunPlanImpl ?? buildRunPlan)(options); const runnable = plan.filter((entry) => entry.providers.length > 0); const skipped = plan.filter((entry) => entry.providers.length === 0); @@ -425,15 +486,19 @@ export async function runCli(argv: string[]): Promise { } if (runnable.length === 0) { console.log("[live:media] nothing to run"); - if (hasExplicitProviderSelection(options)) { - console.error("[live:media] no runnable providers matched the explicit provider selection"); - return 1; + if (options.allowEmpty) { + return 0; } - return 0; + console.error( + hasExplicitProviderSelection(options) + ? "[live:media] no runnable providers matched the explicit provider selection" + : "[live:media] no runnable providers matched available auth; pass --allow-empty to accept an empty live-media run", + ); + return 1; } for (const entry of runnable) { - const exitCode = await runSuite({ + const exitCode = await (deps.runSuiteImpl ?? runSuite)({ plan: entry, quietArgs: options.quietArgs, passthroughArgs: options.passthroughArgs, diff --git a/src/scripts/test-live-media.test.ts b/src/scripts/test-live-media.test.ts index 7fae8785d11..dd0fdbda442 100644 --- a/src/scripts/test-live-media.test.ts +++ b/src/scripts/test-live-media.test.ts @@ -6,16 +6,8 @@ const collectProviderApiKeysMock = vi.fn((provider: string) => process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [], ); -vi.mock("../../src/infra/shell-env.js", () => ({ - loadShellEnvFallback: loadShellEnvFallbackMock, -})); - -vi.mock("../../src/agents/live-auth-keys.js", () => ({ - collectProviderApiKeys: collectProviderApiKeysMock, -})); - function requirePlanEntry( - plan: ReturnType, + plan: Awaited>, suiteId: string, ) { const entry = plan.find((candidate) => candidate.suite.id === suiteId); @@ -40,7 +32,11 @@ describe("test-live-media", () => { vi.stubEnv("TEST_AUTH_VYDRA", "1"); const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); - const plan = buildRunPlan(parseArgs([])); + const plan = await buildRunPlan(parseArgs([]), { + collectProviderApiKeysImpl: collectProviderApiKeysMock, + getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], + loadShellEnvFallbackImpl: loadShellEnvFallbackMock, + }); expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]); expect(requirePlanEntry(plan, "image").providers).toEqual([ @@ -61,8 +57,13 @@ describe("test-live-media", () => { it("supports suite-specific provider filters without auth narrowing", async () => { const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); - const plan = buildRunPlan( + const plan = await buildRunPlan( parseArgs(["video", "--video-providers", "fal,openai,runway", "--all-providers"]), + { + collectProviderApiKeysImpl: collectProviderApiKeysMock, + getProviderEnvVarsImpl: (provider) => [`TEST_AUTH_${provider.toUpperCase()}`], + loadShellEnvFallbackImpl: loadShellEnvFallbackMock, + }, ); expect(plan).toHaveLength(1); diff --git a/test/scripts/test-live-media.test.ts b/test/scripts/test-live-media.test.ts index 6316f3a1e1f..d482476b05e 100644 --- a/test/scripts/test-live-media.test.ts +++ b/test/scripts/test-live-media.test.ts @@ -4,6 +4,7 @@ import { MEDIA_SUITES, findSkippedExplicitProviderSelections, parseArgs, + runCli, } from "../../scripts/test-live-media.ts"; describe("scripts/test-live-media", () => { @@ -44,6 +45,13 @@ describe("scripts/test-live-media", () => { }); }); + it("parses the explicit empty-run escape hatch", () => { + expect(parseArgs(["--allow-empty"])).toMatchObject({ + allowEmpty: true, + requireAuth: true, + }); + }); + it("fails explicit suite selections that auth filtering would skip", () => { const options = parseArgs([ "image", @@ -79,4 +87,32 @@ describe("scripts/test-live-media", () => { expect(skipped).toEqual([]); }); + + it("fails default live media runs when auth filtering leaves no providers", async () => { + await expect( + runCli(["image"], { + buildRunPlanImpl: () => [ + { + providers: [], + skippedReason: "no providers with usable auth", + suite: MEDIA_SUITES.image, + }, + ], + }), + ).resolves.toBe(1); + }); + + it("allows empty live media runs only with an explicit escape hatch", async () => { + await expect( + runCli(["image", "--allow-empty"], { + buildRunPlanImpl: () => [ + { + providers: [], + skippedReason: "no providers with usable auth", + suite: MEDIA_SUITES.image, + }, + ], + }), + ).resolves.toBe(0); + }); });