diff --git a/extensions/qa-lab/src/lab-server.test.ts b/extensions/qa-lab/src/lab-server.test.ts index 3e633b14bc8..1d6b010b6c9 100644 --- a/extensions/qa-lab/src/lab-server.test.ts +++ b/extensions/qa-lab/src/lab-server.test.ts @@ -353,6 +353,58 @@ describe("qa-lab server", () => { ); }); + it("does not eagerly load the runner model catalog before bootstrap is requested", async () => { + const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-lab-lazy-catalog-")); + cleanups.push(async () => { + await rm(repoRoot, { recursive: true, force: true }); + }); + const markerPath = path.join(repoRoot, "runner-catalog-hit.txt"); + + await mkdir(path.join(repoRoot, "dist"), { recursive: true }); + await mkdir(path.join(repoRoot, "extensions/qa-lab/web/dist"), { recursive: true }); + await writeFile( + path.join(repoRoot, "dist/index.js"), + [ + 'const fs = require("node:fs");', + `fs.writeFileSync(${JSON.stringify(markerPath)}, process.argv.slice(2).join(" "), "utf8");`, + "process.stdout.write(JSON.stringify({", + " models: [{", + ' key: "openai/gpt-5.4",', + ' name: "GPT-5.4",', + ' input: "openai/gpt-5.4",', + " available: true,", + " missing: false,", + " }],", + "}));", + ].join("\n"), + "utf8", + ); + await writeFile( + path.join(repoRoot, "extensions/qa-lab/web/dist/index.html"), + "lazy catalog", + "utf8", + ); + + const lab = await startQaLabServer({ + host: "127.0.0.1", + port: 0, + repoRoot, + }); + cleanups.push(async () => { + await lab.stop(); + }); + + await sleep(150); + await expect(readFile(markerPath, "utf8")).rejects.toThrow(); + + const bootstrapResponse = await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`); + expect(bootstrapResponse.status).toBe(200); + + const runnerCatalog = await waitForRunnerCatalog(lab.baseUrl); + expect(runnerCatalog.status).toBe("ready"); + expect(await readFile(markerPath, "utf8")).toContain("models list --all --json"); + }); + it("can disable the embedded echo gateway for real-suite runs", async () => { const lab = await startQaLabServer({ host: "127.0.0.1", diff --git a/extensions/qa-lab/src/lab-server.ts b/extensions/qa-lab/src/lab-server.ts index 08076571297..c742a02b2b5 100644 --- a/extensions/qa-lab/src/lab-server.ts +++ b/extensions/qa-lab/src/lab-server.ts @@ -515,18 +515,25 @@ export async function startQaLabServer(params?: { } | null = null; let publicBaseUrl = ""; - const runnerModelCatalogPromise = (async () => { - try { - const { loadQaRunnerModelOptions } = await import("./model-catalog.runtime.js"); - runnerModelOptions = await loadQaRunnerModelOptions({ - repoRoot, - }); - runnerModelCatalogStatus = "ready"; - } catch { - runnerModelOptions = []; - runnerModelCatalogStatus = "failed"; + let runnerModelCatalogPromise: Promise | null = null; + const ensureRunnerModelCatalog = () => { + if (runnerModelCatalogPromise) { + return runnerModelCatalogPromise; } - })(); + runnerModelCatalogPromise = (async () => { + try { + const { loadQaRunnerModelOptions } = await import("./model-catalog.runtime.js"); + runnerModelOptions = await loadQaRunnerModelOptions({ + repoRoot, + }); + runnerModelCatalogStatus = "ready"; + } catch { + runnerModelOptions = []; + runnerModelCatalogStatus = "failed"; + } + })(); + return runnerModelCatalogPromise; + }; const server = createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); @@ -547,6 +554,7 @@ export async function startQaLabServer(params?: { } if (req.method === "GET" && url.pathname === "/api/bootstrap") { + void ensureRunnerModelCatalog(); const resolvedControlUiUrl = controlUiProxyTarget ? `${publicBaseUrl}/control-ui/` : controlUiUrl; @@ -801,7 +809,6 @@ export async function startQaLabServer(params?: { kickoffTask: scenarioCatalog.kickoffTask, }); } - void runnerModelCatalogPromise; server.on("upgrade", (req, socket, head) => { const url = new URL(req.url ?? "/", "http://127.0.0.1");