import fs from "node:fs"; import { createServer } from "node:http"; import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { acquireDebugProxyCaptureStore, resolveDebugProxySettings, } from "openclaw/plugin-sdk/proxy-capture"; import { closeQaHttpServer, handleQaBusRequest, readQaJsonBody, writeError, writeJson, writeQaRequestBodyLimitError, } from "./bus-server.js"; import { createQaBusState, type QaBusState } from "./bus-state.js"; import { createQaRunnerRuntime } from "./harness-runtime.js"; import { isCaptureQueryPreset, mapCaptureEventForQa, probeTcpReachability, } from "./lab-server-capture.js"; import { detectContentType, isControlUiProxyPath, missingUiHtml, proxyHttpRequest, proxyUpgradeRequest, resolveAdvertisedBaseUrl, resolveUiAssetVersion, tryResolveUiAsset, } from "./lab-server-ui.js"; import type { QaLabLatestReport, QaLabScenarioOutcome, QaLabScenarioRun, QaLabServerHandle, QaLabServerStartParams, } from "./lab-server.types.js"; import type { QaRunnerModelOption } from "./model-catalog.runtime.js"; import { createQaChannelGatewayConfig } from "./qa-channel-transport.js"; import { createIdleQaRunnerSnapshot, createQaRunOutputDir, normalizeQaRunSelection, } from "./run-config.js"; import { qaChannelPlugin, setQaChannelRuntime, type OpenClawConfig } from "./runtime-api.js"; import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js"; type QaLabBootstrapDefaults = { conversationKind: "direct" | "channel"; conversationId: string; senderId: string; senderName: string; }; export type { QaLabLatestReport, QaLabScenarioOutcome, QaLabScenarioRun, QaLabServerHandle, QaLabServerStartParams, } from "./lab-server.types.js"; export function writeQaLabServerError(res: Parameters[0], error: unknown): void { if (writeQaRequestBodyLimitError(res, error)) { return; } writeError(res, 500, error); } function countQaLabScenarioRun(scenarios: QaLabScenarioOutcome[]) { return { total: scenarios.length, pending: scenarios.filter((scenario) => scenario.status === "pending").length, running: scenarios.filter((scenario) => scenario.status === "running").length, passed: scenarios.filter((scenario) => scenario.status === "pass").length, failed: scenarios.filter((scenario) => scenario.status === "fail").length, skipped: scenarios.filter((scenario) => scenario.status === "skip").length, }; } function withQaLabRunCounts(run: Omit): QaLabScenarioRun { return { ...run, counts: countQaLabScenarioRun(run.scenarios), }; } function injectKickoffMessage(params: { state: QaBusState; defaults: QaLabBootstrapDefaults; kickoffTask: string; }) { return params.state.addInboundMessage({ conversation: { id: params.defaults.conversationId, kind: params.defaults.conversationKind, ...(params.defaults.conversationKind === "channel" ? { title: params.defaults.conversationId } : {}), }, senderId: params.defaults.senderId, senderName: params.defaults.senderName, text: params.kickoffTask, }); } function createBootstrapDefaults(autoKickoffTarget?: string): QaLabBootstrapDefaults { if (autoKickoffTarget === "channel") { return { conversationKind: "channel", conversationId: "qa-lab", senderId: "qa-operator", senderName: "QA Operator", }; } return { conversationKind: "direct", conversationId: "qa-operator", senderId: "qa-operator", senderName: "QA Operator", }; } const CONTROL_UI_CREDENTIAL_QUERY_KEYS = new Set([ "access_token", "auth", "devicetoken", "password", "refresh_token", "token", ]); function stripSensitiveQueryParams(rawUrl: string): string { try { const url = new URL(rawUrl); for (const key of Array.from(url.searchParams.keys())) { if (CONTROL_UI_CREDENTIAL_QUERY_KEYS.has(key.toLowerCase())) { url.searchParams.delete(key); } } return url.toString(); } catch { return rawUrl .replace( /([?&])(?:access_token|auth|deviceToken|password|refresh_token|token)=[^&#\s]*&?/gi, (match: string, separator: string) => (match.endsWith("&") ? separator : ""), ) .replace(/[?&]$/, "") .replace("?&", "?"); } } function sanitizeControlUiPublicUrl(url: string | null): string | null { if (!url) { return null; } const fragmentIndex = url.indexOf("#"); const withoutFragment = fragmentIndex === -1 ? url : url.slice(0, fragmentIndex); return stripSensitiveQueryParams(withoutFragment); } function createQaLabConfig(baseUrl: string): OpenClawConfig { return createQaChannelGatewayConfig({ baseUrl }); } async function startQaGatewayLoop(params: { state: QaBusState; baseUrl: string }) { const runtime = createQaRunnerRuntime(); setQaChannelRuntime(runtime); const cfg = createQaLabConfig(params.baseUrl); const account = qaChannelPlugin.config.resolveAccount(cfg, "default"); const abort = new AbortController(); const task = Promise.resolve().then( async () => await qaChannelPlugin.gateway?.startAccount?.({ accountId: account.accountId, account, cfg, runtime: { log: () => undefined, error: () => undefined, exit: () => undefined, }, abortSignal: abort.signal, log: { info: () => undefined, warn: () => undefined, error: () => undefined, debug: () => undefined, }, getStatus: () => ({ accountId: account.accountId, configured: true, enabled: true, running: true, }), setStatus: () => undefined, }), ); return { cfg, async stop() { abort.abort(); await task; }, }; } export async function startQaLabServer( params?: QaLabServerStartParams, ): Promise { const repoRoot = path.resolve(params?.repoRoot ?? process.cwd()); const captureSettings = resolveDebugProxySettings(); const captureStoreLease = acquireDebugProxyCaptureStore( captureSettings.dbPath, captureSettings.blobDir, ); const captureStore = captureStoreLease.store; const state = createQaBusState(); let latestReport: QaLabLatestReport | null = null; let latestScenarioRun: QaLabScenarioRun | null = null; const scenarioCatalog = readQaBootstrapScenarioCatalog(); const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget); let runnerModelOptions: QaRunnerModelOption[] = []; let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading"; let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios); let activeSuiteRun: Promise | null = null; let controlUiProxyTarget = params?.controlUiProxyTarget?.trim() ? new URL(params.controlUiProxyTarget) : null; let controlUiProxyToken = params?.controlUiProxyToken?.trim() || null; let controlUiUrl = sanitizeControlUiPublicUrl(params?.controlUiUrl?.trim() || null); let gateway: | { cfg: OpenClawConfig; stop: () => Promise; } | undefined; const embeddedGatewayEnabled = params?.embeddedGateway !== "disabled"; let labHandle: QaLabServerHandle | null = null; let publicBaseUrl = ""; let runnerModelCatalogPromise: Promise | null = null; let runnerModelCatalogAbort: AbortController | null = null; const ensureRunnerModelCatalog = () => { if (runnerModelCatalogPromise) { return runnerModelCatalogPromise; } runnerModelCatalogAbort = new AbortController(); runnerModelCatalogPromise = (async () => { try { const { loadQaRunnerModelOptions } = await import("./model-catalog.runtime.js"); runnerModelOptions = await loadQaRunnerModelOptions({ repoRoot, signal: runnerModelCatalogAbort?.signal, }); runnerModelCatalogStatus = "ready"; } catch { runnerModelOptions = []; runnerModelCatalogStatus = "failed"; } })().finally(() => { runnerModelCatalogAbort = null; }); return runnerModelCatalogPromise; }; async function runSelfCheck(): Promise { latestScenarioRun = withQaLabRunCounts({ kind: "self-check", status: "running", startedAt: new Date().toISOString(), scenarios: [ { id: "qa-self-check", name: "Synthetic Slack-class roundtrip", status: "running", }, ], }); const result = await runQaSelfCheckAgainstState({ state, cfg: gateway?.cfg ?? createQaLabConfig(listenUrl), transportId: "qa-channel", outputPath: params?.outputPath, repoRoot, waitTimeoutMs: params?.selfCheckWaitTimeoutMs, }); latestScenarioRun = withQaLabRunCounts({ kind: "self-check", status: "completed", startedAt: latestScenarioRun.startedAt, finishedAt: new Date().toISOString(), scenarios: [ { id: "qa-self-check", name: result.scenarioResult.name, status: result.scenarioResult.status, details: result.scenarioResult.details, steps: result.scenarioResult.steps, }, ], }); latestReport = { outputPath: result.outputPath, markdown: result.report, generatedAt: new Date().toISOString(), }; return result; } const server = createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (await handleQaBusRequest({ req, res, state })) { return; } try { if (controlUiProxyTarget && isControlUiProxyPath(url.pathname)) { await proxyHttpRequest({ req, res, target: controlUiProxyTarget, pathname: url.pathname, search: url.search, authorizationToken: controlUiProxyToken, }); return; } if (req.method === "GET" && url.pathname === "/api/bootstrap") { void ensureRunnerModelCatalog(); const resolvedControlUiUrl = controlUiProxyTarget ? `${publicBaseUrl}/control-ui/` : controlUiUrl; const safeControlUiUrl = sanitizeControlUiPublicUrl(resolvedControlUiUrl); writeJson(res, 200, { baseUrl: publicBaseUrl, latestReport, controlUiUrl: safeControlUiUrl, controlUiEmbeddedUrl: safeControlUiUrl, kickoffTask: scenarioCatalog.kickoffTask, scenarios: scenarioCatalog.scenarios, defaults: bootstrapDefaults, runner: runnerSnapshot, runnerCatalog: { status: runnerModelCatalogStatus, real: runnerModelOptions, }, }); return; } if (req.method === "GET" && (url.pathname === "/healthz" || url.pathname === "/readyz")) { writeJson(res, 200, { ok: true, status: "live" }); return; } if (req.method === "GET" && url.pathname === "/api/state") { writeJson(res, 200, state.getSnapshot()); return; } if (req.method === "GET" && url.pathname === "/api/report") { writeJson(res, 200, { report: latestReport }); return; } if (req.method === "GET" && url.pathname === "/api/ui-version") { res.writeHead(200, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", }); res.end(JSON.stringify({ version: resolveUiAssetVersion(params?.uiDistDir) })); return; } if (req.method === "GET" && url.pathname === "/api/outcomes") { writeJson(res, 200, { run: latestScenarioRun }); return; } if (req.method === "GET" && url.pathname === "/api/capture/sessions") { writeJson(res, 200, { sessions: captureStore.listSessions(50), }); return; } if (req.method === "GET" && url.pathname === "/api/capture/startup-status") { const proxyUrl = captureSettings.proxyUrl || "http://127.0.0.1:7799"; const gatewayUrl = controlUiUrl || "http://127.0.0.1:18789/"; const [proxy, gateway] = await Promise.all([ probeTcpReachability(proxyUrl), probeTcpReachability(gatewayUrl), ]); writeJson(res, 200, { status: { proxy: { ...proxy, label: "Proxy", }, gateway: { ...gateway, label: "Gateway", }, qaLab: { label: "QA Lab", url: publicBaseUrl, ok: true, }, }, }); return; } if (req.method === "GET" && url.pathname === "/api/capture/events") { const sessionId = url.searchParams.get("sessionId")?.trim(); writeJson(res, 200, { events: sessionId ? captureStore.getSessionEvents(sessionId, 200).map(mapCaptureEventForQa) : [], }); return; } if (req.method === "GET" && url.pathname === "/api/capture/coverage") { const sessionId = url.searchParams.get("sessionId")?.trim(); if (!sessionId) { writeError(res, 400, "Missing sessionId"); return; } writeJson(res, 200, { coverage: captureStore.summarizeSessionCoverage(sessionId), }); return; } if (req.method === "GET" && url.pathname === "/api/capture/query") { const preset = url.searchParams.get("preset")?.trim(); const sessionId = url.searchParams.get("sessionId")?.trim() || undefined; if (!preset) { writeError(res, 400, "Missing preset"); return; } if (!isCaptureQueryPreset(preset)) { writeError(res, 400, "Unknown preset"); return; } writeJson(res, 200, { rows: captureStore.queryPreset(preset, sessionId), }); return; } if (req.method === "GET" && url.pathname === "/api/capture/blob") { const blobId = url.searchParams.get("id")?.trim(); if (!blobId) { writeError(res, 400, "Missing blob id"); return; } const content = captureStore.readBlob(blobId); if (content == null) { writeError(res, 404, "Blob not found"); return; } writeJson(res, 200, { id: blobId, content }); return; } if (req.method === "POST" && url.pathname === "/api/capture/delete-sessions") { const body = (await readQaJsonBody(req)) as { sessionIds?: unknown }; const sessionIds = Array.isArray(body.sessionIds) ? body.sessionIds.filter((value): value is string => typeof value === "string") : []; writeJson(res, 200, { result: captureStore.deleteSessions(sessionIds), }); return; } if (req.method === "POST" && url.pathname === "/api/capture/purge") { writeJson(res, 200, { result: captureStore.purgeAll(), }); return; } if (req.method === "POST" && url.pathname === "/api/reset") { if (activeSuiteRun) { writeError(res, 409, "QA suite run already in progress"); return; } state.reset(); latestReport = null; latestScenarioRun = null; runnerSnapshot = { ...runnerSnapshot, status: "idle", artifacts: null, error: null, startedAt: undefined, finishedAt: undefined, }; writeJson(res, 200, { ok: true }); return; } if (req.method === "POST" && url.pathname === "/api/inbound/message") { const body = await readQaJsonBody(req); writeJson(res, 200, { message: state.addInboundMessage(body as Parameters[0]), }); return; } if (req.method === "POST" && url.pathname === "/api/kickoff") { writeJson(res, 200, { message: injectKickoffMessage({ state, defaults: bootstrapDefaults, kickoffTask: scenarioCatalog.kickoffTask, }), }); return; } if (req.method === "POST" && url.pathname === "/api/scenario/self-check") { if (activeSuiteRun) { writeError(res, 409, "QA suite run already in progress"); return; } const result = await runSelfCheck(); writeJson(res, 200, serializeSelfCheck(result)); return; } if (req.method === "POST" && url.pathname === "/api/scenario/suite") { if (activeSuiteRun) { writeError(res, 409, "QA suite run already in progress"); return; } const selection = normalizeQaRunSelection( await readQaJsonBody(req), scenarioCatalog.scenarios, ); state.reset(); latestReport = null; latestScenarioRun = null; const startedAt = new Date().toISOString(); runnerSnapshot = { status: "running", selection, startedAt, finishedAt: undefined, artifacts: null, error: null, }; activeSuiteRun = (async () => { try { const { runQaSuite } = await import("./suite.js"); const result = await runQaSuite({ lab: labHandle ?? undefined, startLab: startQaLabServer, outputDir: createQaRunOutputDir(repoRoot), providerMode: selection.providerMode, primaryModel: selection.primaryModel, alternateModel: selection.alternateModel, scenarioIds: selection.scenarioIds, }); runnerSnapshot = { status: "completed", selection, startedAt, finishedAt: new Date().toISOString(), artifacts: { outputDir: result.outputDir, reportPath: result.reportPath, summaryPath: result.summaryPath, watchUrl: result.watchUrl, }, error: null, }; } catch (error) { runnerSnapshot = { status: "failed", selection, startedAt, finishedAt: new Date().toISOString(), artifacts: null, error: formatErrorMessage(error), }; } finally { activeSuiteRun = null; } })(); writeJson(res, 202, { ok: true, runner: runnerSnapshot, }); return; } if (req.method !== "GET" && req.method !== "HEAD") { writeError(res, 404, "not found"); return; } const asset = tryResolveUiAsset(url.pathname, params?.uiDistDir, repoRoot); if (!asset) { const html = missingUiHtml(); res.writeHead(200, { "content-type": "text/html; charset=utf-8", "content-length": Buffer.byteLength(html), }); if (req.method === "HEAD") { res.end(); return; } res.end(html); return; } const body = fs.readFileSync(asset); res.writeHead(200, { "content-type": detectContentType(asset), "content-length": body.byteLength, }); if (req.method === "HEAD") { res.end(); return; } res.end(body); } catch (error) { writeQaLabServerError(res, error); } }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(params?.port ?? 0, params?.host ?? "127.0.0.1", () => resolve()); }); const address = server.address(); if (!address || typeof address === "string") { throw new Error("qa-lab failed to bind"); } const listenUrl = resolveAdvertisedBaseUrl({ bindHost: params?.host ?? "127.0.0.1", bindPort: address.port, }); publicBaseUrl = resolveAdvertisedBaseUrl({ bindHost: params?.host ?? "127.0.0.1", bindPort: address.port, advertiseHost: params?.advertiseHost, advertisePort: params?.advertisePort, }); if (embeddedGatewayEnabled) { gateway = await startQaGatewayLoop({ state, baseUrl: listenUrl }); } if (params?.sendKickoffOnStart) { injectKickoffMessage({ state, defaults: bootstrapDefaults, kickoffTask: scenarioCatalog.kickoffTask, }); } server.on("upgrade", (req, socket, head) => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (!controlUiProxyTarget || !isControlUiProxyPath(url.pathname)) { socket.destroy(); return; } proxyUpgradeRequest({ req, socket, head, target: controlUiProxyTarget, authorizationToken: controlUiProxyToken, }); }); const lab = { baseUrl: publicBaseUrl, listenUrl, state, setControlUi(next: { controlUiUrl?: string | null; controlUiProxyToken?: string | null; controlUiProxyTarget?: string | null; }) { controlUiUrl = sanitizeControlUiPublicUrl(next.controlUiUrl?.trim() || null); controlUiProxyToken = next.controlUiProxyToken?.trim() || null; controlUiProxyTarget = next.controlUiProxyTarget?.trim() ? new URL(next.controlUiProxyTarget) : null; }, setScenarioRun(next: Omit | null) { latestScenarioRun = next ? withQaLabRunCounts(next) : null; }, setLatestReport(next: QaLabLatestReport | null) { latestReport = next; }, runSelfCheck, async stop() { runnerModelCatalogAbort?.abort(); await runnerModelCatalogPromise?.catch(() => undefined); await gateway?.stop(); await closeQaHttpServer(server); captureStoreLease.release(); }, }; labHandle = lab; return lab; } function serializeSelfCheck(result: QaSelfCheckResult) { return { outputPath: result.outputPath, report: result.report, checks: result.checks, scenario: result.scenarioResult, }; }