diff --git a/extensions/discord/src/mentions.test.ts b/extensions/discord/src/mentions.test.ts index 48239efaf117..e622a7a98f44 100644 --- a/extensions/discord/src/mentions.test.ts +++ b/extensions/discord/src/mentions.test.ts @@ -6,7 +6,6 @@ import { } from "./directory-cache.js"; import { discordTextHasBroadcastMention, - discordTextHasTargetedMention, formatMention, rewriteDiscordKnownMentions, } from "./mentions.js"; @@ -129,20 +128,6 @@ describe("rewriteDiscordKnownMentions", () => { }); }); -describe("discordTextHasTargetedMention", () => { - it("detects user and role mentions", () => { - expect(discordTextHasTargetedMention("ping <@123>")).toBe(true); - expect(discordTextHasTargetedMention("ping <@!123>")).toBe(true); - expect(discordTextHasTargetedMention("ping <@&456>")).toBe(true); - }); - - it("ignores plain text, channels, and broadcasts", () => { - expect(discordTextHasTargetedMention("ping @alice")).toBe(false); - expect(discordTextHasTargetedMention("see <#789>")).toBe(false); - expect(discordTextHasTargetedMention("heads up @everyone @here")).toBe(false); - }); -}); - describe("discordTextHasBroadcastMention", () => { it("detects @everyone and @here", () => { expect(discordTextHasBroadcastMention("heads up @everyone")).toBe(true); diff --git a/extensions/discord/src/mentions.ts b/extensions/discord/src/mentions.ts index d5d545a1fc18..57153a8f0725 100644 --- a/extensions/discord/src/mentions.ts +++ b/extensions/discord/src/mentions.ts @@ -12,7 +12,6 @@ type DiscordMentionAliasesConfig = Record; const MENTION_CANDIDATE_PATTERN = /(^|[\s([{"'.,;:!?])@([a-z0-9_.-]{2,32}(?:#[0-9]{4})?)/gi; const DISCORD_RESERVED_MENTIONS = new Set(["everyone", "here"]); const DISCORD_DISCRIMINATOR_SUFFIX = /#\d{4}$/; -const DISCORD_TARGETED_MENTION_PATTERN = /<@!?\d+>|<@&\d+>/; const DISCORD_BROADCAST_MENTION_PATTERN = /@(everyone|here)\b/; function normalizeSnowflake(value: string | number | bigint): string | null { @@ -215,11 +214,6 @@ export function rewriteDiscordKnownMentions( return rewritten; } -/** Whether text carries a Discord user/role mention (`<@id>`, `<@!id>`, `<@&id>`) that pings when sent fresh. */ -export function discordTextHasTargetedMention(text: string): boolean { - return DISCORD_TARGETED_MENTION_PATTERN.test(text); -} - /** Whether text carries an `@everyone`/`@here` broadcast mention. */ export function discordTextHasBroadcastMention(text: string): boolean { return DISCORD_BROADCAST_MENTION_PATTERN.test(text); diff --git a/extensions/discord/src/monitor/provider.ts b/extensions/discord/src/monitor/provider.ts index dd8b0b355710..9d8fec60d339 100644 --- a/extensions/discord/src/monitor/provider.ts +++ b/extensions/discord/src/monitor/provider.ts @@ -686,5 +686,3 @@ export const testing = { shouldLogVerboseForTesting = mock; }, }; - -export const resolveDiscordRuntimeGroupPolicy = resolveOpenProviderRuntimeGroupPolicy; diff --git a/extensions/qa-lab/src/evidence-environment.ts b/extensions/qa-lab/src/evidence-environment.ts new file mode 100644 index 000000000000..88e6ec1e999f --- /dev/null +++ b/extensions/qa-lab/src/evidence-environment.ts @@ -0,0 +1,32 @@ +// Qa Lab plugin module resolves evidence runtime metadata. +import { execFileSync } from "node:child_process"; + +function resolveQaEvidenceCheckoutRef(repoRoot?: string) { + try { + const ref = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { + cwd: repoRoot ?? process.cwd(), + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + return ref || undefined; + } catch { + return undefined; + } +} + +export function resolveQaEvidenceEnvironment(params: { + env?: NodeJS.ProcessEnv; + repoRoot?: string; +}) { + return { + // GitHub's GITHUB_SHA describes the workflow event, not necessarily the + // checked-out ref selected by a manual or remote QA run. + ref: + params.env?.OPENCLAW_QA_REF?.trim() || + resolveQaEvidenceCheckoutRef(params.repoRoot) || + params.env?.GITHUB_SHA?.trim() || + null, + os: process.platform, + nodeVersion: process.version, + }; +} diff --git a/extensions/qa-lab/src/evidence-summary.ts b/extensions/qa-lab/src/evidence-summary.ts index 19d7181a1ae0..3fdff0525209 100644 --- a/extensions/qa-lab/src/evidence-summary.ts +++ b/extensions/qa-lab/src/evidence-summary.ts @@ -1,6 +1,6 @@ // Qa Lab plugin module implements QA evidence summary behavior. -import { execFileSync } from "node:child_process"; import { z } from "zod"; +import { resolveQaEvidenceEnvironment } from "./evidence-environment.js"; import { splitQaModelRef } from "./model-selection.js"; import { getQaProvider, type QaProviderMode } from "./providers/index.js"; import { @@ -398,36 +398,6 @@ function resolveQaEvidenceChannelDriver(params: { env?: NodeJS.ProcessEnv; fallb return id ? { id } : undefined; } -function resolveQaEvidenceCheckoutRef(repoRoot?: string) { - try { - const ref = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { - cwd: repoRoot ?? process.cwd(), - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }).trim(); - return ref || undefined; - } catch { - return undefined; - } -} - -export function resolveQaEvidenceEnvironment(params: { - env?: NodeJS.ProcessEnv; - repoRoot?: string; -}) { - return { - // GitHub's GITHUB_SHA describes the workflow event, not necessarily the - // checked-out ref selected by a manual or remote QA run. - ref: - params.env?.OPENCLAW_QA_REF?.trim() || - resolveQaEvidenceCheckoutRef(params.repoRoot) || - params.env?.GITHUB_SHA?.trim() || - null, - os: process.platform, - nodeVersion: process.version, - }; -} - function resolveQaEvidencePackageSource(env: NodeJS.ProcessEnv | undefined) { const spec = env?.OPENCLAW_QA_PACKAGE_SOURCE?.trim() || undefined; const sha = env?.OPENCLAW_QA_PACKAGE_SOURCE_SHA?.trim() || undefined; diff --git a/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts b/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts index 5f9b7388cc3a..3eeba4da348c 100644 --- a/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts @@ -3,11 +3,11 @@ import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; -import { runDiscordQaLive } from "./discord-live.runtime.js"; +import { discordQaLiveRuntime } from "./discord-live.runtime.js"; export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) { const runOptions = resolveLiveTransportQaRunOptions(opts); - const result = await runDiscordQaLive(runOptions); + const result = await discordQaLiveRuntime.run(runOptions); printLiveTransportQaArtifacts("Discord QA", { report: result.reportPath, summary: result.summaryPath, diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts index 44aad53a9710..a1af9c645298 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -1,11 +1,13 @@ // Qa Lab tests cover discord live plugin behavior. import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { afterEach, describe, expect, it, vi } from "vitest"; import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, findMissingLiveTransportStandardScenarios, -} from "../shared/live-transport-scenarios.js"; -import { testing } from "./discord-live.runtime.js"; +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { discordQaLiveRuntime } from "./discord-live.runtime.js"; + +const { testing } = discordQaLiveRuntime; describe("discord live qa runtime", () => { afterEach(() => { diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts index 1a81e172b425..4a660a249c08 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -355,7 +355,7 @@ const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter( scenario.id !== "discord-thread-reply-filepath-attachment", ); -export function listDiscordQaScenarioCatalog() { +function listDiscordQaScenarioCatalog() { return DISCORD_QA_SCENARIOS.map((scenario) => ({ id: scenario.id })); } @@ -1510,7 +1510,7 @@ async function assertDiscordApplicationCommandsRegistered(params: { ); } -export async function runDiscordQaLive(params: { +async function runDiscordQaLive(params: { repoRoot?: string; outputDir?: string; providerMode?: QaProviderModeInput; @@ -1954,4 +1954,9 @@ export const testing = { resolveDiscordQaRuntimeEnv, waitForDiscordChannelRunning, }; -export { testing as __testing }; + +export const discordQaLiveRuntime = { + listScenarioCatalog: listDiscordQaScenarioCatalog, + run: runDiscordQaLive, + testing, +}; diff --git a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts index ebb9c9728606..d50042506184 100644 --- a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts +++ b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts @@ -18,9 +18,16 @@ vi.mock("./whatsapp/adapter.runtime.js", () => ({ createWhatsAppQaTransportAdapter: createWhatsApp, })); -import { slackQaAdapterFactory } from "./slack/cli.js"; -import { telegramQaAdapterFactory } from "./telegram/cli.js"; -import { whatsappQaAdapterFactory } from "./whatsapp/cli.js"; +import { slackQaCliRegistration } from "./slack/cli.js"; +import { telegramQaCliRegistration } from "./telegram/cli.js"; +import { whatsappQaCliRegistration } from "./whatsapp/cli.js"; + +const slackQaAdapterFactory = slackQaCliRegistration.adapterFactory; +const telegramQaAdapterFactory = telegramQaCliRegistration.adapterFactory; +const whatsappQaAdapterFactory = whatsappQaCliRegistration.adapterFactory; +if (!slackQaAdapterFactory || !telegramQaAdapterFactory || !whatsappQaAdapterFactory) { + throw new Error("expected live transport adapter factories"); +} const factories = [ telegramQaAdapterFactory, diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index a9c7ef1bb651..547d3ac8931e 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -1,20 +1,24 @@ // Qa Lab tests cover live transport scenarios plugin behavior. import fs from "node:fs"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; -import { __testing as discordTesting } from "../discord/discord-live.runtime.js"; -import { __testing as slackTesting } from "../slack/slack-live.runtime.js"; -import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js"; -import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js"; import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + findMissingLiveTransportStandardScenarios, +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; +import { describe, expect, it } from "vitest"; +import { discordQaLiveRuntime } from "../discord/discord-live.runtime.js"; +import { testing as slackTesting } from "../slack/slack-live.runtime.js"; +import { testing as telegramTesting } from "../telegram/telegram-live.runtime.js"; +import { testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js"; +import { buildLiveTransportCoverageLaneSummaries, collectLiveTransportStandardScenarioCoverage, - findMissingLiveTransportStandardScenarios, loadNonYamlScenarioRefs, selectLiveTransportScenarios, } from "./live-transport-scenarios.js"; +const discordTesting = discordQaLiveRuntime.testing; + describe("live transport scenario helpers", () => { it("loads every non-YAML scenario id exactly once", async () => { const refs = await loadNonYamlScenarioRefs(); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index 26539bcc815a..0f00e91c8ab7 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -8,9 +8,7 @@ import { } from "openclaw/plugin-sdk/qa-live-transport-scenarios"; export { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, collectLiveTransportStandardScenarioCoverage, - findMissingLiveTransportStandardScenarios, selectLiveTransportScenarios, type LiveTransportScenarioDefinition, } from "openclaw/plugin-sdk/qa-live-transport-scenarios"; @@ -120,7 +118,7 @@ export async function loadNonYamlScenarioRefs() { return [ ...refs( "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts", - discord.listDiscordQaScenarioCatalog(), + discord.discordQaLiveRuntime.listScenarioCatalog(), ), ...refs( "extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts", diff --git a/extensions/qa-lab/src/live-transports/slack/cli.ts b/extensions/qa-lab/src/live-transports/slack/cli.ts index f84663296e9e..d51b79d30bf1 100644 --- a/extensions/qa-lab/src/live-transports/slack/cli.ts +++ b/extensions/qa-lab/src/live-transports/slack/cli.ts @@ -20,22 +20,21 @@ async function runQaSlack(opts: LiveTransportQaCommandOptions) { await (await loadSlackQaCliRuntime()).runQaSlackCommand(opts); } -export const slackQaAdapterFactory: NonNullable = - { - id: "slack", - scenarioIds: [ - "channel-chat-baseline", - "channel-canary", - "channel-mention-gating", - "channel-top-level-reply-shape", - "thread-follow-up", - "thread-isolation", - ], - matches: ({ channelId, driver }) => driver === "live" && channelId === "slack", - async create(context) { - return await (await loadSlackQaAdapterRuntime()).createSlackQaTransportAdapter(context); - }, - }; +const slackQaAdapterFactory: NonNullable = { + id: "slack", + scenarioIds: [ + "channel-chat-baseline", + "channel-canary", + "channel-mention-gating", + "channel-top-level-reply-shape", + "thread-follow-up", + "thread-isolation", + ], + matches: ({ channelId, driver }) => driver === "live" && channelId === "slack", + async create(context) { + return await (await loadSlackQaAdapterRuntime()).createSlackQaTransportAdapter(context); + }, +}; export const slackQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 5f8529199592..6f8877fb0096 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND } from "../../evidence-summary.js"; -import { testing, runSlackQaLive } from "./slack-live.runtime.js"; +import { runSlackQaLive, testing } from "./slack-live.runtime.js"; function renderExpectedSlackChartAccessibleText(summaryText: string) { return [ diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index fdbf9ab1bc10..beab355b996b 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -21,9 +21,7 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) { await (await loadTelegramQaCliRuntime()).runQaTelegramCommand(opts); } -export const telegramQaAdapterFactory: NonNullable< - LiveTransportQaCliRegistration["adapterFactory"] -> = { +const telegramQaAdapterFactory: NonNullable = { id: "telegram", scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS], matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index 682b307fcfbc..21f60bc10096 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -2,12 +2,12 @@ import { expectDefined } from "@openclaw/normalization-core"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { summarizeLiveTransportRttSamples } from "../shared/live-transport-rtt.js"; import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, findMissingLiveTransportStandardScenarios, -} from "../shared/live-transport-scenarios.js"; +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { summarizeLiveTransportRttSamples } from "../shared/live-transport-rtt.js"; import { testing } from "./telegram-live.runtime.js"; const fetchWithSsrFGuardMock = vi.hoisted(() => diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts index 7293369804f3..b55f5075dbf8 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts @@ -24,9 +24,7 @@ async function runQaWhatsApp(opts: LiveTransportQaCommandOptions) { await (await loadWhatsAppQaCliRuntime()).runQaWhatsAppCommand(opts); } -export const whatsappQaAdapterFactory: NonNullable< - LiveTransportQaCliRegistration["adapterFactory"] -> = { +const whatsappQaAdapterFactory: NonNullable = { id: "whatsapp", scenarioIds: [ "dm-chat-baseline", diff --git a/extensions/qa-lab/src/qa-web-search-provider.shared.ts b/extensions/qa-lab/src/qa-web-search-provider.shared.ts index d52b81ad1ade..5e4ce1951add 100644 --- a/extensions/qa-lab/src/qa-web-search-provider.shared.ts +++ b/extensions/qa-lab/src/qa-web-search-provider.shared.ts @@ -1,7 +1,7 @@ // QA Lab web_search metadata shared by runtime and contract-only loading. import type { WebSearchProviderPlugin } from "openclaw/plugin-sdk/provider-web-search-contract"; -export const QA_LAB_WEB_SEARCH_PROVIDER_ID = "qa-lab-search"; +const QA_LAB_WEB_SEARCH_PROVIDER_ID = "qa-lab-search"; export const QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY = "OPENCLAW_QA_WEB_SEARCH_DENIED_INPUT"; export function createQaLabWebSearchProviderBase(): Omit { diff --git a/extensions/qa-lab/src/qa-web-search-provider.test.ts b/extensions/qa-lab/src/qa-web-search-provider.test.ts index 115428059e31..9aab3071d783 100644 --- a/extensions/qa-lab/src/qa-web-search-provider.test.ts +++ b/extensions/qa-lab/src/qa-web-search-provider.test.ts @@ -4,14 +4,13 @@ import { createQaLabWebSearchProvider as createQaLabWebSearchContractProvider } import { createQaLabWebSearchProvider, QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY, - QA_LAB_WEB_SEARCH_PROVIDER_ID, } from "./qa-web-search-provider.js"; describe("qa-lab web search provider", () => { it("exposes a credential-free QA-only provider", () => { const provider = createQaLabWebSearchProvider(); - expect(provider.id).toBe(QA_LAB_WEB_SEARCH_PROVIDER_ID); + expect(provider.id).toBe("qa-lab-search"); expect(provider.requiresCredential).toBe(false); expect(provider.envVars).toEqual([]); expect(provider.credentialPath).toBe(""); diff --git a/extensions/qa-lab/src/qa-web-search-provider.ts b/extensions/qa-lab/src/qa-web-search-provider.ts index cda7e27b09f4..3906c5c97fe4 100644 --- a/extensions/qa-lab/src/qa-web-search-provider.ts +++ b/extensions/qa-lab/src/qa-web-search-provider.ts @@ -12,10 +12,7 @@ import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY, } from "./qa-web-search-provider.shared.js"; -export { - QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY, - QA_LAB_WEB_SEARCH_PROVIDER_ID, -} from "./qa-web-search-provider.shared.js"; +export { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "./qa-web-search-provider.shared.js"; const QaLabWebSearchSchema = { type: "object", diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index bf4544ccbc06..9eab838b842d 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -361,12 +361,7 @@ export type QaBootstrapScenarioCatalog = { scenarios: QaSeedScenarioWithSource[]; }; -export { - QA_OBSERVABILITY_SCENARIO_IDS, - QA_PERSONAL_AGENT_SCENARIO_IDS, - QA_SCENARIO_PACKS, - resolveQaScenarioPackScenarioIds, -} from "./scenario-packs.js"; +export { QA_SCENARIO_PACKS } from "./scenario-packs.js"; const QA_SCENARIO_PACK_INDEX_PATH = "qa/scenarios/index.yaml"; const QA_SCENARIO_LEGACY_OVERVIEW_PATH = "qa/scenarios.md"; diff --git a/extensions/qa-lab/src/scenario-packs.test.ts b/extensions/qa-lab/src/scenario-packs.test.ts index d32012641037..cfdb607bff09 100644 --- a/extensions/qa-lab/src/scenario-packs.test.ts +++ b/extensions/qa-lab/src/scenario-packs.test.ts @@ -1,12 +1,15 @@ // Qa Lab tests cover scenario packs plugin behavior. import { describe, expect, it } from "vitest"; -import { - QA_OBSERVABILITY_SCENARIO_IDS, - QA_PERSONAL_AGENT_SCENARIO_IDS, - QA_SCENARIO_PACKS, - readQaScenarioById, - resolveQaScenarioPackScenarioIds, -} from "./scenario-catalog.js"; +import { QA_SCENARIO_PACKS, readQaScenarioById } from "./scenario-catalog.js"; +import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js"; + +function scenarioIdsForPack(packId: string): readonly string[] { + const pack = QA_SCENARIO_PACKS.find((candidate) => candidate.id === packId); + if (!pack) { + throw new Error(`missing QA scenario pack: ${packId}`); + } + return pack.scenarioIds; +} describe("qa scenario packs", () => { it("points every pack scenario id at a loadable YAML scenario", () => { @@ -58,13 +61,13 @@ describe("qa scenario packs", () => { it("expands the personal-agent pack in pack order", () => { expect(resolveQaScenarioPackScenarioIds({ pack: "personal-agent" })).toEqual([ - ...QA_PERSONAL_AGENT_SCENARIO_IDS, + ...scenarioIdsForPack("personal-agent"), ]); }); it("expands the observability pack in pack order", () => { expect(resolveQaScenarioPackScenarioIds({ pack: "observability" })).toEqual([ - ...QA_OBSERVABILITY_SCENARIO_IDS, + ...scenarioIdsForPack("observability"), ]); }); @@ -74,7 +77,7 @@ describe("qa scenario packs", () => { pack: "personal-agent", scenarioIds: ["channel-chat-baseline", "personal-reminder-roundtrip"], }), - ).toEqual(["channel-chat-baseline", ...QA_PERSONAL_AGENT_SCENARIO_IDS]); + ).toEqual(["channel-chat-baseline", ...scenarioIdsForPack("personal-agent")]); }); it("rejects unknown scenario packs", () => { diff --git a/extensions/qa-lab/src/scenario-packs.ts b/extensions/qa-lab/src/scenario-packs.ts index 0dc301c8f69d..a506e010a915 100644 --- a/extensions/qa-lab/src/scenario-packs.ts +++ b/extensions/qa-lab/src/scenario-packs.ts @@ -8,7 +8,7 @@ type QaScenarioPackDefinition = { scenarioIds: readonly string[]; }; -export const QA_PERSONAL_AGENT_SCENARIO_IDS = [ +const QA_PERSONAL_AGENT_SCENARIO_IDS = [ "personal-reminder-roundtrip", "personal-channel-thread-reply", "personal-memory-preference-recall", @@ -21,10 +21,7 @@ export const QA_PERSONAL_AGENT_SCENARIO_IDS = [ "personal-failure-recovery", ] as const; -export const QA_OBSERVABILITY_SCENARIO_IDS = [ - "otel-trace-smoke", - "docker-prometheus-smoke", -] as const; +const QA_OBSERVABILITY_SCENARIO_IDS = ["otel-trace-smoke", "docker-prometheus-smoke"] as const; export const QA_SCENARIO_PACKS = [ { diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index 2e19f06c32e7..ea4afab8a774 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -332,9 +332,8 @@ export { readSessionTranscriptSummary, readSkillStatus, seedQaSessionTranscript, - setSessionStoreLockRetryDelaysMsForTests, }; -function setSessionStoreLockRetryDelaysMsForTests(delays?: readonly number[]): void { +export function setSessionStoreLockRetryDelaysMsForTests(delays?: readonly number[]): void { sessionStoreLockRetryDelaysMsForTests = delays; } diff --git a/extensions/qa-lab/src/suite-runtime-gateway.ts b/extensions/qa-lab/src/suite-runtime-gateway.ts index 975db5ff0235..29b02e08830d 100644 --- a/extensions/qa-lab/src/suite-runtime-gateway.ts +++ b/extensions/qa-lab/src/suite-runtime-gateway.ts @@ -403,8 +403,8 @@ export { fetchJson, getGatewayRetryAfterMs, isConfigApplyNoopForSnapshot, - isConfigPatchNoopForSnapshot, isConfigHashConflict, + isConfigPatchNoopForSnapshot, patchConfig, readConfigSnapshot, restartGatewayWithConfigPatch, diff --git a/scripts/deadcode-exports.baseline.mjs b/scripts/deadcode-exports.baseline.mjs index 026bd02f9e06..fd79d2bda412 100644 --- a/scripts/deadcode-exports.baseline.mjs +++ b/scripts/deadcode-exports.baseline.mjs @@ -56,7 +56,6 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "extensions/discord/src/internal/interactions.ts: BaseInteraction", "extensions/discord/src/internal/rest-scheduler.ts: RestSchedulerOptions", "extensions/discord/src/internal/rest.ts: QueuedRequest", - "extensions/discord/src/mentions.ts: discordTextHasTargetedMention", "extensions/discord/src/monitor.gateway.ts: WaitForDiscordGatewayStopParams", "extensions/discord/src/monitor/agent-components.dispatch.ts: resolveDiscordComponentOriginatingTo", "extensions/discord/src/monitor/agent-components.ts: createDiscordComponentButton", @@ -87,7 +86,6 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "extensions/discord/src/monitor/native-command.ts: testing", "extensions/discord/src/monitor/provider.lifecycle.ts: resolveDiscordGatewayReadyTimeoutMs", "extensions/discord/src/monitor/provider.lifecycle.ts: resolveDiscordGatewayRuntimeReadyTimeoutMs", - "extensions/discord/src/monitor/provider.ts: resolveDiscordRuntimeGroupPolicy", "extensions/discord/src/monitor/provider.ts: testing", "extensions/discord/src/monitor/thread-title.ts: normalizeGeneratedThreadTitle", "extensions/discord/src/resolve-allowlist-common.ts: findDiscordGuildByName", @@ -452,7 +450,6 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "extensions/qa-lab/src/child-output.ts: QA_CHILD_STDERR_TAIL_BYTES", "extensions/qa-lab/src/confidence-report.ts: buildQaConfidenceSelfTestSummary", "extensions/qa-lab/src/confidence-report.ts: QaConfidenceManifest", - "extensions/qa-lab/src/evidence-summary.ts: resolveQaEvidenceEnvironment", "extensions/qa-lab/src/gateway-child.ts: buildQaForcedRuntimeEnvPatch", "extensions/qa-lab/src/gateway-process-boundary.ts: __testing", "extensions/qa-lab/src/gateway-process-boundary.ts: parseQaGatewayProcStat", @@ -460,17 +457,10 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "extensions/qa-lab/src/gateway-process-boundary.ts: QA_GATEWAY_PROCESS_BOUNDARY_RETAIN_LEASE_PREFIX", "extensions/qa-lab/src/harness-parity.ts: HarnessVariant", "extensions/qa-lab/src/jsonl-replay.ts: JsonlReplayCellRunner", - "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts: __testing", - "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts: listDiscordQaScenarioCatalog", "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts: testing", "extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts: WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS", - "extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts: findMissingLiveTransportStandardScenarios", - "extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS", - "extensions/qa-lab/src/live-transports/slack/cli.ts: slackQaAdapterFactory", "extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts: testing", - "extensions/qa-lab/src/live-transports/telegram/cli.ts: telegramQaAdapterFactory", "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts: testing", - "extensions/qa-lab/src/live-transports/whatsapp/cli.ts: whatsappQaAdapterFactory", "extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts: testing", "extensions/qa-lab/src/model-selection.runtime.ts: resolveQaPreferredLiveModel", "extensions/qa-lab/src/multipass.runtime.ts: createQaMultipassPlan", @@ -478,15 +468,8 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "extensions/qa-lab/src/providers/mock-openai/server.ts: resolveProviderVariant", "extensions/qa-lab/src/qa-agent-workspace.ts: __testing", "extensions/qa-lab/src/qa-transport-registry.ts: createQaTransportAdapterFactoryRegistry", - "extensions/qa-lab/src/qa-web-search-provider.shared.ts: QA_LAB_WEB_SEARCH_PROVIDER_ID", - "extensions/qa-lab/src/qa-web-search-provider.ts: QA_LAB_WEB_SEARCH_PROVIDER_ID", "extensions/qa-lab/src/run-config.ts: createDefaultQaRunSelection", "extensions/qa-lab/src/runtime-parity.ts: __testing", - "extensions/qa-lab/src/scenario-catalog.ts: QA_OBSERVABILITY_SCENARIO_IDS", - "extensions/qa-lab/src/scenario-catalog.ts: QA_PERSONAL_AGENT_SCENARIO_IDS", - "extensions/qa-lab/src/scenario-catalog.ts: resolveQaScenarioPackScenarioIds", - "extensions/qa-lab/src/scenario-packs.ts: QA_OBSERVABILITY_SCENARIO_IDS", - "extensions/qa-lab/src/scenario-packs.ts: QA_PERSONAL_AGENT_SCENARIO_IDS", "extensions/qa-lab/src/scenario-runtime-api.ts: QaScenarioRuntimeConstants", "extensions/qa-lab/src/scenario-runtime-api.ts: QaScenarioRuntimeDeps", "extensions/qa-lab/src/scorecard-evidence.ts: buildQaProfileScorecardEvidence", diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts index 01ce5c54b44a..badf8c601969 100644 --- a/scripts/qa/ux-matrix-evidence-producer.ts +++ b/scripts/qa/ux-matrix-evidence-producer.ts @@ -6,11 +6,11 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { toRepoRelativePath } from "../../extensions/qa-lab/src/cli-paths.js"; +import { resolveQaEvidenceEnvironment } from "../../extensions/qa-lab/src/evidence-environment.js"; import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND, QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, - resolveQaEvidenceEnvironment, validateQaEvidenceSummaryJson, type QaEvidenceStatus, type QaEvidenceSummaryEntry,