diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index ad2325803f1..e8afbbdbe39 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -1,26 +1,18 @@ -export type LiveTransportStandardScenarioId = - | "canary" - | "mention-gating" - | "allowlist-block" - | "top-level-reply-shape" - | "restart-resume" - | "thread-follow-up" - | "thread-isolation" - | "reaction-observation" - | "help-command"; +import { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + type LiveTransportStandardScenarioId, +} from "openclaw/plugin-sdk/qa-runtime"; -export type LiveTransportScenarioDefinition = { - id: TId; - standardId?: LiveTransportStandardScenarioId; - timeoutMs: number; - title: string; -}; - -type LiveTransportStandardScenarioDefinition = { - description: string; - id: LiveTransportStandardScenarioId; - title: string; -}; +export { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, + type LiveTransportStandardScenarioId, +} from "openclaw/plugin-sdk/qa-runtime"; export type LiveTransportCoverageMember = { scenarioId?: string; @@ -42,63 +34,6 @@ export type LiveTransportCoverageLaneSummary = { transportId: string; }; -const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ - { - id: "canary", - title: "Transport canary", - description: "The lane can trigger one known-good reply on the real transport.", - }, - { - id: "mention-gating", - title: "Mention gating", - description: "Messages without the required mention do not trigger a reply.", - }, - { - id: "allowlist-block", - title: "Sender allowlist block", - description: "Non-allowlisted senders do not trigger a reply.", - }, - { - id: "top-level-reply-shape", - title: "Top-level reply shape", - description: "Top-level replies stay top-level when the lane is configured that way.", - }, - { - id: "restart-resume", - title: "Restart resume", - description: "The lane still responds after a gateway restart.", - }, - { - id: "thread-follow-up", - title: "Thread follow-up", - description: "Threaded prompts receive threaded replies with the expected relation metadata.", - }, - { - id: "thread-isolation", - title: "Thread isolation", - description: "Fresh top-level prompts stay out of prior threads.", - }, - { - id: "reaction-observation", - title: "Reaction observation", - description: "Reaction events are observed and normalized correctly.", - }, - { - id: "help-command", - title: "Help command", - description: "The transport-specific help command path replies successfully.", - }, -] as const; - -export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = - [ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ] as const; - export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [ { transportId: "discord", @@ -141,74 +76,6 @@ export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] }, ] as const; -const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( - LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), -); - -function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { - for (const id of ids) { - if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { - throw new Error(`unknown live transport standard scenario id: ${id}`); - } - } -} - -export function selectLiveTransportScenarios(params: { - ids?: string[]; - laneLabel: string; - scenarios: readonly TDefinition[]; -}) { - if (!params.ids || params.ids.length === 0) { - return [...params.scenarios]; - } - const requested = new Set(params.ids); - const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); - const missingIds = [...requested].filter( - (id) => !selected.some((scenario) => scenario.id === id), - ); - if (missingIds.length > 0) { - throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); - } - return selected; -} - -export function collectLiveTransportStandardScenarioCoverage(params: { - alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; - scenarios: readonly LiveTransportScenarioDefinition[]; -}) { - const coverage: LiveTransportStandardScenarioId[] = []; - const seen = new Set(); - const append = (id: LiveTransportStandardScenarioId | undefined) => { - if (!id || seen.has(id)) { - return; - } - seen.add(id); - coverage.push(id); - }; - - assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); - for (const id of params.alwaysOnStandardScenarioIds ?? []) { - append(id); - } - for (const scenario of params.scenarios) { - if (scenario.standardId) { - assertKnownStandardScenarioIds([scenario.standardId]); - } - append(scenario.standardId); - } - return coverage; -} - -export function findMissingLiveTransportStandardScenarios(params: { - coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; - expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; -}) { - assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); - assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); - const covered = new Set(params.coveredStandardScenarioIds); - return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); -} - export function buildLiveTransportCoverageLaneSummaries( lanes: readonly LiveTransportCoverageLane[] = LIVE_TRANSPORT_COVERAGE_LANES, ): LiveTransportCoverageLaneSummary[] { diff --git a/extensions/qa-matrix/src/shared/live-transport-scenarios.ts b/extensions/qa-matrix/src/shared/live-transport-scenarios.ts index 88bf702b121..c91ad2ebb15 100644 --- a/extensions/qa-matrix/src/shared/live-transport-scenarios.ts +++ b/extensions/qa-matrix/src/shared/live-transport-scenarios.ts @@ -1,148 +1,8 @@ -type LiveTransportStandardScenarioId = - | "canary" - | "mention-gating" - | "allowlist-block" - | "top-level-reply-shape" - | "restart-resume" - | "thread-follow-up" - | "thread-isolation" - | "reaction-observation" - | "help-command"; - -export type LiveTransportScenarioDefinition = { - id: TId; - standardId?: LiveTransportStandardScenarioId; - timeoutMs: number; - title: string; -}; - -type LiveTransportStandardScenarioDefinition = { - description: string; - id: LiveTransportStandardScenarioId; - title: string; -}; - -const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ - { - id: "canary", - title: "Transport canary", - description: "The lane can trigger one known-good reply on the real transport.", - }, - { - id: "mention-gating", - title: "Mention gating", - description: "Messages without the required mention do not trigger a reply.", - }, - { - id: "allowlist-block", - title: "Sender allowlist block", - description: "Non-allowlisted senders do not trigger a reply.", - }, - { - id: "top-level-reply-shape", - title: "Top-level reply shape", - description: "Top-level replies stay top-level when the lane is configured that way.", - }, - { - id: "restart-resume", - title: "Restart resume", - description: "The lane still responds after a gateway restart.", - }, - { - id: "thread-follow-up", - title: "Thread follow-up", - description: "Threaded prompts receive threaded replies with the expected relation metadata.", - }, - { - id: "thread-isolation", - title: "Thread isolation", - description: "Fresh top-level prompts stay out of prior threads.", - }, - { - id: "reaction-observation", - title: "Reaction observation", - description: "Reaction events are observed and normalized correctly.", - }, - { - id: "help-command", - title: "Help command", - description: "The transport-specific help command path replies successfully.", - }, -] as const; - -export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = - [ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ] as const; - -const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( - LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), -); - -function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { - for (const id of ids) { - if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { - throw new Error(`unknown live transport standard scenario id: ${id}`); - } - } -} - -export function selectLiveTransportScenarios(params: { - ids?: string[]; - laneLabel: string; - scenarios: readonly TDefinition[]; -}) { - if (!params.ids || params.ids.length === 0) { - return [...params.scenarios]; - } - const requested = new Set(params.ids); - const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); - const missingIds = [...requested].filter( - (id) => !selected.some((scenario) => scenario.id === id), - ); - if (missingIds.length > 0) { - throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); - } - return selected; -} - -export function collectLiveTransportStandardScenarioCoverage(params: { - alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; - scenarios: readonly LiveTransportScenarioDefinition[]; -}) { - const coverage: LiveTransportStandardScenarioId[] = []; - const seen = new Set(); - const append = (id: LiveTransportStandardScenarioId | undefined) => { - if (!id || seen.has(id)) { - return; - } - seen.add(id); - coverage.push(id); - }; - - assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); - for (const id of params.alwaysOnStandardScenarioIds ?? []) { - append(id); - } - for (const scenario of params.scenarios) { - if (scenario.standardId) { - assertKnownStandardScenarioIds([scenario.standardId]); - } - append(scenario.standardId); - } - return coverage; -} - -export function findMissingLiveTransportStandardScenarios(params: { - coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; - expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; -}) { - assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); - assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); - const covered = new Set(params.coveredStandardScenarioIds); - return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); -} +export { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, + type LiveTransportStandardScenarioId, +} from "openclaw/plugin-sdk/qa-runtime"; diff --git a/src/plugin-sdk/qa-runtime.test.ts b/src/plugin-sdk/qa-runtime.test.ts index 284bb3b7b1f..40c4decf2a1 100644 --- a/src/plugin-sdk/qa-runtime.test.ts +++ b/src/plugin-sdk/qa-runtime.test.ts @@ -95,6 +95,58 @@ describe("plugin-sdk qa-runtime", () => { expect(report).toContain("## Timeline"); }); + it("keeps shared live transport scenario coverage helpers ordered and strict", async () => { + const module = await import("./qa-runtime.js"); + + expect(module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([ + "canary", + "mention-gating", + "allowlist-block", + "top-level-reply-shape", + "restart-resume", + ]); + + const definitions = [ + { id: "alpha", timeoutMs: 1_000, title: "alpha" }, + { id: "beta", timeoutMs: 1_000, title: "beta" }, + ] as const; + expect( + module.selectLiveTransportScenarios({ + ids: ["beta"], + laneLabel: "Demo", + scenarios: definitions, + }), + ).toEqual([definitions[1]]); + expect(() => + module.selectLiveTransportScenarios({ + ids: ["missing"], + laneLabel: "Demo", + scenarios: definitions, + }), + ).toThrow("unknown Demo QA scenario id(s): missing"); + + const covered = module.collectLiveTransportStandardScenarioCoverage({ + alwaysOnStandardScenarioIds: ["canary"], + scenarios: [ + { id: "scenario-1", standardId: "mention-gating", timeoutMs: 1_000, title: "mention" }, + { + id: "scenario-2", + standardId: "mention-gating", + timeoutMs: 1_000, + title: "mention again", + }, + { id: "scenario-3", standardId: "restart-resume", timeoutMs: 1_000, title: "restart" }, + ], + }); + expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]); + expect( + module.findMissingLiveTransportStandardScenarios({ + coveredStandardScenarioIds: covered, + expectedStandardScenarioIds: module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + }), + ).toEqual(["allowlist-block", "top-level-reply-shape"]); + }); + it("builds shared live-lane artifact errors", async () => { const module = await import("./qa-runtime.js"); diff --git a/src/plugin-sdk/qa-runtime.ts b/src/plugin-sdk/qa-runtime.ts index 1392e503a31..915536d8a55 100644 --- a/src/plugin-sdk/qa-runtime.ts +++ b/src/plugin-sdk/qa-runtime.ts @@ -62,6 +62,155 @@ export type QaReportScenario = { steps?: QaReportCheck[]; }; +export type LiveTransportStandardScenarioId = + | "canary" + | "mention-gating" + | "allowlist-block" + | "top-level-reply-shape" + | "restart-resume" + | "thread-follow-up" + | "thread-isolation" + | "reaction-observation" + | "help-command"; + +export type LiveTransportScenarioDefinition = { + id: TId; + standardId?: LiveTransportStandardScenarioId; + timeoutMs: number; + title: string; +}; + +type LiveTransportStandardScenarioDefinition = { + description: string; + id: LiveTransportStandardScenarioId; + title: string; +}; + +const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ + { + id: "canary", + title: "Transport canary", + description: "The lane can trigger one known-good reply on the real transport.", + }, + { + id: "mention-gating", + title: "Mention gating", + description: "Messages without the required mention do not trigger a reply.", + }, + { + id: "allowlist-block", + title: "Sender allowlist block", + description: "Non-allowlisted senders do not trigger a reply.", + }, + { + id: "top-level-reply-shape", + title: "Top-level reply shape", + description: "Top-level replies stay top-level when the lane is configured that way.", + }, + { + id: "restart-resume", + title: "Restart resume", + description: "The lane still responds after a gateway restart.", + }, + { + id: "thread-follow-up", + title: "Thread follow-up", + description: "Threaded prompts receive threaded replies with the expected relation metadata.", + }, + { + id: "thread-isolation", + title: "Thread isolation", + description: "Fresh top-level prompts stay out of prior threads.", + }, + { + id: "reaction-observation", + title: "Reaction observation", + description: "Reaction events are observed and normalized correctly.", + }, + { + id: "help-command", + title: "Help command", + description: "The transport-specific help command path replies successfully.", + }, +] as const; + +export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = + [ + "canary", + "mention-gating", + "allowlist-block", + "top-level-reply-shape", + "restart-resume", + ] as const; + +const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( + LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), +); + +function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { + for (const id of ids) { + if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { + throw new Error(`unknown live transport standard scenario id: ${id}`); + } + } +} + +export function selectLiveTransportScenarios(params: { + ids?: string[]; + laneLabel: string; + scenarios: readonly TDefinition[]; +}) { + if (!params.ids || params.ids.length === 0) { + return [...params.scenarios]; + } + const requested = new Set(params.ids); + const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); + const missingIds = [...requested].filter( + (id) => !selected.some((scenario) => scenario.id === id), + ); + if (missingIds.length > 0) { + throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); + } + return selected; +} + +export function collectLiveTransportStandardScenarioCoverage(params: { + alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; + scenarios: readonly LiveTransportScenarioDefinition[]; +}) { + const coverage: LiveTransportStandardScenarioId[] = []; + const seen = new Set(); + const append = (id: LiveTransportStandardScenarioId | undefined) => { + if (!id || seen.has(id)) { + return; + } + seen.add(id); + coverage.push(id); + }; + + assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); + for (const id of params.alwaysOnStandardScenarioIds ?? []) { + append(id); + } + for (const scenario of params.scenarios) { + if (scenario.standardId) { + assertKnownStandardScenarioIds([scenario.standardId]); + } + append(scenario.standardId); + } + return coverage; +} + +export function findMissingLiveTransportStandardScenarios(params: { + coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; + expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; +}) { + assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); + assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); + const covered = new Set(params.coveredStandardScenarioIds); + return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); +} + export type QaDockerRunCommand = ( command: string, args: string[],