vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-31 14:08:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: share live transport scenario helpers

Browse Source
This commit is contained in:
Vincent Koc
2026-05-29 02:24:14 +02:00
parent 7bf100b028
commit ca41fa293f
4 changed files with 223 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts
View File

@@ -1,26 +1,18 @@
export type LiveTransportStandardScenarioId =
| "canary"
| "mention-gating"
| "allowlist-block"
| "top-level-reply-shape"
| "restart-resume"
| "thread-follow-up"
| "thread-isolation"
| "reaction-observation"
| "help-command";
import {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
collectLiveTransportStandardScenarioCoverage,
findMissingLiveTransportStandardScenarios,
type LiveTransportStandardScenarioId,
} from "openclaw/plugin-sdk/qa-runtime";
export type LiveTransportScenarioDefinition<TId extends string = string> = {
id: TId;
standardId?: LiveTransportStandardScenarioId;
timeoutMs: number;
title: string;
};
type LiveTransportStandardScenarioDefinition = {
description: string;
id: LiveTransportStandardScenarioId;
title: string;
};
export {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
collectLiveTransportStandardScenarioCoverage,
findMissingLiveTransportStandardScenarios,
selectLiveTransportScenarios,
type LiveTransportScenarioDefinition,
type LiveTransportStandardScenarioId,
} from "openclaw/plugin-sdk/qa-runtime";
export type LiveTransportCoverageMember = {
scenarioId?: string;
@@ -42,63 +34,6 @@ export type LiveTransportCoverageLaneSummary = {
transportId: string;
};
const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [
{
id: "canary",
title: "Transport canary",
description: "The lane can trigger one known-good reply on the real transport.",
},
{
id: "mention-gating",
title: "Mention gating",
description: "Messages without the required mention do not trigger a reply.",
},
{
id: "allowlist-block",
title: "Sender allowlist block",
description: "Non-allowlisted senders do not trigger a reply.",
},
{
id: "top-level-reply-shape",
title: "Top-level reply shape",
description: "Top-level replies stay top-level when the lane is configured that way.",
},
{
id: "restart-resume",
title: "Restart resume",
description: "The lane still responds after a gateway restart.",
},
{
id: "thread-follow-up",
title: "Thread follow-up",
description: "Threaded prompts receive threaded replies with the expected relation metadata.",
},
{
id: "thread-isolation",
title: "Thread isolation",
description: "Fresh top-level prompts stay out of prior threads.",
},
{
id: "reaction-observation",
title: "Reaction observation",
description: "Reaction events are observed and normalized correctly.",
},
{
id: "help-command",
title: "Help command",
description: "The transport-specific help command path replies successfully.",
},
] as const;
export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
[
"canary",
"mention-gating",
"allowlist-block",
"top-level-reply-shape",
"restart-resume",
] as const;
export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [
{
transportId: "discord",
@@ -141,74 +76,6 @@ export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[]
},
] as const;
const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
);
function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) {
for (const id of ids) {
if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) {
throw new Error(`unknown live transport standard scenario id: ${id}`);
}
}
}
export function selectLiveTransportScenarios<TDefinition extends { id: string }>(params: {
ids?: string[];
laneLabel: string;
scenarios: readonly TDefinition[];
}) {
if (!params.ids || params.ids.length === 0) {
return [...params.scenarios];
}
const requested = new Set(params.ids);
const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id));
const missingIds = [...requested].filter(
(id) => !selected.some((scenario) => scenario.id === id),
);
if (missingIds.length > 0) {
throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
}
return selected;
}
export function collectLiveTransportStandardScenarioCoverage<TId extends string>(params: {
alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[];
scenarios: readonly LiveTransportScenarioDefinition<TId>[];
}) {
const coverage: LiveTransportStandardScenarioId[] = [];
const seen = new Set<LiveTransportStandardScenarioId>();
const append = (id: LiveTransportStandardScenarioId | undefined) => {
if (!id || seen.has(id)) {
return;
}
seen.add(id);
coverage.push(id);
};
assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []);
for (const id of params.alwaysOnStandardScenarioIds ?? []) {
append(id);
}
for (const scenario of params.scenarios) {
if (scenario.standardId) {
assertKnownStandardScenarioIds([scenario.standardId]);
}
append(scenario.standardId);
}
return coverage;
}
export function findMissingLiveTransportStandardScenarios(params: {
coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
}) {
assertKnownStandardScenarioIds(params.coveredStandardScenarioIds);
assertKnownStandardScenarioIds(params.expectedStandardScenarioIds);
const covered = new Set(params.coveredStandardScenarioIds);
return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
}
export function buildLiveTransportCoverageLaneSummaries(
lanes: readonly LiveTransportCoverageLane[] = LIVE_TRANSPORT_COVERAGE_LANES,
): LiveTransportCoverageLaneSummary[] {

156
extensions/qa-matrix/src/shared/live-transport-scenarios.ts
View File

@@ -1,148 +1,8 @@
type LiveTransportStandardScenarioId =
| "canary"
| "mention-gating"
| "allowlist-block"
| "top-level-reply-shape"
| "restart-resume"
| "thread-follow-up"
| "thread-isolation"
| "reaction-observation"
| "help-command";
export type LiveTransportScenarioDefinition<TId extends string = string> = {
id: TId;
standardId?: LiveTransportStandardScenarioId;
timeoutMs: number;
title: string;
};
type LiveTransportStandardScenarioDefinition = {
description: string;
id: LiveTransportStandardScenarioId;
title: string;
};
const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [
{
id: "canary",
title: "Transport canary",
description: "The lane can trigger one known-good reply on the real transport.",
},
{
id: "mention-gating",
title: "Mention gating",
description: "Messages without the required mention do not trigger a reply.",
},
{
id: "allowlist-block",
title: "Sender allowlist block",
description: "Non-allowlisted senders do not trigger a reply.",
},
{
id: "top-level-reply-shape",
title: "Top-level reply shape",
description: "Top-level replies stay top-level when the lane is configured that way.",
},
{
id: "restart-resume",
title: "Restart resume",
description: "The lane still responds after a gateway restart.",
},
{
id: "thread-follow-up",
title: "Thread follow-up",
description: "Threaded prompts receive threaded replies with the expected relation metadata.",
},
{
id: "thread-isolation",
title: "Thread isolation",
description: "Fresh top-level prompts stay out of prior threads.",
},
{
id: "reaction-observation",
title: "Reaction observation",
description: "Reaction events are observed and normalized correctly.",
},
{
id: "help-command",
title: "Help command",
description: "The transport-specific help command path replies successfully.",
},
] as const;
export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
[
"canary",
"mention-gating",
"allowlist-block",
"top-level-reply-shape",
"restart-resume",
] as const;
const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
);
function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) {
for (const id of ids) {
if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) {
throw new Error(`unknown live transport standard scenario id: ${id}`);
}
}
}
export function selectLiveTransportScenarios<TDefinition extends { id: string }>(params: {
ids?: string[];
laneLabel: string;
scenarios: readonly TDefinition[];
}) {
if (!params.ids || params.ids.length === 0) {
return [...params.scenarios];
}
const requested = new Set(params.ids);
const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id));
const missingIds = [...requested].filter(
(id) => !selected.some((scenario) => scenario.id === id),
);
if (missingIds.length > 0) {
throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
}
return selected;
}
export function collectLiveTransportStandardScenarioCoverage<TId extends string>(params: {
alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[];
scenarios: readonly LiveTransportScenarioDefinition<TId>[];
}) {
const coverage: LiveTransportStandardScenarioId[] = [];
const seen = new Set<LiveTransportStandardScenarioId>();
const append = (id: LiveTransportStandardScenarioId | undefined) => {
if (!id || seen.has(id)) {
return;
}
seen.add(id);
coverage.push(id);
};
assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []);
for (const id of params.alwaysOnStandardScenarioIds ?? []) {
append(id);
}
for (const scenario of params.scenarios) {
if (scenario.standardId) {
assertKnownStandardScenarioIds([scenario.standardId]);
}
append(scenario.standardId);
}
return coverage;
}
export function findMissingLiveTransportStandardScenarios(params: {
coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
}) {
assertKnownStandardScenarioIds(params.coveredStandardScenarioIds);
assertKnownStandardScenarioIds(params.expectedStandardScenarioIds);
const covered = new Set(params.coveredStandardScenarioIds);
return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
}
export {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
collectLiveTransportStandardScenarioCoverage,
findMissingLiveTransportStandardScenarios,
selectLiveTransportScenarios,
type LiveTransportScenarioDefinition,
type LiveTransportStandardScenarioId,
} from "openclaw/plugin-sdk/qa-runtime";

52
src/plugin-sdk/qa-runtime.test.ts
View File

@@ -95,6 +95,58 @@ describe("plugin-sdk qa-runtime", () => {
expect(report).toContain("## Timeline");
});
it("keeps shared live transport scenario coverage helpers ordered and strict", async () => {
const module = await import("./qa-runtime.js");
expect(module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([
"canary",
"mention-gating",
"allowlist-block",
"top-level-reply-shape",
"restart-resume",
]);
const definitions = [
{ id: "alpha", timeoutMs: 1_000, title: "alpha" },
{ id: "beta", timeoutMs: 1_000, title: "beta" },
] as const;
expect(
module.selectLiveTransportScenarios({
ids: ["beta"],
laneLabel: "Demo",
scenarios: definitions,
}),
).toEqual([definitions[1]]);
expect(() =>
module.selectLiveTransportScenarios({
ids: ["missing"],
laneLabel: "Demo",
scenarios: definitions,
}),
).toThrow("unknown Demo QA scenario id(s): missing");
const covered = module.collectLiveTransportStandardScenarioCoverage({
alwaysOnStandardScenarioIds: ["canary"],
scenarios: [
{ id: "scenario-1", standardId: "mention-gating", timeoutMs: 1_000, title: "mention" },
{
id: "scenario-2",
standardId: "mention-gating",
timeoutMs: 1_000,
title: "mention again",
},
{ id: "scenario-3", standardId: "restart-resume", timeoutMs: 1_000, title: "restart" },
],
});
expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]);
expect(
module.findMissingLiveTransportStandardScenarios({
coveredStandardScenarioIds: covered,
expectedStandardScenarioIds: module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
}),
).toEqual(["allowlist-block", "top-level-reply-shape"]);
});
it("builds shared live-lane artifact errors", async () => {
const module = await import("./qa-runtime.js");

149
src/plugin-sdk/qa-runtime.ts
View File

@@ -62,6 +62,155 @@ export type QaReportScenario = {
steps?: QaReportCheck[];
};
export type LiveTransportStandardScenarioId =
| "canary"
| "mention-gating"
| "allowlist-block"
| "top-level-reply-shape"
| "restart-resume"
| "thread-follow-up"
| "thread-isolation"
| "reaction-observation"
| "help-command";
export type LiveTransportScenarioDefinition<TId extends string = string> = {
id: TId;
standardId?: LiveTransportStandardScenarioId;
timeoutMs: number;
title: string;
};
type LiveTransportStandardScenarioDefinition = {
description: string;
id: LiveTransportStandardScenarioId;
title: string;
};
const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [
{
id: "canary",
title: "Transport canary",
description: "The lane can trigger one known-good reply on the real transport.",
},
{
id: "mention-gating",
title: "Mention gating",
description: "Messages without the required mention do not trigger a reply.",
},
{
id: "allowlist-block",
title: "Sender allowlist block",
description: "Non-allowlisted senders do not trigger a reply.",
},
{
id: "top-level-reply-shape",
title: "Top-level reply shape",
description: "Top-level replies stay top-level when the lane is configured that way.",
},
{
id: "restart-resume",
title: "Restart resume",
description: "The lane still responds after a gateway restart.",
},
{
id: "thread-follow-up",
title: "Thread follow-up",
description: "Threaded prompts receive threaded replies with the expected relation metadata.",
},
{
id: "thread-isolation",
title: "Thread isolation",
description: "Fresh top-level prompts stay out of prior threads.",
},
{
id: "reaction-observation",
title: "Reaction observation",
description: "Reaction events are observed and normalized correctly.",
},
{
id: "help-command",
title: "Help command",
description: "The transport-specific help command path replies successfully.",
},
] as const;
export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] =
[
"canary",
"mention-gating",
"allowlist-block",
"top-level-reply-shape",
"restart-resume",
] as const;
const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
);
function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) {
for (const id of ids) {
if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) {
throw new Error(`unknown live transport standard scenario id: ${id}`);
}
}
}
export function selectLiveTransportScenarios<TDefinition extends { id: string }>(params: {
ids?: string[];
laneLabel: string;
scenarios: readonly TDefinition[];
}) {
if (!params.ids || params.ids.length === 0) {
return [...params.scenarios];
}
const requested = new Set(params.ids);
const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id));
const missingIds = [...requested].filter(
(id) => !selected.some((scenario) => scenario.id === id),
);
if (missingIds.length > 0) {
throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
}
return selected;
}
export function collectLiveTransportStandardScenarioCoverage<TId extends string>(params: {
alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[];
scenarios: readonly LiveTransportScenarioDefinition<TId>[];
}) {
const coverage: LiveTransportStandardScenarioId[] = [];
const seen = new Set<LiveTransportStandardScenarioId>();
const append = (id: LiveTransportStandardScenarioId | undefined) => {
if (!id || seen.has(id)) {
return;
}
seen.add(id);
coverage.push(id);
};
assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []);
for (const id of params.alwaysOnStandardScenarioIds ?? []) {
append(id);
}
for (const scenario of params.scenarios) {
if (scenario.standardId) {
assertKnownStandardScenarioIds([scenario.standardId]);
}
append(scenario.standardId);
}
return coverage;
}
export function findMissingLiveTransportStandardScenarios(params: {
coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[];
}) {
assertKnownStandardScenarioIds(params.coveredStandardScenarioIds);
assertKnownStandardScenarioIds(params.expectedStandardScenarioIds);
const covered = new Set(params.coveredStandardScenarioIds);
return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
}
export type QaDockerRunCommand = (
command: string,
args: string[],