From 2d9b78fff73afa4ff6449be49487545c2fe789ac Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 15 Jul 2026 16:50:41 -0700 Subject: [PATCH] refactor(qa): canonicalize live scenario coverage (#108464) --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/concepts/qa-e2e-automation.md | 36 ++-- docs/docs_map.md | 2 +- docs/plugins/sdk-migration.md | 14 +- docs/plugins/sdk-subpaths.md | 1 - extensions/qa-lab/src/cli.runtime.test.ts | 9 +- extensions/qa-lab/src/cli.runtime.ts | 10 +- extensions/qa-lab/src/coverage-report.test.ts | 10 -- extensions/qa-lab/src/coverage-report.ts | 44 +---- .../discord/cli.runtime.test.ts | 6 +- .../qa-lab/src/live-transports/discord/cli.ts | 2 - .../discord/discord-live.runtime.test.ts | 14 -- .../discord/discord-live.runtime.ts | 43 ++--- .../src/live-transports/discord/profiles.ts | 20 +-- .../live-transport-adapters.test.ts | 31 +++- .../src/live-transports/matrix/cli.test.ts | 18 -- .../qa-lab/src/live-transports/matrix/cli.ts | 16 -- .../src/live-transports/matrix/profiles.ts | 168 ++---------------- .../shared/live-transport-result.ts | 1 - .../shared/live-transport-scenarios.test.ts | 138 -------------- .../shared/live-transport-scenarios.ts | 107 ----------- .../live-transports/slack/cli.runtime.test.ts | 6 +- .../qa-lab/src/live-transports/slack/cli.ts | 2 - .../src/live-transports/slack/profiles.ts | 37 +--- .../slack/slack-live.contracts.ts | 7 +- .../slack/slack-live.scenarios.ts | 10 -- .../src/live-transports/telegram/cli.test.ts | 13 -- .../src/live-transports/telegram/cli.ts | 2 - .../live-transports/telegram/profiles.test.ts | 23 ++- .../src/live-transports/telegram/profiles.ts | 64 ++----- .../src/live-transports/whatsapp/cli.ts | 2 - .../src/live-transports/whatsapp/profiles.ts | 58 +----- .../whatsapp/whatsapp-live.contracts.ts | 11 +- .../whatsapp/whatsapp-live.runtime.test.ts | 13 +- .../whatsapp-live.scenarios.capabilities.ts | 10 -- .../whatsapp-live.scenarios.conversation.ts | 10 -- .../whatsapp-live.scenarios.delivery.ts | 5 - .../whatsapp-live.scenarios.user-path.ts | 9 - .../qa-lab/src/scenario-catalog.test.ts | 35 ++-- extensions/qa-lab/src/scenario-catalog.ts | 27 +++ extensions/qa-lab/src/scenario-lane.ts | 6 +- extensions/qa-lab/src/suite-planning.test.ts | 6 +- extensions/qa-lab/src/suite-test-helpers.ts | 2 + package.json | 4 - ...struction-followthrough-repo-contract.yaml | 2 +- .../agents/subagent-forked-context.yaml | 2 +- qa/scenarios/agents/subagent-handoff.yaml | 2 +- .../a2a-message-tool-mirror-dedupe.yaml | 2 +- qa/scenarios/channels/channel-canary.yaml | 10 ++ .../channels/channel-chat-baseline.yaml | 9 + .../channels/channel-dm-group-routing.yaml | 1 + .../channels/channel-mention-gating.yaml | 13 ++ .../channel-multi-actor-ordering.yaml | 1 + ...nnel-secondary-conversation-isolation.yaml | 1 + .../channels/channel-sender-allowlist.yaml | 1 + .../channel-top-level-reply-shape.yaml | 9 + qa/scenarios/channels/discord-canary.yaml | 1 + .../channels/discord-mention-gating.yaml | 1 + ...cord-native-help-command-registration.yaml | 1 + qa/scenarios/channels/dm-chat-baseline.yaml | 1 + .../channels/dm-per-room-session.yaml | 1 + qa/scenarios/channels/dm-shared-session.yaml | 1 + ...oup-message-tool-unavailable-fallback.yaml | 2 +- .../matrix-allowbots-default-block.yaml | 3 +- ...rix-allowbots-mentions-dm-unmentioned.yaml | 3 +- ...rix-allowbots-mentions-mentioned-room.yaml | 3 +- ...-mentions-unmentioned-open-room-block.yaml | 3 +- ...ots-room-override-blocks-account-true.yaml | 3 +- ...ots-room-override-enables-account-off.yaml | 3 +- .../matrix-allowbots-self-sender-ignored.yaml | 3 +- ...-allowbots-true-unmentioned-open-room.yaml | 3 +- .../channels/matrix-allowlist-hot-reload.yaml | 3 +- .../matrix-approval-channel-target-both.yaml | 3 +- .../matrix-approval-deny-reaction.yaml | 3 +- ...matrix-approval-exec-metadata-chunked.yaml | 3 +- ...x-approval-exec-metadata-single-event.yaml | 3 +- ...approval-plugin-metadata-single-event.yaml | 3 +- .../matrix-approval-thread-target.yaml | 3 +- .../matrix-attachment-only-ignored.yaml | 3 +- .../matrix-dm-thread-reply-override.yaml | 3 +- .../matrix-e2ee-artifact-redaction.yaml | 3 +- .../channels/matrix-e2ee-basic-reply.yaml | 3 +- .../matrix-e2ee-bootstrap-success.yaml | 3 +- ...trix-e2ee-cli-account-add-enable-e2ee.yaml | 3 +- ...li-encryption-setup-bootstrap-failure.yaml | 3 +- ...-e2ee-cli-encryption-setup-idempotent.yaml | 3 +- ...ee-cli-encryption-setup-multi-account.yaml | 3 +- .../matrix-e2ee-cli-encryption-setup.yaml | 3 +- .../matrix-e2ee-cli-recovery-key-invalid.yaml | 3 +- .../matrix-e2ee-cli-recovery-key-setup.yaml | 3 +- .../matrix-e2ee-cli-self-verification.yaml | 3 +- ...rix-e2ee-cli-setup-then-gateway-reply.yaml | 3 +- ...trix-e2ee-corrupt-crypto-idb-snapshot.yaml | 3 +- .../matrix-e2ee-device-sas-verification.yaml | 3 +- .../matrix-e2ee-dm-sas-verification.yaml | 3 +- ...trix-e2ee-history-exists-backup-empty.yaml | 3 +- .../matrix-e2ee-key-bootstrap-failure.yaml | 3 +- .../channels/matrix-e2ee-media-image.yaml | 3 +- .../channels/matrix-e2ee-qr-verification.yaml | 3 +- .../matrix-e2ee-recovery-key-lifecycle.yaml | 3 +- ...-recovery-owner-verification-required.yaml | 3 +- .../channels/matrix-e2ee-restart-resume.yaml | 3 +- ...ackup-deleted-local-reupload-restores.yaml | 3 +- ...ver-backup-deleted-local-state-intact.yaml | 3 +- ...ver-device-deleted-local-state-intact.yaml | 3 +- ...erver-device-deleted-relogin-recovers.yaml | 3 +- .../matrix-e2ee-stale-device-hygiene.yaml | 3 +- ...stale-recovery-key-after-backup-reset.yaml | 3 +- ...x-e2ee-state-after-missing-encryption.yaml | 3 +- ...e2ee-state-loss-external-recovery-key.yaml | 3 +- ...atrix-e2ee-state-loss-no-recovery-key.yaml | 3 +- ...x-e2ee-state-loss-stored-recovery-key.yaml | 3 +- ...ix-e2ee-sync-state-loss-crypto-intact.yaml | 3 +- .../matrix-e2ee-thread-follow-up.yaml | 3 +- ...x-e2ee-verification-notice-no-trigger.yaml | 3 +- ...atrix-e2ee-wrong-account-recovery-key.yaml | 3 +- .../matrix-homeserver-restart-resume.yaml | 3 +- .../channels/matrix-inbound-edit-ignored.yaml | 4 +- ...rix-inbound-edit-no-duplicate-trigger.yaml | 4 +- ...trix-initial-catchup-then-incremental.yaml | 3 +- .../channels/matrix-media-type-coverage.yaml | 3 +- .../matrix-mention-metadata-spoof-block.yaml | 4 +- .../matrix-mxid-prefixed-command-block.yaml | 3 +- .../channels/matrix-reaction-not-a-reply.yaml | 3 +- .../matrix-reaction-notification.yaml | 3 +- .../matrix-reaction-redaction-observed.yaml | 3 +- .../channels/matrix-reaction-threaded.yaml | 3 +- .../channels/matrix-room-autojoin-invite.yaml | 3 +- .../channels/matrix-room-block-streaming.yaml | 2 +- .../matrix-room-generated-image-delivery.yaml | 3 +- ...x-room-image-understanding-attachment.yaml | 3 +- .../channels/matrix-room-membership-loss.yaml | 3 +- ...matrix-room-partial-streaming-preview.yaml | 3 +- .../matrix-room-quiet-streaming-preview.yaml | 3 +- ...ix-room-tool-progress-command-preview.yaml | 3 +- .../matrix-room-tool-progress-error.yaml | 3 +- ...rix-room-tool-progress-mention-safety.yaml | 3 +- ...ix-room-tool-progress-preview-opt-out.yaml | 3 +- .../matrix-room-tool-progress-preview.yaml | 3 +- .../matrix-secondary-room-open-trigger.yaml | 3 +- .../matrix-stale-sync-replay-dedupe.yaml | 3 +- .../matrix-thread-nested-reply-shape.yaml | 4 +- .../matrix-thread-root-preservation.yaml | 4 +- .../matrix-unsupported-media-safe.yaml | 3 +- .../matrix-voice-preflight-mention.yaml | 3 +- .../channels/qa-channel-reconnect-dedupe.yaml | 2 +- .../channels/reaction-edit-delete.yaml | 2 +- .../channels/slack-allowlist-block.yaml | 1 + .../channels/slack-approval-exec-native.yaml | 1 + .../slack-approval-plugin-native.yaml | 1 + qa/scenarios/channels/slack-canary.yaml | 1 + .../slack-chart-presentation-native.yaml | 1 + .../slack-codex-approval-exec-native.yaml | 1 + .../slack-codex-approval-plugin-native.yaml | 1 + .../channels/slack-mention-gating.yaml | 1 + .../channels/slack-reaction-glyph-native.yaml | 1 + .../channels/slack-restart-resume.yaml | 2 +- .../slack-table-presentation-native.yaml | 1 + .../channels/slack-top-level-reply-shape.yaml | 1 + .../channels/subagent-thread-spawn.yaml | 2 +- .../channels/telegram-commands-command.yaml | 7 + .../channels/telegram-context-command.yaml | 7 + .../telegram-current-session-status-tool.yaml | 3 +- .../channels/telegram-help-command.yaml | 7 + .../telegram-long-final-reuses-preview.yaml | 3 +- .../telegram-long-final-three-chunks.yaml | 3 +- .../telegram-other-bot-command-gating.yaml | 10 +- ...legram-repeated-command-authorization.yaml | 10 +- .../telegram-reply-chain-exact-marker.yaml | 1 + .../channels/telegram-status-command.yaml | 7 + .../telegram-stream-final-single-message.yaml | 3 +- .../telegram-tool-only-usage-footer.yaml | 3 +- .../telegram-tools-compact-command.yaml | 7 + .../channels/telegram-whoami-command.yaml | 7 + qa/scenarios/channels/thread-follow-up.yaml | 9 +- qa/scenarios/channels/thread-isolation.yaml | 9 +- .../channels/thread-reply-override.yaml | 3 +- .../whatsapp-access-control-dm-disabled.yaml | 2 +- .../whatsapp-access-control-dm-open.yaml | 2 +- ...hatsapp-access-control-group-disabled.yaml | 2 +- .../whatsapp-access-control-group-open.yaml | 2 +- .../whatsapp-agent-message-action-react.yaml | 1 + ...sapp-agent-message-action-upload-file.yaml | 1 + .../channels/whatsapp-audio-preflight.yaml | 1 + .../whatsapp-broadcast-group-fanout.yaml | 1 + qa/scenarios/channels/whatsapp-canary.yaml | 1 + .../channels/whatsapp-commands-command.yaml | 3 +- .../channels/whatsapp-context-command.yaml | 3 +- .../whatsapp-group-activation-always.yaml | 1 + ...sapp-group-agent-message-action-react.yaml | 1 + ...roup-agent-message-action-upload-file.yaml | 1 + .../whatsapp-group-allowlist-block.yaml | 1 + .../channels/whatsapp-group-audio-gating.yaml | 1 + .../whatsapp-group-outbound-audio.yaml | 1 + .../whatsapp-group-outbound-media.yaml | 1 + .../whatsapp-group-outbound-poll.yaml | 1 + ...hatsapp-group-pending-history-context.yaml | 1 + .../whatsapp-group-reply-to-bot-triggers.yaml | 1 + .../whatsapp-group-reply-to-message.yaml | 1 + .../channels/whatsapp-help-command.yaml | 4 +- .../whatsapp-inbound-image-caption.yaml | 1 + .../whatsapp-inbound-reaction-no-trigger.yaml | 1 + .../whatsapp-inbound-structured-messages.yaml | 1 + .../channels/whatsapp-mention-gating.yaml | 1 + .../channels/whatsapp-message-actions.yaml | 1 + .../channels/whatsapp-native-new-command.yaml | 3 +- ...-outbound-document-preserves-filename.yaml | 1 + .../whatsapp-outbound-media-matrix.yaml | 1 + .../channels/whatsapp-outbound-poll.yaml | 1 + .../channels/whatsapp-pairing-block.yaml | 2 +- .../whatsapp-reply-context-isolation.yaml | 1 + .../whatsapp-reply-delivery-shape.yaml | 1 + .../channels/whatsapp-reply-to-message.yaml | 1 + .../whatsapp-reply-to-mode-batched.yaml | 1 + .../channels/whatsapp-restart-resume.yaml | 2 +- .../channels/whatsapp-status-command.yaml | 3 +- .../whatsapp-status-reaction-lifecycle.yaml | 1 + .../channels/whatsapp-status-reactions.yaml | 1 + ...tsapp-stream-final-message-accounting.yaml | 1 + .../whatsapp-tool-only-usage-footer.yaml | 3 +- .../whatsapp-tools-compact-command.yaml | 3 +- .../whatsapp-top-level-reply-shape.yaml | 1 + .../channels/whatsapp-whoami-command.yaml | 3 +- .../config/config-apply-restart-wakeup.yaml | 2 +- .../goals/goal-context-next-turn.yaml | 2 +- .../goal-context-survives-compaction.yaml | 2 +- .../goals/goal-followthrough-live.yaml | 2 +- .../media/image-generation-roundtrip.yaml | 2 +- .../media/image-understanding-attachment.yaml | 2 +- .../media/native-image-generation.yaml | 2 +- .../active-memory-preprompt-recall.yaml | 2 +- qa/scenarios/memory/memory-recall.yaml | 2 +- .../memory/session-memory-ranking.yaml | 2 +- .../memory/thread-memory-isolation.yaml | 2 +- .../personal/channel-thread-reply.yaml | 2 +- .../personal/memory-preference-recall.yaml | 2 +- qa/scenarios/personal/reminder-roundtrip.yaml | 2 +- .../scheduling/cron-condition-watcher.yaml | 3 +- .../cron-natural-fire-no-duplicate.yaml | 2 +- .../scheduling/cron-one-minute-ping.yaml | 2 +- .../cron-single-run-no-duplicate.yaml | 2 +- ...ui-assistant-transcript-role-boundary.yaml | 2 +- ...control-ui-qa-channel-image-roundtrip.yaml | 2 +- scripts/lib/plugin-sdk-doc-metadata.ts | 3 - scripts/lib/plugin-sdk-entrypoints.json | 1 - scripts/plugin-sdk-surface-report.mjs | 6 +- scripts/qa/ux-matrix-evidence-producer.ts | 34 ++-- src/plugin-sdk/qa-live-transport-scenarios.ts | 166 ----------------- src/plugin-sdk/qa-runner-runtime.ts | 1 - src/plugin-sdk/qa-runtime.test.ts | 52 ------ src/plugin-sdk/qa-runtime.ts | 9 - src/plugins/compat/registry.ts | 1 - src/plugins/sdk-alias.test.ts | 21 +-- 253 files changed, 618 insertions(+), 1271 deletions(-) delete mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts delete mode 100644 extensions/qa-lab/src/live-transports/telegram/cli.test.ts delete mode 100644 src/plugin-sdk/qa-live-transport-scenarios.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index f9f7117ec76d..0226fba4299e 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -505c586d2ac5e556d9128d90f9dba2b3d47b98e5ef7dd81aefb6208da5643095 plugin-sdk-api-baseline.json -588cdf7f285e2d50c6a03410b67c30ac99886c3e4eaebbfb2c503893578da979 plugin-sdk-api-baseline.jsonl +25f96bb0ea9201fa0a2b47156911297de8fd0a83af806c63bd6679c60511b1b8 plugin-sdk-api-baseline.json +eb7359b5c19b6543da999194471f576fe715f099aa1c44e81cfe5d307275f2b6 plugin-sdk-api-baseline.jsonl diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 79f6c32b8c9c..d360220050e9 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -194,10 +194,8 @@ precedence over `--profile`. | `e2ee-deep` | 18 | State-loss, backup, key recovery, device hygiene, and SAS/QR/DM verification. | | `e2ee-cli` | 9 | `openclaw matrix encryption setup`, recovery-key, multi-account, gateway round-trip, and self-verification commands through the harness. | -The profile mapping lives in -`extensions/qa-lab/src/live-transports/matrix/profiles.ts`. The declarative -Matrix scenarios live under `qa/scenarios/channels/`; their live -implementations live under +Profile membership and channel/driver requirements live with the declarative +Matrix scenarios under `qa/scenarios/channels/`. Their live implementations live under `extensions/qa-lab/src/live-transports/matrix/scenarios/`. The adapter provisions a disposable Tuwunel homeserver in Docker (default @@ -393,27 +391,19 @@ only set/missing status for `OPENCLAW_QA_CONVEX_SECRET_CI` and `OPENCLAW_QA_CONVEX_SECRET_MAINTAINER`, and verifies admin/list reachability when the maintainer secret is present. -## Live transport coverage +## Canonical scenario coverage -Live transport lanes share one contract instead of each inventing their own -scenario list shape. `qa-channel` is the broad synthetic product-behavior -suite and is not part of the live transport coverage matrix. +The root `taxonomy.yaml` defines semantic coverage IDs. Scenario YAML files +under `qa/scenarios/` map each scenario to those IDs and own execution +metadata: `channel` is the only channel requirement, `driver` constrains the +runner when needed, and `profiles` declare named run membership. TypeScript +runners query that catalog; they do not maintain parallel scenario or coverage +inventories. -Live transport runners import the shared scenario ids, baseline coverage -helpers, and scenario-selection helper from -`openclaw/plugin-sdk/qa-live-transport-scenarios`. - -| Lane | Canary | Mention gating | Bot-to-bot | Allowlist block | Top-level reply | Quote reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration | -| -------- | ------ | -------------- | ---------- | --------------- | --------------- | ----------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- | -| Discord | x | x | x | | | | | | | | | x | -| Matrix | x | x | x | x | x | | x | x | x | x | | | -| Slack | x | x | x | x | x | | x | x | x | | | | -| Telegram | x | x | x | | | | | | | | x | | -| WhatsApp | x | x | | x | x | x | x | | | x | x | | - -This keeps `qa-channel` as the broad product-behavior suite while Matrix, -Telegram, and the other live transports share one explicit transport-contract -checklist. +Static `qa coverage` output reports the taxonomy-to-scenario mapping. Actual +proof comes from `qa-evidence.json`, which records the executed scenario, +coverage IDs, channel, driver, and result. Channel and driver are report +dimensions, not additional coverage-ID vocabularies. For a disposable Linux VM lane without bringing Docker into the QA path, run: diff --git a/docs/docs_map.md b/docs/docs_map.md index 65e723190608..aa46315d672d 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -2820,7 +2820,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H3: Discord Mantis scenarios - H3: Mantis Slack desktop and visual-task runners - H3: Credential pool health check - - H2: Live transport coverage + - H2: Canonical scenario coverage - H2: Discord, Slack, Telegram, and WhatsApp QA reference - H3: Shared CLI flags - H3: Telegram QA diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 3b9e108097a0..8b44e7bf359d 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -1050,13 +1050,13 @@ known external consumers; only its public package export will be retired. Its module will remain available to bundled plugins as a private-local-only subpath. -| `removeAfter` | Tier | SDK subpaths | -| ------------- | -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `2026-07-30` | Earlier compatibility deprecations | `agent-dir-compat`, `channel-envelope`, `channel-inbound-roots`, `channel-location`, `channel-message-runtime`, `channel-pairing-paths`, `channel-reply-options-runtime`, `config-schema`, `config-types`, `direct-dm`, `direct-dm-access`, `mattermost`, `media-generation-runtime-shared`, `memory-core`, `memory-core-engine-runtime`, `memory-core-host-events`, `memory-core-host-multimodal`, `memory-core-host-query`, `memory-host-files`, `memory-host-status`, `music-generation-core`, `outbound-runtime`, `outbound-send-deps`, `provider-auth-login`, `provider-zai-endpoint`, `reply-dedupe`, `runtime-logger`, `runtime-secret-resolution`, `self-hosted-provider-setup`, `setup-adapter-runtime`, `telegram-command-config`, `webhook-path`, `zalouser` | -| `2026-07-30` | θ1: fully unused; remove subpath | `command-gating`, `lmstudio`, `lmstudio-runtime`, `secret-provider-integration`, `skills-runtime` | -| `2026-07-30` | θ2: bundled-only; retire public export | `access-groups`, `account-resolution-runtime`, `acp-binding-resolve-runtime`, `acp-binding-runtime`, `acp-runtime`, `acp-runtime-backend`, `agent-core`, `agent-harness-exec-review-runtime`, `agent-harness-task-runtime`, `agent-harness-tool-runtime`, `agent-media-payload`, `agent-sessions`, `approval-reaction-runtime`, `approval-reference-runtime`, `async-lock-runtime`, `browser-config`, `bundled-channel-config-schema`, `channel-activity-runtime`, `channel-config-writes`, `channel-mention-gating`, `channel-route`, `channel-secret-tts-runtime`, `channel-targets`, `chat-channel-ids`, `cli-backend`, `cli-runtime`, `codex-mcp-projection`, `command-status-runtime`, `command-surface`, `concurrency-runtime`, `context-visibility-runtime`, `conversation-binding-runtime`, `cron-store-runtime`, `dangerous-name-runtime`, `delivery-queue-runtime`, `direct-dm-guard-policy`, `directory-config-runtime`, `document-extractor`, `embedding-providers`, `exec-approvals-runtime`, `expect-runtime`, `fetch-runtime`, `file-access-runtime`, `file-lock`, `global-singleton`, `group-activation`, `heartbeat-runtime`, `host-runtime`, `html-entity-runtime`, `image-generation`, `image-generation-core`, `image-generation-runtime`, `inline-image-data-url-runtime`, `json-schema-runtime`, `json-unsafe-integers`, `keyed-async-queue`, `llm`, `markdown-table-runtime`, `media-generation-runtime`, `media-understanding`, `memory-core-host-embedding-registry`, `memory-core-host-engine-embeddings`, `memory-core-host-engine-qmd`, `memory-core-host-engine-storage`, `memory-core-host-runtime-cli`, `memory-core-host-runtime-core`, `memory-core-host-runtime-files`, `memory-core-host-secret`, `memory-core-host-status`, `memory-host-core`, `memory-host-events`, `memory-host-markdown`, `memory-host-search`, `message-tool-delivery-hints`, `migration`, `migration-runtime`, `music-generation`, `node-host`, `number-runtime`, `outbound-media`, `pair-loop-guard-runtime`, `plugin-config-runtime`, `plugin-state-runtime`, `poll-runtime`, `process-runtime`, `provider-auth-api-key`, `provider-auth-login-flow-runtime`, `provider-auth-result`, `provider-auth-runtime`, `provider-catalog-live-runtime`, `provider-catalog-shared`, `provider-entry`, `provider-env-vars`, `provider-http`, `provider-model-shared`, `provider-model-types`, `provider-oauth-runtime`, `provider-onboard`, `provider-selection-runtime`, `provider-setup`, `provider-stream`, `provider-stream-family`, `provider-stream-shared`, `provider-tools`, `provider-transport-runtime`, `provider-usage`, `provider-web-fetch`, `provider-web-fetch-contract`, `provider-web-search`, `provider-web-search-config-contract`, `provider-web-search-contract`, `qa-live-transport-scenarios`, `qa-runner-runtime`, `realtime-bootstrap-context`, `realtime-transcription`, `realtime-voice`, `reply-reference`, `request-url`, `response-limit-runtime`, `retry-runtime`, `runtime-doctor`, `runtime-fetch`, `sandbox`, `secret-file-runtime`, `secure-random-runtime`, `session-binding-runtime`, `session-catalog`, `session-key-runtime`, `session-transcript-hit`, `session-transcript-runtime`, `session-visibility`, `simple-completion-runtime`, `speech`, `speech-core`, `sqlite-runtime`, `ssrf-dispatcher`, `string-normalization-runtime`, `system-event-runtime`, `talk-config-runtime`, `target-resolver-runtime`, `text-autolink-runtime`, `text-utility-runtime`, `thread-bindings-runtime`, `thread-bindings-session-runtime`, `time-runtime`, `tool-payload`, `tool-plugin`, `tool-results`, `transcripts`, `transport-ready-runtime`, `tts-runtime`, `types`, `video-generation`, `video-generation-core`, `video-generation-runtime`, `web-content-extractor`, `webhook-targets`, `windows-spawn` | -| `2026-08-15` | Earlier compatibility deprecations | `agent-config-primitives`, `channel-logging`, `channel-secret-runtime`, `channel-streaming`, `group-access`, `inbound-reply-dispatch`, `matrix`, `text-runtime`, `zod` | -| `2026-09-01` | Earlier compatibility deprecations | `channel-lifecycle`, `channel-message`, `channel-reply-pipeline`, `config-runtime`, `infra-runtime` | +| `removeAfter` | Tier | SDK subpaths | +| ------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `2026-07-30` | Earlier compatibility deprecations | `agent-dir-compat`, `channel-envelope`, `channel-inbound-roots`, `channel-location`, `channel-message-runtime`, `channel-pairing-paths`, `channel-reply-options-runtime`, `config-schema`, `config-types`, `direct-dm`, `direct-dm-access`, `mattermost`, `media-generation-runtime-shared`, `memory-core`, `memory-core-engine-runtime`, `memory-core-host-events`, `memory-core-host-multimodal`, `memory-core-host-query`, `memory-host-files`, `memory-host-status`, `music-generation-core`, `outbound-runtime`, `outbound-send-deps`, `provider-auth-login`, `provider-zai-endpoint`, `reply-dedupe`, `runtime-logger`, `runtime-secret-resolution`, `self-hosted-provider-setup`, `setup-adapter-runtime`, `telegram-command-config`, `webhook-path`, `zalouser` | +| `2026-07-30` | θ1: fully unused; remove subpath | `command-gating`, `lmstudio`, `lmstudio-runtime`, `secret-provider-integration`, `skills-runtime` | +| `2026-07-30` | θ2: bundled-only; retire public export | `access-groups`, `account-resolution-runtime`, `acp-binding-resolve-runtime`, `acp-binding-runtime`, `acp-runtime`, `acp-runtime-backend`, `agent-core`, `agent-harness-exec-review-runtime`, `agent-harness-task-runtime`, `agent-harness-tool-runtime`, `agent-media-payload`, `agent-sessions`, `approval-reaction-runtime`, `approval-reference-runtime`, `async-lock-runtime`, `browser-config`, `bundled-channel-config-schema`, `channel-activity-runtime`, `channel-config-writes`, `channel-mention-gating`, `channel-route`, `channel-secret-tts-runtime`, `channel-targets`, `chat-channel-ids`, `cli-backend`, `cli-runtime`, `codex-mcp-projection`, `command-status-runtime`, `command-surface`, `concurrency-runtime`, `context-visibility-runtime`, `conversation-binding-runtime`, `cron-store-runtime`, `dangerous-name-runtime`, `delivery-queue-runtime`, `direct-dm-guard-policy`, `directory-config-runtime`, `document-extractor`, `embedding-providers`, `exec-approvals-runtime`, `expect-runtime`, `fetch-runtime`, `file-access-runtime`, `file-lock`, `global-singleton`, `group-activation`, `heartbeat-runtime`, `host-runtime`, `html-entity-runtime`, `image-generation`, `image-generation-core`, `image-generation-runtime`, `inline-image-data-url-runtime`, `json-schema-runtime`, `json-unsafe-integers`, `keyed-async-queue`, `llm`, `markdown-table-runtime`, `media-generation-runtime`, `media-understanding`, `memory-core-host-embedding-registry`, `memory-core-host-engine-embeddings`, `memory-core-host-engine-qmd`, `memory-core-host-engine-storage`, `memory-core-host-runtime-cli`, `memory-core-host-runtime-core`, `memory-core-host-runtime-files`, `memory-core-host-secret`, `memory-core-host-status`, `memory-host-core`, `memory-host-events`, `memory-host-markdown`, `memory-host-search`, `message-tool-delivery-hints`, `migration`, `migration-runtime`, `music-generation`, `node-host`, `number-runtime`, `outbound-media`, `pair-loop-guard-runtime`, `plugin-config-runtime`, `plugin-state-runtime`, `poll-runtime`, `process-runtime`, `provider-auth-api-key`, `provider-auth-login-flow-runtime`, `provider-auth-result`, `provider-auth-runtime`, `provider-catalog-live-runtime`, `provider-catalog-shared`, `provider-entry`, `provider-env-vars`, `provider-http`, `provider-model-shared`, `provider-model-types`, `provider-oauth-runtime`, `provider-onboard`, `provider-selection-runtime`, `provider-setup`, `provider-stream`, `provider-stream-family`, `provider-stream-shared`, `provider-tools`, `provider-transport-runtime`, `provider-usage`, `provider-web-fetch`, `provider-web-fetch-contract`, `provider-web-search`, `provider-web-search-config-contract`, `provider-web-search-contract`, `qa-runner-runtime`, `realtime-bootstrap-context`, `realtime-transcription`, `realtime-voice`, `reply-reference`, `request-url`, `response-limit-runtime`, `retry-runtime`, `runtime-doctor`, `runtime-fetch`, `sandbox`, `secret-file-runtime`, `secure-random-runtime`, `session-binding-runtime`, `session-catalog`, `session-key-runtime`, `session-transcript-hit`, `session-transcript-runtime`, `session-visibility`, `simple-completion-runtime`, `speech`, `speech-core`, `sqlite-runtime`, `ssrf-dispatcher`, `string-normalization-runtime`, `system-event-runtime`, `talk-config-runtime`, `target-resolver-runtime`, `text-autolink-runtime`, `text-utility-runtime`, `thread-bindings-runtime`, `thread-bindings-session-runtime`, `time-runtime`, `tool-payload`, `tool-plugin`, `tool-results`, `transcripts`, `transport-ready-runtime`, `tts-runtime`, `types`, `video-generation`, `video-generation-core`, `video-generation-runtime`, `web-content-extractor`, `webhook-targets`, `windows-spawn` | +| `2026-08-15` | Earlier compatibility deprecations | `agent-config-primitives`, `channel-logging`, `channel-secret-runtime`, `channel-streaming`, `group-access`, `inbound-reply-dispatch`, `matrix`, `text-runtime`, `zod` | +| `2026-09-01` | Earlier compatibility deprecations | `channel-lifecycle`, `channel-message`, `channel-reply-pipeline`, `config-runtime`, `infra-runtime` | All core plugins have already migrated. External plugins should migrate before the next major release. Run `pnpm plugins:boundary-report` to see which diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 0c83b9d96ea6..eda187d1e213 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -257,7 +257,6 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/process-runtime` | Process exec helpers | | `plugin-sdk/node-host` | Node-host executable resolution and PTY resume helpers | | `plugin-sdk/cli-runtime` | Deprecated broad barrel for CLI formatting, wait, version, argument-invocation, and lazy command-group helpers; prefer focused CLI/runtime subpaths | - | `plugin-sdk/qa-live-transport-scenarios` | Shared live transport QA scenario ids, baseline coverage helpers, and scenario-selection helper | | `plugin-sdk/qa-runner-runtime` | Supported facade exposing plugin QA scenarios through the CLI command surface | | `plugin-sdk/tts-runtime` | Supported facade for text-to-speech config schemas and runtime helpers | | `plugin-sdk/gateway-method-runtime` | Reserved Gateway method dispatch helper for plugin HTTP routes that declare `contracts.gatewayMethodDispatch: ["authenticated-request"]` | diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index eb21d6a02c31..0a78d7455530 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -3,7 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { QaScenarioPack } from "./scenario-catalog.js"; +import { listQaScenariosForExecutionProfile, type QaScenarioPack } from "./scenario-catalog.js"; const { runQaManualLane, @@ -283,7 +283,6 @@ describe("qa cli runtime", () => { listLiveTransportQaAdapterFactories.mockReturnValue([ { id: "telegram", - scenarioIds: ["channel-chat-baseline"], matches: vi.fn(), create: vi.fn(), }, @@ -678,7 +677,7 @@ describe("qa cli runtime", () => { ); }); - it("uses the selected live adapter's declared scenarios by default", async () => { + it("uses the selected live adapter's YAML profile by default", async () => { await runQaSuiteCommand({ channelDriver: "live", channel: "telegram", @@ -687,7 +686,9 @@ describe("qa cli runtime", () => { expect(runQaSuite).toHaveBeenCalledWith( expect.objectContaining({ adapterOptions: expect.objectContaining({ explicitScenarioSelection: false }), - scenarioIds: ["channel-chat-baseline"], + scenarioIds: listQaScenariosForExecutionProfile("telegram:adapter").map( + (scenario) => scenario.id, + ), }), ); }); diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index e9bd4630b228..137e94ccb899 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -70,6 +70,7 @@ import { } from "./run-config.js"; import type { RuntimeId } from "./runtime-parity.js"; import { + listQaScenariosForExecutionProfile, QA_RUNTIME_PARITY_TIERS, readQaScenarioPack, type QaRuntimeParityTier, @@ -986,13 +987,14 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { if (liveChannelId && !liveAdapterFactory) { throw new Error(`unknown live QA adapter: ${liveChannelId}`); } + // liveChannelId exists only for the live driver, and explicit IDs always win. + // This keeps adapter profiles out of non-live and explicit-selection paths. const liveScenarioIds = liveAdapterFactory && scenarioIds.length === 0 - ? [...(liveAdapterFactory.scenarioIds ?? [])] + ? listQaScenariosForExecutionProfile(`${liveChannelId}:adapter`).map( + (scenario) => scenario.id, + ) : scenarioIds; - if (liveAdapterFactory && liveScenarioIds.length === 0) { - throw new Error(`live QA adapter ${liveChannelId} does not declare default scenarios`); - } if (runner !== "host" && runner !== "multipass") { throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`); } diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index 68b011898ea2..278f8278507b 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -165,12 +165,6 @@ describe("qa coverage report", () => { expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0); expect(inventory.overlappingCoverage.length).toBeGreaterThan(0); expect(inventory.missingCoverage).toStrictEqual([]); - expect(inventory.liveTransportLanes.map((lane) => lane.transportId)).toEqual([ - "discord", - "slack", - "telegram", - "whatsapp", - ]); expect(inventory.scorecardTaxonomy.profileCount).toBe(3); expect( inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "smoke-ci"), @@ -306,10 +300,6 @@ describe("qa coverage report", () => { ); expect(report).toContain("otel-trace-smoke, docker-prometheus-smoke"); expect(report).toContain("personal-share-safe-diagnostics-artifact"); - expect(report).toContain("## Live Transport Lanes"); - expect(report).toContain( - "- telegram (telegram): canary: channel-canary, help-command: telegram-help-command, mention-gating: channel-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume", - ); expect(report).toContain("## Scorecard Taxonomy"); expect(report).toContain("- Taxonomy: taxonomy.yaml"); expect(report).toContain("- Fulfilled taxonomy categories:"); diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index a51d120b5805..e3e89c0d215e 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -1,9 +1,5 @@ // Qa Lab plugin module implements coverage report behavior. import { normalizeStringEntriesLower } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { - buildLiveTransportCoverageLaneSummaries, - type LiveTransportCoverageLaneSummary, -} from "./live-transports/shared/live-transport-scenarios.js"; import { QA_SCENARIO_PACKS, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { readQaScorecardTaxonomyReport, @@ -28,7 +24,7 @@ type QaScenarioSearchMatch = QaCoverageScenarioSummary & { executionPath?: string; runtimeParityTier?: string; requiredProviderMode?: string; - requiredChannelDriver?: string; + driver?: string; requiredProvider?: string; requiredModel?: string; }; @@ -63,7 +59,6 @@ type QaCoverageInventory = { byTheme: Record; bySurface: Record; scenarioPacks: QaCoverageScenarioPackSummary[]; - liveTransportLanes: LiveTransportCoverageLaneSummary[]; scorecardTaxonomy: QaScorecardTaxonomyReport; }; @@ -168,7 +163,7 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}), runtimeParityTier: scenario.runtimeParityTier, requiredProviderMode: stringifyConfigValue(config.requiredProviderMode), - requiredChannelDriver: stringifyConfigValue(config.requiredChannelDriver), + driver: scenario.execution.driver, requiredProvider: stringifyConfigValue(config.requiredProvider), requiredModel: stringifyConfigValue(config.requiredModel), }; @@ -300,7 +295,6 @@ export function buildQaCoverageInventory( byTheme, bySurface, scenarioPacks: buildScenarioPackSummaries(scenarios), - liveTransportLanes: buildLiveTransportCoverageLaneSummaries(), scorecardTaxonomy: readQaScorecardTaxonomyReport(scenarios), }; } @@ -314,28 +308,6 @@ function pushCoverageIdLines(lines: string[], coverageIds: readonly QaCoverageId } } -function pushLiveTransportLines( - lines: string[], - lanes: readonly LiveTransportCoverageLaneSummary[], -) { - for (const lane of lanes) { - const members = lane.members - .map((member) => - member.scenarioId - ? `${member.standardId}: ${member.scenarioId}` - : `${member.standardId}: always-on`, - ) - .join(", "); - const missing = - lane.baselineMissingStandardScenarioIds.length > 0 - ? lane.baselineMissingStandardScenarioIds.join(", ") - : "none"; - lines.push( - `- ${lane.transportId} (${lane.commandName}): ${members}; missing baseline: ${missing}`, - ); - } -} - function pushScenarioPackLines(lines: string[], packs: readonly QaCoverageScenarioPackSummary[]) { for (const pack of packs) { const missing = @@ -443,12 +415,6 @@ export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): lines.push(""); } - if (inventory.liveTransportLanes.length > 0) { - lines.push("## Live Transport Lanes", ""); - pushLiveTransportLines(lines, inventory.liveTransportLanes); - lines.push(""); - } - pushScorecardTaxonomyLines(lines, inventory.scorecardTaxonomy); if (inventory.overlappingCoverage.length > 0) { @@ -474,7 +440,7 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) { const metadata = [ match.runtimeParityTier ? `runtimeParityTier=${match.runtimeParityTier}` : "", match.requiredProviderMode ? `providerMode=${match.requiredProviderMode}` : "", - match.requiredChannelDriver ? `channelDriver=${match.requiredChannelDriver}` : "", + match.driver ? `channelDriver=${match.driver}` : "", match.requiredProvider ? `provider=${match.requiredProvider}` : "", match.requiredModel ? `model=${match.requiredModel}` : "", ].filter(Boolean); @@ -487,7 +453,7 @@ function uniqueScenarioValues(values: (string | undefined)[]) { function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) { const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" "); - const requiredDrivers = uniqueScenarioValues(matches.map((match) => match.requiredChannelDriver)); + const requiredDrivers = uniqueScenarioValues(matches.map((match) => match.driver)); const channels = uniqueScenarioValues(matches.map((match) => match.channel)); const selectedDriver = requiredDrivers.length === 1 ? requiredDrivers[0] : channels.length === 1 ? "live" : undefined; @@ -500,7 +466,7 @@ function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) { function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) { const groups = new Map(); for (const match of matches) { - const key = JSON.stringify([match.executionKind, match.channel, match.requiredChannelDriver]); + const key = JSON.stringify([match.executionKind, match.channel, match.driver]); const group = groups.get(key) ?? []; group.push(match); groups.set(key, group); diff --git a/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts index 6461b8880944..4aae1e4a2cca 100644 --- a/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts @@ -13,8 +13,8 @@ vi.mock("../shared/live-transport-cli.runtime.js", () => ({ }), })); +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; import { runQaDiscordCommand } from "./cli.runtime.js"; -import { DISCORD_QA_DEFAULT_SCENARIO_IDS } from "./profiles.js"; describe("QA Lab Discord CLI runtime", () => { beforeEach(() => { @@ -48,7 +48,9 @@ describe("QA Lab Discord CLI runtime", () => { channelDriver: "live", channel: "discord", concurrency: 1, - scenarioIds: [...DISCORD_QA_DEFAULT_SCENARIO_IDS], + scenarioIds: listQaScenariosForExecutionProfile("discord:adapter").map( + (scenario) => scenario.id, + ), sutAccountId: "discord-sut", credentialSource: "convex", credentialRole: "ci", diff --git a/extensions/qa-lab/src/live-transports/discord/cli.ts b/extensions/qa-lab/src/live-transports/discord/cli.ts index 6faf0df0513b..be933b57a76f 100644 --- a/extensions/qa-lab/src/live-transports/discord/cli.ts +++ b/extensions/qa-lab/src/live-transports/discord/cli.ts @@ -5,7 +5,6 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { DISCORD_QA_DEFAULT_SCENARIO_IDS } from "./profiles.js"; type DiscordQaCliRuntime = typeof import("./cli.runtime.js"); type DiscordQaAdapterRuntime = typeof import("./adapter.runtime.js"); @@ -27,7 +26,6 @@ export const discordQaCliRegistration: LiveTransportQaCliRegistration = commandName: "discord", adapterFactory: { id: "discord", - scenarioIds: DISCORD_QA_DEFAULT_SCENARIO_IDS, matches: ({ channelId, driver }) => driver === "live" && channelId === "discord", async create(context) { return await (await loadDiscordQaAdapterRuntime()).createDiscordQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts index 8e7f6e6e753f..ff2369d26b87 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -1,9 +1,5 @@ // Qa Lab tests cover discord live plugin behavior. import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - findMissingLiveTransportStandardScenarios, -} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; import { afterEach, describe, expect, it, vi } from "vitest"; import { discordQaScenarioSupport } from "./discord-live.runtime.js"; @@ -474,16 +470,6 @@ describe("discord live qa runtime", () => { ); }); - it("tracks Discord live coverage against the shared transport contract", () => { - expect(testing.DISCORD_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]); - expect( - findMissingLiveTransportStandardScenarios({ - coveredStandardScenarioIds: testing.DISCORD_QA_STANDARD_SCENARIO_IDS, - expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - }), - ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); - }); - it("lists Discord application commands through the REST API", async () => { vi.stubGlobal( "fetch", diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts index 20f33c033010..48a0912ce70c 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -18,12 +18,8 @@ import { chromium } from "playwright-core"; import { z } from "zod"; import { startQaGatewayChild } from "../../gateway-child.js"; import { isTruthyOptIn } from "../../mantis-options.runtime.js"; +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; import { assertLiveScenarioReply as assertDiscordScenarioReply } from "../shared/live-scenario-reply.js"; -import { - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, -} from "../shared/live-transport-scenarios.js"; type DiscordQaRuntimeEnv = { guildId: string; @@ -69,7 +65,10 @@ type DiscordQaScenarioRun = replyContent: string; }; -type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition & { +type DiscordQaScenarioDefinition = { + id: DiscordQaScenarioId; + title: string; + timeoutMs: number; buildRun: (sutApplicationId: string) => DiscordQaScenarioRun; }; @@ -171,7 +170,6 @@ type DiscordQaScenarioResult = { artifactPaths?: Record; id: string; title: string; - standardId?: string; status: "pass" | "fail"; details: string; requestStartedAt?: string; @@ -239,7 +237,6 @@ const DISCORD_QA_ENV_KEYS = [ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ { id: "discord-canary", - standardId: "canary", title: "Discord canary echo", timeoutMs: 45_000, buildRun: (sutApplicationId) => { @@ -255,7 +252,6 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ }, { id: "discord-mention-gating", - standardId: "mention-gating", title: "Discord unmentioned message does not trigger", timeoutMs: 8_000, buildRun: () => { @@ -318,17 +314,6 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ }, ]; -const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter( - (scenario) => - scenario.id !== "discord-status-reactions-tool-only" && - scenario.id !== "discord-voice-autojoin" && - scenario.id !== "discord-thread-reply-filepath-attachment", -); - -const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ - scenarios: DISCORD_QA_SCENARIOS, -}); - const discordQaCredentialPayloadSchema = z.object({ guildId: z.string().trim().min(1), channelId: z.string().trim().min(1), @@ -1233,7 +1218,6 @@ async function runDiscordThreadReplyFilePathAttachmentScenario(params: { return { id: params.scenario.id, title: params.scenario.title, - standardId: params.scenario.standardId, status, details: status === "pass" @@ -1365,12 +1349,16 @@ function buildObservedMessagesArtifact(params: { } function findScenario(ids?: string[]) { - const scenarios = ids && ids.length > 0 ? DISCORD_QA_SCENARIOS : DISCORD_QA_DEFAULT_SCENARIOS; - return selectLiveTransportScenarios({ - ids, - laneLabel: "Discord", - scenarios, - }); + const requestedIds = + ids && ids.length > 0 + ? ids + : listQaScenariosForExecutionProfile("discord:adapter").map((scenario) => scenario.id); + const scenariosById = new Map(DISCORD_QA_SCENARIOS.map((scenario) => [scenario.id, scenario])); + const missingIds = requestedIds.filter((id) => !scenariosById.has(id as DiscordQaScenarioId)); + if (missingIds.length > 0) { + throw new Error(`unknown Discord QA scenario id(s): ${missingIds.join(", ")}`); + } + return requestedIds.map((id) => scenariosById.get(id as DiscordQaScenarioId)!); } function matchesDiscordScenarioReply(params: { @@ -1421,7 +1409,6 @@ async function assertDiscordApplicationCommandsRegistered(params: { const testing = { DISCORD_QA_SCENARIOS, - DISCORD_QA_STANDARD_SCENARIO_IDS, collectSeenReactionSequence, assertDiscordScenarioReply, assertDiscordApplicationCommandsRegistered, diff --git a/extensions/qa-lab/src/live-transports/discord/profiles.ts b/extensions/qa-lab/src/live-transports/discord/profiles.ts index 6663cdad33d1..c396a28a90dc 100644 --- a/extensions/qa-lab/src/live-transports/discord/profiles.ts +++ b/extensions/qa-lab/src/live-transports/discord/profiles.ts @@ -1,19 +1,7 @@ -const DISCORD_QA_ALL_SCENARIO_IDS = [ - "discord-canary", - "discord-mention-gating", - "discord-native-help-command-registration", - "discord-voice-autojoin", - "discord-status-reactions-tool-only", - "discord-thread-reply-filepath-attachment", -] as const; - -export const DISCORD_QA_DEFAULT_SCENARIO_IDS = DISCORD_QA_ALL_SCENARIO_IDS.filter( - (scenarioId) => - scenarioId !== "discord-voice-autojoin" && - scenarioId !== "discord-status-reactions-tool-only" && - scenarioId !== "discord-thread-reply-filepath-attachment", -); +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; export function resolveDiscordQaScenarioIds(scenarioIds?: readonly string[]) { - return scenarioIds?.length ? [...scenarioIds] : [...DISCORD_QA_DEFAULT_SCENARIO_IDS]; + return scenarioIds?.length + ? [...scenarioIds] + : listQaScenariosForExecutionProfile("discord:adapter").map((scenario) => scenario.id); } diff --git a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts index 004d664c4495..8ae3b03439a2 100644 --- a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts +++ b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { createQaBusState } from "../bus-state.js"; import { createQaChannelTransport } from "../qa-channel-transport.js"; import { createQaTransportAdapter } from "../qa-transport-registry.js"; +import { listQaScenariosForExecutionProfile } from "../scenario-catalog.js"; const { createSlack, createTelegram, createWhatsApp } = vi.hoisted(() => ({ createSlack: vi.fn(), @@ -19,10 +20,8 @@ vi.mock("./whatsapp/adapter.runtime.js", () => ({ })); import { slackQaCliRegistration } from "./slack/cli.js"; -import { SLACK_QA_DEFAULT_SCENARIO_IDS } from "./slack/profiles.js"; import { telegramQaCliRegistration } from "./telegram/cli.js"; import { whatsappQaCliRegistration } from "./whatsapp/cli.js"; -import { resolveWhatsAppQaScenarioIds } from "./whatsapp/profiles.js"; const slackQaAdapterFactory = slackQaCliRegistration.adapterFactory; const telegramQaAdapterFactory = telegramQaCliRegistration.adapterFactory; @@ -38,14 +37,30 @@ const factories = [ ] as const; describe("live transport adapter factories", () => { - it("assigns the canonical live scenario defaults to Slack", () => { - expect(slackQaAdapterFactory.scenarioIds).toEqual(SLACK_QA_DEFAULT_SCENARIO_IDS); + it("selects Slack generic defaults from the YAML adapter profile", () => { + expect( + listQaScenariosForExecutionProfile("slack:adapter").map((scenario) => scenario.id), + ).toEqual([ + "channel-chat-baseline", + "channel-canary", + "channel-mention-gating", + "channel-top-level-reply-shape", + "thread-follow-up", + "thread-isolation", + ]); }); - it("assigns the canonical live-frontier scenario defaults to WhatsApp", () => { - expect(whatsappQaAdapterFactory.scenarioIds).toEqual( - resolveWhatsAppQaScenarioIds({ providerMode: "live-frontier" }), - ); + it("selects WhatsApp DM-safe defaults from the YAML adapter profile", () => { + expect( + listQaScenariosForExecutionProfile("whatsapp:adapter").map((scenario) => scenario.id), + ).toEqual([ + "dm-chat-baseline", + "channel-canary", + "channel-dm-group-routing", + "channel-mention-gating", + "channel-top-level-reply-shape", + "whatsapp-help-command", + ]); }); it.each([ diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts index 53d050d97306..c4ed575d36e7 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts @@ -50,24 +50,6 @@ describe("QA Lab Matrix CLI registration", () => { stdoutSpy.mockRestore(); }); - it("keeps generic Matrix suite defaults in the repo-backed flow catalog", () => { - expect(matrixQaCliRegistration.adapterFactory?.scenarioIds).toEqual([ - "channel-chat-baseline", - "channel-canary", - "channel-dm-group-routing", - "channel-mention-gating", - "channel-sender-allowlist", - "channel-top-level-reply-shape", - "channel-secondary-conversation-isolation", - "channel-multi-actor-ordering", - "thread-follow-up", - "thread-isolation", - "thread-reply-override", - "dm-shared-session", - "dm-per-room-session", - ]); - }); - it("exposes only QA Lab selector flags", () => { const qa = new Command(); matrixQaCliRegistration.register(qa); diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.ts b/extensions/qa-lab/src/live-transports/matrix/cli.ts index bc65e56449eb..8fd0b873efcb 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.ts @@ -8,21 +8,6 @@ import { } from "../shared/live-transport-cli.js"; const DISABLE_MATRIX_QA_FORCE_EXIT_ENV = "OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT"; -const MATRIX_QA_REPO_FLOW_SCENARIO_IDS = [ - "channel-chat-baseline", - "channel-canary", - "channel-dm-group-routing", - "channel-mention-gating", - "channel-sender-allowlist", - "channel-top-level-reply-shape", - "channel-secondary-conversation-isolation", - "channel-multi-actor-ordering", - "thread-follow-up", - "thread-isolation", - "thread-reply-override", - "dm-shared-session", - "dm-per-room-session", -] as const; const loadMatrixQaCliRuntime = createLazyCliRuntimeLoader( () => import("./cli.runtime.js"), @@ -71,7 +56,6 @@ async function runQaMatrix(opts: LiveTransportQaCommandOptions) { const matrixQaAdapterFactory: NonNullable = { id: "matrix", - scenarioIds: MATRIX_QA_REPO_FLOW_SCENARIO_IDS, matches: ({ channelId, driver }) => driver === "live" && channelId === "matrix", async create(context) { return await (await loadMatrixQaAdapterRuntime()).createMatrixQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/matrix/profiles.ts b/extensions/qa-lab/src/live-transports/matrix/profiles.ts index 30b1653877a1..80844c052349 100644 --- a/extensions/qa-lab/src/live-transports/matrix/profiles.ts +++ b/extensions/qa-lab/src/live-transports/matrix/profiles.ts @@ -1,166 +1,28 @@ -const MATRIX_QA_RELEASE_SCENARIO_IDS = [ - "channel-chat-baseline", - "matrix-allowlist-hot-reload", +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; + +const MATRIX_QA_PROFILES = [ + "all", + "fast", + "release", + "transport", + "media", + "e2ee-smoke", + "e2ee-deep", + "e2ee-cli", ] as const; -const MATRIX_QA_FAST_SCENARIO_IDS = [ - "channel-chat-baseline", - "thread-follow-up", - "thread-isolation", - "channel-top-level-reply-shape", - "matrix-reaction-notification", - "matrix-approval-exec-metadata-single-event", - "matrix-approval-exec-metadata-chunked", - "channel-mention-gating", - "matrix-allowbots-default-block", - "matrix-allowbots-mentions-mentioned-room", - "channel-sender-allowlist", - "matrix-e2ee-basic-reply", -] as const; - -const MATRIX_QA_TRANSPORT_SCENARIO_IDS = [ - "thread-follow-up", - "matrix-thread-root-preservation", - "matrix-thread-nested-reply-shape", - "thread-isolation", - "channel-top-level-reply-shape", - "thread-reply-override", - "matrix-room-partial-streaming-preview", - "matrix-room-quiet-streaming-preview", - "matrix-room-tool-progress-preview", - "matrix-room-tool-progress-command-preview", - "matrix-room-tool-progress-preview-opt-out", - "matrix-room-tool-progress-error", - "matrix-room-tool-progress-mention-safety", - "dm-chat-baseline", - "dm-shared-session", - "matrix-dm-thread-reply-override", - "dm-per-room-session", - "matrix-room-autojoin-invite", - "channel-secondary-conversation-isolation", - "matrix-secondary-room-open-trigger", - "matrix-reaction-notification", - "matrix-reaction-threaded", - "matrix-reaction-not-a-reply", - "matrix-reaction-redaction-observed", - "matrix-approval-exec-metadata-single-event", - "matrix-approval-exec-metadata-chunked", - "matrix-approval-plugin-metadata-single-event", - "matrix-approval-deny-reaction", - "matrix-approval-thread-target", - "matrix-approval-channel-target-both", - "matrix-initial-catchup-then-incremental", - "matrix-stale-sync-replay-dedupe", - "matrix-room-membership-loss", - "matrix-homeserver-restart-resume", - "channel-mention-gating", - "matrix-allowbots-default-block", - "matrix-allowbots-true-unmentioned-open-room", - "matrix-allowbots-mentions-mentioned-room", - "matrix-allowbots-mentions-unmentioned-open-room-block", - "matrix-allowbots-mentions-dm-unmentioned", - "matrix-allowbots-room-override-blocks-account-true", - "matrix-allowbots-room-override-enables-account-off", - "matrix-allowbots-self-sender-ignored", - "matrix-mxid-prefixed-command-block", - "matrix-mention-metadata-spoof-block", - "matrix-allowlist-hot-reload", - "channel-sender-allowlist", - "channel-multi-actor-ordering", - "matrix-inbound-edit-ignored", - "matrix-inbound-edit-no-duplicate-trigger", -] as const; - -const MATRIX_QA_MEDIA_SCENARIO_IDS = [ - "matrix-room-image-understanding-attachment", - "matrix-room-generated-image-delivery", - "matrix-media-type-coverage", - "matrix-voice-preflight-mention", - "matrix-attachment-only-ignored", - "matrix-unsupported-media-safe", - "matrix-e2ee-media-image", -] as const; - -const MATRIX_QA_E2EE_SMOKE_SCENARIO_IDS = [ - "matrix-e2ee-basic-reply", - "matrix-e2ee-thread-follow-up", - "matrix-e2ee-bootstrap-success", - "matrix-e2ee-recovery-key-lifecycle", - "matrix-e2ee-recovery-owner-verification-required", - "matrix-e2ee-restart-resume", - "matrix-e2ee-artifact-redaction", - "matrix-e2ee-key-bootstrap-failure", -] as const; - -const MATRIX_QA_E2EE_DEEP_SCENARIO_IDS = [ - "matrix-e2ee-state-after-missing-encryption", - "matrix-e2ee-state-loss-external-recovery-key", - "matrix-e2ee-state-loss-stored-recovery-key", - "matrix-e2ee-state-loss-no-recovery-key", - "matrix-e2ee-stale-recovery-key-after-backup-reset", - "matrix-e2ee-server-backup-deleted-local-state-intact", - "matrix-e2ee-server-backup-deleted-local-reupload-restores", - "matrix-e2ee-corrupt-crypto-idb-snapshot", - "matrix-e2ee-server-device-deleted-local-state-intact", - "matrix-e2ee-server-device-deleted-relogin-recovers", - "matrix-e2ee-sync-state-loss-crypto-intact", - "matrix-e2ee-history-exists-backup-empty", - "matrix-e2ee-device-sas-verification", - "matrix-e2ee-qr-verification", - "matrix-e2ee-stale-device-hygiene", - "matrix-e2ee-dm-sas-verification", - "matrix-e2ee-verification-notice-no-trigger", - "matrix-e2ee-wrong-account-recovery-key", -] as const; - -const MATRIX_QA_E2EE_CLI_SCENARIO_IDS = [ - "matrix-e2ee-cli-account-add-enable-e2ee", - "matrix-e2ee-cli-encryption-setup", - "matrix-e2ee-cli-encryption-setup-idempotent", - "matrix-e2ee-cli-encryption-setup-bootstrap-failure", - "matrix-e2ee-cli-recovery-key-setup", - "matrix-e2ee-cli-recovery-key-invalid", - "matrix-e2ee-cli-encryption-setup-multi-account", - "matrix-e2ee-cli-setup-then-gateway-reply", - "matrix-e2ee-cli-self-verification", -] as const; - -const MATRIX_QA_ALL_SCENARIO_IDS = Array.from( - new Set([ - "channel-chat-baseline", - ...MATRIX_QA_TRANSPORT_SCENARIO_IDS, - ...MATRIX_QA_MEDIA_SCENARIO_IDS, - ...MATRIX_QA_E2EE_SMOKE_SCENARIO_IDS, - ...MATRIX_QA_E2EE_DEEP_SCENARIO_IDS, - ...MATRIX_QA_E2EE_CLI_SCENARIO_IDS, - ]), -); - -const MATRIX_QA_PROFILE_SCENARIO_IDS = { - all: MATRIX_QA_ALL_SCENARIO_IDS, - fast: MATRIX_QA_FAST_SCENARIO_IDS, - release: MATRIX_QA_RELEASE_SCENARIO_IDS, - transport: MATRIX_QA_TRANSPORT_SCENARIO_IDS, - media: MATRIX_QA_MEDIA_SCENARIO_IDS, - "e2ee-smoke": MATRIX_QA_E2EE_SMOKE_SCENARIO_IDS, - "e2ee-deep": MATRIX_QA_E2EE_DEEP_SCENARIO_IDS, - "e2ee-cli": MATRIX_QA_E2EE_CLI_SCENARIO_IDS, -} as const; - export function resolveMatrixQaScenarioIds(params: { profile?: string; scenarioIds?: readonly string[]; }): string[] { - if (params.scenarioIds && params.scenarioIds.length > 0) { + if (params.scenarioIds?.length) { return [...params.scenarioIds]; } const profile = params.profile?.trim() || "all"; - if (!(profile in MATRIX_QA_PROFILE_SCENARIO_IDS)) { + if (!MATRIX_QA_PROFILES.some((candidate) => candidate === profile)) { throw new Error( - `Unknown QA Lab Matrix profile "${profile}". Expected one of: all, fast, release, transport, media, e2ee-smoke, e2ee-deep, e2ee-cli.`, + `Unknown QA Lab Matrix profile "${profile}". Expected one of: ${MATRIX_QA_PROFILES.join(", ")}.`, ); } - return [ - ...MATRIX_QA_PROFILE_SCENARIO_IDS[profile as keyof typeof MATRIX_QA_PROFILE_SCENARIO_IDS], - ]; + return listQaScenariosForExecutionProfile(`matrix:${profile}`).map((scenario) => scenario.id); } diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts index e14590dfa98c..351fa0006059 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts @@ -13,7 +13,6 @@ export type LiveTransportCheckResult = { title: string; status: "pass" | "fail"; details: string; - coverageIds?: readonly string[]; timing?: QaEvidenceTiming; rttMs?: number; requestStartedAt?: string; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts deleted file mode 100644 index e62ed6b23c99..000000000000 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -// Qa Lab tests cover live transport scenarios plugin behavior. -import fs from "node:fs"; -import { fileURLToPath } from "node:url"; -import { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - findMissingLiveTransportStandardScenarios, -} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; -import { describe, expect, it } from "vitest"; -import { - buildLiveTransportCoverageLaneSummaries, - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, -} from "./live-transport-scenarios.js"; - -describe("live transport scenario helpers", () => { - it("uses the public live transport scenario SDK seam", () => { - const source = fs.readFileSync( - fileURLToPath(new URL("./live-transport-scenarios.ts", import.meta.url)), - "utf8", - ); - - expect(source).toContain("openclaw/plugin-sdk/qa-live-transport-scenarios"); - expect(source).not.toContain("openclaw/plugin-sdk/qa-runtime"); - }); - - it("keeps the repo-wide baseline contract ordered", () => { - expect(LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ]); - }); - - it("selects requested scenarios and reports unknown ids with the lane label", () => { - const definitions = [ - { id: "alpha", timeoutMs: 1_000, title: "alpha" }, - { id: "beta", timeoutMs: 1_000, title: "beta" }, - ] as const; - - expect( - selectLiveTransportScenarios({ - ids: ["beta"], - laneLabel: "Demo", - scenarios: definitions, - }), - ).toEqual([definitions[1]]); - - expect(() => - selectLiveTransportScenarios({ - ids: ["alpha", "missing"], - laneLabel: "Demo", - scenarios: definitions, - }), - ).toThrow("unknown Demo QA scenario id(s): missing"); - }); - - it("dedupes always-on and scenario-backed standard coverage", () => { - const covered = collectLiveTransportStandardScenarioCoverage({ - alwaysOnStandardScenarioIds: ["canary"], - scenarios: [ - { - id: "scenario-1", - standardId: "mention-gating", - timeoutMs: 1_000, - title: "mention", - }, - { - id: "scenario-2", - standardId: "mention-gating", - timeoutMs: 1_000, - title: "mention again", - }, - { - id: "scenario-3", - standardId: "restart-resume", - timeoutMs: 1_000, - title: "restart", - }, - ], - }); - - expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]); - expect( - findMissingLiveTransportStandardScenarios({ - coveredStandardScenarioIds: covered, - expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - }), - ).toEqual(["allowlist-block", "top-level-reply-shape"]); - }); - - it("summarizes live transport lane membership for coverage reports", () => { - const lanes = buildLiveTransportCoverageLaneSummaries(); - - expect(lanes.map((lane) => lane.transportId)).toEqual([ - "discord", - "slack", - "telegram", - "whatsapp", - ]); - expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({ - standardId: "canary", - scenarioId: "channel-canary", - }); - expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({ - standardId: "restart-resume", - scenarioId: "slack-restart-resume", - }); - expect(lanes.find((lane) => lane.transportId === "whatsapp")?.members).toContainEqual({ - standardId: "allowlist-block", - scenarioId: "whatsapp-group-allowlist-block", - }); - expect(lanes.find((lane) => lane.transportId === "whatsapp")?.members).toContainEqual({ - standardId: "restart-resume", - scenarioId: "whatsapp-restart-resume", - }); - expect( - lanes.find((lane) => lane.transportId === "discord")?.baselineMissingStandardScenarioIds, - ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); - expect( - lanes.find((lane) => lane.transportId === "whatsapp")?.baselineMissingStandardScenarioIds, - ).toEqual([]); - }); - - it("keeps Telegram runtime coverage represented in its mixed-owner lane", () => { - const lanes = new Map( - buildLiveTransportCoverageLaneSummaries().map((lane) => [ - lane.transportId, - lane.standardScenarioIds, - ]), - ); - - expect(lanes.get("telegram")).toEqual( - expect.arrayContaining(["canary", "help-command", "mention-gating"]), - ); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts deleted file mode 100644 index 9b863195e1fc..000000000000 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ /dev/null @@ -1,107 +0,0 @@ -// Qa Lab plugin module implements live transport scenarios behavior. -import { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - collectLiveTransportStandardScenarioCoverage, - findMissingLiveTransportStandardScenarios, - type LiveTransportScenarioDefinition, - type LiveTransportStandardScenarioId, -} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; - -export { - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, -} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; - -type LiveTransportCoverageMember = { - scenarioId?: string; - standardId: LiveTransportStandardScenarioId; -}; - -type LiveTransportCoverageLane = { - commandName: string; - members: readonly LiveTransportCoverageMember[]; - transportId: string; -}; - -export type LiveTransportCoverageLaneSummary = { - baselineMissingStandardScenarioIds: LiveTransportStandardScenarioId[]; - commandName: string; - memberCount: number; - members: LiveTransportCoverageMember[]; - standardScenarioIds: LiveTransportStandardScenarioId[]; - transportId: string; -}; - -const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [ - { - transportId: "discord", - commandName: "discord", - members: [ - { standardId: "canary", scenarioId: "discord-canary" }, - { standardId: "mention-gating", scenarioId: "discord-mention-gating" }, - ], - }, - { - transportId: "slack", - commandName: "slack", - members: [ - { standardId: "canary", scenarioId: "slack-canary" }, - { standardId: "mention-gating", scenarioId: "slack-mention-gating" }, - { standardId: "allowlist-block", scenarioId: "slack-allowlist-block" }, - { standardId: "top-level-reply-shape", scenarioId: "slack-top-level-reply-shape" }, - { standardId: "restart-resume", scenarioId: "slack-restart-resume" }, - ], - }, - { - transportId: "telegram", - commandName: "telegram", - members: [ - { standardId: "canary", scenarioId: "channel-canary" }, - { standardId: "help-command", scenarioId: "telegram-help-command" }, - { standardId: "mention-gating", scenarioId: "channel-mention-gating" }, - ], - }, - { - transportId: "whatsapp", - commandName: "whatsapp", - members: [ - { standardId: "canary", scenarioId: "whatsapp-canary" }, - { standardId: "mention-gating", scenarioId: "whatsapp-mention-gating" }, - { standardId: "top-level-reply-shape", scenarioId: "whatsapp-top-level-reply-shape" }, - { standardId: "restart-resume", scenarioId: "whatsapp-restart-resume" }, - { standardId: "help-command", scenarioId: "whatsapp-help-command" }, - { standardId: "quote-reply", scenarioId: "whatsapp-reply-to-message" }, - { standardId: "quote-reply", scenarioId: "whatsapp-group-reply-to-message" }, - { standardId: "reaction-observation", scenarioId: "whatsapp-status-reactions" }, - { standardId: "allowlist-block", scenarioId: "whatsapp-group-allowlist-block" }, - ], - }, -] as const; - -export function buildLiveTransportCoverageLaneSummaries( - lanes: readonly LiveTransportCoverageLane[] = LIVE_TRANSPORT_COVERAGE_LANES, -): LiveTransportCoverageLaneSummary[] { - return lanes - .map((lane) => { - const scenarios: LiveTransportScenarioDefinition[] = lane.members.map((member) => ({ - id: member.scenarioId ?? `${lane.transportId}:${member.standardId}`, - standardId: member.standardId, - timeoutMs: 0, - title: member.standardId, - })); - const standardScenarioIds = collectLiveTransportStandardScenarioCoverage({ scenarios }); - return { - baselineMissingStandardScenarioIds: findMissingLiveTransportStandardScenarios({ - coveredStandardScenarioIds: standardScenarioIds, - expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - }), - commandName: lane.commandName, - memberCount: lane.members.length, - members: [...lane.members], - standardScenarioIds, - transportId: lane.transportId, - }; - }) - .toSorted((left, right) => left.transportId.localeCompare(right.transportId)); -} diff --git a/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts index 26029322364f..08839801c7fe 100644 --- a/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts @@ -13,8 +13,8 @@ vi.mock("../shared/live-transport-cli.runtime.js", () => ({ }), })); +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; import { runQaSlackCommand } from "./cli.runtime.js"; -import { SLACK_QA_DEFAULT_SCENARIO_IDS } from "./profiles.js"; describe("QA Lab Slack CLI runtime", () => { beforeEach(() => { @@ -48,7 +48,9 @@ describe("QA Lab Slack CLI runtime", () => { channelDriver: "live", channel: "slack", concurrency: 1, - scenarioIds: [...SLACK_QA_DEFAULT_SCENARIO_IDS], + scenarioIds: listQaScenariosForExecutionProfile("slack:default").map( + (scenario) => scenario.id, + ), sutAccountId: "slack-sut", credentialSource: "convex", credentialRole: "ci", diff --git a/extensions/qa-lab/src/live-transports/slack/cli.ts b/extensions/qa-lab/src/live-transports/slack/cli.ts index d86a1f3fb00c..f12b43b60071 100644 --- a/extensions/qa-lab/src/live-transports/slack/cli.ts +++ b/extensions/qa-lab/src/live-transports/slack/cli.ts @@ -5,7 +5,6 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { SLACK_QA_DEFAULT_SCENARIO_IDS } from "./profiles.js"; type SlackQaAdapterRuntime = typeof import("./adapter.runtime.js"); type SlackQaCliRuntime = typeof import("./cli.runtime.js"); @@ -23,7 +22,6 @@ async function runQaSlack(opts: LiveTransportQaCommandOptions) { const slackQaAdapterFactory: NonNullable = { id: "slack", - scenarioIds: SLACK_QA_DEFAULT_SCENARIO_IDS, matches: ({ channelId, driver }) => driver === "live" && channelId === "slack", async create(context) { return await (await loadSlackQaAdapterRuntime()).createSlackQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/slack/profiles.ts b/extensions/qa-lab/src/live-transports/slack/profiles.ts index d5360d402cc7..3fe8cbdb14e9 100644 --- a/extensions/qa-lab/src/live-transports/slack/profiles.ts +++ b/extensions/qa-lab/src/live-transports/slack/profiles.ts @@ -1,36 +1,7 @@ -const SLACK_QA_ALL_SCENARIO_IDS = [ - "slack-canary", - "slack-mention-gating", - "slack-allowlist-block", - "slack-channel-disabled-warning", - "slack-top-level-reply-shape", - "slack-progress-commentary-true", - "slack-progress-commentary-false", - "slack-progress-commentary-omitted", - "slack-progress-commentary-verbose-dedupe", - "slack-chart-presentation-native", - "slack-table-presentation-native", - "slack-table-invalid-blocks-fallback", - "slack-reaction-glyph-native", - "slack-approval-exec-native", - "slack-approval-plugin-native", - "slack-codex-approval-exec-native", - "slack-codex-approval-plugin-native", -] as const; - -const SLACK_QA_EXPLICIT_SCENARIO_IDS = new Set([ - "slack-channel-disabled-warning", - "slack-progress-commentary-true", - "slack-progress-commentary-false", - "slack-progress-commentary-omitted", - "slack-progress-commentary-verbose-dedupe", - "slack-table-invalid-blocks-fallback", -]); - -export const SLACK_QA_DEFAULT_SCENARIO_IDS = SLACK_QA_ALL_SCENARIO_IDS.filter( - (scenarioId) => !SLACK_QA_EXPLICIT_SCENARIO_IDS.has(scenarioId), -); +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; export function resolveSlackQaScenarioIds(scenarioIds?: readonly string[]) { - return scenarioIds?.length ? [...scenarioIds] : [...SLACK_QA_DEFAULT_SCENARIO_IDS]; + return scenarioIds?.length + ? [...scenarioIds] + : listQaScenariosForExecutionProfile("slack:default").map((scenario) => scenario.id); } diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts index 3556d6a4478e..c3f4124e8536 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts @@ -5,7 +5,6 @@ import { z } from "zod"; import type { startQaGatewayChild } from "../../gateway-child.js"; import { splitQaModelRef } from "../../model-selection.js"; import type { RuntimeId } from "../../runtime-parity.js"; -import type { LiveTransportScenarioDefinition } from "../shared/live-transport-scenarios.js"; export type SlackQaRuntimeEnv = { channelId: string; @@ -227,10 +226,12 @@ export type SlackQaScenarioContext = { waitForReady: () => Promise; }; -export type SlackQaScenarioDefinition = LiveTransportScenarioDefinition & { +export type SlackQaScenarioDefinition = { + id: SlackQaScenarioId; + title: string; + timeoutMs: number; buildRun: (sutUserId: string) => SlackQaScenarioRun; configOverrides?: SlackQaConfigOverrides; - defaultEnabled?: boolean; forcedRuntime?: RuntimeId; }; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts index 58d43ef20619..2384489030d1 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts @@ -24,7 +24,6 @@ import { const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-canary", - standardId: "canary", title: "Slack canary echo", timeoutMs: 45_000, buildRun: (sutUserId) => { @@ -38,7 +37,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ }, { id: "slack-mention-gating", - standardId: "mention-gating", title: "Slack unmentioned bot message does not trigger", timeoutMs: 8_000, buildRun: () => { @@ -52,7 +50,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ }, { id: "slack-allowlist-block", - standardId: "allowlist-block", title: "Slack non-allowlisted sender does not trigger", timeoutMs: 8_000, configOverrides: { @@ -72,7 +69,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ id: "slack-channel-disabled-warning", title: "Slack disabled channel warns and does not trigger", timeoutMs: 8_000, - defaultEnabled: false, configOverrides: { channelEnabled: false }, buildRun: (sutUserId) => { const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -116,7 +112,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ }, { id: "slack-top-level-reply-shape", - standardId: "top-level-reply-shape", title: "Slack top-level reply stays top-level", timeoutMs: 45_000, configOverrides: { replyToMode: "off" }, @@ -139,7 +134,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-progress-commentary-true", title: "Slack progress commentary true is independent from tool progress", - defaultEnabled: false, timeoutMs: 90_000, configOverrides: { progress: { commentary: true, toolProgress: false }, @@ -153,7 +147,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-progress-commentary-false", title: "Slack progress commentary false stays out of the progress draft", - defaultEnabled: false, timeoutMs: 90_000, configOverrides: { progress: { commentary: false, toolProgress: false }, @@ -167,7 +160,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-progress-commentary-omitted", title: "Slack omitted progress commentary preserves the tool-progress default", - defaultEnabled: false, timeoutMs: 90_000, configOverrides: { progress: { toolProgress: true }, @@ -181,7 +173,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-progress-commentary-verbose-dedupe", title: "Slack explicit commentary yields to durable verbose progress", - defaultEnabled: false, timeoutMs: 90_000, configOverrides: { progress: { commentary: true, toolProgress: false, verboseDefault: "on" }, @@ -269,7 +260,6 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ { id: "slack-table-invalid-blocks-fallback", title: "Slack rejects an over-limit native table and stores its complete fallback", - defaultEnabled: false, timeoutMs: 45_000, buildRun: () => ({ kind: "direct-transport", diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.test.ts b/extensions/qa-lab/src/live-transports/telegram/cli.test.ts deleted file mode 100644 index 6acf708979bf..000000000000 --- a/extensions/qa-lab/src/live-transports/telegram/cli.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { telegramQaCliRegistration } from "./cli.js"; -import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js"; - -describe("Telegram QA CLI registration", () => { - it("keeps the generic live baseline outside command profile selection", () => { - expect(telegramQaCliRegistration.adapterFactory?.scenarioIds).toEqual([ - "channel-chat-baseline", - ...TELEGRAM_QA_ALL_SCENARIO_IDS, - ]); - expect(TELEGRAM_QA_ALL_SCENARIO_IDS).not.toContain("channel-chat-baseline"); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index 84281567783d..8837ba288aea 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -5,7 +5,6 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js"; type TelegramQaAdapterRuntime = typeof import("./adapter.runtime.js"); type TelegramQaCliRuntime = typeof import("./cli.runtime.js"); @@ -23,7 +22,6 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) { const telegramQaAdapterFactory: NonNullable = { id: "telegram", - scenarioIds: ["channel-chat-baseline", ...TELEGRAM_QA_ALL_SCENARIO_IDS], matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", async create(context) { return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts index e9d142bceebe..f4e2479b879e 100644 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts @@ -1,9 +1,6 @@ import { describe, expect, it } from "vitest"; -import { - listTelegramQaScenarios, - resolveTelegramQaScenarioIds, - TELEGRAM_QA_ALL_SCENARIO_IDS, -} from "./profiles.js"; +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; +import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js"; describe("Telegram QA profiles", () => { it("keeps release focused and adds the scripted long-final check for mock runs", () => { @@ -16,19 +13,19 @@ describe("Telegram QA profiles", () => { }); it("selects every migrated Telegram scenario through all", () => { - expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual([ - ...TELEGRAM_QA_ALL_SCENARIO_IDS, - ]); + expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual( + listQaScenariosForExecutionProfile("telegram:all").map((scenario) => scenario.id), + ); }); it("lets explicit scenarios override profile selection", () => { expect( resolveTelegramQaScenarioIds({ - profile: "all", + profile: "release", providerMode: "live-frontier", - scenarioIds: ["telegram-status-command"], + scenarioIds: ["thread-follow-up"], }), - ).toEqual(["telegram-status-command"]); + ).toEqual(["thread-follow-up"]); }); it("rejects unknown profiles and scenarios before gateway startup", () => { @@ -47,7 +44,9 @@ describe("Telegram QA profiles", () => { const scenarios = listTelegramQaScenarios("mock-openai"); expect(scenarios.map(({ id }) => id).toSorted()).toEqual( - [...TELEGRAM_QA_ALL_SCENARIO_IDS].toSorted(), + listQaScenariosForExecutionProfile("telegram:all") + .map((scenario) => scenario.id) + .toSorted(), ); expect( scenarios.find(({ id }) => id === "telegram-long-final-reuses-preview")?.defaultEnabled, diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.ts index 5f95f774fba8..d1838cb6d9dc 100644 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.ts +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.ts @@ -1,32 +1,4 @@ -import { readQaScenarioPack } from "../../scenario-catalog.js"; - -const TELEGRAM_QA_RELEASE_SCENARIO_IDS = [ - "channel-canary", - "channel-mention-gating", - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", - "telegram-context-command", - "telegram-other-bot-command-gating", -] as const; - -const TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS = [ - ...TELEGRAM_QA_RELEASE_SCENARIO_IDS, - "telegram-long-final-reuses-preview", -] as const; - -export const TELEGRAM_QA_ALL_SCENARIO_IDS = [ - ...TELEGRAM_QA_RELEASE_SCENARIO_IDS, - "telegram-current-session-status-tool", - "telegram-tool-only-usage-footer", - "telegram-reply-chain-exact-marker", - "telegram-stream-final-single-message", - "telegram-long-final-reuses-preview", - "telegram-long-final-three-chunks", -] as const; +import { listQaScenariosForExecutionProfile, readQaScenarioPack } from "../../scenario-catalog.js"; type TelegramQaProfile = "all" | "release"; @@ -40,13 +12,17 @@ function resolveTelegramQaProfile(profile: string | undefined): TelegramQaProfil ); } +function listTelegramQaProfileScenarios(profile: string) { + return listQaScenariosForExecutionProfile(`telegram:${profile}`); +} + export function resolveTelegramQaScenarioIds(params: { profile?: string; providerMode: string; scenarioIds?: readonly string[]; }): string[] { - const knownIds = new Set(TELEGRAM_QA_ALL_SCENARIO_IDS); - if (params.scenarioIds && params.scenarioIds.length > 0) { + if (params.scenarioIds?.length) { + const knownIds = new Set(readQaScenarioPack().scenarios.map((scenario) => scenario.id)); const unknownIds = params.scenarioIds.filter((id) => !knownIds.has(id)); if (unknownIds.length > 0) { throw new Error(`unknown Telegram QA scenario id(s): ${unknownIds.join(", ")}`); @@ -54,24 +30,18 @@ export function resolveTelegramQaScenarioIds(params: { return [...params.scenarioIds]; } const profile = resolveTelegramQaProfile(params.profile); - if (profile === "all") { - return [...TELEGRAM_QA_ALL_SCENARIO_IDS]; - } - return params.providerMode === "mock-openai" - ? [...TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS] - : [...TELEGRAM_QA_RELEASE_SCENARIO_IDS]; + const executionProfile = + profile === "release" && params.providerMode === "mock-openai" ? "mock-release" : profile; + return listTelegramQaProfileScenarios(executionProfile).map((scenario) => scenario.id); } export function listTelegramQaScenarios(providerMode: string) { const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode, profile: "release" })); - const allIds = new Set(TELEGRAM_QA_ALL_SCENARIO_IDS); - return readQaScenarioPack() - .scenarios.filter((scenario) => allIds.has(scenario.id)) - .map((scenario) => ({ - id: scenario.id, - title: scenario.title, - rationale: scenario.objective, - regressionRefs: scenario.regressionRefs ?? [], - defaultEnabled: defaultIds.has(scenario.id), - })); + return listTelegramQaProfileScenarios("all").map((scenario) => ({ + id: scenario.id, + title: scenario.title, + rationale: scenario.objective, + regressionRefs: scenario.regressionRefs ?? [], + defaultEnabled: defaultIds.has(scenario.id), + })); } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts index 09505fbb6f95..3b5f304e45ce 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts @@ -5,7 +5,6 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { resolveWhatsAppQaScenarioIds } from "./profiles.js"; type WhatsAppQaAdapterRuntime = typeof import("./adapter.runtime.js"); type WhatsAppQaCliRuntime = typeof import("./cli.runtime.js"); @@ -23,7 +22,6 @@ async function runQaWhatsApp(opts: LiveTransportQaCommandOptions) { const whatsappQaAdapterFactory: NonNullable = { id: "whatsapp", - scenarioIds: resolveWhatsAppQaScenarioIds({ providerMode: "live-frontier" }), matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp", async create(context) { return await (await loadWhatsAppQaAdapterRuntime()).createWhatsAppQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/profiles.ts b/extensions/qa-lab/src/live-transports/whatsapp/profiles.ts index 870e3926d071..88a78e2a3cb3 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/profiles.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/profiles.ts @@ -1,55 +1,4 @@ -const WHATSAPP_QA_LIVE_DEFAULT_SCENARIO_IDS = [ - "whatsapp-canary", - "whatsapp-mention-gating", - "whatsapp-top-level-reply-shape", - "whatsapp-reply-to-message", - "whatsapp-group-reply-to-message", - "whatsapp-status-reactions", - "whatsapp-group-allowlist-block", - "whatsapp-help-command", -] as const; - -const WHATSAPP_QA_MOCK_DEFAULT_SCENARIO_IDS = [ - "whatsapp-canary", - "whatsapp-mention-gating", - "whatsapp-group-pending-history-context", - "whatsapp-broadcast-group-fanout", - "whatsapp-group-activation-always", - "whatsapp-group-reply-to-bot-triggers", - "whatsapp-top-level-reply-shape", - "whatsapp-reply-to-message", - "whatsapp-group-reply-to-message", - "whatsapp-reply-to-mode-batched", - "whatsapp-agent-message-action-react", - "whatsapp-agent-message-action-upload-file", - "whatsapp-group-agent-message-action-react", - "whatsapp-group-agent-message-action-upload-file", - "whatsapp-inbound-reaction-no-trigger", - "whatsapp-reply-context-isolation", - "whatsapp-inbound-image-caption", - "whatsapp-audio-preflight", - "whatsapp-outbound-media-matrix", - "whatsapp-outbound-document-preserves-filename", - "whatsapp-outbound-poll", - "whatsapp-group-outbound-media", - "whatsapp-group-outbound-audio", - "whatsapp-group-outbound-poll", - "whatsapp-message-actions", - "whatsapp-inbound-structured-messages", - "whatsapp-group-audio-gating", - "whatsapp-reply-delivery-shape", - "whatsapp-stream-final-message-accounting", - "whatsapp-status-reactions", - "whatsapp-status-reaction-lifecycle", - "whatsapp-group-allowlist-block", - "whatsapp-help-command", - "whatsapp-commands-command", - "whatsapp-tools-compact-command", - "whatsapp-whoami-command", - "whatsapp-context-command", - "whatsapp-tool-only-usage-footer", - "whatsapp-native-new-command", -] as const; +import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; export function resolveWhatsAppQaScenarioIds(params: { providerMode: string; @@ -58,7 +7,6 @@ export function resolveWhatsAppQaScenarioIds(params: { if (params.scenarioIds?.length) { return [...params.scenarioIds]; } - return params.providerMode === "mock-openai" - ? [...WHATSAPP_QA_MOCK_DEFAULT_SCENARIO_IDS] - : [...WHATSAPP_QA_LIVE_DEFAULT_SCENARIO_IDS]; + const profile = params.providerMode === "mock-openai" ? "mock-default" : "default"; + return listQaScenariosForExecutionProfile(`whatsapp:${profile}`).map((scenario) => scenario.id); } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts index 13db97963e67..814379713a65 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.contracts.ts @@ -6,8 +6,6 @@ import type { import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import type { startQaGatewayChild } from "../../gateway-child.js"; -import type { QaProviderMode } from "../../run-config.js"; -import type { LiveTransportScenarioDefinition } from "../shared/live-transport-scenarios.js"; export type WhatsAppQaRuntimeEnv = { driverAuthArchiveBase64: string; @@ -271,11 +269,12 @@ export type WhatsAppQaConfigOverrides = { }; }; -export type WhatsAppQaScenarioDefinition = LiveTransportScenarioDefinition & { +export type WhatsAppQaScenarioDefinition = { + id: WhatsAppQaScenarioId; + title: string; + timeoutMs: number; buildRun: () => WhatsAppQaScenarioRun; configOverrides?: WhatsAppQaConfigOverrides; - defaultEnabled?: boolean; - defaultProviderModes?: readonly QaProviderMode[]; requiresGroupJid?: boolean; requiredPluginIds?: readonly string[]; }; @@ -300,7 +299,6 @@ export type WhatsAppQaScenarioResult = { responseObservedAt: string; source: "approval-request-to-resolution" | "request-to-observed-message"; }; - standardId?: string; status: "fail" | "pass" | "skip"; title: string; }; @@ -309,7 +307,6 @@ export function buildWhatsAppQaScenarioResultBase(scenario: WhatsAppQaScenarioDe return { id: scenario.id, title: scenario.title, - standardId: scenario.standardId, posture: WHATSAPP_QA_SCENARIO_POSTURES[scenario.id], }; } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index e1573f2d87f7..840ed7ac8f74 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -188,7 +188,8 @@ const WHATSAPP_GROUP_CAPABILITY_SCENARIO_IDS = [ function findMockWhatsAppScenario(id: WhatsAppScenarioIdFilter) { const scenario = getWhatsAppQaScenarioDefinition(id); - if (!scenario.standardId && !scenario.defaultProviderModes?.includes("mock-openai")) { + const mockScenarioIds = new Set(resolveWhatsAppQaScenarioIds({ providerMode: "mock-openai" })); + if (!mockScenarioIds.has(id)) { throw new Error(`missing WhatsApp mock-openai scenario ${id}`); } return scenario; @@ -306,7 +307,6 @@ describe("WhatsApp QA live runtime", () => { } expect(scenario.id).toBe("whatsapp-agent-message-action-react"); - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(scenario.configOverrides).toMatchObject({ actions: true }); expect(run.target).toBe("dm"); expect(run.input).toMatch(/React to this WhatsApp message/i); @@ -386,7 +386,6 @@ describe("WhatsApp QA live runtime", () => { throw new Error(`${scenario.id} unexpectedly built an approval run`); } - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(run.target).toBe("dm"); } }); @@ -403,7 +402,6 @@ describe("WhatsApp QA live runtime", () => { throw new Error(`${scenario.id} unexpectedly built an approval run`); } - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(scenario.requiresGroupJid).toBe(true); expect(run.target).toBe("group"); } @@ -1059,7 +1057,6 @@ describe("WhatsApp QA live runtime", () => { } expect(scenario.requiresGroupJid).toBe(true); - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(run.target).toBe("group"); expect(run.configMode).toBe("open"); expect(run.input).toContain("openclawqa"); @@ -1078,7 +1075,6 @@ describe("WhatsApp QA live runtime", () => { } expect(scenario.requiresGroupJid).toBe(true); - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(scenario.configOverrides).toMatchObject({ groupPolicy: "open" }); expect(run.target).toBe("group"); expect(run.configMode).toBe("allowlist"); @@ -1405,20 +1401,17 @@ describe("WhatsApp QA live runtime", () => { runs.map(({ scenario, run }) => ({ id: scenario.id, requiresGroupJid: scenario.requiresGroupJid, - standardId: scenario.standardId, target: run.target, })), ).toEqual([ { id: "whatsapp-reply-to-message", requiresGroupJid: undefined, - standardId: "quote-reply", target: "dm", }, { id: "whatsapp-group-reply-to-message", requiresGroupJid: true, - standardId: "quote-reply", target: "group", }, ]); @@ -1607,7 +1600,6 @@ describe("WhatsApp QA live runtime", () => { throw new Error("whatsapp-stream-final-message-accounting unexpectedly built approval run"); } - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(run.input).toContain("WhatsApp long final QA check"); expect(run.matchText).toBe("WHATSAPP-LONG-FINAL-BEGIN"); expect(run.expectedJoinedSutTextIncludes).toEqual([ @@ -1936,7 +1928,6 @@ describe("WhatsApp QA live runtime", () => { } expect(scenario.requiredPluginIds).toEqual(["openai"]); - expect(scenario.defaultProviderModes).toEqual(["mock-openai"]); expect(scenarioRun.expectReply).toBe(true); expect(scenarioRun.matchText).toBe("WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"); expect(scenarioRun.sendMode).toMatchObject({ diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts index 525d99836397..5bd05ef84809 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.capabilities.ts @@ -24,7 +24,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-outbound-document-preserves-filename", title: "WhatsApp direct Gateway document preserves filename and caption", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, buildRun: () => { const token = `WHATSAPP_QA_DOCUMENT_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -62,8 +61,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-outbound-send-serialization", title: "WhatsApp parallel Gateway sends deliver every outbound message", - defaultEnabled: false, - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, buildRun: () => { const token = `WHATSAPP_QA_SERIAL_SEND_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -99,7 +96,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-outbound-poll", title: "WhatsApp direct Gateway poll delivers outbound native poll", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, buildRun: () => { const token = `WHATSAPP_QA_OUTBOUND_POLL_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -133,7 +129,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-group-outbound-media", title: "WhatsApp direct Gateway send delivers media to a group", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, requiresGroupJid: true, buildRun: () => { @@ -196,7 +191,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-group-outbound-audio", title: "WhatsApp direct Gateway send delivers audio to a group", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, requiresGroupJid: true, buildRun: () => { @@ -240,7 +234,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-group-outbound-poll", title: "WhatsApp direct Gateway poll delivers native poll to a group", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, requiresGroupJid: true, buildRun: () => { @@ -276,7 +269,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-message-actions", title: "WhatsApp direct Gateway message.action react and upload-file execute", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, configOverrides: { actions: true, @@ -331,7 +323,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-inbound-structured-messages", title: "WhatsApp inbound structured messages reach the agent", - defaultProviderModes: ["mock-openai"], timeoutMs: 240_000, buildRun: () => { const token = `WHATSAPP_QA_STRUCTURED_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -396,7 +387,6 @@ export const WHATSAPP_QA_CAPABILITY_SCENARIOS: WhatsAppQaScenarioDefinition[] = { id: "whatsapp-group-audio-gating", title: "WhatsApp group audio mention gating", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, configOverrides: { audioPreflight: true, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts index 75fd5382c8a4..d4c98794ebf6 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.conversation.ts @@ -44,7 +44,6 @@ function buildWhatsAppQuoteReplyRun(target: "dm" | "group"): WhatsAppQaMessageSc export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-canary", - standardId: "canary", title: "WhatsApp DM canary", timeoutMs: 60_000, buildRun: () => { @@ -60,7 +59,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] }, { id: "whatsapp-mention-gating", - standardId: "mention-gating", title: "WhatsApp group mention gating", timeoutMs: 60_000, requiresGroupJid: true, @@ -82,7 +80,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] { id: "whatsapp-group-pending-history-context", title: "WhatsApp group pending history reaches mentioned turns", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { groupHistoryLimit: 50, @@ -116,7 +113,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] { id: "whatsapp-broadcast-group-fanout", title: "WhatsApp group broadcast fans out to multiple agents", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, configOverrides: { broadcast: { @@ -155,7 +151,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] { id: "whatsapp-group-activation-always", title: "WhatsApp group activation always wakes unmentioned messages", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, configOverrides: { groupPolicy: "open", @@ -242,7 +237,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] { id: "whatsapp-group-reply-to-bot-triggers", title: "WhatsApp group reply to bot wakes without an explicit mention", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, configOverrides: { groupPolicy: "open", @@ -300,7 +294,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] }, { id: "whatsapp-top-level-reply-shape", - standardId: "top-level-reply-shape", title: "WhatsApp DM top-level reply shape", timeoutMs: 60_000, configOverrides: { @@ -326,7 +319,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] }, { id: "whatsapp-reply-to-message", - standardId: "quote-reply", title: "WhatsApp DM reply-to mode quotes the triggering message", timeoutMs: 60_000, configOverrides: { @@ -336,7 +328,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] }, { id: "whatsapp-group-reply-to-message", - standardId: "quote-reply", title: "WhatsApp group reply-to mode quotes the triggering message", timeoutMs: 60_000, configOverrides: { @@ -348,7 +339,6 @@ export const WHATSAPP_QA_CONVERSATION_SCENARIOS: WhatsAppQaScenarioDefinition[] { id: "whatsapp-reply-to-mode-batched", title: "WhatsApp batched reply-to mode quotes the queued message", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { inboundDebounceMs: 250, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts index 527f20086388..bd57e9b9703c 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.delivery.ts @@ -12,7 +12,6 @@ export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-reply-delivery-shape", title: "WhatsApp direct Gateway send chunks long replies", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, buildRun: () => { const token = `WHATSAPP_QA_REPLY_SHAPE_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -75,7 +74,6 @@ export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-stream-final-message-accounting", title: "WhatsApp streamed final response has exactly the final chunks", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, buildRun: () => ({ configMode: "allowlist", @@ -106,7 +104,6 @@ export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ }, { id: "whatsapp-status-reactions", - standardId: "reaction-observation", title: "WhatsApp status reactions are observable", timeoutMs: 60_000, configOverrides: { @@ -133,7 +130,6 @@ export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-status-reaction-lifecycle", title: "WhatsApp status reaction lifecycle updates the triggering message", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { statusReactions: { @@ -170,7 +166,6 @@ export const WHATSAPP_QA_DELIVERY_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ }, { id: "whatsapp-group-allowlist-block", - standardId: "allowlist-block", title: "WhatsApp group outside allowlist stays quiet", timeoutMs: 8_000, configOverrides: { diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts index 10ef57888cd3..6785865fb558 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.scenarios.user-path.ts @@ -22,7 +22,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-agent-message-action-react", title: "WhatsApp user-path agent reaction uses the message tool", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { actions: true, @@ -53,7 +52,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-agent-message-action-upload-file", title: "WhatsApp user-path agent upload-file sends media", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { actions: true, @@ -91,7 +89,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-group-agent-message-action-react", title: "WhatsApp group user-path agent reaction uses the message tool", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { actions: true, @@ -123,7 +120,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-group-agent-message-action-upload-file", title: "WhatsApp group user-path agent upload-file sends media", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { actions: true, @@ -163,7 +159,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-inbound-reaction-no-trigger", title: "WhatsApp inbound user reaction does not start a fresh run", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, buildRun: () => { const token = `WHATSAPP_QA_INBOUND_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -197,7 +192,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-reply-context-isolation", title: "WhatsApp direct Gateway send does not reuse prior quote context", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, buildRun: () => { const token = `WHATSAPP_QA_REPLY_ISOLATION_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -256,7 +250,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-inbound-image-caption", title: "WhatsApp inbound image caption reaches the agent", - defaultProviderModes: ["mock-openai"], timeoutMs: 60_000, buildRun: () => { const token = `WHATSAPP_QA_IMAGE_${randomUUID().slice(0, 8).toUpperCase()}`; @@ -278,7 +271,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-audio-preflight", title: "WhatsApp inbound audio preflight transcript reaches the agent", - defaultProviderModes: ["mock-openai"], timeoutMs: 90_000, configOverrides: { audioPreflight: true, @@ -301,7 +293,6 @@ export const WHATSAPP_QA_USER_PATH_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ { id: "whatsapp-outbound-media-matrix", title: "WhatsApp direct Gateway send delivers outbound media variants", - defaultProviderModes: ["mock-openai"], timeoutMs: 120_000, buildRun: () => { const token = `WHATSAPP_QA_OUTBOUND_MEDIA_${randomUUID().slice(0, 8).toUpperCase()}`; diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 40e5e0b1cd07..3abaeda307f3 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -731,7 +731,7 @@ describe("qa scenario catalog", () => { it("keeps provider-sensitive QA flow scenarios on their supported lanes", () => { const strandedConfig = readQaScenarioExecutionConfig("message-tool-stranded-final-reply") as - | { requiredChannelDriver?: string; requiredProviderMode?: string } + | { requiredProviderMode?: string } | undefined; const retryFailureConfig = readQaScenarioExecutionConfig( "message-tool-stranded-final-retry-failure", @@ -834,7 +834,6 @@ describe("qa scenario catalog", () => { | { workspaceFiles?: Record; prompt?: string; - requiredChannelDriver?: string; expectedReplyAll?: string[]; expectedArtifactAll?: string[]; expectedArtifactAny?: string[]; @@ -847,7 +846,7 @@ describe("qa scenario catalog", () => { "Mission: prove you followed the repo contract.", ); expect(config?.prompt).toContain("Repo contract followthrough check."); - expect(config?.requiredChannelDriver).toBe("qa-channel"); + expect(scenario.execution.driver).toBe("qa-channel"); expect(config?.expectedReplyAll).toEqual(["read:", "wrote:", "status:"]); expect(config?.expectedArtifactAll).toEqual(["repo contract"]); expect(config?.expectedArtifactAny).toContain("evidence path"); @@ -879,46 +878,39 @@ describe("qa scenario catalog", () => { ]; for (const scenarioId of scenarioIds) { - const config = readQaScenarioExecutionConfig(scenarioId) as - | { requiredChannelDriver?: string } - | undefined; - expect(config?.requiredChannelDriver, scenarioId).toBe("qa-channel"); + expect(readQaScenarioById(scenarioId).execution.driver, scenarioId).toBe("qa-channel"); } }); - it("routes canonical thread relation flows through Crabline Matrix", () => { + it("keeps portable thread relation flows free of a channel requirement", () => { for (const scenarioId of ["thread-follow-up", "thread-isolation"]) { const scenario = readQaScenarioById(scenarioId); - const config = readQaScenarioExecutionConfig(scenarioId) as - | { requiredChannelDriver?: string } - | undefined; - expect(scenario.execution.channel, scenarioId).toBe("matrix"); - expect(config?.requiredChannelDriver, scenarioId).toBeUndefined(); + expect(scenario.execution.channel, scenarioId).toBeUndefined(); + expect(scenario.execution.driver, scenarioId).toBeUndefined(); + expect(Object.keys(scenario.execution.profiles ?? {}), scenarioId).toEqual( + expect.arrayContaining(["matrix:adapter", "slack:adapter"]), + ); } }); it("keeps Matrix subagent thread spawn explicitly selectable", () => { const scenario = readQaScenarioById("subagent-thread-spawn"); - const config = readQaScenarioExecutionConfig("subagent-thread-spawn") as - | { requiredChannelDriver?: string } - | undefined; expect(scenario.execution.channel).toBe("matrix"); - expect(config?.requiredChannelDriver).toBe("live"); + expect(scenario.execution.driver).toBe("live"); }); it("routes native command session targeting through Crabline Telegram", () => { const scenario = readQaScenarioById("native-command-session-target"); const config = readQaScenarioExecutionConfig("native-command-session-target") as | { - requiredChannelDriver?: string; requiredProviderMode?: string; } | undefined; expect(scenario.execution.channel).toBe("telegram"); - expect(config?.requiredChannelDriver).toBeUndefined(); + expect(scenario.execution.driver).toBeUndefined(); expect(config?.requiredProviderMode).toBe("mock-openai"); }); @@ -935,10 +927,7 @@ describe("qa scenario catalog", () => { ]; for (const scenarioId of liveScenarioIds) { - const config = readQaScenarioExecutionConfig(scenarioId) as - | { requiredChannelDriver?: string } - | undefined; - expect(config?.requiredChannelDriver, scenarioId).toBe("live"); + expect(readQaScenarioById(scenarioId).execution.driver, scenarioId).toBe("live"); } }); diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 895df1c10e16..bdd66b40c2d2 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -66,6 +66,14 @@ const qaScenarioChannelSchema = z message: "scenario execution channel ids must use lowercase dotted or dashed tokens", }); +const qaScenarioDriverSchema = z.enum(["qa-channel", "crabline", "live"]); +const qaScenarioProfileSchema = z + .string() + .trim() + .regex(/^[a-z0-9]+(?:[.:/-][a-z0-9]+)*$/, { + message: "scenario execution profiles must use lowercase namespaced tokens", + }); + const qaScenarioTransportPolicySchema = z.object({ requireGroupMention: z.literal(true).optional(), senderAllowlist: z.array(z.string().trim().min(1)).min(1).optional(), @@ -77,6 +85,8 @@ const qaFlowScenarioExecutionSchema = z kind: z.literal("flow").default("flow"), summary: z.string().trim().min(1).optional(), channel: qaScenarioChannelSchema.optional(), + driver: qaScenarioDriverSchema.optional(), + profiles: z.record(qaScenarioProfileSchema, z.number().int().nonnegative()).optional(), suiteIsolation: z.literal("isolated").optional(), isolationReason: z.string().trim().min(1).optional(), transportPolicy: qaScenarioTransportPolicySchema.optional(), @@ -87,6 +97,8 @@ const qaFlowScenarioExecutionSchema = z const qaTestFileScenarioExecutionBaseSchema = z.object({ summary: z.string().trim().min(1).optional(), channel: qaScenarioChannelSchema.optional(), + driver: qaScenarioDriverSchema.optional(), + profiles: z.record(qaScenarioProfileSchema, z.number().int().nonnegative()).optional(), path: qaScenarioRepoRefSchema, config: qaScenarioConfigSchema.optional(), }); @@ -546,6 +558,21 @@ export function readQaScenarioExecutionConfig(id: string): Record candidate.id === id)?.execution?.config; } +export function listQaScenariosForExecutionProfile(profile: string): QaSeedScenarioWithSource[] { + const normalized = qaScenarioProfileSchema.parse(profile); + const scenarios = readQaScenarioPack() + .scenarios.filter((scenario) => scenario.execution.profiles?.[normalized] !== undefined) + .toSorted((left, right) => { + const orderDelta = + left.execution.profiles![normalized]! - right.execution.profiles![normalized]!; + return orderDelta || left.sourcePath.localeCompare(right.sourcePath); + }); + if (scenarios.length === 0) { + throw new Error(`unknown QA scenario execution profile: ${normalized}`); + } + return scenarios; +} + export function validateQaScenarioExecutionConfig(config: Record) { return qaScenarioConfigSchema.parse(config); } diff --git a/extensions/qa-lab/src/scenario-lane.ts b/extensions/qa-lab/src/scenario-lane.ts index f6e15ee52d76..d0bbfd4549ab 100644 --- a/extensions/qa-lab/src/scenario-lane.ts +++ b/extensions/qa-lab/src/scenario-lane.ts @@ -28,10 +28,10 @@ export function describeQaProviderLaneMismatches(params: { if (requiredProviderMode && params.providerMode !== requiredProviderMode) { mismatches.push(`providerMode=${requiredProviderMode}`); } - const requiredChannelDriver = normalizeQaConfigString(config.requiredChannelDriver); + const requiredDriver = params.scenario.execution.driver; const effectiveChannelDriver = params.channelDriver ?? "qa-channel"; - if (requiredChannelDriver && effectiveChannelDriver !== requiredChannelDriver) { - mismatches.push(`channelDriver=${requiredChannelDriver}`); + if (requiredDriver && effectiveChannelDriver !== requiredDriver) { + mismatches.push(`channelDriver=${requiredDriver}`); } const scenarioChannel = params.scenario.execution.channel?.trim().toLowerCase(); if ( diff --git a/extensions/qa-lab/src/suite-planning.test.ts b/extensions/qa-lab/src/suite-planning.test.ts index 77091cf551f5..1cbbeea186a4 100644 --- a/extensions/qa-lab/src/suite-planning.test.ts +++ b/extensions/qa-lab/src/suite-planning.test.ts @@ -685,10 +685,10 @@ describe("qa suite planning helpers", () => { const scenarios = [ makeQaSuiteTestScenario("generic"), makeQaSuiteTestScenario("qa-channel-only", { - config: { requiredChannelDriver: "qa-channel" }, + driver: "qa-channel", }), makeQaSuiteTestScenario("crabline-only", { - config: { requiredChannelDriver: "crabline" }, + driver: "crabline", }), ]; @@ -714,7 +714,7 @@ describe("qa suite planning helpers", () => { const scenarios = [ makeQaSuiteTestScenario("generic"), makeQaSuiteTestScenario("qa-channel-only", { - config: { requiredChannelDriver: "qa-channel" }, + driver: "qa-channel", }), ]; diff --git a/extensions/qa-lab/src/suite-test-helpers.ts b/extensions/qa-lab/src/suite-test-helpers.ts index 7f98a95a2068..834dc57f59e6 100644 --- a/extensions/qa-lab/src/suite-test-helpers.ts +++ b/extensions/qa-lab/src/suite-test-helpers.ts @@ -8,6 +8,7 @@ export function makeQaSuiteTestScenario( id: string, params: { channel?: string; + driver?: QaSuiteTestScenario["execution"]["driver"]; config?: Record; plugins?: string[]; gatewayConfigPatch?: Record; @@ -32,6 +33,7 @@ export function makeQaSuiteTestScenario( execution: { kind: "flow", ...(params.channel ? { channel: params.channel } : {}), + ...(params.driver ? { driver: params.driver } : {}), ...(params.suiteIsolation ? { suiteIsolation: params.suiteIsolation } : {}), ...(params.transportPolicy ? { transportPolicy: params.transportPolicy } : {}), ...(params.config ? { config: params.config } : {}), diff --git a/package.json b/package.json index dcb66437f334..6cb2be8efa08 100644 --- a/package.json +++ b/package.json @@ -1149,10 +1149,6 @@ "types": "./dist/plugin-sdk/qa-runner-runtime.d.ts", "default": "./dist/plugin-sdk/qa-runner-runtime.js" }, - "./plugin-sdk/qa-live-transport-scenarios": { - "types": "./dist/plugin-sdk/qa-live-transport-scenarios.d.ts", - "default": "./dist/plugin-sdk/qa-live-transport-scenarios.js" - }, "./plugin-sdk/memory-core": { "types": "./dist/plugin-sdk/memory-core.d.ts", "default": "./dist/plugin-sdk/memory-core.js" diff --git a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml index b7e371ebdf1e..8b25e243b0e8 100644 --- a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml +++ b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml @@ -24,8 +24,8 @@ scenario: execution: kind: flow summary: Verify the agent reads repo instructions first, then completes the first bounded followthrough task without stalling. + driver: qa-channel config: - requiredChannelDriver: qa-channel workspaceFiles: AGENT.md: |- # Repo contract diff --git a/qa/scenarios/agents/subagent-forked-context.yaml b/qa/scenarios/agents/subagent-forked-context.yaml index 45958ed36ed1..477c5490751a 100644 --- a/qa/scenarios/agents/subagent-forked-context.yaml +++ b/qa/scenarios/agents/subagent-forked-context.yaml @@ -21,8 +21,8 @@ scenario: execution: kind: flow summary: Ask the agent to delegate work that depends on the current transcript and assert sessions_spawn carries context=fork. + driver: qa-channel config: - requiredChannelDriver: qa-channel contextNeedle: FORKED-CONTEXT-ALPHA prompt: "Forked subagent context QA check. The visible code in this current conversation is FORKED-CONTEXT-ALPHA. Delegate to a native subagent to report the visible code from the requester transcript. Do not include the visible code in the child task text; the child must recover it from forked transcript context. Use forked context if the child needs the current transcript; otherwise it will not know the code. A spawn-accepted result is not the answer. Wait for the child completion, then make sure user-visible output includes the visible code." diff --git a/qa/scenarios/agents/subagent-handoff.yaml b/qa/scenarios/agents/subagent-handoff.yaml index 71f71219aec6..e69f69cea267 100644 --- a/qa/scenarios/agents/subagent-handoff.yaml +++ b/qa/scenarios/agents/subagent-handoff.yaml @@ -20,8 +20,8 @@ scenario: execution: kind: flow summary: Verify the agent can delegate a bounded task to a subagent and fold the result back into the main thread. + driver: qa-channel config: - requiredChannelDriver: qa-channel prompt: "Delegate one bounded QA task to a subagent. Wait for the subagent to finish. Then reply with three labeled sections exactly once: Delegated task, Result, Evidence. Include the child result itself, not 'waiting'." flow: diff --git a/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml b/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml index 7bc30db54d31..82ae2972daf9 100644 --- a/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml +++ b/qa/scenarios/channels/a2a-message-tool-mirror-dedupe.yaml @@ -37,10 +37,10 @@ scenario: - src/shared/transcript-only-openclaw-assistant.ts execution: kind: flow + driver: qa-channel summary: Run a real QA Gateway/qa-channel A2A source-reply turn and assert the message-tool delivery mirror is not announced again. config: requiredProviderMode: mock-openai - requiredChannelDriver: qa-channel conversationId: qa-a2a-mirror-room conversationTitle: QA A2A Mirror Room promptSnippet: qa a2a message-tool mirror check diff --git a/qa/scenarios/channels/channel-canary.yaml b/qa/scenarios/channels/channel-canary.yaml index b0f559949320..0ffeaca8b9f3 100644 --- a/qa/scenarios/channels/channel-canary.yaml +++ b/qa/scenarios/channels/channel-canary.yaml @@ -20,6 +20,16 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/suite.ts execution: + profiles: + { + "matrix:adapter": 1, + "slack:adapter": 1, + "telegram:adapter": 1, + "telegram:all": 0, + "telegram:mock-release": 0, + "telegram:release": 0, + "whatsapp:adapter": 1, + } kind: flow summary: Run the shared channel canary through QA Channel, Crabline, or a live adapter. transportPolicy: diff --git a/qa/scenarios/channels/channel-chat-baseline.yaml b/qa/scenarios/channels/channel-chat-baseline.yaml index 944bca4de4f1..bc66ef3e8cdf 100644 --- a/qa/scenarios/channels/channel-chat-baseline.yaml +++ b/qa/scenarios/channels/channel-chat-baseline.yaml @@ -21,6 +21,15 @@ scenario: - extensions/qa-channel/src/inbound.ts - extensions/qa-lab/src/bus-state.ts execution: + profiles: + { + "matrix:adapter": 0, + "matrix:all": 0, + "matrix:fast": 0, + "matrix:release": 0, + "slack:adapter": 0, + "telegram:adapter": 0, + } kind: flow suiteIsolation: isolated summary: Verify the QA agent can respond correctly in a shared channel and respect mention-driven group semantics. diff --git a/qa/scenarios/channels/channel-dm-group-routing.yaml b/qa/scenarios/channels/channel-dm-group-routing.yaml index 56f8cd404c0f..77daec347671 100644 --- a/qa/scenarios/channels/channel-dm-group-routing.yaml +++ b/qa/scenarios/channels/channel-dm-group-routing.yaml @@ -20,6 +20,7 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - src/routing/resolve-route.ts execution: + profiles: { "matrix:adapter": 2, "whatsapp:adapter": 2 } kind: flow summary: Send sequential DM and group turns and assert route separation. transportPolicy: diff --git a/qa/scenarios/channels/channel-mention-gating.yaml b/qa/scenarios/channels/channel-mention-gating.yaml index 1ab550c3d069..1ec8a5cea6e7 100644 --- a/qa/scenarios/channels/channel-mention-gating.yaml +++ b/qa/scenarios/channels/channel-mention-gating.yaml @@ -19,6 +19,19 @@ scenario: - extensions/qa-channel/src/inbound.ts - extensions/qa-lab/src/qa-transport.ts execution: + profiles: + { + "matrix:adapter": 3, + "matrix:all": 35, + "matrix:fast": 7, + "matrix:transport": 34, + "slack:adapter": 2, + "telegram:adapter": 2, + "telegram:all": 1, + "telegram:mock-release": 1, + "telegram:release": 1, + "whatsapp:adapter": 3, + } kind: flow summary: Prove the shared mention gate with a quiet turn followed by a mentioned turn. transportPolicy: diff --git a/qa/scenarios/channels/channel-multi-actor-ordering.yaml b/qa/scenarios/channels/channel-multi-actor-ordering.yaml index 15e3ec291dd7..7d4d62bb7532 100644 --- a/qa/scenarios/channels/channel-multi-actor-ordering.yaml +++ b/qa/scenarios/channels/channel-multi-actor-ordering.yaml @@ -19,6 +19,7 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:adapter": 7, "matrix:all": 48, "matrix:transport": 47 } kind: flow summary: Preserve actor order across a blocked observer turn and a successful driver turn. transportPolicy: diff --git a/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml b/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml index 5af4755e3ace..ed4a3c54cfcd 100644 --- a/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml +++ b/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml @@ -19,6 +19,7 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:adapter": 6, "matrix:all": 19, "matrix:transport": 18 } kind: flow summary: Send sequential turns to primary and secondary channel conversations and assert isolation. transportPolicy: diff --git a/qa/scenarios/channels/channel-sender-allowlist.yaml b/qa/scenarios/channels/channel-sender-allowlist.yaml index 4e586f46b318..c5cbd56a565a 100644 --- a/qa/scenarios/channels/channel-sender-allowlist.yaml +++ b/qa/scenarios/channels/channel-sender-allowlist.yaml @@ -19,6 +19,7 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - src/channels/turn/kernel.ts execution: + profiles: { "matrix:adapter": 4, "matrix:all": 47, "matrix:fast": 10, "matrix:transport": 46 } kind: flow summary: Exercise sender allowlist rejection and the configured driver override in one ordered flow. transportPolicy: diff --git a/qa/scenarios/channels/channel-top-level-reply-shape.yaml b/qa/scenarios/channels/channel-top-level-reply-shape.yaml index 649db6439b5c..8b97d46b1371 100644 --- a/qa/scenarios/channels/channel-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/channel-top-level-reply-shape.yaml @@ -19,6 +19,15 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/runtime-api.ts execution: + profiles: + { + "matrix:adapter": 5, + "matrix:all": 5, + "matrix:fast": 3, + "matrix:transport": 4, + "slack:adapter": 3, + "whatsapp:adapter": 4, + } kind: flow summary: Assert portable top-level reply shape while native adapters normalize transport relations. transportPolicy: diff --git a/qa/scenarios/channels/discord-canary.yaml b/qa/scenarios/channels/discord-canary.yaml index b432377056b4..973e92a61ecd 100644 --- a/qa/scenarios/channels/discord-canary.yaml +++ b/qa/scenarios/channels/discord-canary.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "discord:adapter": 0, "discord:default": 0 } kind: flow channel: discord timeoutMs: 45000 diff --git a/qa/scenarios/channels/discord-mention-gating.yaml b/qa/scenarios/channels/discord-mention-gating.yaml index 195eb774e60a..0296ea757182 100644 --- a/qa/scenarios/channels/discord-mention-gating.yaml +++ b/qa/scenarios/channels/discord-mention-gating.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "discord:adapter": 1, "discord:default": 1 } kind: flow channel: discord timeoutMs: 8000 diff --git a/qa/scenarios/channels/discord-native-help-command-registration.yaml b/qa/scenarios/channels/discord-native-help-command-registration.yaml index 68acac3a51d7..61d3ecdc6677 100644 --- a/qa/scenarios/channels/discord-native-help-command-registration.yaml +++ b/qa/scenarios/channels/discord-native-help-command-registration.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [discord.native-slash-command-registration] execution: + profiles: { "discord:adapter": 2, "discord:default": 2 } kind: flow channel: discord timeoutMs: 45000 diff --git a/qa/scenarios/channels/dm-chat-baseline.yaml b/qa/scenarios/channels/dm-chat-baseline.yaml index 89fa67f087cf..bd57e642d643 100644 --- a/qa/scenarios/channels/dm-chat-baseline.yaml +++ b/qa/scenarios/channels/dm-chat-baseline.yaml @@ -21,6 +21,7 @@ scenario: - extensions/qa-channel/src/gateway.ts - extensions/qa-lab/src/lab-server.ts execution: + profiles: { "matrix:all": 14, "matrix:transport": 13, "whatsapp:adapter": 0 } kind: flow summary: Verify the QA agent can chat coherently in a DM, explain the QA setup, and stay in character. config: diff --git a/qa/scenarios/channels/dm-per-room-session.yaml b/qa/scenarios/channels/dm-per-room-session.yaml index aa0aa9dd7229..0d87ed27e9df 100644 --- a/qa/scenarios/channels/dm-per-room-session.yaml +++ b/qa/scenarios/channels/dm-per-room-session.yaml @@ -25,6 +25,7 @@ scenario: - extensions/matrix/src/matrix/monitor/handler.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:adapter": 12, "matrix:all": 17, "matrix:transport": 16 } kind: flow channel: matrix summary: Send consecutive DMs in two Matrix rooms and require per-room isolation. diff --git a/qa/scenarios/channels/dm-shared-session.yaml b/qa/scenarios/channels/dm-shared-session.yaml index b3a7b57d1a44..25a7a88262e0 100644 --- a/qa/scenarios/channels/dm-shared-session.yaml +++ b/qa/scenarios/channels/dm-shared-session.yaml @@ -17,6 +17,7 @@ scenario: - extensions/matrix/src/matrix/monitor/handler.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:adapter": 11, "matrix:all": 15, "matrix:transport": 14 } kind: flow channel: matrix summary: Send consecutive DMs from one Matrix user in two rooms and require the shared-session notice. diff --git a/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml b/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml index 5501c0f4211f..d426ce539722 100644 --- a/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml +++ b/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml @@ -34,8 +34,8 @@ scenario: execution: kind: flow summary: Verify message_tool visible replies degrade to automatic delivery when the active group policy removes message. + driver: qa-channel config: - requiredChannelDriver: qa-channel conversationId: qa-fallback-room promptSnippet: qa group message unavailable fallback check prompt: "@openclaw qa group message unavailable fallback check. exact marker: `QA-GROUP-FALLBACK-OK`" diff --git a/qa/scenarios/channels/matrix-allowbots-default-block.yaml b/qa/scenarios/channels/matrix-allowbots-default-block.yaml index 6b5c81b37142..fd464ee5219d 100644 --- a/qa/scenarios/channels/matrix-allowbots-default-block.yaml +++ b/qa/scenarios/channels/matrix-allowbots-default-block.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 36, "matrix:fast": 8, "matrix:transport": 35 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: configuredBotRoles: - observer diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml index 44f07bfc6f38..4dde7bc203fc 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.dm execution: + profiles: { "matrix:all": 40, "matrix:transport": 39 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: mentions configuredBotRoles: diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml index fec2a402f98f..c88a10033d16 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 38, "matrix:fast": 9, "matrix:transport": 37 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: mentions configuredBotRoles: diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml index 55c6987a2ced..2ff105a28f94 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 39, "matrix:transport": 38 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: mentions configuredBotRoles: diff --git a/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml b/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml index 5f29862e7b4c..21c84fddb9ef 100644 --- a/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml +++ b/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 41, "matrix:transport": 40 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: true configuredBotRoles: diff --git a/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml b/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml index bc01f0e8ca30..ea3168b9e6c2 100644 --- a/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml +++ b/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 42, "matrix:transport": 41 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: configuredBotRoles: - observer diff --git a/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml b/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml index db0b1687b15a..45fed1dfc2de 100644 --- a/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml +++ b/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 43, "matrix:transport": 42 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: true groupAllowRoles: diff --git a/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml b/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml index d1dd15b1a955..719c37bae062 100644 --- a/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml +++ b/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 37, "matrix:transport": 36 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: allowBots: true configuredBotRoles: diff --git a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml index 3830c489a25d..bab1662ba9b7 100644 --- a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml +++ b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml @@ -15,14 +15,15 @@ scenario: codeRefs: - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:all": 46, "matrix:release": 1, "matrix:transport": 45 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 suiteIsolation: isolated summary: Exercise Matrix observer-role allowlist hot reload through normalized transport state. + driver: live config: - requiredChannelDriver: live accountId: sut flow: diff --git a/qa/scenarios/channels/matrix-approval-channel-target-both.yaml b/qa/scenarios/channels/matrix-approval-channel-target-both.yaml index 34d9113d7870..306994104b48 100644 --- a/qa/scenarios/channels/matrix-approval-channel-target-both.yaml +++ b/qa/scenarios/channels/matrix-approval-channel-target-both.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 30, "matrix:transport": 29 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: exec: true diff --git a/qa/scenarios/channels/matrix-approval-deny-reaction.yaml b/qa/scenarios/channels/matrix-approval-deny-reaction.yaml index 4d744e640232..237dd726b4a6 100644 --- a/qa/scenarios/channels/matrix-approval-deny-reaction.yaml +++ b/qa/scenarios/channels/matrix-approval-deny-reaction.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 28, "matrix:transport": 27 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: exec: true diff --git a/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml b/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml index 70f9f94091a7..d93d99b7a8ba 100644 --- a/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml +++ b/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 26, "matrix:fast": 6, "matrix:transport": 25 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: exec: true diff --git a/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml b/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml index fb319722c43a..466b3bb57fad 100644 --- a/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml +++ b/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 25, "matrix:fast": 5, "matrix:transport": 24 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: exec: true diff --git a/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml b/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml index 5fe1fd27ff1b..97e479e3f799 100644 --- a/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml +++ b/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 27, "matrix:transport": 26 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: plugin: true diff --git a/qa/scenarios/channels/matrix-approval-thread-target.yaml b/qa/scenarios/channels/matrix-approval-thread-target.yaml index ebd458f3045e..f6d35fecc057 100644 --- a/qa/scenarios/channels/matrix-approval-thread-target.yaml +++ b/qa/scenarios/channels/matrix-approval-thread-target.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.native-approval-prompts execution: + profiles: { "matrix:all": 29, "matrix:transport": 28 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: approvalForwarding: exec: true diff --git a/qa/scenarios/channels/matrix-attachment-only-ignored.yaml b/qa/scenarios/channels/matrix-attachment-only-ignored.yaml index d7b9e04de676..359cd5de7928 100644 --- a/qa/scenarios/channels/matrix-attachment-only-ignored.yaml +++ b/qa/scenarios/channels/matrix-attachment-only-ignored.yaml @@ -8,12 +8,13 @@ scenario: secondary: - channels.inbound-media-normalization execution: + profiles: { "matrix:all": 55, "matrix:media": 4 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml b/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml index 820f70ff236f..81c31441914f 100644 --- a/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml +++ b/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml @@ -8,12 +8,13 @@ scenario: secondary: - channels.dm execution: + profiles: { "matrix:all": 16, "matrix:transport": 15 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: dm: threadReplies: always diff --git a/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml b/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml index 0308e1dfcc01..9e0ebdd996fb 100644 --- a/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml +++ b/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 64, "matrix:e2ee-smoke": 6 } kind: flow channel: matrix timeoutMs: 150000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml b/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml index 6c69ce10b201..693d6d9d6307 100644 --- a/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml +++ b/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 58, "matrix:e2ee-smoke": 0, "matrix:fast": 11 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml b/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml index 62e870c1f49b..45c9d3602c14 100644 --- a/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml +++ b/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 60, "matrix:e2ee-smoke": 2 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml b/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml index 49b9e402cfdf..64a0330134f4 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 84, "matrix:e2ee-cli": 0 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml index 7af0e0471824..412297eca646 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 87, "matrix:e2ee-cli": 3 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml index 20d10f5eac57..b8274a611b66 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 86, "matrix:e2ee-cli": 2 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml index 48a6dcfd6839..e8e56c2669d9 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 90, "matrix:e2ee-cli": 6 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml index efa90ee96b89..af40e59e38eb 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 85, "matrix:e2ee-cli": 1 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml index 3f50b5530b35..5ae930ad6c6e 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 89, "matrix:e2ee-cli": 5 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml index 38fa4f0f29bc..50480a9de412 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 88, "matrix:e2ee-cli": 4 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml b/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml index 41899fd8a695..356b71b16cb3 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 92, "matrix:e2ee-cli": 8 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml b/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml index 2ab0d27dd0a6..c8e693d26c57 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 91, "matrix:e2ee-cli": 7 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: false startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml b/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml index 1b56a6b251cc..ef84ed772956 100644 --- a/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml +++ b/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 73, "matrix:e2ee-deep": 7 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml b/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml index 7feb55a9a9f7..488f87915f0c 100644 --- a/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 78, "matrix:e2ee-deep": 12 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml b/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml index 06602db69041..e5e08dafaa34 100644 --- a/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.dm execution: + profiles: { "matrix:all": 81, "matrix:e2ee-deep": 15 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml b/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml index 15fe3e7e786f..fa741407acd4 100644 --- a/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml +++ b/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 77, "matrix:e2ee-deep": 11 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml b/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml index af07c863ba93..14edfecb8281 100644 --- a/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml +++ b/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 65, "matrix:e2ee-smoke": 7 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-media-image.yaml b/qa/scenarios/channels/matrix-e2ee-media-image.yaml index 0cd7d18618f6..7aef89dc1112 100644 --- a/qa/scenarios/channels/matrix-e2ee-media-image.yaml +++ b/qa/scenarios/channels/matrix-e2ee-media-image.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.media-rich-content execution: + profiles: { "matrix:all": 57, "matrix:media": 6 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml b/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml index bf0553b9b4a9..8855ce000291 100644 --- a/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 79, "matrix:e2ee-deep": 13 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml b/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml index ab9837b72e03..f7ea029136bd 100644 --- a/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml +++ b/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 61, "matrix:e2ee-smoke": 3 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml b/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml index a5b3e2cf9efc..e938ceaf5f12 100644 --- a/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml +++ b/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 62, "matrix:e2ee-smoke": 4 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml b/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml index cccba6ce83ee..f3dcf321e9e3 100644 --- a/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml +++ b/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 63, "matrix:e2ee-smoke": 5 } kind: flow channel: matrix timeoutMs: 150000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml index a8ad9abe4386..7c0ccd9ff8e5 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 72, "matrix:e2ee-deep": 6 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml index a25769dff532..fea3b174c710 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 71, "matrix:e2ee-deep": 5 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml index 6cf5f97bdab4..08037f8a971e 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 74, "matrix:e2ee-deep": 8 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml index f3e551f8e492..54e6c8b71d72 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 75, "matrix:e2ee-deep": 9 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml b/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml index 98adfcc95ebd..3b1408a4e060 100644 --- a/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml +++ b/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 80, "matrix:e2ee-deep": 14 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml b/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml index 6b72bf7a7d6d..b71345377215 100644 --- a/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml +++ b/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 70, "matrix:e2ee-deep": 4 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml b/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml index b4c1e0b958e4..baf72a774506 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 66, "matrix:e2ee-deep": 0 } kind: flow channel: matrix timeoutMs: 150000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml index 96cd5092833d..9d7ac46a0042 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 67, "matrix:e2ee-deep": 1 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml index 85b7381cc8d8..8ee5e0278bda 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 69, "matrix:e2ee-deep": 3 } kind: flow channel: matrix timeoutMs: 120000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml index 146e348d31ca..ee6abba69196 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 68, "matrix:e2ee-deep": 2 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml b/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml index 57aa2821d2d3..26b2d6e0c43d 100644 --- a/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 76, "matrix:e2ee-deep": 10 } kind: flow channel: matrix timeoutMs: 150000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml index 9205f1c533ea..16c5f6e7c608 100644 --- a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml +++ b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.threads execution: + profiles: { "matrix:all": 59, "matrix:e2ee-smoke": 1 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml b/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml index 2300ce3a77de..bf2fd9164e46 100644 --- a/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml +++ b/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 82, "matrix:e2ee-deep": 16 } kind: flow channel: matrix timeoutMs: 30000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml index c60863c30355..821655cf70d5 100644 --- a/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.access-and-identity execution: + profiles: { "matrix:all": 83, "matrix:e2ee-deep": 17 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: encryption: true startupVerification: off diff --git a/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml b/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml index 5ef9869ca18d..bd5da561e574 100644 --- a/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml +++ b/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 34, "matrix:transport": 33 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml b/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml index e8eefeca251f..6cffc480ebf2 100644 --- a/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml +++ b/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml @@ -6,12 +6,12 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 49, "matrix:transport": 48 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 - config: - requiredChannelDriver: live + driver: live flow: module: ./live-transports/matrix/scenarios/scenario-runtime-edit.js diff --git a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml index 9181cacb8d62..76ca960dc62f 100644 --- a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml +++ b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml @@ -6,12 +6,12 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 50, "matrix:transport": 49 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 - config: - requiredChannelDriver: live + driver: live flow: module: ./live-transports/matrix/scenarios/scenario-runtime-edit.js diff --git a/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml b/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml index 822c7ad6235d..6daa07e3b1d7 100644 --- a/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml +++ b/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 31, "matrix:transport": 30 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-media-type-coverage.yaml b/qa/scenarios/channels/matrix-media-type-coverage.yaml index d1430abef320..3f3a1b3cc3c9 100644 --- a/qa/scenarios/channels/matrix-media-type-coverage.yaml +++ b/qa/scenarios/channels/matrix-media-type-coverage.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.media-rich-content execution: + profiles: { "matrix:all": 53, "matrix:media": 2 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml b/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml index 94c1514119e7..c003607feebd 100644 --- a/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml +++ b/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml @@ -6,12 +6,12 @@ scenario: primary: - channels.access-policy execution: + profiles: { "matrix:all": 45, "matrix:transport": 44 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 - config: - requiredChannelDriver: live + driver: live flow: module: ./live-transports/matrix/scenarios/scenario-runtime-policy.js diff --git a/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml b/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml index e001d9647a69..9d2e4ba95abf 100644 --- a/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml +++ b/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml @@ -8,12 +8,13 @@ scenario: secondary: - channels.group-messages execution: + profiles: { "matrix:all": 44, "matrix:transport": 43 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: groupPolicy: open diff --git a/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml b/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml index 19cba88fdf89..2ffdd4b38e52 100644 --- a/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml +++ b/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 23, "matrix:transport": 22 } kind: flow channel: matrix timeoutMs: 8000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixRequireCanary: true flow: diff --git a/qa/scenarios/channels/matrix-reaction-notification.yaml b/qa/scenarios/channels/matrix-reaction-notification.yaml index 266b3428ab68..91c23090e25d 100644 --- a/qa/scenarios/channels/matrix-reaction-notification.yaml +++ b/qa/scenarios/channels/matrix-reaction-notification.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 21, "matrix:fast": 4, "matrix:transport": 20 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixRequireCanary: true flow: diff --git a/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml b/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml index 1dfd227572b4..b6335d1aaf3e 100644 --- a/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml +++ b/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 24, "matrix:transport": 23 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixRequireCanary: true flow: diff --git a/qa/scenarios/channels/matrix-reaction-threaded.yaml b/qa/scenarios/channels/matrix-reaction-threaded.yaml index 6005cb407340..30fd82cb6444 100644 --- a/qa/scenarios/channels/matrix-reaction-threaded.yaml +++ b/qa/scenarios/channels/matrix-reaction-threaded.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.message-actions execution: + profiles: { "matrix:all": 22, "matrix:transport": 21 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixRequireCanary: true flow: diff --git a/qa/scenarios/channels/matrix-room-autojoin-invite.yaml b/qa/scenarios/channels/matrix-room-autojoin-invite.yaml index 0361bdf02bca..d2eeeef2d054 100644 --- a/qa/scenarios/channels/matrix-room-autojoin-invite.yaml +++ b/qa/scenarios/channels/matrix-room-autojoin-invite.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 18, "matrix:transport": 17 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: autoJoin: always groupPolicy: open diff --git a/qa/scenarios/channels/matrix-room-block-streaming.yaml b/qa/scenarios/channels/matrix-room-block-streaming.yaml index c80fcc37a205..840d2e2216d5 100644 --- a/qa/scenarios/channels/matrix-room-block-streaming.yaml +++ b/qa/scenarios/channels/matrix-room-block-streaming.yaml @@ -11,8 +11,8 @@ scenario: providerMode: mock-openai timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: agentDefaults: blockStreamingChunk: diff --git a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml index b35666c63cf8..ea340c19407b 100644 --- a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml +++ b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.media-rich-content execution: + profiles: { "matrix:all": 52, "matrix:media": 1 } kind: flow channel: matrix timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: quiet matrixTopology: diff --git a/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml b/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml index f87f8e7726e3..c7b7878f166d 100644 --- a/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml +++ b/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml @@ -10,12 +10,13 @@ scenario: docsRefs: - docs/tools/media-overview.md execution: + profiles: { "matrix:all": 51, "matrix:media": 0 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-room-membership-loss.yaml b/qa/scenarios/channels/matrix-room-membership-loss.yaml index 98d9ed638884..28ab49323607 100644 --- a/qa/scenarios/channels/matrix-room-membership-loss.yaml +++ b/qa/scenarios/channels/matrix-room-membership-loss.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 33, "matrix:transport": 32 } kind: flow channel: matrix timeoutMs: 75000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml b/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml index 6432a01025df..e83c01e68e77 100644 --- a/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml +++ b/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml @@ -8,12 +8,13 @@ scenario: docsRefs: - docs/concepts/streaming.md execution: + profiles: { "matrix:all": 7, "matrix:transport": 6 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: partial diff --git a/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml b/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml index 10e7675bffc8..071ae7f78a49 100644 --- a/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml +++ b/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml @@ -8,12 +8,13 @@ scenario: docsRefs: - docs/concepts/streaming.md execution: + profiles: { "matrix:all": 8, "matrix:transport": 7 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: quiet diff --git a/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml b/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml index 6168424290b8..052ad27a7f75 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.streaming execution: + profiles: { "matrix:all": 10, "matrix:transport": 9 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: quiet toolProfile: coding diff --git a/qa/scenarios/channels/matrix-room-tool-progress-error.yaml b/qa/scenarios/channels/matrix-room-tool-progress-error.yaml index 1a96515f7a7b..ad787a3b6bfa 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-error.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-error.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.streaming execution: + profiles: { "matrix:all": 12, "matrix:transport": 11 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: quiet toolProfile: coding diff --git a/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml b/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml index 269470198416..0be233931418 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.streaming execution: + profiles: { "matrix:all": 13, "matrix:transport": 12 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: partial toolProfile: coding diff --git a/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml b/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml index 6e402b19e25f..4d5af3afd54d 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.streaming execution: + profiles: { "matrix:all": 11, "matrix:transport": 10 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: mode: quiet diff --git a/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml b/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml index 490c2d851bac..f7cb8a344119 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.streaming execution: + profiles: { "matrix:all": 9, "matrix:transport": 8 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: streaming: quiet toolProfile: coding diff --git a/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml b/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml index 54f03bc1b324..765197a993d7 100644 --- a/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml +++ b/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.group-messages execution: + profiles: { "matrix:all": 20, "matrix:transport": 19 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: groupsByKey: secondary: diff --git a/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml b/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml index 2df8f5e271ed..74e0a5595b46 100644 --- a/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml +++ b/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml @@ -6,12 +6,13 @@ scenario: primary: - channels.reconnect execution: + profiles: { "matrix:all": 32, "matrix:transport": 31 } kind: flow channel: matrix timeoutMs: 90000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml index ac59dc1aed71..b014dbac4fb8 100644 --- a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml +++ b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml @@ -6,12 +6,12 @@ scenario: primary: - channels.threads execution: + profiles: { "matrix:all": 3, "matrix:transport": 2 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 - config: - requiredChannelDriver: live + driver: live flow: module: ./live-transports/matrix/scenarios/scenario-runtime-room.js diff --git a/qa/scenarios/channels/matrix-thread-root-preservation.yaml b/qa/scenarios/channels/matrix-thread-root-preservation.yaml index 6ef5af61b2ba..ccdd257faec1 100644 --- a/qa/scenarios/channels/matrix-thread-root-preservation.yaml +++ b/qa/scenarios/channels/matrix-thread-root-preservation.yaml @@ -6,12 +6,12 @@ scenario: primary: - channels.threads execution: + profiles: { "matrix:all": 2, "matrix:transport": 1 } kind: flow channel: matrix timeoutMs: 60000 retryCount: 0 - config: - requiredChannelDriver: live + driver: live flow: module: ./live-transports/matrix/scenarios/scenario-runtime-room.js diff --git a/qa/scenarios/channels/matrix-unsupported-media-safe.yaml b/qa/scenarios/channels/matrix-unsupported-media-safe.yaml index 651486b4f71c..a8e56e81e961 100644 --- a/qa/scenarios/channels/matrix-unsupported-media-safe.yaml +++ b/qa/scenarios/channels/matrix-unsupported-media-safe.yaml @@ -8,12 +8,13 @@ scenario: secondary: - runtime.delivery execution: + profiles: { "matrix:all": 56, "matrix:media": 5 } kind: flow channel: matrix timeoutMs: 45000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixTopology: defaultRoomKey: main rooms: diff --git a/qa/scenarios/channels/matrix-voice-preflight-mention.yaml b/qa/scenarios/channels/matrix-voice-preflight-mention.yaml index e3c8446197cd..b98dd9d1b317 100644 --- a/qa/scenarios/channels/matrix-voice-preflight-mention.yaml +++ b/qa/scenarios/channels/matrix-voice-preflight-mention.yaml @@ -6,13 +6,14 @@ scenario: primary: - channels.media-rich-content execution: + profiles: { "matrix:all": 54, "matrix:media": 3 } kind: flow channel: matrix providerMode: live-frontier timeoutMs: 180000 retryCount: 0 + driver: live config: - requiredChannelDriver: live matrixConfigOverrides: audio: enabled: true diff --git a/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml b/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml index a5a996c1846a..302a86cbcb96 100644 --- a/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml +++ b/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml @@ -24,8 +24,8 @@ scenario: execution: kind: flow summary: Verify qa-channel readiness recovery does not duplicate old outbound delivery. + driver: qa-channel config: - requiredChannelDriver: qa-channel firstPrompt: "@openclaw Reconnect dedupe setup marker. Reply exactly: RECONNECT-FIRST-OK" secondPrompt: "@openclaw Reconnect dedupe follow-up marker. Reply exactly: RECONNECT-SECOND-OK" firstMarker: RECONNECT-FIRST-OK diff --git a/qa/scenarios/channels/reaction-edit-delete.yaml b/qa/scenarios/channels/reaction-edit-delete.yaml index e6353bea894b..2eb6ceb0ee75 100644 --- a/qa/scenarios/channels/reaction-edit-delete.yaml +++ b/qa/scenarios/channels/reaction-edit-delete.yaml @@ -21,8 +21,8 @@ scenario: execution: kind: flow summary: Verify the agent can use channel-owned message actions and that the QA transcript reflects them. + driver: qa-channel config: - requiredChannelDriver: qa-channel target: "channel:qa-room" seedText: "seed message" editedText: "seed message (edited)" diff --git a/qa/scenarios/channels/slack-allowlist-block.yaml b/qa/scenarios/channels/slack-allowlist-block.yaml index 9a51cf3bb5c0..f03b56d94e1c 100644 --- a/qa/scenarios/channels/slack-allowlist-block.yaml +++ b/qa/scenarios/channels/slack-allowlist-block.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.access-and-identity] execution: + profiles: { "slack:default": 2 } kind: flow channel: slack timeoutMs: 8000 diff --git a/qa/scenarios/channels/slack-approval-exec-native.yaml b/qa/scenarios/channels/slack-approval-exec-native.yaml index 62000b7b8987..de578a4d77cb 100644 --- a/qa/scenarios/channels/slack-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-approval-exec-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.native-approval-prompts] execution: + profiles: { "slack:default": 7 } kind: flow channel: slack timeoutMs: 60000 diff --git a/qa/scenarios/channels/slack-approval-plugin-native.yaml b/qa/scenarios/channels/slack-approval-plugin-native.yaml index 027adfb0e2c1..74e7025505fe 100644 --- a/qa/scenarios/channels/slack-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-approval-plugin-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.native-approval-prompts] execution: + profiles: { "slack:default": 8 } kind: flow channel: slack timeoutMs: 60000 diff --git a/qa/scenarios/channels/slack-canary.yaml b/qa/scenarios/channels/slack-canary.yaml index 64145d020120..bbfbc12c172d 100644 --- a/qa/scenarios/channels/slack-canary.yaml +++ b/qa/scenarios/channels/slack-canary.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "slack:default": 0 } kind: flow channel: slack timeoutMs: 45000 diff --git a/qa/scenarios/channels/slack-chart-presentation-native.yaml b/qa/scenarios/channels/slack-chart-presentation-native.yaml index d830d8bc011d..7fe13832b3ad 100644 --- a/qa/scenarios/channels/slack-chart-presentation-native.yaml +++ b/qa/scenarios/channels/slack-chart-presentation-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.media-rich-content] execution: + profiles: { "slack:default": 4 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml index f34b9de5290a..811b969c50ea 100644 --- a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.native-approval-prompts] execution: + profiles: { "slack:default": 9 } kind: flow channel: slack runtime: codex diff --git a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml index 1e0e52e73062..4830c330f27f 100644 --- a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.native-approval-prompts] execution: + profiles: { "slack:default": 10 } kind: flow channel: slack runtime: codex diff --git a/qa/scenarios/channels/slack-mention-gating.yaml b/qa/scenarios/channels/slack-mention-gating.yaml index 5c5e0973c9cb..7cf4fce8c69b 100644 --- a/qa/scenarios/channels/slack-mention-gating.yaml +++ b/qa/scenarios/channels/slack-mention-gating.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "slack:default": 1 } kind: flow channel: slack timeoutMs: 8000 diff --git a/qa/scenarios/channels/slack-reaction-glyph-native.yaml b/qa/scenarios/channels/slack-reaction-glyph-native.yaml index 54c1ca73ef94..c6de652b2bc3 100644 --- a/qa/scenarios/channels/slack-reaction-glyph-native.yaml +++ b/qa/scenarios/channels/slack-reaction-glyph-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.message-actions] execution: + profiles: { "slack:default": 6 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-restart-resume.yaml b/qa/scenarios/channels/slack-restart-resume.yaml index 2d0884b4f1f4..94b65a9092e7 100644 --- a/qa/scenarios/channels/slack-restart-resume.yaml +++ b/qa/scenarios/channels/slack-restart-resume.yaml @@ -20,8 +20,8 @@ scenario: channel: slack suiteIsolation: isolated summary: Exercise normalized restart and continuation delivery. + driver: live config: - requiredChannelDriver: live conversationKind: group conversationId: C0123456789 senderId: U0123456789 diff --git a/qa/scenarios/channels/slack-table-presentation-native.yaml b/qa/scenarios/channels/slack-table-presentation-native.yaml index 5297eba19d84..8b2777937b99 100644 --- a/qa/scenarios/channels/slack-table-presentation-native.yaml +++ b/qa/scenarios/channels/slack-table-presentation-native.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.media-rich-content] execution: + profiles: { "slack:default": 5 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-top-level-reply-shape.yaml b/qa/scenarios/channels/slack-top-level-reply-shape.yaml index 2c0d3cfedcba..2c321d0631d6 100644 --- a/qa/scenarios/channels/slack-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/slack-top-level-reply-shape.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-visible-replies] execution: + profiles: { "slack:default": 3 } kind: flow channel: slack timeoutMs: 45000 diff --git a/qa/scenarios/channels/subagent-thread-spawn.yaml b/qa/scenarios/channels/subagent-thread-spawn.yaml index dc132f4eae17..41df89fde19f 100644 --- a/qa/scenarios/channels/subagent-thread-spawn.yaml +++ b/qa/scenarios/channels/subagent-thread-spawn.yaml @@ -38,8 +38,8 @@ scenario: kind: flow channel: matrix summary: Spawn a bound Matrix child session and require native thread relation evidence on completion. + driver: live config: - requiredChannelDriver: live conversationId: main childMarker: QA-MATRIX-SUBAGENT-THREAD-OK prompt: 'Use sessions_spawn for this QA check. task="Finish with exactly QA-MATRIX-SUBAGENT-THREAD-OK." label=qa-thread-subagent thread=true mode=session. After spawn returns status="accepted", wait for the child session reply in the spawned Matrix thread. Do not write QA-MATRIX-SUBAGENT-THREAD-OK in the parent response.' diff --git a/qa/scenarios/channels/telegram-commands-command.yaml b/qa/scenarios/channels/telegram-commands-command.yaml index 52a22aa5c6bf..a8c64eaff0f8 100644 --- a/qa/scenarios/channels/telegram-commands-command.yaml +++ b/qa/scenarios/channels/telegram-commands-command.yaml @@ -14,6 +14,13 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 4, + "telegram:all": 3, + "telegram:mock-release": 3, + "telegram:release": 3, + } kind: flow channel: telegram summary: Request the Telegram command catalog and verify representative entries. diff --git a/qa/scenarios/channels/telegram-context-command.yaml b/qa/scenarios/channels/telegram-context-command.yaml index 560a8ca6d319..e21fedb9e97b 100644 --- a/qa/scenarios/channels/telegram-context-command.yaml +++ b/qa/scenarios/channels/telegram-context-command.yaml @@ -14,6 +14,13 @@ scenario: - src/auto-reply/reply/commands-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 9, + "telegram:all": 8, + "telegram:mock-release": 8, + "telegram:release": 8, + } kind: flow channel: telegram summary: Run Telegram context help and verify the list shortcut. diff --git a/qa/scenarios/channels/telegram-current-session-status-tool.yaml b/qa/scenarios/channels/telegram-current-session-status-tool.yaml index d75846b008e8..e9559b8cb15d 100644 --- a/qa/scenarios/channels/telegram-current-session-status-tool.yaml +++ b/qa/scenarios/channels/telegram-current-session-status-tool.yaml @@ -15,11 +15,12 @@ scenario: - src/agents/tools/session-status-tool.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 11, "telegram:all": 10 } kind: flow channel: telegram summary: Ask the model to resolve the current Telegram group session through session_status. + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai flow: diff --git a/qa/scenarios/channels/telegram-help-command.yaml b/qa/scenarios/channels/telegram-help-command.yaml index 1475a4740710..b0ed97791a3c 100644 --- a/qa/scenarios/channels/telegram-help-command.yaml +++ b/qa/scenarios/channels/telegram-help-command.yaml @@ -15,6 +15,13 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 3, + "telegram:all": 2, + "telegram:mock-release": 2, + "telegram:release": 2, + } kind: flow channel: telegram summary: Send Telegram help and verify the concise native command guide. diff --git a/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml index 7b6517e2c020..43aa68afb0ec 100644 --- a/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml +++ b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml @@ -16,11 +16,12 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 15, "telegram:all": 14, "telegram:mock-release": 10 } kind: flow channel: telegram summary: Run the scripted long final and verify preview reuse and chunk count. + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai flow: diff --git a/qa/scenarios/channels/telegram-long-final-three-chunks.yaml b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml index db946b7eea22..d1fc997cabda 100644 --- a/qa/scenarios/channels/telegram-long-final-three-chunks.yaml +++ b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml @@ -16,11 +16,12 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 16, "telegram:all": 15 } kind: flow channel: telegram summary: Run the scripted three-chunk final and verify final chunk accounting. + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai flow: diff --git a/qa/scenarios/channels/telegram-other-bot-command-gating.yaml b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml index d437a4c90792..7c512a6042d1 100644 --- a/qa/scenarios/channels/telegram-other-bot-command-gating.yaml +++ b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml @@ -14,11 +14,17 @@ scenario: - extensions/telegram/src/bot-message-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 10, + "telegram:all": 9, + "telegram:mock-release": 9, + "telegram:release": 9, + } kind: flow channel: telegram summary: Address a command to another bot and verify OpenClaw ignores it. - config: - requiredChannelDriver: live + driver: live flow: steps: diff --git a/qa/scenarios/channels/telegram-repeated-command-authorization.yaml b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml index d20ce4fe14c7..949125f50788 100644 --- a/qa/scenarios/channels/telegram-repeated-command-authorization.yaml +++ b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml @@ -14,13 +14,19 @@ scenario: - extensions/telegram/src/bot-message-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 8, + "telegram:all": 7, + "telegram:mock-release": 7, + "telegram:release": 7, + } kind: flow channel: telegram suiteIsolation: isolated isolationReason: Mutates the live Telegram group allowlist and restarts the QA gateway. summary: Remove and restore the Telegram operator allowlist, then run repeated commands. - config: - requiredChannelDriver: live + driver: live flow: steps: diff --git a/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml index 8a71a7221858..2ce6f533b107 100644 --- a/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml +++ b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml @@ -14,6 +14,7 @@ scenario: - src/auto-reply/reply/reply-payload.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 13, "telegram:all": 12 } kind: flow channel: telegram summary: Request one exact Telegram marker and verify single-message delivery. diff --git a/qa/scenarios/channels/telegram-status-command.yaml b/qa/scenarios/channels/telegram-status-command.yaml index 9425b338e7e1..7b304cb5395e 100644 --- a/qa/scenarios/channels/telegram-status-command.yaml +++ b/qa/scenarios/channels/telegram-status-command.yaml @@ -16,6 +16,13 @@ scenario: - src/auto-reply/reply/commands-status.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 7, + "telegram:all": 6, + "telegram:mock-release": 6, + "telegram:release": 6, + } kind: flow channel: telegram summary: Run Telegram status and verify representative status fields. diff --git a/qa/scenarios/channels/telegram-stream-final-single-message.yaml b/qa/scenarios/channels/telegram-stream-final-single-message.yaml index 4eb3803f91cb..1d36f0d00429 100644 --- a/qa/scenarios/channels/telegram-stream-final-single-message.yaml +++ b/qa/scenarios/channels/telegram-stream-final-single-message.yaml @@ -16,12 +16,13 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 14, "telegram:all": 13 } kind: flow channel: telegram + driver: live summary: Run the scripted quiet-streaming reply and verify one final message. config: marker: QA-TELEGRAM-STREAM-SINGLE-OK - requiredChannelDriver: live requiredProviderMode: mock-openai flow: diff --git a/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml index f68bf8858222..0f9b2c9713f9 100644 --- a/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml +++ b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml @@ -17,11 +17,12 @@ scenario: - src/auto-reply/reply/commands-session.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: { "telegram:adapter": 12, "telegram:all": 11 } kind: flow channel: telegram summary: Enable token usage mode, then verify the next Telegram reply footer. + driver: live config: - requiredChannelDriver: live marker: QA-TELEGRAM-USAGE-FOOTER-OK flow: diff --git a/qa/scenarios/channels/telegram-tools-compact-command.yaml b/qa/scenarios/channels/telegram-tools-compact-command.yaml index 6bfb8e6ae0aa..7a3a9e4b9a63 100644 --- a/qa/scenarios/channels/telegram-tools-compact-command.yaml +++ b/qa/scenarios/channels/telegram-tools-compact-command.yaml @@ -14,6 +14,13 @@ scenario: - src/auto-reply/reply/commands-tools.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 5, + "telegram:all": 4, + "telegram:mock-release": 4, + "telegram:release": 4, + } kind: flow channel: telegram summary: Request compact tools through Telegram and verify inventory text. diff --git a/qa/scenarios/channels/telegram-whoami-command.yaml b/qa/scenarios/channels/telegram-whoami-command.yaml index 1acf3f398c21..cacd0f412a7a 100644 --- a/qa/scenarios/channels/telegram-whoami-command.yaml +++ b/qa/scenarios/channels/telegram-whoami-command.yaml @@ -14,6 +14,13 @@ scenario: - src/auto-reply/reply/commands-whoami.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: + profiles: + { + "telegram:adapter": 6, + "telegram:all": 5, + "telegram:mock-release": 5, + "telegram:release": 5, + } kind: flow channel: telegram summary: Run whoami and verify channel identity context. diff --git a/qa/scenarios/channels/thread-follow-up.yaml b/qa/scenarios/channels/thread-follow-up.yaml index 0d1a858fdf99..30c0b7b52d12 100644 --- a/qa/scenarios/channels/thread-follow-up.yaml +++ b/qa/scenarios/channels/thread-follow-up.yaml @@ -22,8 +22,15 @@ scenario: - extensions/qa-lab/src/live-transports/slack/adapter.runtime.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: + { + "matrix:adapter": 8, + "matrix:all": 1, + "matrix:fast": 1, + "matrix:transport": 0, + "slack:adapter": 4, + } kind: flow - channel: matrix summary: Send a deterministic follow-up through the shared host and require native thread relation evidence. config: rootMarker: QA-THREAD-ROOT-OK diff --git a/qa/scenarios/channels/thread-isolation.yaml b/qa/scenarios/channels/thread-isolation.yaml index bbc2e9c7e95b..d1d7abd5b709 100644 --- a/qa/scenarios/channels/thread-isolation.yaml +++ b/qa/scenarios/channels/thread-isolation.yaml @@ -20,8 +20,15 @@ scenario: - extensions/qa-lab/src/live-transports/slack/adapter.runtime.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: + { + "matrix:adapter": 9, + "matrix:all": 4, + "matrix:fast": 2, + "matrix:transport": 3, + "slack:adapter": 5, + } kind: flow - channel: matrix summary: Establish a thread, then require a fresh top-level reply without stale relation metadata. config: threadMarker: QA-THREAD-ISOLATION-THREAD-OK diff --git a/qa/scenarios/channels/thread-reply-override.yaml b/qa/scenarios/channels/thread-reply-override.yaml index 706ccd3843fc..c1d8884b7acd 100644 --- a/qa/scenarios/channels/thread-reply-override.yaml +++ b/qa/scenarios/channels/thread-reply-override.yaml @@ -23,11 +23,12 @@ scenario: - extensions/matrix/src/matrix/monitor/threads.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: + profiles: { "matrix:adapter": 10, "matrix:all": 6, "matrix:transport": 5 } kind: flow channel: matrix summary: Require the canonical Matrix adapter to surface the native m.thread relation through threadId. + driver: live config: - requiredChannelDriver: live conversationId: main marker: QA-MATRIX-THREAD-OVERRIDE-OK diff --git a/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml b/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml index c2fc6390ddb2..ab222add4877 100644 --- a/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml @@ -18,8 +18,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Verify WhatsApp dmPolicy disabled behavior through the canonical adapter. + driver: live config: - requiredChannelDriver: live policyKey: dmPolicy policyValue: disabled conversationKind: direct diff --git a/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml b/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml index ca3a96c46b01..4bff554d41f0 100644 --- a/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml @@ -18,8 +18,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Verify WhatsApp dmPolicy open behavior through the canonical adapter. + driver: live config: - requiredChannelDriver: live policyKey: dmPolicy policyValue: open conversationKind: direct diff --git a/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml b/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml index 14976407fb4b..bfe6f8db7de2 100644 --- a/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml @@ -18,8 +18,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Verify WhatsApp groupPolicy disabled behavior through the canonical adapter. + driver: live config: - requiredChannelDriver: live policyKey: groupPolicy policyValue: disabled conversationKind: group diff --git a/qa/scenarios/channels/whatsapp-access-control-group-open.yaml b/qa/scenarios/channels/whatsapp-access-control-group-open.yaml index 5567784d275e..d1e6048075e8 100644 --- a/qa/scenarios/channels/whatsapp-access-control-group-open.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-group-open.yaml @@ -18,8 +18,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Verify WhatsApp groupPolicy open behavior through the canonical adapter. + driver: live config: - requiredChannelDriver: live policyKey: groupPolicy policyValue: open conversationKind: group diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml index 1567c484a6ea..21b6dea30fc0 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 10 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml index a8c544b81db0..261a4e7e13f4 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 11 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-audio-preflight.yaml b/qa/scenarios/channels/whatsapp-audio-preflight.yaml index 56090288c079..2cbf7d3c59e0 100644 --- a/qa/scenarios/channels/whatsapp-audio-preflight.yaml +++ b/qa/scenarios/channels/whatsapp-audio-preflight.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 17 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml index a648e8db5ddf..29e64ffc7bcd 100644 --- a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml +++ b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 3 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-canary.yaml b/qa/scenarios/channels/whatsapp-canary.yaml index b96b68df7b0d..d64218d49e75 100644 --- a/qa/scenarios/channels/whatsapp-canary.yaml +++ b/qa/scenarios/channels/whatsapp-canary.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 0, "whatsapp:mock-default": 0 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-commands-command.yaml b/qa/scenarios/channels/whatsapp-commands-command.yaml index 8b2b04890af8..6a864c771458 100644 --- a/qa/scenarios/channels/whatsapp-commands-command.yaml +++ b/qa/scenarios/channels/whatsapp-commands-command.yaml @@ -14,10 +14,11 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 33 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live expectedAny: [/session, /verbose] summary: Request the WhatsApp command catalog and verify representative entries. diff --git a/qa/scenarios/channels/whatsapp-context-command.yaml b/qa/scenarios/channels/whatsapp-context-command.yaml index 0299f482dce2..db00918afbae 100644 --- a/qa/scenarios/channels/whatsapp-context-command.yaml +++ b/qa/scenarios/channels/whatsapp-context-command.yaml @@ -14,10 +14,11 @@ scenario: - src/auto-reply/reply/commands-context.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 36 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai expectedAny: [Context breakdown, "Workspace:", Tool schemas] summary: Run WhatsApp context list and verify the context breakdown. diff --git a/qa/scenarios/channels/whatsapp-group-activation-always.yaml b/qa/scenarios/channels/whatsapp-group-activation-always.yaml index d68b06b08e37..9035645bb69e 100644 --- a/qa/scenarios/channels/whatsapp-group-activation-always.yaml +++ b/qa/scenarios/channels/whatsapp-group-activation-always.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 4 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml index 87977709bc75..2e2e47b85cfc 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 12 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml index 5063acfd6651..26968cbfe465 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 13 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml index ee571bd062fa..fc5c4193ee42 100644 --- a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml +++ b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 6, "whatsapp:mock-default": 31 } kind: flow channel: whatsapp timeoutMs: 8000 diff --git a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml index f867cf651eec..4f1342f50dcf 100644 --- a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml +++ b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 26 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml index 5947199e9885..e30c01c2cd7d 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 22 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml index 3e8d609a0d83..61d5b58b61e1 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 21 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml index 06813ea7e9cd..65a60bd546fd 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 23 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml index 31b80716ae80..0c17dbf26d26 100644 --- a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml +++ b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 2 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml index f3b951bd4a67..f99a961f7a2a 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 5 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml index bc88fc801f0e..291b30d480af 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 4, "whatsapp:mock-default": 8 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-help-command.yaml b/qa/scenarios/channels/whatsapp-help-command.yaml index 53ffdcbed163..fa9de264f2fc 100644 --- a/qa/scenarios/channels/whatsapp-help-command.yaml +++ b/qa/scenarios/channels/whatsapp-help-command.yaml @@ -14,10 +14,10 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:adapter": 5, "whatsapp:default": 7, "whatsapp:mock-default": 32 } kind: flow channel: whatsapp - config: - requiredChannelDriver: live + driver: live summary: Send WhatsApp help and verify command guidance. flow: diff --git a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml index 069d78099c86..5699ae1a34b9 100644 --- a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 16 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml index c75be902983d..706182f42096 100644 --- a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 14 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml index e5da4648aa26..637afa34a739 100644 --- a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 25 } kind: flow channel: whatsapp timeoutMs: 240000 diff --git a/qa/scenarios/channels/whatsapp-mention-gating.yaml b/qa/scenarios/channels/whatsapp-mention-gating.yaml index b4443fb39632..d0d4bca0b4c1 100644 --- a/qa/scenarios/channels/whatsapp-mention-gating.yaml +++ b/qa/scenarios/channels/whatsapp-mention-gating.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 1, "whatsapp:mock-default": 1 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-message-actions.yaml b/qa/scenarios/channels/whatsapp-message-actions.yaml index 2b76bab5bbad..942f23dfbc68 100644 --- a/qa/scenarios/channels/whatsapp-message-actions.yaml +++ b/qa/scenarios/channels/whatsapp-message-actions.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 24 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-native-new-command.yaml b/qa/scenarios/channels/whatsapp-native-new-command.yaml index f5b78b399b39..602af5e3bc85 100644 --- a/qa/scenarios/channels/whatsapp-native-new-command.yaml +++ b/qa/scenarios/channels/whatsapp-native-new-command.yaml @@ -14,10 +14,11 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 38 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai summary: Run `/new` through WhatsApp and verify the new-session acknowledgement. diff --git a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml index 1f392d138d68..ff5e9367a1bc 100644 --- a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 19 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml index ba9e7a582910..fa215875371b 100644 --- a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 18 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-outbound-poll.yaml index 2011f329b223..12009be903dc 100644 --- a/qa/scenarios/channels/whatsapp-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-poll.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 20 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-pairing-block.yaml b/qa/scenarios/channels/whatsapp-pairing-block.yaml index b2dca31097c4..133adf5c98c1 100644 --- a/qa/scenarios/channels/whatsapp-pairing-block.yaml +++ b/qa/scenarios/channels/whatsapp-pairing-block.yaml @@ -19,8 +19,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Verify pairing gate output through the canonical WhatsApp adapter. + driver: live config: - requiredChannelDriver: live accountId: sut conversationId: 15550000001@s.whatsapp.net senderId: 15550000002@s.whatsapp.net diff --git a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml index 6cb01aa084a0..c8bf751f54e7 100644 --- a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml +++ b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 15 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml index b1c7de5bf0b7..9f0cab4ccce6 100644 --- a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml +++ b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 27 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-reply-to-message.yaml index afc00e2dad4f..b2d45b2191dd 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-message.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 3, "whatsapp:mock-default": 7 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml index 9d14eea850c1..c390a4ea6191 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 9 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-restart-resume.yaml b/qa/scenarios/channels/whatsapp-restart-resume.yaml index 15ecefa4cf97..04aadc84cd92 100644 --- a/qa/scenarios/channels/whatsapp-restart-resume.yaml +++ b/qa/scenarios/channels/whatsapp-restart-resume.yaml @@ -20,8 +20,8 @@ scenario: channel: whatsapp suiteIsolation: isolated summary: Exercise normalized restart and continuation delivery. + driver: live config: - requiredChannelDriver: live conversationKind: direct conversationId: 15550000001@s.whatsapp.net senderId: 15550000002@s.whatsapp.net diff --git a/qa/scenarios/channels/whatsapp-status-command.yaml b/qa/scenarios/channels/whatsapp-status-command.yaml index 96530940a7b1..0091e6bf9474 100644 --- a/qa/scenarios/channels/whatsapp-status-command.yaml +++ b/qa/scenarios/channels/whatsapp-status-command.yaml @@ -16,8 +16,7 @@ scenario: execution: kind: flow channel: whatsapp - config: - requiredChannelDriver: live + driver: live summary: Run WhatsApp status and verify session text. flow: diff --git a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml index ba59bc0bd27c..c116a9164a6d 100644 --- a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml +++ b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 30 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-status-reactions.yaml b/qa/scenarios/channels/whatsapp-status-reactions.yaml index 4a03be6d13c9..19dae841ed5e 100644 --- a/qa/scenarios/channels/whatsapp-status-reactions.yaml +++ b/qa/scenarios/channels/whatsapp-status-reactions.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 5, "whatsapp:mock-default": 29 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml index fbf66bdd4c41..dfdd5c6de87c 100644 --- a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml +++ b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:mock-default": 28 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml index d044940014ec..4c3a189ebc15 100644 --- a/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml +++ b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml @@ -15,10 +15,11 @@ scenario: - src/auto-reply/reply/commands-session.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 37 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai marker: WHATSAPP-QA-USAGE-FOOTER-OK summary: Enable token usage mode, then verify the next WhatsApp reply footer. diff --git a/qa/scenarios/channels/whatsapp-tools-compact-command.yaml b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml index b4850cc77a29..00bfde801918 100644 --- a/qa/scenarios/channels/whatsapp-tools-compact-command.yaml +++ b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml @@ -14,10 +14,11 @@ scenario: - src/auto-reply/reply/commands-tools.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 34 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai expectedAny: [exec, Use /tools verbose for descriptions] summary: Request compact tools through WhatsApp and verify inventory text. diff --git a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml index 0ba9fed93599..3f2a890a4154 100644 --- a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml @@ -5,6 +5,7 @@ scenario: coverage: primary: [channels.group-messages] execution: + profiles: { "whatsapp:default": 2, "whatsapp:mock-default": 6 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-whoami-command.yaml b/qa/scenarios/channels/whatsapp-whoami-command.yaml index 46b4928c2882..8deae49d13fd 100644 --- a/qa/scenarios/channels/whatsapp-whoami-command.yaml +++ b/qa/scenarios/channels/whatsapp-whoami-command.yaml @@ -14,10 +14,11 @@ scenario: - src/auto-reply/reply/commands-whoami.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: + profiles: { "whatsapp:mock-default": 35 } kind: flow channel: whatsapp + driver: live config: - requiredChannelDriver: live requiredProviderMode: mock-openai expectedAny: [Identity, "Channel: whatsapp", "AllowFrom:"] summary: Run WhatsApp whoami and verify channel identity context. diff --git a/qa/scenarios/config/config-apply-restart-wakeup.yaml b/qa/scenarios/config/config-apply-restart-wakeup.yaml index 71712d1f567a..92faa72c1f79 100644 --- a/qa/scenarios/config/config-apply-restart-wakeup.yaml +++ b/qa/scenarios/config/config-apply-restart-wakeup.yaml @@ -22,8 +22,8 @@ scenario: execution: kind: flow summary: Verify a restart-required config.apply restarts cleanly and delivers the post-restart wake message back into the QA channel. + driver: qa-channel config: - requiredChannelDriver: qa-channel channelId: qa-room announcePrompt: "Acknowledge restart wake-up setup in qa-room." diff --git a/qa/scenarios/goals/goal-context-next-turn.yaml b/qa/scenarios/goals/goal-context-next-turn.yaml index c701b3a949c5..bdfac7084ec8 100644 --- a/qa/scenarios/goals/goal-context-next-turn.yaml +++ b/qa/scenarios/goals/goal-context-next-turn.yaml @@ -25,8 +25,8 @@ scenario: execution: kind: flow summary: Start a goal, send an unrelated next turn, and inspect the exact mock-provider request payload. + driver: qa-channel config: - requiredChannelDriver: qa-channel requiredProviderMode: mock-openai conversationId: goal-context-next-turn senderId: qa-goal-operator diff --git a/qa/scenarios/goals/goal-context-survives-compaction.yaml b/qa/scenarios/goals/goal-context-survives-compaction.yaml index 74ae0d69dc55..799a0d46871d 100644 --- a/qa/scenarios/goals/goal-context-survives-compaction.yaml +++ b/qa/scenarios/goals/goal-context-survives-compaction.yaml @@ -33,8 +33,8 @@ scenario: execution: kind: flow summary: Start a goal, compact the session, then inspect the next mock-provider request for active-goal context. + driver: qa-channel config: - requiredChannelDriver: qa-channel requiredProviderMode: mock-openai conversationId: goal-context-compaction senderId: qa-goal-operator diff --git a/qa/scenarios/goals/goal-followthrough-live.yaml b/qa/scenarios/goals/goal-followthrough-live.yaml index 2bb44261badc..ed9c3eb2b46e 100644 --- a/qa/scenarios/goals/goal-followthrough-live.yaml +++ b/qa/scenarios/goals/goal-followthrough-live.yaml @@ -28,8 +28,8 @@ scenario: execution: kind: flow summary: Stage a two-step active goal, then require a bare continue turn to write the next artifact without replanning. + driver: qa-channel config: - requiredChannelDriver: qa-channel conversationId: goal-followthrough-live senderId: qa-goal-operator artifactFilePrefix: goal-continuance-live diff --git a/qa/scenarios/media/image-generation-roundtrip.yaml b/qa/scenarios/media/image-generation-roundtrip.yaml index e67f231b0f58..55d263cce30e 100644 --- a/qa/scenarios/media/image-generation-roundtrip.yaml +++ b/qa/scenarios/media/image-generation-roundtrip.yaml @@ -24,8 +24,8 @@ scenario: execution: kind: flow summary: Verify a generated image is saved as media, reattached on the next turn, and described correctly through the vision path. + driver: qa-channel config: - requiredChannelDriver: qa-channel generatePrompt: "Image generation check: generate a QA lighthouse image and summarize it in one short sentence." generatePromptSnippet: "Image generation check" inspectPrompt: "Roundtrip image inspection check: describe the generated lighthouse attachment in one short sentence." diff --git a/qa/scenarios/media/image-understanding-attachment.yaml b/qa/scenarios/media/image-understanding-attachment.yaml index 91e21199a026..bad3ea50e08f 100644 --- a/qa/scenarios/media/image-understanding-attachment.yaml +++ b/qa/scenarios/media/image-understanding-attachment.yaml @@ -23,8 +23,8 @@ scenario: execution: kind: flow summary: Verify an attached image reaches the agent model and the agent can describe what it sees. + driver: qa-channel config: - requiredChannelDriver: qa-channel prompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence." requiredColorGroups: - [red, scarlet, crimson] diff --git a/qa/scenarios/media/native-image-generation.yaml b/qa/scenarios/media/native-image-generation.yaml index a347fdb684cd..7efabc2e6b42 100644 --- a/qa/scenarios/media/native-image-generation.yaml +++ b/qa/scenarios/media/native-image-generation.yaml @@ -22,8 +22,8 @@ scenario: execution: kind: flow summary: Verify image_generate appears when configured and returns a real saved media artifact. + driver: qa-channel config: - requiredChannelDriver: qa-channel prompt: "Image generation check: generate a QA lighthouse image and summarize it in one short sentence." promptSnippet: "Image generation check" generatedNeedle: "QA lighthouse" diff --git a/qa/scenarios/memory/active-memory-preprompt-recall.yaml b/qa/scenarios/memory/active-memory-preprompt-recall.yaml index 9cecf36c4a0e..2ebe4a3b17e4 100644 --- a/qa/scenarios/memory/active-memory-preprompt-recall.yaml +++ b/qa/scenarios/memory/active-memory-preprompt-recall.yaml @@ -42,8 +42,8 @@ scenario: execution: kind: flow summary: Verify Active Memory stays off when session-toggled off, runs memory search/get when enabled, and helps a live model answer with the recalled preference in the first visible reply. + driver: qa-channel config: - requiredChannelDriver: qa-channel baselineConversationId: qa-active-memory-off activeConversationId: qa-active-memory-on memoryFact: "Stable QA movie night usual favorite snack preference: lemon pepper wings with blue cheese." diff --git a/qa/scenarios/memory/memory-recall.yaml b/qa/scenarios/memory/memory-recall.yaml index 54d92612eff3..068fe8ce9de2 100644 --- a/qa/scenarios/memory/memory-recall.yaml +++ b/qa/scenarios/memory/memory-recall.yaml @@ -47,8 +47,8 @@ scenario: execution: kind: flow summary: Verify the agent can store a fact, switch topics, then recall the fact accurately later. + driver: qa-channel config: - requiredChannelDriver: qa-channel resetDurableMemory: true rememberPrompt: "Please remember this fact for later: the QA canary code is ALPHA-7. Use your normal memory mechanism, avoid manual repo cleanup, and reply exactly `Remembered ALPHA-7.` once stored." rememberAckAny: diff --git a/qa/scenarios/memory/session-memory-ranking.yaml b/qa/scenarios/memory/session-memory-ranking.yaml index cd67c540d355..70287b3fb721 100644 --- a/qa/scenarios/memory/session-memory-ranking.yaml +++ b/qa/scenarios/memory/session-memory-ranking.yaml @@ -23,8 +23,8 @@ scenario: execution: kind: flow summary: Verify session-transcript memory can outrank stale durable notes and drive the final answer toward the newer fact. + driver: qa-channel config: - requiredChannelDriver: qa-channel staleFact: ORBIT-9 currentFact: ORBIT-10 transcriptId: qa-session-memory-ranking diff --git a/qa/scenarios/memory/thread-memory-isolation.yaml b/qa/scenarios/memory/thread-memory-isolation.yaml index 875ab9dcedc9..8447481118cc 100644 --- a/qa/scenarios/memory/thread-memory-isolation.yaml +++ b/qa/scenarios/memory/thread-memory-isolation.yaml @@ -24,8 +24,8 @@ scenario: execution: kind: flow summary: Verify a memory-backed answer requested inside a thread stays in-thread and does not leak into the root channel. + driver: qa-channel config: - requiredChannelDriver: qa-channel memoryFact: "Thread-hidden codename: ORBIT-22." memoryQuery: "hidden thread codename ORBIT-22" expectedNeedle: "ORBIT-22" diff --git a/qa/scenarios/personal/channel-thread-reply.yaml b/qa/scenarios/personal/channel-thread-reply.yaml index 4287953e9f6d..67c74b6ad788 100644 --- a/qa/scenarios/personal/channel-thread-reply.yaml +++ b/qa/scenarios/personal/channel-thread-reply.yaml @@ -29,8 +29,8 @@ scenario: execution: kind: flow summary: Verify fake personal replies stay routed to the requested QA conversation and thread. + driver: qa-channel config: - requiredChannelDriver: qa-channel dmUserId: qa-alice dmUserName: QA Alice dmMarker: PERSONAL-DM-OK diff --git a/qa/scenarios/personal/memory-preference-recall.yaml b/qa/scenarios/personal/memory-preference-recall.yaml index fb79c7664dfd..80e35c335d5b 100644 --- a/qa/scenarios/personal/memory-preference-recall.yaml +++ b/qa/scenarios/personal/memory-preference-recall.yaml @@ -29,8 +29,8 @@ scenario: execution: kind: flow summary: Verify fake personal preference recall through the local QA memory path. + driver: qa-channel config: - requiredChannelDriver: qa-channel sessionKey: agent:qa:personal-memory rememberPrompt: "Please remember this fact for later: my fake personal QA preference is that my preferred reminder label code is ORBIT-9. Use your normal memory mechanism and reply exactly `Remembered ORBIT-9.` once stored." rememberAckAny: diff --git a/qa/scenarios/personal/reminder-roundtrip.yaml b/qa/scenarios/personal/reminder-roundtrip.yaml index 4e71a93e34de..d9f853cce6b9 100644 --- a/qa/scenarios/personal/reminder-roundtrip.yaml +++ b/qa/scenarios/personal/reminder-roundtrip.yaml @@ -29,8 +29,8 @@ scenario: execution: kind: flow summary: Verify a fake personal reminder roundtrip stays local to the QA channel. + driver: qa-channel config: - requiredChannelDriver: qa-channel channelId: qa-personal-room channelTitle: QA Personal Room reminderPromptTemplate: "A local personal QA reminder fired. Reply in one short sentence containing this exact marker: {{marker}}" diff --git a/qa/scenarios/scheduling/cron-condition-watcher.yaml b/qa/scenarios/scheduling/cron-condition-watcher.yaml index 2b8d6b95ea70..49c705267e82 100644 --- a/qa/scenarios/scheduling/cron-condition-watcher.yaml +++ b/qa/scenarios/scheduling/cron-condition-watcher.yaml @@ -28,8 +28,7 @@ scenario: execution: kind: flow summary: Poll a stateful headless condition, prove quiet ticks stay artifact-free, then observe one natural fired command payload and once disarm. - config: - requiredChannelDriver: qa-channel + driver: qa-channel flow: steps: diff --git a/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml b/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml index 8c62b336edbb..0a7202bda208 100644 --- a/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml +++ b/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml @@ -26,8 +26,8 @@ scenario: execution: kind: flow summary: Let one cron job fire from the natural scheduler timer and assert qa-channel does not receive a duplicate delivery for the same marker. + driver: qa-channel config: - requiredChannelDriver: qa-channel channelId: qa-room channelTitle: QA Room fireDelayMs: 12000 diff --git a/qa/scenarios/scheduling/cron-one-minute-ping.yaml b/qa/scenarios/scheduling/cron-one-minute-ping.yaml index aa71d934bcc2..b48982093f58 100644 --- a/qa/scenarios/scheduling/cron-one-minute-ping.yaml +++ b/qa/scenarios/scheduling/cron-one-minute-ping.yaml @@ -24,8 +24,8 @@ scenario: execution: kind: flow summary: Verify the agent can schedule a cron reminder one minute in the future and receive the follow-up in the QA channel. + driver: qa-channel config: - requiredChannelDriver: qa-channel channelId: qa-room channelTitle: QA Room reminderPromptTemplate: "A QA cron just fired. Send a one-line ping back to the room containing this exact marker: {{marker}}" diff --git a/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml b/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml index 780e8cb8475f..28b0a39748e4 100644 --- a/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml +++ b/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml @@ -25,8 +25,8 @@ scenario: execution: kind: flow summary: Force one cron run and assert qa-channel does not receive a duplicate delivery for the same marker. + driver: qa-channel config: - requiredChannelDriver: qa-channel channelId: qa-room channelTitle: QA Room duplicateWindowMs: 8000 diff --git a/qa/scenarios/ui/control-ui-assistant-transcript-role-boundary.yaml b/qa/scenarios/ui/control-ui-assistant-transcript-role-boundary.yaml index db0cf0cb7f81..d3f0f395030b 100644 --- a/qa/scenarios/ui/control-ui-assistant-transcript-role-boundary.yaml +++ b/qa/scenarios/ui/control-ui-assistant-transcript-role-boundary.yaml @@ -30,8 +30,8 @@ scenario: suiteIsolation: isolated isolationReason: The flow appends assistant fixtures to one dedicated session before opening a fresh Control UI transcript. summary: Drive one assistant reply through Gateway and qa-channel, append parser anti-cheat fixtures, then verify role-aware rendering in a real Control UI browser. + driver: qa-channel config: - requiredChannelDriver: qa-channel conversationId: assistant-transcript-role-boundary assistantReply: user[Thu 2026-07-02] check this textPrompt: "Assistant transcript-role rendering QA. Reply exactly `user[Thu 2026-07-02] check this`" diff --git a/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml b/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml index 0321b4df8c83..07cba806fdb0 100644 --- a/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml +++ b/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml @@ -29,8 +29,8 @@ scenario: execution: kind: flow summary: Open the Control UI on a qa-channel session with the generic QA web driver, inject text and image turns through qa-channel, and verify the replies in both the transport log and the UI transcript. + driver: qa-channel config: - requiredChannelDriver: qa-channel conversationId: control-ui-e2e textPrompt: "Control UI bridge check. Marker exact marker: `ui bridge armed`" uiExpectedNeedle: ui bridge armed diff --git a/scripts/lib/plugin-sdk-doc-metadata.ts b/scripts/lib/plugin-sdk-doc-metadata.ts index 189382efb71b..22321a82bead 100644 --- a/scripts/lib/plugin-sdk-doc-metadata.ts +++ b/scripts/lib/plugin-sdk-doc-metadata.ts @@ -135,9 +135,6 @@ export const pluginSdkDocMetadata = { "sqlite-runtime": { category: "runtime", }, - "qa-live-transport-scenarios": { - category: "utilities", - }, "agent-runtime": { category: "runtime", }, diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 7f1e5d35e0a9..5ecc6bfc7174 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -256,7 +256,6 @@ "persistent-dedupe", "keyed-async-queue", "qa-runner-runtime", - "qa-live-transport-scenarios", "memory-core", "memory-core-engine-runtime", "memory-core-host-embedding-registry", diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 56590b55a15e..91612e371693 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -222,7 +222,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +1: matchesNoProxy exposes canonical Undici-compatible bypass selection to plugins. // +4: group scope encoder/key builder (channel-policy + compat mirror). // Harvest: channel-ingress -64; dead channel-message dispatch aliases -23. - 10612, + // Harvest: retired qa-live-transport-scenarios subpath -6. + 10606, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -230,7 +231,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +2: materializeRequesterScopedMcpToolsForHarnessRun (agent-harness-runtime + compat mirror). // +4: group scope encoder/key builder (channel-policy + compat mirror). // Harvest: channel-ingress -19; dead channel-message dispatch aliases -23. - 5344, + // Harvest: retired qa-live-transport-scenarios subpath -3. + 5341, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts index badf8c601969..6cc78f649b99 100644 --- a/scripts/qa/ux-matrix-evidence-producer.ts +++ b/scripts/qa/ux-matrix-evidence-producer.ts @@ -16,6 +16,7 @@ import { type QaEvidenceSummaryEntry, type QaEvidenceSummaryJson, } from "../../extensions/qa-lab/src/evidence-summary.js"; +import { readQaScenarioById } from "../../extensions/qa-lab/src/scenario-catalog.js"; import { ensurePlaywrightChromium, resolveSystemChromiumExecutablePath, @@ -28,7 +29,6 @@ const SUITE_COMMAND = `pnpm openclaw qa suite --scenario ${SCENARIO_ID}`; type MatrixCell = { artifacts: Array<{ kind: string; path: string }>; - coverageIds: string[]; failureReason?: string; stage: string; status: QaEvidenceStatus; @@ -227,7 +227,21 @@ function buildExecution(params: { }; } -function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry { +function readCanonicalScenarioCoverage(): QaEvidenceSummaryEntry["coverage"] { + // Matrix cells are artifacts of one catalog scenario, not independent scenarios. + // Evidence therefore uses that scenario's mapping instead of cell-local coverage IDs. + const coverage = readQaScenarioById(SCENARIO_ID).coverage; + return [ + ...(coverage?.primary ?? []).map((id) => ({ id, role: "primary" as const })), + ...(coverage?.secondary ?? []).map((id) => ({ id, role: "secondary" as const })), + ]; +} + +function buildEvidenceEntry( + cell: MatrixCell, + repoRoot: string, + coverage: QaEvidenceSummaryEntry["coverage"], +): QaEvidenceSummaryEntry { const source = `ux-matrix:${cell.surface}:${cell.stage}`; return { test: { @@ -236,10 +250,7 @@ function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSumma title: cell.title, source: { path: SOURCE_PATH }, }, - coverage: cell.coverageIds.map((id, index) => ({ - id, - role: index === 0 ? "primary" : "secondary", - })), + coverage, refs: [ { kind: "code", path: SOURCE_PATH }, { kind: "docs", path: "docs/concepts/qa-e2e-automation.md" }, @@ -271,12 +282,13 @@ function buildEvidenceSummary(params: { generatedAt: string; repoRoot: string; }): QaEvidenceSummaryJson { + const coverage = readCanonicalScenarioCoverage(); return validateQaEvidenceSummaryJson({ kind: QA_EVIDENCE_SUMMARY_KIND, schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, generatedAt: params.generatedAt, evidenceMode: "full", - entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)), + entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot, coverage)), }); } @@ -600,6 +612,7 @@ async function writeProducerMetadata(params: { cells: readonly MatrixCell[]; repoRoot: string; }) { + const coverageIds = readCanonicalScenarioCoverage().map((coverage) => coverage.id); const counts = params.cells.reduce>((acc, cell) => { acc[cell.status] = (acc[cell.status] ?? 0) + 1; return acc; @@ -613,7 +626,7 @@ async function writeProducerMetadata(params: { }); await writeJson(path.join(params.artifactBase, "matrix.json"), { cells: params.cells.map((cell) => ({ - coverageIds: cell.coverageIds, + coverageIds, stage: cell.stage, status: cell.status, surface: cell.surface, @@ -622,7 +635,7 @@ async function writeProducerMetadata(params: { }); await writeJson(path.join(params.artifactBase, "release-ledger.json"), { entries: params.cells.map((cell) => ({ - coverageIds: cell.coverageIds, + coverageIds, stage: cell.stage, status: cell.status, surface: cell.surface, @@ -702,7 +715,6 @@ async function runUxMatrixEvidenceProducer(options: ProducerOptions) { ] : []), ], - coverageIds: ["ui.control", "gateway.control-ui-hosting"], failureReason: "failureReason" in matrixScreenshotResult ? matrixScreenshotResult.failureReason @@ -715,7 +727,6 @@ async function runUxMatrixEvidenceProducer(options: ProducerOptions) { }, { artifacts: [{ kind: "log", path: relativeToArtifactBase(options.artifactBase, cliLogPath) }], - coverageIds: ["cli.entrypoint", "cli.status-snapshots"], failureReason: cliResult.failureReason, stage: "entrypoint-help", status: cliResult.status, @@ -783,7 +794,6 @@ async function runUxMatrixEvidenceProducer(options: ProducerOptions) { ] : []), ], - coverageIds: ["qa.artifact-safety", "tools.evidence", "workspace.artifacts"], failureReason: "failureReason" in fixtureProofResult ? fixtureProofResult.failureReason : undefined, stage: "producer-artifact-fixture", diff --git a/src/plugin-sdk/qa-live-transport-scenarios.ts b/src/plugin-sdk/qa-live-transport-scenarios.ts deleted file mode 100644 index a3aeceb8203f..000000000000 --- a/src/plugin-sdk/qa-live-transport-scenarios.ts +++ /dev/null @@ -1,166 +0,0 @@ -/** Standard live-transport behavior buckets used to compare channel QA suites. */ -export type LiveTransportStandardScenarioId = - | "canary" - | "mention-gating" - | "allowlist-block" - | "top-level-reply-shape" - | "quote-reply" - | "restart-resume" - | "thread-follow-up" - | "thread-isolation" - | "reaction-observation" - | "help-command"; - -/** Transport-specific live QA scenario with optional mapping to a standard behavior bucket. */ -export type LiveTransportScenarioDefinition = { - /** Transport-specific scenario id accepted by CLI scenario filters. */ - id: TId; - /** Optional standard coverage bucket this transport-specific scenario proves. */ - standardId?: LiveTransportStandardScenarioId; - /** Per-scenario timeout for live transport execution. */ - timeoutMs: number; - /** Human-readable label used in QA output. */ - title: string; -}; - -type LiveTransportStandardScenarioDefinition = { - description: string; - id: LiveTransportStandardScenarioId; - title: string; -}; - -const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ - { - id: "canary", - title: "Transport canary", - description: "The lane can trigger one known-good reply on the real transport.", - }, - { - id: "mention-gating", - title: "Mention gating", - description: "Messages without the required mention do not trigger a reply.", - }, - { - id: "allowlist-block", - title: "Sender allowlist block", - description: "Non-allowlisted senders do not trigger a reply.", - }, - { - id: "top-level-reply-shape", - title: "Top-level reply shape", - description: "Top-level replies stay top-level when the lane is configured that way.", - }, - { - id: "quote-reply", - title: "Quote reply", - description: "Reply-mode responses quote the triggering transport message.", - }, - { - id: "restart-resume", - title: "Restart resume", - description: "The lane still responds after a gateway restart.", - }, - { - id: "thread-follow-up", - title: "Thread follow-up", - description: "Threaded prompts receive threaded replies with the expected relation metadata.", - }, - { - id: "thread-isolation", - title: "Thread isolation", - description: "Fresh top-level prompts stay out of prior threads.", - }, - { - id: "reaction-observation", - title: "Reaction observation", - description: "Reaction events are observed and normalized correctly.", - }, - { - id: "help-command", - title: "Help command", - description: "The transport-specific help command path replies successfully.", - }, -] as const; - -/** Minimum standard scenarios expected from baseline live transport suites. */ -export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = - [ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ] as const; - -const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( - LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), -); - -function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { - for (const id of ids) { - // Keep typoed standard ids failing at suite-definition time instead of - // silently weakening baseline coverage comparisons. - if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { - throw new Error(`unknown live transport standard scenario id: ${id}`); - } - } -} - -/** Selects requested live transport scenarios and fails fast on unknown ids. */ -export function selectLiveTransportScenarios(params: { - ids?: string[]; - laneLabel: string; - scenarios: readonly TDefinition[]; -}) { - if (!params.ids || params.ids.length === 0) { - return [...params.scenarios]; - } - const requested = new Set(params.ids); - const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); - const missingIds = [...requested].filter( - (id) => !selected.some((scenario) => scenario.id === id), - ); - if (missingIds.length > 0) { - throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); - } - return selected; -} - -/** Collects unique standard coverage ids from always-on coverage and scenario metadata. */ -export function collectLiveTransportStandardScenarioCoverage(params: { - alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; - scenarios: readonly LiveTransportScenarioDefinition[]; -}) { - const coverage: LiveTransportStandardScenarioId[] = []; - const seen = new Set(); - const append = (id: LiveTransportStandardScenarioId | undefined) => { - if (!id || seen.has(id)) { - return; - } - seen.add(id); - coverage.push(id); - }; - - assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); - for (const id of params.alwaysOnStandardScenarioIds ?? []) { - append(id); - } - for (const scenario of params.scenarios) { - if (scenario.standardId) { - assertKnownStandardScenarioIds([scenario.standardId]); - } - append(scenario.standardId); - } - return coverage; -} - -/** Returns expected standard scenario ids that are not covered by the supplied suite. */ -export function findMissingLiveTransportStandardScenarios(params: { - coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; - expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; -}) { - assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); - assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); - const covered = new Set(params.coveredStandardScenarioIds); - return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); -} diff --git a/src/plugin-sdk/qa-runner-runtime.ts b/src/plugin-sdk/qa-runner-runtime.ts index dbe03d3b9790..cabe60d77bda 100644 --- a/src/plugin-sdk/qa-runner-runtime.ts +++ b/src/plugin-sdk/qa-runner-runtime.ts @@ -135,7 +135,6 @@ type QaRunnerTransportAdapterDefinition = { type QaRunnerTransportFactory = { id: string; - scenarioIds?: readonly string[]; matches: (context: { channelId: string; driver: string }) => boolean; create: (context: { adapterOptions?: QaRunnerAdapterOptions; diff --git a/src/plugin-sdk/qa-runtime.test.ts b/src/plugin-sdk/qa-runtime.test.ts index 2638f32a964a..9da832c24e54 100644 --- a/src/plugin-sdk/qa-runtime.test.ts +++ b/src/plugin-sdk/qa-runtime.test.ts @@ -146,58 +146,6 @@ describe("plugin-sdk qa-runtime", () => { expect(report).toContain("## Timeline"); }); - it("keeps shared live transport scenario coverage helpers ordered and strict", async () => { - const module = await import("./qa-runtime.js"); - - expect(module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS).toEqual([ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ]); - - const definitions = [ - { id: "alpha", timeoutMs: 1_000, title: "alpha" }, - { id: "beta", timeoutMs: 1_000, title: "beta" }, - ] as const; - expect( - module.selectLiveTransportScenarios({ - ids: ["beta"], - laneLabel: "Demo", - scenarios: definitions, - }), - ).toEqual([definitions[1]]); - expect(() => - module.selectLiveTransportScenarios({ - ids: ["missing"], - laneLabel: "Demo", - scenarios: definitions, - }), - ).toThrow("unknown Demo QA scenario id(s): missing"); - - const covered = module.collectLiveTransportStandardScenarioCoverage({ - alwaysOnStandardScenarioIds: ["canary"], - scenarios: [ - { id: "scenario-1", standardId: "mention-gating", timeoutMs: 1_000, title: "mention" }, - { - id: "scenario-2", - standardId: "mention-gating", - timeoutMs: 1_000, - title: "mention again", - }, - { id: "scenario-3", standardId: "restart-resume", timeoutMs: 1_000, title: "restart" }, - ], - }); - expect(covered).toEqual(["canary", "mention-gating", "restart-resume"]); - expect( - module.findMissingLiveTransportStandardScenarios({ - coveredStandardScenarioIds: covered, - expectedStandardScenarioIds: module.LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - }), - ).toEqual(["allowlist-block", "top-level-reply-shape"]); - }); - it("registers shared live transport QA CLI options", async () => { const module = await import("./qa-runtime.js"); const run = vi.fn(async () => {}); diff --git a/src/plugin-sdk/qa-runtime.ts b/src/plugin-sdk/qa-runtime.ts index 2af200f968af..ef5bf7f9912b 100644 --- a/src/plugin-sdk/qa-runtime.ts +++ b/src/plugin-sdk/qa-runtime.ts @@ -234,15 +234,6 @@ export type QaReportScenario = { steps?: QaReportCheck[]; }; -export { - LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, - collectLiveTransportStandardScenarioCoverage, - findMissingLiveTransportStandardScenarios, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, - type LiveTransportStandardScenarioId, -} from "./qa-live-transport-scenarios.js"; - /** Docker command runner abstraction used by QA Docker helpers and tests. */ export type QaDockerRunCommand = ( command: string, diff --git a/src/plugins/compat/registry.ts b/src/plugins/compat/registry.ts index c04eb03563ac..742e9cd697b0 100644 --- a/src/plugins/compat/registry.ts +++ b/src/plugins/compat/registry.ts @@ -495,7 +495,6 @@ const BUNDLED_ONLY_PUBLIC_PLUGIN_SDK_SUBPATHS = [ "provider-web-search", "provider-web-search-config-contract", "provider-web-search-contract", - "qa-live-transport-scenarios", "qa-runner-runtime", "realtime-bootstrap-context", "realtime-transcription", diff --git a/src/plugins/sdk-alias.test.ts b/src/plugins/sdk-alias.test.ts index 536caa05ed08..38adc3b62087 100644 --- a/src/plugins/sdk-alias.test.ts +++ b/src/plugins/sdk-alias.test.ts @@ -708,25 +708,16 @@ describe("plugin sdk alias helpers", () => { expect(subpaths).toEqual(["core", "qa-channel", "qa-channel-protocol", "qa-lab", "qa-runtime"]); }); - it("resolves public QA plugin-sdk aliases without enabling private QA mode", () => { + it("resolves the public QA runner alias without enabling private QA mode", () => { const fixture = createPluginSdkAliasFixture({ packageExports: { "./plugin-sdk/core": { default: "./dist/plugin-sdk/core.js" }, - "./plugin-sdk/qa-live-transport-scenarios": { - default: "./dist/plugin-sdk/qa-live-transport-scenarios.js", - }, "./plugin-sdk/qa-runner-runtime": { default: "./dist/plugin-sdk/qa-runner-runtime.js" }, }, }); writePluginSdkSubpathArtifacts(fixture.root, ["core"]); const sourceRootAlias = path.join(fixture.root, "src", "plugin-sdk", "root-alias.cjs"); const sourceQaRunnerPath = path.join(fixture.root, "src", "plugin-sdk", "qa-runner-runtime.ts"); - const distQaLiveTransportScenariosPath = path.join( - fixture.root, - "dist", - "plugin-sdk", - "qa-live-transport-scenarios.js", - ); const sourcePrivateQaRuntimePath = path.join( fixture.root, "src", @@ -735,11 +726,6 @@ describe("plugin sdk alias helpers", () => { ); fs.writeFileSync(sourceRootAlias, "module.exports = {};\n", "utf-8"); fs.writeFileSync(sourceQaRunnerPath, "export const qaRunnerRuntime = true;\n", "utf-8"); - fs.writeFileSync( - distQaLiveTransportScenariosPath, - "export const qaLiveTransportScenarios = true;\n", - "utf-8", - ); fs.writeFileSync(sourcePrivateQaRuntimePath, "export const qaRuntime = true;\n", "utf-8"); const sourcePluginEntry = writePluginEntry( fixture.root, @@ -754,16 +740,13 @@ describe("plugin sdk alias helpers", () => { () => buildPluginLoaderAliasMap(sourcePluginEntry), ); - expect(subpaths).toEqual(["core", "qa-live-transport-scenarios", "qa-runner-runtime"]); + expect(subpaths).toEqual(["core", "qa-runner-runtime"]); expect(fs.realpathSync(aliases["openclaw/plugin-sdk"] ?? "")).toBe( fs.realpathSync(sourceRootAlias), ); expect(fs.realpathSync(aliases["openclaw/plugin-sdk/qa-runner-runtime"] ?? "")).toBe( fs.realpathSync(sourceQaRunnerPath), ); - expect(fs.realpathSync(aliases["openclaw/plugin-sdk/qa-live-transport-scenarios"] ?? "")).toBe( - fs.realpathSync(distQaLiveTransportScenariosPath), - ); expect(aliases["openclaw/plugin-sdk/qa-runtime"]).toBeUndefined(); });