From b58f9c52580ddc09bafc5b2bd1238e93b4fadd0f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 5 Apr 2026 08:22:09 +0100 Subject: [PATCH] feat: add qa channel foundation --- .github/labeler.yml | 5 + .gitignore | 1 + docs/channels/qa-channel.md | 89 ++ docs/concepts/qa-e2e-automation.md | 834 +++++++++++++++++++ extensions/qa-channel/api.ts | 4 + extensions/qa-channel/index.ts | 15 + extensions/qa-channel/openclaw.plugin.json | 9 + extensions/qa-channel/package.json | 45 + extensions/qa-channel/runtime-api.ts | 1 + extensions/qa-channel/setup-entry.ts | 4 + extensions/qa-channel/src/accounts.ts | 61 ++ extensions/qa-channel/src/bus-client.ts | 224 +++++ extensions/qa-channel/src/channel-actions.ts | 193 +++++ extensions/qa-channel/src/channel.test.ts | 225 +++++ extensions/qa-channel/src/channel.ts | 112 +++ extensions/qa-channel/src/config-schema.ts | 32 + extensions/qa-channel/src/gateway.ts | 55 ++ extensions/qa-channel/src/inbound.ts | 124 +++ extensions/qa-channel/src/outbound.ts | 34 + extensions/qa-channel/src/protocol.ts | 180 ++++ extensions/qa-channel/src/runtime-api.ts | 23 + extensions/qa-channel/src/runtime.ts | 7 + extensions/qa-channel/src/setup.ts | 40 + extensions/qa-channel/src/status.ts | 23 + extensions/qa-channel/src/types.ts | 44 + extensions/qa-channel/test-api.ts | 2 + package.json | 1 + scripts/qa-e2e.ts | 6 + src/qa-e2e/bus-queries.ts | 136 +++ src/qa-e2e/bus-server.ts | 170 ++++ src/qa-e2e/bus-state.test.ts | 65 ++ src/qa-e2e/bus-state.ts | 256 ++++++ src/qa-e2e/bus-waiters.ts | 87 ++ src/qa-e2e/harness-runtime.ts | 75 ++ src/qa-e2e/report.ts | 91 ++ src/qa-e2e/runner.ts | 124 +++ src/qa-e2e/scenario.ts | 65 ++ src/qa-e2e/self-check-scenario.ts | 122 +++ 38 files changed, 3584 insertions(+) create mode 100644 docs/channels/qa-channel.md create mode 100644 docs/concepts/qa-e2e-automation.md create mode 100644 extensions/qa-channel/api.ts create mode 100644 extensions/qa-channel/index.ts create mode 100644 extensions/qa-channel/openclaw.plugin.json create mode 100644 extensions/qa-channel/package.json create mode 100644 extensions/qa-channel/runtime-api.ts create mode 100644 extensions/qa-channel/setup-entry.ts create mode 100644 extensions/qa-channel/src/accounts.ts create mode 100644 extensions/qa-channel/src/bus-client.ts create mode 100644 extensions/qa-channel/src/channel-actions.ts create mode 100644 extensions/qa-channel/src/channel.test.ts create mode 100644 extensions/qa-channel/src/channel.ts create mode 100644 extensions/qa-channel/src/config-schema.ts create mode 100644 extensions/qa-channel/src/gateway.ts create mode 100644 extensions/qa-channel/src/inbound.ts create mode 100644 extensions/qa-channel/src/outbound.ts create mode 100644 extensions/qa-channel/src/protocol.ts create mode 100644 extensions/qa-channel/src/runtime-api.ts create mode 100644 extensions/qa-channel/src/runtime.ts create mode 100644 extensions/qa-channel/src/setup.ts create mode 100644 extensions/qa-channel/src/status.ts create mode 100644 extensions/qa-channel/src/types.ts create mode 100644 extensions/qa-channel/test-api.ts create mode 100644 scripts/qa-e2e.ts create mode 100644 src/qa-e2e/bus-queries.ts create mode 100644 src/qa-e2e/bus-server.ts create mode 100644 src/qa-e2e/bus-state.test.ts create mode 100644 src/qa-e2e/bus-state.ts create mode 100644 src/qa-e2e/bus-waiters.ts create mode 100644 src/qa-e2e/harness-runtime.ts create mode 100644 src/qa-e2e/report.ts create mode 100644 src/qa-e2e/runner.ts create mode 100644 src/qa-e2e/scenario.ts create mode 100644 src/qa-e2e/self-check-scenario.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index 8a254ac93f8..8c39d170ffc 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -64,6 +64,11 @@ - any-glob-to-any-file: - "extensions/qqbot/**" - "docs/channels/qqbot.md" +"channel: qa-channel": + - changed-files: + - any-glob-to-any-file: + - "extensions/qa-channel/**" + - "docs/channels/qa-channel.md" "channel: signal": - changed-files: - any-glob-to-any-file: diff --git a/.gitignore b/.gitignore index 4999fbefaac..ffa92691b18 100644 --- a/.gitignore +++ b/.gitignore @@ -148,3 +148,4 @@ changelog/fragments/ .artifacts/ test/fixtures/openclaw-vitest-unit-report.json analysis/ +.artifacts/qa-e2e/ diff --git a/docs/channels/qa-channel.md b/docs/channels/qa-channel.md new file mode 100644 index 00000000000..3b060c40569 --- /dev/null +++ b/docs/channels/qa-channel.md @@ -0,0 +1,89 @@ +--- +title: "QA Channel" +summary: "Synthetic Slack-class channel plugin for deterministic OpenClaw QA scenarios" +read_when: + - You are wiring the synthetic QA transport into a local or CI test run + - You need the bundled qa-channel config surface + - You are iterating on end-to-end QA automation +--- + +# QA Channel + +`qa-channel` is a bundled synthetic message transport for automated OpenClaw QA. + +It is not a production channel. It exists to exercise the same channel plugin +boundary used by real transports while keeping state deterministic and fully +inspectable. + +## What it does today + +- Slack-class target grammar: + - `dm:` + - `channel:` + - `thread:/` +- HTTP-backed synthetic bus for: + - inbound message injection + - outbound transcript capture + - thread creation + - reactions + - edits + - deletes + - search and read actions +- Bundled host-side self-check runner that writes a Markdown report + +## Config + +```json +{ + "channels": { + "qa-channel": { + "baseUrl": "http://127.0.0.1:43123", + "botUserId": "openclaw", + "botDisplayName": "OpenClaw QA", + "allowFrom": ["*"], + "pollTimeoutMs": 1000 + } + } +} +``` + +Supported account keys: + +- `baseUrl` +- `botUserId` +- `botDisplayName` +- `pollTimeoutMs` +- `allowFrom` +- `defaultTo` +- `actions.messages` +- `actions.reactions` +- `actions.search` +- `actions.threads` + +## Runner + +Current vertical slice: + +```bash +pnpm qa:e2e +``` + +This starts the in-repo QA bus, boots the bundled `qa-channel` runtime slice, +runs a deterministic self-check, and writes a Markdown report under +`.artifacts/qa-e2e/`. + +## Scope + +Current scope is intentionally narrow: + +- bus + plugin transport +- threaded routing grammar +- channel-owned message actions +- Markdown reporting + +Follow-up work will add: + +- Dockerized OpenClaw orchestration +- provider/model matrix execution +- richer scenario discovery +- OpenClaw-native orchestration later diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md new file mode 100644 index 00000000000..ca11227db20 --- /dev/null +++ b/docs/concepts/qa-e2e-automation.md @@ -0,0 +1,834 @@ +--- +title: "QA E2E Automation" +summary: "Design note for a full end-to-end QA system built on a synthetic message-channel plugin, Dockerized OpenClaw, and subagent-driven scenario execution" +read_when: + - You are designing a true end-to-end QA harness for OpenClaw + - You want a synthetic message channel for automated feature verification + - You want subagents to discover features, run scenarios, and propose fixes +--- + +# QA E2E Automation + +This note proposes a true end-to-end QA system for OpenClaw built around a +real channel plugin dedicated to testing. + +The core idea: + +- run OpenClaw inside Docker in a realistic gateway configuration +- expose a synthetic but full-featured message channel as a normal plugin +- let a QA harness inject inbound traffic and inspect outbound state +- let OpenClaw agents and subagents explore, verify, and report on behavior +- optionally escalate failing scenarios into host-side fix workflows that open PRs + +This is not a unit-test replacement. It is a product-level system test layer. + +## Chosen direction + +The initial direction for this project is: + +- build the full system inside this repo +- test against a matrix, not a single model/provider pair +- use Markdown reports as the first output artifact +- defer auto-PR and auto-fix work until later +- treat Slack-class semantics as the MVP transport target +- keep orchestration simple in v1, with a host-side controller that exercises + the moving parts directly +- evolve toward OpenClaw becoming the orchestration layer later, once the + transport, scenario, and reporting model are proven + +## Goals + +- Test OpenClaw through a real messaging-channel boundary, not only `chat.send` + or embedded mocks. +- Verify channel semantics that matter for real use: + - DMs + - channels/groups + - threads + - edits + - deletes + - reactions + - polls + - attachments +- Verify agent behavior across realistic user flows: + - memory + - thread binding + - model switching + - cron jobs + - subagents + - approvals + - routing + - channel-specific `message` actions +- Make the QA runner capable of feature discovery: + - read docs + - inspect plugin capability discovery + - inspect code and config + - generate a scenario protocol +- Support deterministic protocol tests and best-effort real-model tests as + separate lanes. +- Allow automated bug triage artifacts that can feed a host-side fix worker. + +## Non-goals + +- Not a replacement for existing unit, contract, or live tests. +- Not a production channel. +- Not a requirement that all bug fixing happen from inside the Dockerized + OpenClaw runtime. +- Not a reason to add test-only core branches for one channel. + +## Why a channel plugin + +OpenClaw already has the right boundary: + +- core owns the shared `message` tool, prompt wiring, outer session + bookkeeping, and dispatch +- channel plugins own: + - config + - pairing + - security + - session grammar + - threading + - outbound delivery + - channel-owned actions and capability discovery + +That means the cleanest design is: + +- a real channel plugin for QA transport semantics +- a separate QA control plane for injection and inspection + +This keeps the test transport inside the same architecture used by Slack, +Discord, Teams, and similar channels. + +## System overview + +The system has five pieces. + +1. `qa-channel` plugin + +- Bundled extension under `extensions/qa-channel` +- Normal `ChannelPlugin` +- Behaves like a Slack/Discord/Teams-class channel +- Registers channel-owned message actions through the shared `message` tool + +2. `qa-bus` sidecar + +- Small HTTP and/or WS service +- Canonical state store for synthetic conversations, messages, threads, + reactions, edits, and event history +- Accepts inbound events from the harness +- Exposes inspection and wait APIs for assertions + +3. Dockerized OpenClaw gateway + +- Runs as close to real deployment as practical +- Loads `qa-channel` +- Uses normal config, routing, session, cron, and plugin loading + +4. QA orchestrator + +- Host-side runner or dedicated OpenClaw-driven controller +- Provisions scenario environments +- Seeds config +- Resets state +- Executes test matrix +- Collects structured outcomes + +5. Auto-fix worker + +- Host-side workflow +- Creates a worktree +- launches a coding agent +- runs scoped verification +- opens a PR + +The auto-fix worker should start outside the container. It needs direct repo +and GitHub access, clean worktree control, and better isolation from the +runtime under test. + +## High-level flow + +1. Start `qa-bus`. +2. Start OpenClaw in Docker with `qa-channel` enabled. +3. QA orchestrator injects inbound messages into `qa-bus`. +4. `qa-channel` receives them as normal inbound traffic. +5. OpenClaw runs the agent loop normally. +6. Outbound replies and channel actions flow back through `qa-channel` into + `qa-bus`. +7. QA orchestrator inspects state or waits on events. +8. Orchestrator records pass/fail/flaky/unknown plus artifacts. +9. Severe failures optionally emit a bug packet for the host-side fix worker. + +## Lanes + +The system should have two distinct lanes. + +### Lane A: deterministic protocol lane + +Use a deterministic or tightly controlled model setup. + +Preferred options: + +- a canned provider fixture +- the bundled `synthetic` provider when useful +- fixed prompts with exact assertions + +Purpose: + +- verify transport and product semantics +- keep flakiness low +- catch regressions in routing, memory plumbing, thread binding, cron, and tool + invocation + +### Lane B: quality lane + +Use real providers and real models in a matrix. + +Purpose: + +- verify that the agent can still do good work end to end +- evaluate feature discoverability and instruction following +- surface model-specific breakage or degraded behavior + +Expected result type: + +- best-effort +- rubric-based +- more tolerant of wording variation + +Matrix guidance for v1: + +- start with a small curated matrix, not "everything configured" +- keep deterministic protocol runs separate from quality runs +- report matrix cells independently so one provider/model failure does not hide + transport correctness + +Do not mix these lanes. Protocol correctness and model quality should fail +independently. + +## Use existing bootstrap seam first + +Before the custom channel exists, OpenClaw already has a useful bootstrap path: + +- admin-scoped synthetic originating-route fields on `chat.send` +- synthetic message-channel headers for HTTP flows + +That is enough to build a first QA controller for: + +- thread/session routing +- ACP bind flows +- subagent delivery +- cron wake paths +- memory persistence checks + +This should be Phase 0 because it de-risks the scenario protocol before the +full channel lands. + +## `qa-channel` plugin design + +## Package layout + +Suggested package: + +- `extensions/qa-channel/` + +Suggested file layout: + +- `package.json` +- `openclaw.plugin.json` +- `index.ts` +- `setup-entry.ts` +- `api.ts` +- `runtime-api.ts` +- `src/channel.ts` +- `src/channel-api.ts` +- `src/config-schema.ts` +- `src/setup-core.ts` +- `src/setup-surface.ts` +- `src/runtime.ts` +- `src/channel.runtime.ts` +- `src/inbound.ts` +- `src/outbound.ts` +- `src/state-client.ts` +- `src/targets.ts` +- `src/threading.ts` +- `src/message-actions.ts` +- `src/probe.ts` +- `src/doctor.ts` +- `src/*.test.ts` + +Model it after Slack, Discord, Teams, or Google Chat packaging, not as a one-off +test helper. + +## Capabilities + +MVP capabilities: + +- one account +- DMs +- channels +- threads +- send text +- reply in thread +- read +- edit +- delete +- react +- search +- upload-file +- download-file + +Phase 2 capabilities: + +- polls +- member-info +- channel-info +- channel-list +- pin and unpin +- permissions +- topic create and edit + +These map naturally onto the shared `message` tool action model already used by +channel plugins. + +## Conversation model + +Use a stable synthetic grammar that supports both simplicity and realistic +coverage. + +Suggested ids: + +- DM conversation: `dm:` +- channel: `chan:` +- thread: `thread::` +- message id: `msg:` + +Suggested target forms: + +- `qa:dm:` +- `qa:chan:` +- `qa:thread::` + +The plugin should own translation between external target strings and canonical +conversation ids. + +## Pairing and security + +Even though this is a QA channel, it should still implement real policy +surfaces: + +- DM allowlist / pairing flow +- group policy +- mention gating where relevant +- trusted sender ids + +Reason: + +- these are product features and should be testable through the QA transport +- the QA lane should be able to verify policy failures, not only happy paths + +## Threading model + +Threading is one of the main reasons to build this channel. + +Required semantics: + +- create thread from a top-level message +- reply inside an existing thread +- list thread messages +- preserve parent message linkage +- let OpenClaw thread binding attach a session to a thread + +The QA bus must preserve: + +- conversation id +- thread id +- parent message id +- sender id +- timestamps + +## Channel-owned message actions + +The plugin should implement `actions.describeMessageTool(...)` and +`actions.handleAction(...)`. + +MVP action list: + +- `send` +- `read` +- `reply` +- `react` +- `edit` +- `delete` +- `thread-create` +- `thread-reply` +- `search` +- `upload-file` +- `download-file` + +This is enough to test the shared `message` tool end to end with real channel +semantics. + +## `qa-bus` design + +`qa-bus` is the transport simulator and assertion backend. + +It should not know OpenClaw internals. It should know channel state. + +For v1, keep `qa-bus` in this repo so: + +- fixtures and scenarios evolve with product code +- the transport contract can change in lock-step with the plugin +- CI and local dev do not need another repo checkout + +## Responsibilities + +- accept inbound user/platform events +- persist canonical conversation state +- persist append-only event log +- expose inspection APIs +- expose blocking wait APIs +- support reset per scenario or per suite + +## Transport + +HTTP is enough for MVP. + +Suggested endpoints: + +- `POST /reset` +- `POST /inbound/message` +- `POST /inbound/edit` +- `POST /inbound/delete` +- `POST /inbound/reaction` +- `POST /inbound/thread/create` +- `GET /state/conversations` +- `GET /state/messages` +- `GET /state/threads` +- `GET /events` +- `POST /wait` + +Optional WS stream: + +- `/stream` + +Useful for live event taps and debugging. + +## State model + +Persist three layers. + +1. Conversation snapshot + +- participants +- type +- thread topology +- latest message pointers + +2. Message snapshot + +- sender +- content +- attachments +- edit history +- reactions +- parent and thread linkage + +3. Append-only event log + +- canonical timestamp +- causal ordering +- source: inbound, outbound, action, system +- payload + +The append-only log matters because many QA assertions are event-oriented, not +just state-oriented. + +## Assertion API + +The harness needs waiters, not just snapshots. + +Suggested `POST /wait` contract: + +- `kind` +- `match` +- `timeoutMs` + +Examples: + +- wait for outbound message matching text regex +- wait for thread creation +- wait for reaction added +- wait for message edit +- wait for no event of type X within Y ms + +This gives stable tests without custom polling code in every scenario. + +## QA orchestrator design + +The orchestrator should own scenario planning and artifact collection. + +Start host-side. Later, OpenClaw can orchestrate parts of it. + +This is the chosen v1 direction. + +Why: + +- simpler to iterate while the transport and scenario protocol are still moving +- easier access to the repo, logs, Docker, and test fixtures +- easier artifact collection and report generation +- avoids over-coupling the first version to subagent behavior before the QA + protocol itself is stable + +## Inputs + +- docs pages +- channel capability discovery +- configured provider/model lane +- scenario catalog +- repo/test metadata + +## Outputs + +- structured protocol report +- scenario transcript +- captured channel state +- gateway logs +- failure packets + +For v1, the primary output is a Markdown report. + +Suggested report sections: + +- suite summary +- environment +- provider/model matrix +- scenarios passed +- scenarios failed +- flaky or inconclusive scenarios +- captured evidence links or inline excerpts +- suspected ownership or file hints +- follow-up recommendations + +## Scenario format + +Use a data-driven scenario spec. + +Suggested shape: + +```json +{ + "id": "thread-memory-recall", + "lane": "deterministic", + "preconditions": ["qa-channel", "memory-enabled"], + "steps": [ + { + "type": "injectMessage", + "to": "qa:dm:user-a", + "text": "Remember that the deploy key is kiwi." + }, + { "type": "waitForOutbound", "match": { "textIncludes": "kiwi" } }, + { "type": "injectMessage", "to": "qa:dm:user-a", "text": "What was the deploy key?" }, + { "type": "waitForOutbound", "match": { "textIncludes": "kiwi" } } + ], + "assertions": [{ "type": "outboundTextIncludes", "value": "kiwi" }] +} +``` + +Keep the execution engine generic and the scenario catalog declarative. + +## Feature discovery + +The orchestrator can discover candidate scenarios from three sources. + +1. Docs + +- channel docs +- testing docs +- gateway docs +- subagents docs +- cron docs + +2. Runtime capability discovery + +- channel `message` action discovery +- plugin status and channel capabilities +- configured providers/models + +3. Code hints + +- known action names +- channel-specific feature flags +- config schema + +This should produce a proposed protocol with: + +- must-test +- can-test +- blocked +- unsupported + +## Scenario classes + +Recommended catalog: + +- transport basics + - DM send and reply + - channel send + - thread create and reply + - reaction add and read + - edit and delete +- policy + - allowlist + - pairing + - group mention gating +- shared `message` tool + - read + - search + - reply + - react + - upload and download +- agent quality + - follows channel context + - obeys thread semantics + - uses memory across turns + - switches model when instructed +- automation + - cron add and run + - cron delivery into channel + - scheduled reminders +- subagents + - spawn + - announce + - threaded follow-up + - nested orchestration when enabled +- failure handling + - unsupported action + - timeout + - malformed target + - policy denial + +## OpenClaw as orchestrator + +Longer-term, OpenClaw itself can coordinate the QA run. + +Suggested architecture: + +- one controller session +- N worker subagents +- each worker owns one scenario or scenario shard +- workers report structured results back to controller + +Good fits for existing OpenClaw primitives: + +- `sessions_spawn` +- `subagents` +- cron-based wakeups for long-running suites +- thread-bound sessions for scenario-local follow-up + +Best near-term use: + +- controller generates the plan +- workers execute scenarios in parallel +- controller synthesizes report + +Avoid making the controller also own host Git operations in the first version. + +Chosen direction: + +- v1: host-side controller +- v2+: OpenClaw-native orchestration once the scenario protocol and transport + model are stable + +## Auto-fix workflow + +The system should emit a structured bug packet when a scenario fails. + +Suggested bug packet: + +- scenario id +- lane +- failure kind +- minimal repro steps +- channel event transcript +- gateway transcript +- logs +- suspected files +- confidence + +Host-side fix worker flow: + +1. receive bug packet +2. create detached worktree +3. launch coding agent in worktree +4. write failing regression first when practical +5. implement fix +6. run scoped verification +7. open PR + +This should remain host-side at first because it needs: + +- repo write access +- worktree hygiene +- git credentials +- GitHub auth + +Chosen direction: + +- do not auto-open PRs in v1 +- emit Markdown reports and structured failure packets first +- add host-side worktree + PR automation later + +## Rollout plan + +## Phase 0: bootstrap on existing synthetic ingress + +Build a first QA runner without a new channel: + +- use `chat.send` with admin-scoped synthetic originating-route fields +- run deterministic scenarios against routing, memory, cron, subagents, and ACP +- validate protocol format and artifact collection + +Exit criteria: + +- scenario runner exists +- structured protocol report exists +- failure artifacts exist + +## Phase 1: MVP `qa-channel` + +Build the plugin and bus with: + +- DM +- channels +- threads +- read +- reply +- react +- edit +- delete +- search + +Target semantics: + +- Slack-class transport behavior +- not full Teams-class parity yet + +Exit criteria: + +- OpenClaw in Docker can talk to `qa-bus` +- harness can inject + inspect +- one green end-to-end suite across message transport and agent behavior + +## Phase 2: protocol expansion + +Add: + +- attachments +- polls +- pins +- richer policy tests +- quality lane with real provider/model matrix + +Exit criteria: + +- scenario matrix covers major built-in features +- deterministic and quality lanes are separated + +## Phase 3: subagent-driven QA + +Add: + +- controller agent +- worker subagents +- scenario discovery from docs + capability discovery +- parallel execution + +Exit criteria: + +- one controller can fan out and synthesize a suite report + +## Phase 4: auto-fix loop + +Add: + +- bug packet emission +- host-side worktree runner +- PR creation + +Exit criteria: + +- selected failures can auto-produce draft PRs + +## Risks + +## Risk: too much magic in one layer + +If the QA channel, bus, and orchestrator all become smart at once, debugging +will be painful. + +Mitigation: + +- keep `qa-channel` transport-focused +- keep `qa-bus` state-focused +- keep orchestrator separate + +## Risk: flaky assertions from model variance + +Mitigation: + +- deterministic lane +- quality lane +- different pass criteria + +## Risk: test-only branches leaking into core + +Mitigation: + +- no core special cases for `qa-channel` +- use normal plugin seams +- use admin synthetic ingress only as bootstrap + +## Risk: auto-fix overreach + +Mitigation: + +- keep fix worker host-side +- require explicit policy for when PRs can open automatically +- gate with scoped tests + +## Risk: building a fake platform nobody uses + +Mitigation: + +- emulate Slack/Discord/Teams semantics, not an abstract transport +- prioritize features that stress shared OpenClaw boundaries + +## MVP recommendation + +If building this now, start with this exact order. + +1. Host-side scenario runner using existing synthetic originating-route support. +2. `qa-bus` sidecar with state, events, reset, and wait APIs. +3. `extensions/qa-channel` MVP with DMs, channels, threads, reply, read, react, + edit, delete, and search. +4. Markdown report generator for suite + matrix output. +5. One deterministic end-to-end suite: + - inject inbound DM + - verify reply + - create thread + - verify follow-up in thread + - verify memory recall on later turn +6. Add curated real-model matrix quality lane. +7. Add controller subagent orchestration. +8. Add host-side auto-fix worktree runner. + +This order gets real value quickly without requiring the full grand design to +land before the first useful signal appears. + +## Current product decisions + +- `qa-bus` lives inside this repo +- the first controller is host-side +- Slack-class behavior is the MVP target +- the quality lane uses a curated matrix +- first version produces Markdown reports, not PRs +- OpenClaw-native orchestration is a later phase, not a v1 requirement diff --git a/extensions/qa-channel/api.ts b/extensions/qa-channel/api.ts new file mode 100644 index 00000000000..d3f235c7226 --- /dev/null +++ b/extensions/qa-channel/api.ts @@ -0,0 +1,4 @@ +export * from "./src/accounts.js"; +export * from "./src/channel.js"; +export * from "./src/channel-actions.js"; +export * from "./src/runtime.js"; diff --git a/extensions/qa-channel/index.ts b/extensions/qa-channel/index.ts new file mode 100644 index 00000000000..f444df63c19 --- /dev/null +++ b/extensions/qa-channel/index.ts @@ -0,0 +1,15 @@ +import type { ChannelPlugin } from "openclaw/plugin-sdk/core"; +import { defineChannelPluginEntry } from "openclaw/plugin-sdk/core"; +import { qaChannelPlugin } from "./src/channel.js"; +import { setQaChannelRuntime } from "./src/runtime.js"; + +export { qaChannelPlugin } from "./src/channel.js"; +export { setQaChannelRuntime } from "./src/runtime.js"; + +export default defineChannelPluginEntry({ + id: "qa-channel", + name: "QA Channel", + description: "Synthetic QA channel plugin", + plugin: qaChannelPlugin as ChannelPlugin, + setRuntime: setQaChannelRuntime, +}); diff --git a/extensions/qa-channel/openclaw.plugin.json b/extensions/qa-channel/openclaw.plugin.json new file mode 100644 index 00000000000..4fb9b26ba8d --- /dev/null +++ b/extensions/qa-channel/openclaw.plugin.json @@ -0,0 +1,9 @@ +{ + "id": "qa-channel", + "channels": ["qa-channel"], + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/qa-channel/package.json b/extensions/qa-channel/package.json new file mode 100644 index 00000000000..166e7fa6ee5 --- /dev/null +++ b/extensions/qa-channel/package.json @@ -0,0 +1,45 @@ +{ + "name": "@openclaw/qa-channel", + "version": "2026.4.4", + "private": true, + "description": "OpenClaw QA synthetic channel plugin", + "type": "module", + "devDependencies": { + "openclaw": "workspace:*" + }, + "peerDependencies": { + "openclaw": ">=2026.4.4" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ], + "setupEntry": "./setup-entry.ts", + "channel": { + "id": "qa-channel", + "label": "QA Channel", + "selectionLabel": "QA Channel (Synthetic)", + "detailLabel": "QA Channel", + "docsPath": "/channels/qa-channel", + "docsLabel": "qa-channel", + "blurb": "Synthetic Slack-class transport for automated OpenClaw QA scenarios.", + "systemImage": "checklist", + "order": 999, + "exposure": { + "configured": false, + "setup": false, + "docs": false + } + }, + "install": { + "npmSpec": "@openclaw/qa-channel", + "defaultChoice": "npm", + "minHostVersion": ">=2026.4.4" + } + } +} diff --git a/extensions/qa-channel/runtime-api.ts b/extensions/qa-channel/runtime-api.ts new file mode 100644 index 00000000000..801051438fb --- /dev/null +++ b/extensions/qa-channel/runtime-api.ts @@ -0,0 +1 @@ +export * from "./src/runtime-api.js"; diff --git a/extensions/qa-channel/setup-entry.ts b/extensions/qa-channel/setup-entry.ts new file mode 100644 index 00000000000..4166d48abbe --- /dev/null +++ b/extensions/qa-channel/setup-entry.ts @@ -0,0 +1,4 @@ +import { defineSetupPluginEntry } from "openclaw/plugin-sdk/core"; +import { qaChannelPlugin } from "./src/channel.js"; + +export default defineSetupPluginEntry(qaChannelPlugin); diff --git a/extensions/qa-channel/src/accounts.ts b/extensions/qa-channel/src/accounts.ts new file mode 100644 index 00000000000..c310a5ed922 --- /dev/null +++ b/extensions/qa-channel/src/accounts.ts @@ -0,0 +1,61 @@ +import { createAccountListHelpers } from "openclaw/plugin-sdk/account-helpers"; +import { DEFAULT_ACCOUNT_ID, normalizeAccountId } from "openclaw/plugin-sdk/account-id"; +import { resolveMergedAccountConfig } from "openclaw/plugin-sdk/account-resolution"; +import type { CoreConfig, QaChannelAccountConfig, ResolvedQaChannelAccount } from "./types.js"; + +const DEFAULT_POLL_TIMEOUT_MS = 1_000; + +const { + listAccountIds: listQaChannelAccountIds, + resolveDefaultAccountId: resolveDefaultQaChannelAccountId, +} = createAccountListHelpers("qa-channel", { normalizeAccountId }); + +export { listQaChannelAccountIds, resolveDefaultQaChannelAccountId }; + +function resolveMergedQaAccountConfig(cfg: CoreConfig, accountId: string): QaChannelAccountConfig { + return resolveMergedAccountConfig({ + channelConfig: cfg.channels?.["qa-channel"] as QaChannelAccountConfig | undefined, + accounts: cfg.channels?.["qa-channel"]?.accounts as + | Record> + | undefined, + accountId, + omitKeys: ["defaultAccount"], + normalizeAccountId, + }); +} + +export function resolveQaChannelAccount(params: { + cfg: CoreConfig; + accountId?: string | null; +}): ResolvedQaChannelAccount { + const accountId = normalizeAccountId(params.accountId); + const merged = resolveMergedQaAccountConfig(params.cfg, accountId); + const baseEnabled = params.cfg.channels?.["qa-channel"]?.enabled !== false; + const enabled = baseEnabled && merged.enabled !== false; + const baseUrl = merged.baseUrl?.trim() ?? ""; + const botUserId = merged.botUserId?.trim() || "openclaw"; + const botDisplayName = merged.botDisplayName?.trim() || "OpenClaw QA"; + return { + accountId, + enabled, + configured: Boolean(baseUrl), + name: merged.name?.trim() || undefined, + baseUrl, + botUserId, + botDisplayName, + pollTimeoutMs: merged.pollTimeoutMs ?? DEFAULT_POLL_TIMEOUT_MS, + config: { + ...merged, + allowFrom: merged.allowFrom ?? ["*"], + }, + }; +} + +export function listEnabledQaChannelAccounts(cfg: CoreConfig): ResolvedQaChannelAccount[] { + return listQaChannelAccountIds(cfg) + .map((accountId) => resolveQaChannelAccount({ cfg, accountId })) + .filter((account) => account.enabled); +} + +export { DEFAULT_ACCOUNT_ID }; +export type { ResolvedQaChannelAccount } from "./types.js"; diff --git a/extensions/qa-channel/src/bus-client.ts b/extensions/qa-channel/src/bus-client.ts new file mode 100644 index 00000000000..ca109cee65a --- /dev/null +++ b/extensions/qa-channel/src/bus-client.ts @@ -0,0 +1,224 @@ +import type { + QaBusConversation, + QaBusEvent, + QaBusInboundMessageInput, + QaBusMessage, + QaBusPollResult, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, +} from "./protocol.js"; + +export type { + QaBusConversation, + QaBusConversationKind, + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusEvent, + QaBusInboundMessageInput, + QaBusMessage, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusPollResult, + QaBusReactToMessageInput, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, + QaBusWaitForInput, +} from "./protocol.js"; + +type JsonResult = Promise; + +async function postJson( + baseUrl: string, + path: string, + body: unknown, + signal?: AbortSignal, +): JsonResult { + const response = await fetch(`${baseUrl}${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify(body), + signal, + }); + const payload = (await response.json()) as T | { error?: string }; + if (!response.ok) { + const error = + typeof payload === "object" && payload && "error" in payload ? payload.error : undefined; + throw new Error(error || `qa-bus request failed: ${response.status}`); + } + return payload as T; +} + +export function normalizeQaTarget(raw: string): string | undefined { + const trimmed = raw.trim(); + if (!trimmed) { + return undefined; + } + return trimmed; +} + +export function parseQaTarget(raw: string): { + chatType: "direct" | "channel"; + conversationId: string; + threadId?: string; +} { + const normalized = normalizeQaTarget(raw); + if (!normalized) { + throw new Error("qa-channel target is required"); + } + if (normalized.startsWith("thread:")) { + const rest = normalized.slice("thread:".length); + const slashIndex = rest.indexOf("/"); + if (slashIndex <= 0 || slashIndex === rest.length - 1) { + throw new Error(`invalid qa-channel thread target: ${normalized}`); + } + return { + chatType: "channel", + conversationId: rest.slice(0, slashIndex), + threadId: rest.slice(slashIndex + 1), + }; + } + if (normalized.startsWith("channel:")) { + return { + chatType: "channel", + conversationId: normalized.slice("channel:".length), + }; + } + if (normalized.startsWith("dm:")) { + return { + chatType: "direct", + conversationId: normalized.slice("dm:".length), + }; + } + return { + chatType: "direct", + conversationId: normalized, + }; +} + +export function buildQaTarget(params: { + chatType: "direct" | "channel"; + conversationId: string; + threadId?: string | null; +}) { + if (params.threadId) { + return `thread:${params.conversationId}/${params.threadId}`; + } + return `${params.chatType === "direct" ? "dm" : "channel"}:${params.conversationId}`; +} + +export async function pollQaBus(params: { + baseUrl: string; + accountId: string; + cursor: number; + timeoutMs: number; + signal?: AbortSignal; +}): Promise { + return await postJson( + params.baseUrl, + "/v1/poll", + { + accountId: params.accountId, + cursor: params.cursor, + timeoutMs: params.timeoutMs, + }, + params.signal, + ); +} + +export async function sendQaBusMessage(params: { + baseUrl: string; + accountId: string; + to: string; + text: string; + senderId?: string; + senderName?: string; + threadId?: string; + replyToId?: string; +}) { + return await postJson<{ message: QaBusMessage }>(params.baseUrl, "/v1/outbound/message", params); +} + +export async function createQaBusThread(params: { + baseUrl: string; + accountId: string; + conversationId: string; + title: string; + createdBy?: string; +}) { + return await postJson<{ thread: QaBusThread }>( + params.baseUrl, + "/v1/actions/thread-create", + params, + ); +} + +export async function reactToQaBusMessage(params: { + baseUrl: string; + accountId: string; + messageId: string; + emoji: string; + senderId?: string; +}) { + return await postJson<{ message: QaBusMessage }>(params.baseUrl, "/v1/actions/react", params); +} + +export async function editQaBusMessage(params: { + baseUrl: string; + accountId: string; + messageId: string; + text: string; +}) { + return await postJson<{ message: QaBusMessage }>(params.baseUrl, "/v1/actions/edit", params); +} + +export async function deleteQaBusMessage(params: { + baseUrl: string; + accountId: string; + messageId: string; +}) { + return await postJson<{ message: QaBusMessage }>(params.baseUrl, "/v1/actions/delete", params); +} + +export async function readQaBusMessage(params: { + baseUrl: string; + accountId: string; + messageId: string; +}) { + return await postJson<{ message: QaBusMessage }>(params.baseUrl, "/v1/actions/read", params); +} + +export async function searchQaBusMessages(params: { + baseUrl: string; + input: QaBusSearchMessagesInput; +}) { + return await postJson<{ messages: QaBusMessage[] }>( + params.baseUrl, + "/v1/actions/search", + params.input, + ); +} + +export async function injectQaBusInboundMessage(params: { + baseUrl: string; + input: QaBusInboundMessageInput; +}) { + return await postJson<{ message: QaBusMessage }>( + params.baseUrl, + "/v1/inbound/message", + params.input, + ); +} + +export async function getQaBusState(baseUrl: string): Promise { + const response = await fetch(`${baseUrl}/v1/state`); + if (!response.ok) { + throw new Error(`qa-bus request failed: ${response.status}`); + } + return (await response.json()) as QaBusStateSnapshot; +} diff --git a/extensions/qa-channel/src/channel-actions.ts b/extensions/qa-channel/src/channel-actions.ts new file mode 100644 index 00000000000..6af09a329d5 --- /dev/null +++ b/extensions/qa-channel/src/channel-actions.ts @@ -0,0 +1,193 @@ +import { Type } from "@sinclair/typebox"; +import { jsonResult, readStringParam } from "openclaw/plugin-sdk/core"; +import { resolveQaChannelAccount } from "./accounts.js"; +import { + createQaBusThread, + deleteQaBusMessage, + editQaBusMessage, + parseQaTarget, + reactToQaBusMessage, + readQaBusMessage, + searchQaBusMessages, + sendQaBusMessage, +} from "./bus-client.js"; +import type { ChannelMessageActionAdapter, ChannelMessageActionName } from "./runtime-api.js"; +import type { CoreConfig } from "./types.js"; + +function listQaChannelActions( + cfg: CoreConfig, + accountId?: string | null, +): ChannelMessageActionName[] { + const account = resolveQaChannelAccount({ cfg, accountId }); + if (!account.enabled || !account.configured) { + return []; + } + const actions = new Set(["send"]); + if (account.config.actions?.messages !== false) { + actions.add("read"); + actions.add("edit"); + actions.add("delete"); + } + if (account.config.actions?.reactions !== false) { + actions.add("react"); + actions.add("reactions"); + } + if (account.config.actions?.threads !== false) { + actions.add("thread-create"); + actions.add("thread-reply"); + } + if (account.config.actions?.search !== false) { + actions.add("search"); + } + return Array.from(actions); +} + +export const qaChannelMessageActions: ChannelMessageActionAdapter = { + describeMessageTool: (context) => ({ + actions: listQaChannelActions(context.cfg as CoreConfig, context.accountId), + capabilities: [], + schema: { + properties: { + channelId: Type.Optional(Type.String()), + threadId: Type.Optional(Type.String()), + messageId: Type.Optional(Type.String()), + emoji: Type.Optional(Type.String()), + title: Type.Optional(Type.String()), + query: Type.Optional(Type.String()), + }, + }, + }), + extractToolSend: ({ args }: { args: Record }) => { + const action = typeof args.action === "string" ? args.action.trim() : ""; + if (action === "sendMessage") { + const to = typeof args.to === "string" ? args.to : undefined; + return to ? { to } : null; + } + if (action === "threadReply") { + const channelId = typeof args.channelId === "string" ? args.channelId.trim() : ""; + const threadId = typeof args.threadId === "string" ? args.threadId.trim() : ""; + return channelId && threadId ? { to: `thread:${channelId}/${threadId}` } : null; + } + return null; + }, + handleAction: async (context) => { + const { action, cfg, accountId, params } = context; + const account = resolveQaChannelAccount({ cfg: cfg as CoreConfig, accountId }); + const baseUrl = account.baseUrl; + + switch (action) { + case "thread-create": { + const channelId = + readStringParam(params, "channelId") ?? + (() => { + const to = readStringParam(params, "to"); + return to ? parseQaTarget(to).conversationId : undefined; + })(); + const title = readStringParam(params, "title") ?? "QA thread"; + if (!channelId) { + throw new Error("qa-channel thread-create requires channelId"); + } + const { thread } = await createQaBusThread({ + baseUrl, + accountId: account.accountId, + conversationId: channelId, + title, + createdBy: account.botUserId, + }); + return jsonResult({ + thread, + target: `thread:${channelId}/${thread.id}`, + }); + } + case "thread-reply": { + const channelId = readStringParam(params, "channelId"); + const threadId = readStringParam(params, "threadId"); + const text = readStringParam(params, "text"); + if (!channelId || !threadId || !text) { + throw new Error("qa-channel thread-reply requires channelId, threadId, and text"); + } + const { message } = await sendQaBusMessage({ + baseUrl, + accountId: account.accountId, + to: `thread:${channelId}/${threadId}`, + text, + senderId: account.botUserId, + senderName: account.botDisplayName, + threadId, + }); + return jsonResult({ message }); + } + case "react": { + const messageId = readStringParam(params, "messageId"); + const emoji = readStringParam(params, "emoji"); + if (!messageId || !emoji) { + throw new Error("qa-channel react requires messageId and emoji"); + } + const { message } = await reactToQaBusMessage({ + baseUrl, + accountId: account.accountId, + messageId, + emoji, + senderId: account.botUserId, + }); + return jsonResult({ message }); + } + case "reactions": + case "read": { + const messageId = readStringParam(params, "messageId"); + if (!messageId) { + throw new Error(`qa-channel ${action} requires messageId`); + } + const { message } = await readQaBusMessage({ + baseUrl, + accountId: account.accountId, + messageId, + }); + return jsonResult({ message }); + } + case "edit": { + const messageId = readStringParam(params, "messageId"); + const text = readStringParam(params, "text"); + if (!messageId || !text) { + throw new Error("qa-channel edit requires messageId and text"); + } + const { message } = await editQaBusMessage({ + baseUrl, + accountId: account.accountId, + messageId, + text, + }); + return jsonResult({ message }); + } + case "delete": { + const messageId = readStringParam(params, "messageId"); + if (!messageId) { + throw new Error("qa-channel delete requires messageId"); + } + const { message } = await deleteQaBusMessage({ + baseUrl, + accountId: account.accountId, + messageId, + }); + return jsonResult({ message }); + } + case "search": { + const query = readStringParam(params, "query"); + const channelId = readStringParam(params, "channelId"); + const threadId = readStringParam(params, "threadId"); + const { messages } = await searchQaBusMessages({ + baseUrl, + input: { + accountId: account.accountId, + query, + conversationId: channelId, + threadId, + }, + }); + return jsonResult({ messages }); + } + default: + throw new Error(`qa-channel action not implemented: ${action}`); + } + }, +}; diff --git a/extensions/qa-channel/src/channel.test.ts b/extensions/qa-channel/src/channel.test.ts new file mode 100644 index 00000000000..ebf08517544 --- /dev/null +++ b/extensions/qa-channel/src/channel.test.ts @@ -0,0 +1,225 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; +import { describe, expect, it } from "vitest"; +import { extractToolPayload } from "../../../src/infra/outbound/tool-payload.js"; +import { startQaBusServer } from "../../../src/qa-e2e/bus-server.js"; +import { createQaBusState } from "../../../src/qa-e2e/bus-state.js"; +import { createStartAccountContext } from "../../../test/helpers/plugins/start-account-context.js"; +import { qaChannelPlugin } from "../api.js"; +import { setQaChannelRuntime } from "../api.js"; + +function createMockQaRuntime(): PluginRuntime { + const sessionUpdatedAt = new Map(); + return { + channel: { + routing: { + resolveAgentRoute({ + accountId, + peer, + }: { + accountId?: string | null; + peer?: { kind?: string; id?: string } | null; + }) { + return { + agentId: "qa-agent", + channel: "qa-channel", + accountId: accountId ?? "default", + sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`, + mainSessionKey: "qa-agent:main", + lastRoutePolicy: "session", + matchedBy: "default", + }; + }, + }, + session: { + resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) { + return agentId; + }, + readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) { + return sessionUpdatedAt.get(sessionKey); + }, + recordInboundSession({ sessionKey }: { sessionKey: string }) { + sessionUpdatedAt.set(sessionKey, Date.now()); + }, + }, + reply: { + resolveEnvelopeFormatOptions() { + return {}; + }, + formatAgentEnvelope({ body }: { body: string }) { + return body; + }, + finalizeInboundContext(ctx: Record) { + return ctx as typeof ctx & { CommandAuthorized: boolean }; + }, + async dispatchReplyWithBufferedBlockDispatcher({ + ctx, + dispatcherOptions, + }: { + ctx: { BodyForAgent?: string; Body?: string }; + dispatcherOptions: { deliver: (payload: { text: string }) => Promise }; + }) { + await dispatcherOptions.deliver({ + text: `qa-echo: ${String(ctx.BodyForAgent ?? ctx.Body ?? "")}`, + }); + }, + }, + }, + } as unknown as PluginRuntime; +} + +describe("qa-channel plugin", () => { + it("roundtrips inbound DM traffic through the qa bus", async () => { + const state = createQaBusState(); + const bus = await startQaBusServer({ state }); + setQaChannelRuntime(createMockQaRuntime()); + + const cfg = { + channels: { + "qa-channel": { + baseUrl: bus.baseUrl, + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + allowFrom: ["*"], + }, + }, + }; + const account = qaChannelPlugin.config.resolveAccount(cfg, "default"); + const abort = new AbortController(); + const startAccount = qaChannelPlugin.gateway?.startAccount; + expect(startAccount).toBeDefined(); + const task = startAccount!( + createStartAccountContext({ + account, + cfg, + abortSignal: abort.signal, + }), + ); + + try { + state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + senderName: "Alice", + text: "hello", + }); + + const outbound = await state.waitFor({ + kind: "message-text", + textIncludes: "qa-echo: hello", + direction: "outbound", + timeoutMs: 5_000, + }); + expect("text" in outbound && outbound.text).toContain("qa-echo: hello"); + } finally { + abort.abort(); + await task; + await bus.stop(); + } + }); + + it("exposes thread and message actions against the qa bus", async () => { + const state = createQaBusState(); + const bus = await startQaBusServer({ state }); + + try { + const cfg = { + channels: { + "qa-channel": { + baseUrl: bus.baseUrl, + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + }, + }, + }; + + const handleAction = qaChannelPlugin.actions?.handleAction; + expect(handleAction).toBeDefined(); + + const threadResult = await handleAction!({ + channel: "qa-channel", + action: "thread-create", + cfg, + accountId: "default", + params: { + channelId: "qa-room", + title: "QA thread", + }, + }); + const threadPayload = extractToolPayload(threadResult) as { + thread: { id: string }; + target: string; + }; + expect(threadPayload.thread.id).toBeTruthy(); + expect(threadPayload.target).toContain(threadPayload.thread.id); + + const outbound = state.addOutboundMessage({ + to: threadPayload.target, + text: "message", + threadId: threadPayload.thread.id, + }); + + await handleAction!({ + channel: "qa-channel", + action: "react", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + emoji: "white_check_mark", + }, + }); + + await handleAction!({ + channel: "qa-channel", + action: "edit", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + text: "message (edited)", + }, + }); + + const readResult = await handleAction!({ + channel: "qa-channel", + action: "read", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + }, + }); + const readPayload = extractToolPayload(readResult) as { message: { text: string } }; + expect(readPayload.message.text).toContain("(edited)"); + + const searchResult = await handleAction!({ + channel: "qa-channel", + action: "search", + cfg, + accountId: "default", + params: { + query: "edited", + channelId: "qa-room", + threadId: threadPayload.thread.id, + }, + }); + const searchPayload = extractToolPayload(searchResult) as { + messages: Array<{ id: string }>; + }; + expect(searchPayload.messages.some((message) => message.id === outbound.id)).toBe(true); + + await handleAction!({ + channel: "qa-channel", + action: "delete", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + }, + }); + expect(state.readMessage({ messageId: outbound.id }).deleted).toBe(true); + } finally { + await bus.stop(); + } + }); +}); diff --git a/extensions/qa-channel/src/channel.ts b/extensions/qa-channel/src/channel.ts new file mode 100644 index 00000000000..cbe9e6e45e6 --- /dev/null +++ b/extensions/qa-channel/src/channel.ts @@ -0,0 +1,112 @@ +import { + buildChannelOutboundSessionRoute, + createChatChannelPlugin, + getChatChannelMeta, +} from "openclaw/plugin-sdk/core"; +import { + DEFAULT_ACCOUNT_ID, + listQaChannelAccountIds, + resolveDefaultQaChannelAccountId, + resolveQaChannelAccount, +} from "./accounts.js"; +import { buildQaTarget, normalizeQaTarget, parseQaTarget } from "./bus-client.js"; +import { qaChannelMessageActions } from "./channel-actions.js"; +import { qaChannelPluginConfigSchema } from "./config-schema.js"; +import { startQaGatewayAccount } from "./gateway.js"; +import { sendQaChannelText } from "./outbound.js"; +import type { ChannelPlugin } from "./runtime-api.js"; +import { applyQaSetup } from "./setup.js"; +import { qaChannelStatus } from "./status.js"; +import type { CoreConfig, ResolvedQaChannelAccount } from "./types.js"; + +const CHANNEL_ID = "qa-channel" as const; +const meta = { ...getChatChannelMeta(CHANNEL_ID) }; + +export const qaChannelPlugin: ChannelPlugin = createChatChannelPlugin({ + base: { + id: CHANNEL_ID, + meta, + capabilities: { + chatTypes: ["direct", "group"], + }, + reload: { configPrefixes: ["channels.qa-channel"] }, + configSchema: qaChannelPluginConfigSchema, + setup: { + applyAccountConfig: ({ cfg, accountId, input }) => + applyQaSetup({ + cfg, + accountId, + input: input as Record, + }), + }, + config: { + listAccountIds: (cfg) => listQaChannelAccountIds(cfg as CoreConfig), + resolveAccount: (cfg, accountId) => + resolveQaChannelAccount({ cfg: cfg as CoreConfig, accountId }), + defaultAccountId: (cfg) => resolveDefaultQaChannelAccountId(cfg as CoreConfig), + isConfigured: (account) => account.configured, + resolveAllowFrom: ({ cfg, accountId }) => + resolveQaChannelAccount({ cfg: cfg as CoreConfig, accountId }).config.allowFrom, + resolveDefaultTo: ({ cfg, accountId }) => + resolveQaChannelAccount({ cfg: cfg as CoreConfig, accountId }).config.defaultTo, + }, + messaging: { + normalizeTarget: normalizeQaTarget, + parseExplicitTarget: ({ raw }) => { + const parsed = parseQaTarget(raw); + return { + to: buildQaTarget(parsed), + threadId: parsed.threadId, + chatType: parsed.chatType, + }; + }, + inferTargetChatType: ({ to }) => parseQaTarget(to).chatType, + targetResolver: { + looksLikeId: (raw) => + /^((dm|channel):|thread:[^/]+\/)/i.test(raw.trim()) || raw.trim().length > 0, + hint: "", + }, + resolveOutboundSessionRoute: ({ cfg, agentId, accountId, target, threadId }) => { + const parsed = parseQaTarget(target); + return buildChannelOutboundSessionRoute({ + cfg, + agentId, + channel: CHANNEL_ID, + accountId, + peer: { + kind: parsed.chatType === "direct" ? "direct" : "channel", + id: buildQaTarget(parsed), + }, + chatType: parsed.chatType, + from: `qa-channel:${accountId ?? DEFAULT_ACCOUNT_ID}`, + to: buildQaTarget(parsed), + threadId: threadId ?? parsed.threadId, + }); + }, + }, + status: qaChannelStatus, + gateway: { + startAccount: async (ctx) => { + await startQaGatewayAccount(CHANNEL_ID, meta.label, ctx); + }, + }, + actions: qaChannelMessageActions, + }, + outbound: { + base: { + deliveryMode: "direct", + }, + attachedResults: { + channel: CHANNEL_ID, + sendText: async ({ cfg, to, text, accountId, threadId, replyToId }) => + await sendQaChannelText({ + cfg: cfg as CoreConfig, + accountId, + to, + text, + threadId, + replyToId, + }), + }, + }, +}); diff --git a/extensions/qa-channel/src/config-schema.ts b/extensions/qa-channel/src/config-schema.ts new file mode 100644 index 00000000000..545da4f7c97 --- /dev/null +++ b/extensions/qa-channel/src/config-schema.ts @@ -0,0 +1,32 @@ +import { buildChannelConfigSchema } from "openclaw/plugin-sdk/channel-config-schema"; +import { z } from "openclaw/plugin-sdk/zod"; + +const QaChannelActionConfigSchema = z + .object({ + messages: z.boolean().optional(), + reactions: z.boolean().optional(), + search: z.boolean().optional(), + threads: z.boolean().optional(), + }) + .strict(); + +export const QaChannelAccountConfigSchema = z + .object({ + name: z.string().optional(), + enabled: z.boolean().optional(), + baseUrl: z.string().url().optional(), + botUserId: z.string().optional(), + botDisplayName: z.string().optional(), + pollTimeoutMs: z.number().int().min(100).max(30_000).optional(), + allowFrom: z.array(z.union([z.string(), z.number()])).optional(), + defaultTo: z.string().optional(), + actions: QaChannelActionConfigSchema.optional(), + }) + .strict(); + +export const QaChannelConfigSchema = QaChannelAccountConfigSchema.extend({ + accounts: z.record(z.string(), QaChannelAccountConfigSchema.partial()).optional(), + defaultAccount: z.string().optional(), +}).strict(); + +export const qaChannelPluginConfigSchema = buildChannelConfigSchema(QaChannelConfigSchema); diff --git a/extensions/qa-channel/src/gateway.ts b/extensions/qa-channel/src/gateway.ts new file mode 100644 index 00000000000..37bc1c9aed1 --- /dev/null +++ b/extensions/qa-channel/src/gateway.ts @@ -0,0 +1,55 @@ +import { pollQaBus } from "./bus-client.js"; +import { handleQaInbound } from "./inbound.js"; +import type { ChannelGatewayContext } from "./runtime-api.js"; +import type { CoreConfig, ResolvedQaChannelAccount } from "./types.js"; + +export async function startQaGatewayAccount( + channelId: string, + channelLabel: string, + ctx: ChannelGatewayContext, +) { + const account = ctx.account; + if (!account.configured) { + throw new Error(`QA channel is not configured for account "${account.accountId}"`); + } + ctx.setStatus({ + accountId: account.accountId, + running: true, + configured: true, + enabled: account.enabled, + baseUrl: account.baseUrl, + }); + let cursor = 0; + try { + while (!ctx.abortSignal.aborted) { + const result = await pollQaBus({ + baseUrl: account.baseUrl, + accountId: account.accountId, + cursor, + timeoutMs: account.pollTimeoutMs, + signal: ctx.abortSignal, + }); + cursor = result.cursor; + for (const event of result.events) { + if (event.kind !== "inbound-message") { + continue; + } + await handleQaInbound({ + channelId, + channelLabel, + account, + config: ctx.cfg as CoreConfig, + message: event.message, + }); + } + } + } catch (error) { + if (!(error instanceof Error) || error.name !== "AbortError") { + throw error; + } + } + ctx.setStatus({ + accountId: account.accountId, + running: false, + }); +} diff --git a/extensions/qa-channel/src/inbound.ts b/extensions/qa-channel/src/inbound.ts new file mode 100644 index 00000000000..c235c8f063b --- /dev/null +++ b/extensions/qa-channel/src/inbound.ts @@ -0,0 +1,124 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/core"; +import { dispatchInboundReplyWithBase } from "openclaw/plugin-sdk/inbound-reply-dispatch"; +import { buildQaTarget, sendQaBusMessage, type QaBusMessage } from "./bus-client.js"; +import { getQaChannelRuntime } from "./runtime.js"; +import type { CoreConfig, ResolvedQaChannelAccount } from "./types.js"; + +export async function handleQaInbound(params: { + channelId: string; + channelLabel: string; + account: ResolvedQaChannelAccount; + config: CoreConfig; + message: QaBusMessage; +}) { + const runtime = getQaChannelRuntime(); + const inbound = params.message; + const target = buildQaTarget({ + chatType: inbound.conversation.kind, + conversationId: inbound.conversation.id, + threadId: inbound.threadId, + }); + const route = runtime.channel.routing.resolveAgentRoute({ + cfg: params.config as OpenClawConfig, + channel: params.channelId, + accountId: params.account.accountId, + peer: { + kind: inbound.conversation.kind === "direct" ? "direct" : "channel", + id: target, + }, + }); + const storePath = runtime.channel.session.resolveStorePath(params.config.session?.store, { + agentId: route.agentId, + }); + const previousTimestamp = runtime.channel.session.readSessionUpdatedAt({ + storePath, + sessionKey: route.sessionKey, + }); + const body = runtime.channel.reply.formatAgentEnvelope({ + channel: params.channelLabel, + from: inbound.senderName || inbound.senderId, + timestamp: inbound.timestamp, + previousTimestamp, + envelope: runtime.channel.reply.resolveEnvelopeFormatOptions(params.config as OpenClawConfig), + body: inbound.text, + }); + + const ctxPayload = runtime.channel.reply.finalizeInboundContext({ + Body: body, + BodyForAgent: inbound.text, + RawBody: inbound.text, + CommandBody: inbound.text, + From: buildQaTarget({ + chatType: inbound.conversation.kind, + conversationId: inbound.senderId, + }), + To: target, + SessionKey: route.sessionKey, + AccountId: route.accountId ?? params.account.accountId, + ChatType: inbound.conversation.kind === "direct" ? "direct" : "group", + ConversationLabel: + inbound.threadTitle || + inbound.conversation.title || + inbound.senderName || + inbound.conversation.id, + GroupSubject: + inbound.conversation.kind === "channel" + ? inbound.threadTitle || inbound.conversation.title || inbound.conversation.id + : undefined, + GroupChannel: inbound.conversation.kind === "channel" ? inbound.conversation.id : undefined, + NativeChannelId: inbound.conversation.id, + MessageThreadId: inbound.threadId, + ThreadLabel: inbound.threadTitle, + ThreadParentId: inbound.threadId ? inbound.conversation.id : undefined, + SenderName: inbound.senderName, + SenderId: inbound.senderId, + Provider: params.channelId, + Surface: params.channelId, + MessageSid: inbound.id, + MessageSidFull: inbound.id, + ReplyToId: inbound.replyToId, + Timestamp: inbound.timestamp, + OriginatingChannel: params.channelId, + OriginatingTo: target, + CommandAuthorized: true, + }); + + await dispatchInboundReplyWithBase({ + cfg: params.config as OpenClawConfig, + channel: params.channelId, + accountId: params.account.accountId, + route, + storePath, + ctxPayload, + core: runtime, + deliver: async (payload) => { + const text = + payload && typeof payload === "object" && "text" in payload + ? String((payload as { text?: string }).text ?? "") + : ""; + if (!text.trim()) { + return; + } + await sendQaBusMessage({ + baseUrl: params.account.baseUrl, + accountId: params.account.accountId, + to: target, + text, + senderId: params.account.botUserId, + senderName: params.account.botDisplayName, + threadId: inbound.threadId, + replyToId: inbound.id, + }); + }, + onRecordError: (error) => { + throw error instanceof Error + ? error + : new Error(`qa-channel session record failed: ${String(error)}`); + }, + onDispatchError: (error) => { + throw error instanceof Error + ? error + : new Error(`qa-channel dispatch failed: ${String(error)}`); + }, + }); +} diff --git a/extensions/qa-channel/src/outbound.ts b/extensions/qa-channel/src/outbound.ts new file mode 100644 index 00000000000..956a93a96bb --- /dev/null +++ b/extensions/qa-channel/src/outbound.ts @@ -0,0 +1,34 @@ +import { resolveQaChannelAccount } from "./accounts.js"; +import { buildQaTarget, parseQaTarget, sendQaBusMessage } from "./bus-client.js"; +import type { CoreConfig } from "./types.js"; + +export async function sendQaChannelText(params: { + cfg: CoreConfig; + accountId?: string | null; + to: string; + text: string; + threadId?: string | number | null; + replyToId?: string | number | null; +}) { + const account = resolveQaChannelAccount({ cfg: params.cfg, accountId: params.accountId }); + const parsed = parseQaTarget(params.to); + const resolvedThreadId = params.threadId == null ? parsed.threadId : String(params.threadId); + const { message } = await sendQaBusMessage({ + baseUrl: account.baseUrl, + accountId: account.accountId, + to: buildQaTarget({ + chatType: parsed.chatType, + conversationId: parsed.conversationId, + threadId: resolvedThreadId, + }), + text: params.text, + senderId: account.botUserId, + senderName: account.botDisplayName, + threadId: resolvedThreadId, + replyToId: params.replyToId == null ? undefined : String(params.replyToId), + }); + return { + to: params.to, + messageId: message.id, + }; +} diff --git a/extensions/qa-channel/src/protocol.ts b/extensions/qa-channel/src/protocol.ts new file mode 100644 index 00000000000..ce00a37d057 --- /dev/null +++ b/extensions/qa-channel/src/protocol.ts @@ -0,0 +1,180 @@ +export type QaBusConversationKind = "direct" | "channel"; + +export type QaBusConversation = { + id: string; + kind: QaBusConversationKind; + title?: string; +}; + +export type QaBusMessage = { + id: string; + accountId: string; + direction: "inbound" | "outbound"; + conversation: QaBusConversation; + senderId: string; + senderName?: string; + text: string; + timestamp: number; + threadId?: string; + threadTitle?: string; + replyToId?: string; + deleted?: boolean; + editedAt?: number; + reactions: Array<{ + emoji: string; + senderId: string; + timestamp: number; + }>; +}; + +export type QaBusThread = { + id: string; + accountId: string; + conversationId: string; + title: string; + createdAt: number; + createdBy: string; +}; + +export type QaBusEvent = + | { + cursor: number; + kind: "inbound-message"; + accountId: string; + message: QaBusMessage; + } + | { + cursor: number; + kind: "outbound-message"; + accountId: string; + message: QaBusMessage; + } + | { + cursor: number; + kind: "thread-created"; + accountId: string; + thread: QaBusThread; + } + | { + cursor: number; + kind: "message-edited"; + accountId: string; + message: QaBusMessage; + } + | { + cursor: number; + kind: "message-deleted"; + accountId: string; + message: QaBusMessage; + } + | { + cursor: number; + kind: "reaction-added"; + accountId: string; + message: QaBusMessage; + emoji: string; + senderId: string; + }; + +export type QaBusInboundMessageInput = { + accountId?: string; + conversation: QaBusConversation; + senderId: string; + senderName?: string; + text: string; + timestamp?: number; + threadId?: string; + threadTitle?: string; + replyToId?: string; +}; + +export type QaBusOutboundMessageInput = { + accountId?: string; + to: string; + senderId?: string; + senderName?: string; + text: string; + timestamp?: number; + threadId?: string; + replyToId?: string; +}; + +export type QaBusCreateThreadInput = { + accountId?: string; + conversationId: string; + title: string; + createdBy?: string; + timestamp?: number; +}; + +export type QaBusReactToMessageInput = { + accountId?: string; + messageId: string; + emoji: string; + senderId?: string; + timestamp?: number; +}; + +export type QaBusEditMessageInput = { + accountId?: string; + messageId: string; + text: string; + timestamp?: number; +}; + +export type QaBusDeleteMessageInput = { + accountId?: string; + messageId: string; + timestamp?: number; +}; + +export type QaBusSearchMessagesInput = { + accountId?: string; + query?: string; + conversationId?: string; + threadId?: string; + limit?: number; +}; + +export type QaBusReadMessageInput = { + accountId?: string; + messageId: string; +}; + +export type QaBusPollInput = { + accountId?: string; + cursor?: number; + timeoutMs?: number; + limit?: number; +}; + +export type QaBusPollResult = { + cursor: number; + events: QaBusEvent[]; +}; + +export type QaBusStateSnapshot = { + cursor: number; + conversations: QaBusConversation[]; + threads: QaBusThread[]; + messages: QaBusMessage[]; + events: QaBusEvent[]; +}; + +export type QaBusWaitForInput = + | { + timeoutMs?: number; + kind: "event-kind"; + eventKind: QaBusEvent["kind"]; + } + | { + timeoutMs?: number; + kind: "message-text"; + textIncludes: string; + direction?: QaBusMessage["direction"]; + } + | { + timeoutMs?: number; + kind: "thread-id"; + threadId: string; + }; diff --git a/extensions/qa-channel/src/runtime-api.ts b/extensions/qa-channel/src/runtime-api.ts new file mode 100644 index 00000000000..8fc1f531ffa --- /dev/null +++ b/extensions/qa-channel/src/runtime-api.ts @@ -0,0 +1,23 @@ +export type { + ChannelMessageActionAdapter, + ChannelMessageActionName, +} from "openclaw/plugin-sdk/channel-contract"; +export type { PluginRuntime } from "openclaw/plugin-sdk/core"; +export type { ChannelGatewayContext } from "openclaw/plugin-sdk/channel-contract"; +export type { RuntimeEnv } from "openclaw/plugin-sdk/runtime"; +export type { ChannelPlugin } from "openclaw/plugin-sdk/core"; +export { + buildChannelConfigSchema, + buildChannelOutboundSessionRoute, + createChatChannelPlugin, + defineChannelPluginEntry, + getChatChannelMeta, + jsonResult, + readStringParam, +} from "openclaw/plugin-sdk/core"; +export { + createComputedAccountStatusAdapter, + createDefaultChannelRuntimeState, +} from "openclaw/plugin-sdk/status-helpers"; +export { createPluginRuntimeStore } from "openclaw/plugin-sdk/runtime-store"; +export { dispatchInboundReplyWithBase } from "openclaw/plugin-sdk/inbound-reply-dispatch"; diff --git a/extensions/qa-channel/src/runtime.ts b/extensions/qa-channel/src/runtime.ts new file mode 100644 index 00000000000..353e2dcc29c --- /dev/null +++ b/extensions/qa-channel/src/runtime.ts @@ -0,0 +1,7 @@ +import { createPluginRuntimeStore } from "openclaw/plugin-sdk/runtime-store"; +import type { PluginRuntime } from "./runtime-api.js"; + +const { setRuntime: setQaChannelRuntime, getRuntime: getQaChannelRuntime } = + createPluginRuntimeStore("QA channel runtime not initialized"); + +export { getQaChannelRuntime, setQaChannelRuntime }; diff --git a/extensions/qa-channel/src/setup.ts b/extensions/qa-channel/src/setup.ts new file mode 100644 index 00000000000..7b67e1adcd3 --- /dev/null +++ b/extensions/qa-channel/src/setup.ts @@ -0,0 +1,40 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/core"; +import { DEFAULT_ACCOUNT_ID } from "./accounts.js"; +import type { CoreConfig } from "./types.js"; + +export function applyQaSetup(params: { + cfg: OpenClawConfig; + accountId: string; + input: Record; +}): OpenClawConfig { + const nextCfg = structuredClone(params.cfg) as CoreConfig; + const section = nextCfg.channels?.["qa-channel"] ?? {}; + const accounts = { ...(section.accounts ?? {}) }; + const target = + params.accountId === DEFAULT_ACCOUNT_ID + ? { ...section } + : { ...(accounts[params.accountId] ?? {}) }; + if (typeof params.input.baseUrl === "string") { + target.baseUrl = params.input.baseUrl; + } + if (typeof params.input.botUserId === "string") { + target.botUserId = params.input.botUserId; + } + if (typeof params.input.botDisplayName === "string") { + target.botDisplayName = params.input.botDisplayName; + } + nextCfg.channels ??= {}; + if (params.accountId === DEFAULT_ACCOUNT_ID) { + nextCfg.channels["qa-channel"] = { + ...section, + ...target, + }; + } else { + accounts[params.accountId] = target; + nextCfg.channels["qa-channel"] = { + ...section, + accounts, + }; + } + return nextCfg as OpenClawConfig; +} diff --git a/extensions/qa-channel/src/status.ts b/extensions/qa-channel/src/status.ts new file mode 100644 index 00000000000..e0470955003 --- /dev/null +++ b/extensions/qa-channel/src/status.ts @@ -0,0 +1,23 @@ +import { DEFAULT_ACCOUNT_ID } from "./accounts.js"; +import { + createComputedAccountStatusAdapter, + createDefaultChannelRuntimeState, +} from "./runtime-api.js"; +import type { ResolvedQaChannelAccount } from "./types.js"; + +export const qaChannelStatus = createComputedAccountStatusAdapter({ + defaultRuntime: createDefaultChannelRuntimeState(DEFAULT_ACCOUNT_ID), + buildChannelSummary: ({ snapshot }) => ({ + baseUrl: snapshot.baseUrl ?? "[missing]", + }), + resolveAccountSnapshot: ({ account }) => ({ + accountId: account.accountId, + name: account.name, + enabled: account.enabled, + configured: account.configured, + extra: { + baseUrl: account.baseUrl || "[missing]", + botUserId: account.botUserId, + }, + }), +}); diff --git a/extensions/qa-channel/src/types.ts b/extensions/qa-channel/src/types.ts new file mode 100644 index 00000000000..0a1c4a2538f --- /dev/null +++ b/extensions/qa-channel/src/types.ts @@ -0,0 +1,44 @@ +export type QaChannelActionConfig = { + messages?: boolean; + reactions?: boolean; + search?: boolean; + threads?: boolean; +}; + +export type QaChannelAccountConfig = { + name?: string; + enabled?: boolean; + baseUrl?: string; + botUserId?: string; + botDisplayName?: string; + pollTimeoutMs?: number; + allowFrom?: Array; + defaultTo?: string; + actions?: QaChannelActionConfig; +}; + +export type QaChannelConfig = QaChannelAccountConfig & { + accounts?: Record>; + defaultAccount?: string; +}; + +export type CoreConfig = { + channels?: { + "qa-channel"?: QaChannelConfig; + }; + session?: { + store?: string; + }; +}; + +export type ResolvedQaChannelAccount = { + accountId: string; + enabled: boolean; + configured: boolean; + name?: string; + baseUrl: string; + botUserId: string; + botDisplayName: string; + pollTimeoutMs: number; + config: QaChannelAccountConfig; +}; diff --git a/extensions/qa-channel/test-api.ts b/extensions/qa-channel/test-api.ts new file mode 100644 index 00000000000..dc896f10fc4 --- /dev/null +++ b/extensions/qa-channel/test-api.ts @@ -0,0 +1,2 @@ +export * from "./src/protocol.js"; +export * from "./src/bus-client.js"; diff --git a/package.json b/package.json index ed9b9545132..12eca03ceec 100644 --- a/package.json +++ b/package.json @@ -1029,6 +1029,7 @@ "protocol:check": "pnpm protocol:gen && pnpm protocol:gen:swift && git diff --exit-code -- dist/protocol.schema.json apps/macos/Sources/OpenClawProtocol/GatewayModels.swift apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift", "protocol:gen": "node --import tsx scripts/protocol-gen.ts", "protocol:gen:swift": "node --import tsx scripts/protocol-gen-swift.ts", + "qa:e2e": "node --import tsx scripts/qa-e2e.ts", "release:check": "pnpm check:base-config-schema && pnpm check:bundled-channel-config-metadata && pnpm config:docs:check && pnpm plugin-sdk:check-exports && pnpm plugin-sdk:api:check && node --import tsx scripts/release-check.ts", "release:openclaw:npm:check": "node --import tsx scripts/openclaw-npm-release-check.ts", "release:openclaw:npm:verify-published": "node --import tsx scripts/openclaw-npm-postpublish-verify.ts", diff --git a/scripts/qa-e2e.ts b/scripts/qa-e2e.ts new file mode 100644 index 00000000000..c329ddccb67 --- /dev/null +++ b/scripts/qa-e2e.ts @@ -0,0 +1,6 @@ +import { runQaE2eSelfCheck } from "../src/qa-e2e/runner.js"; + +const outputPath = process.argv[2]?.trim() || ".artifacts/qa-e2e/self-check.md"; + +const result = await runQaE2eSelfCheck({ outputPath }); +process.stdout.write(`QA self-check report: ${result.outputPath}\n`); diff --git a/src/qa-e2e/bus-queries.ts b/src/qa-e2e/bus-queries.ts new file mode 100644 index 00000000000..fe92541e2d9 --- /dev/null +++ b/src/qa-e2e/bus-queries.ts @@ -0,0 +1,136 @@ +import type { + QaBusConversation, + QaBusEvent, + QaBusMessage, + QaBusPollInput, + QaBusPollResult, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, +} from "../../extensions/qa-channel/test-api.js"; + +export const DEFAULT_ACCOUNT_ID = "default"; + +export function normalizeAccountId(raw?: string): string { + const trimmed = raw?.trim(); + return trimmed || DEFAULT_ACCOUNT_ID; +} + +export function normalizeConversationFromTarget(target: string): { + conversation: QaBusConversation; + threadId?: string; +} { + const trimmed = target.trim(); + if (trimmed.startsWith("thread:")) { + const rest = trimmed.slice("thread:".length); + const slash = rest.indexOf("/"); + if (slash > 0) { + return { + conversation: { id: rest.slice(0, slash), kind: "channel" }, + threadId: rest.slice(slash + 1), + }; + } + } + if (trimmed.startsWith("channel:")) { + return { + conversation: { id: trimmed.slice("channel:".length), kind: "channel" }, + }; + } + if (trimmed.startsWith("dm:")) { + return { + conversation: { id: trimmed.slice("dm:".length), kind: "direct" }, + }; + } + return { + conversation: { id: trimmed, kind: "direct" }, + }; +} + +export function cloneMessage(message: QaBusMessage): QaBusMessage { + return { + ...message, + conversation: { ...message.conversation }, + reactions: message.reactions.map((reaction) => ({ ...reaction })), + }; +} + +export function cloneEvent(event: QaBusEvent): QaBusEvent { + switch (event.kind) { + case "inbound-message": + case "outbound-message": + case "message-edited": + case "message-deleted": + case "reaction-added": + return { ...event, message: cloneMessage(event.message) }; + case "thread-created": + return { ...event, thread: { ...event.thread } }; + } +} + +export function buildQaBusSnapshot(params: { + cursor: number; + conversations: Map; + threads: Map; + messages: Map; + events: QaBusEvent[]; +}): QaBusStateSnapshot { + return { + cursor: params.cursor, + conversations: Array.from(params.conversations.values()).map((conversation) => ({ + ...conversation, + })), + threads: Array.from(params.threads.values()).map((thread) => ({ ...thread })), + messages: Array.from(params.messages.values()).map((message) => cloneMessage(message)), + events: params.events.map((event) => cloneEvent(event)), + }; +} + +export function readQaBusMessage(params: { + messages: Map; + input: QaBusReadMessageInput; +}) { + const message = params.messages.get(params.input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${params.input.messageId}`); + } + return cloneMessage(message); +} + +export function searchQaBusMessages(params: { + messages: Map; + input: QaBusSearchMessagesInput; +}) { + const accountId = normalizeAccountId(params.input.accountId); + const limit = Math.max(1, Math.min(params.input.limit ?? 20, 100)); + const query = params.input.query?.trim().toLowerCase(); + return Array.from(params.messages.values()) + .filter((message) => message.accountId === accountId) + .filter((message) => + params.input.conversationId ? message.conversation.id === params.input.conversationId : true, + ) + .filter((message) => + params.input.threadId ? message.threadId === params.input.threadId : true, + ) + .filter((message) => (query ? message.text.toLowerCase().includes(query) : true)) + .slice(-limit) + .map((message) => cloneMessage(message)); +} + +export function pollQaBusEvents(params: { + events: QaBusEvent[]; + cursor: number; + input?: QaBusPollInput; +}): QaBusPollResult { + const accountId = normalizeAccountId(params.input?.accountId); + const startCursor = params.input?.cursor ?? 0; + const limit = Math.max(1, Math.min(params.input?.limit ?? 100, 500)); + const matches = params.events + .filter((event) => event.accountId === accountId && event.cursor > startCursor) + .slice(0, limit) + .map((event) => cloneEvent(event)); + return { + cursor: params.cursor, + events: matches, + }; +} diff --git a/src/qa-e2e/bus-server.ts b/src/qa-e2e/bus-server.ts new file mode 100644 index 00000000000..efad93e647d --- /dev/null +++ b/src/qa-e2e/bus-server.ts @@ -0,0 +1,170 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusInboundMessageInput, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusReactToMessageInput, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusWaitForInput, +} from "../../extensions/qa-channel/test-api.js"; +import type { QaBusState } from "./bus-state.js"; + +async function readJson(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const text = Buffer.concat(chunks).toString("utf8").trim(); + return text ? (JSON.parse(text) as unknown) : {}; +} + +function writeJson(res: ServerResponse, statusCode: number, body: unknown) { + const payload = JSON.stringify(body); + res.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(payload), + }); + res.end(payload); +} + +function writeError(res: ServerResponse, statusCode: number, error: unknown) { + writeJson(res, statusCode, { + error: error instanceof Error ? error.message : String(error), + }); +} + +async function handleRequest(params: { + req: IncomingMessage; + res: ServerResponse; + state: QaBusState; +}) { + const method = params.req.method ?? "GET"; + const url = new URL(params.req.url ?? "/", "http://127.0.0.1"); + + if (method === "GET" && url.pathname === "/health") { + writeJson(params.res, 200, { ok: true }); + return; + } + + if (method === "GET" && url.pathname === "/v1/state") { + writeJson(params.res, 200, params.state.getSnapshot()); + return; + } + + if (method !== "POST") { + writeError(params.res, 405, "method not allowed"); + return; + } + + const body = (await readJson(params.req)) as Record; + + try { + switch (url.pathname) { + case "/v1/reset": + params.state.reset(); + writeJson(params.res, 200, { ok: true }); + return; + case "/v1/inbound/message": + writeJson(params.res, 200, { + message: params.state.addInboundMessage(body as unknown as QaBusInboundMessageInput), + }); + return; + case "/v1/outbound/message": + writeJson(params.res, 200, { + message: params.state.addOutboundMessage(body as unknown as QaBusOutboundMessageInput), + }); + return; + case "/v1/actions/thread-create": + writeJson(params.res, 200, { + thread: params.state.createThread(body as unknown as QaBusCreateThreadInput), + }); + return; + case "/v1/actions/react": + writeJson(params.res, 200, { + message: params.state.reactToMessage(body as unknown as QaBusReactToMessageInput), + }); + return; + case "/v1/actions/edit": + writeJson(params.res, 200, { + message: params.state.editMessage(body as unknown as QaBusEditMessageInput), + }); + return; + case "/v1/actions/delete": + writeJson(params.res, 200, { + message: params.state.deleteMessage(body as unknown as QaBusDeleteMessageInput), + }); + return; + case "/v1/actions/read": + writeJson(params.res, 200, { + message: params.state.readMessage(body as unknown as QaBusReadMessageInput), + }); + return; + case "/v1/actions/search": + writeJson(params.res, 200, { + messages: params.state.searchMessages(body as unknown as QaBusSearchMessagesInput), + }); + return; + case "/v1/poll": { + const input = body as unknown as QaBusPollInput; + const timeoutMs = Math.max(0, Math.min(input.timeoutMs ?? 0, 30_000)); + const initial = params.state.poll(input); + if (initial.events.length > 0 || timeoutMs === 0) { + writeJson(params.res, 200, initial); + return; + } + try { + await params.state.waitFor({ + kind: "event-kind", + eventKind: "inbound-message", + timeoutMs, + }); + } catch { + // timeout is fine for long-poll. + } + writeJson(params.res, 200, params.state.poll(input)); + return; + } + case "/v1/wait": + writeJson(params.res, 200, { + match: await params.state.waitFor(body as unknown as QaBusWaitForInput), + }); + return; + default: + writeError(params.res, 404, "not found"); + } + } catch (error) { + writeError(params.res, 400, error); + } +} + +export function createQaBusServer(state: QaBusState): Server { + return createServer(async (req, res) => { + await handleRequest({ req, res, state }); + }); +} + +export async function startQaBusServer(params: { state: QaBusState; port?: number }) { + const server = createQaBusServer(params.state); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(params.port ?? 0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("qa-bus failed to bind"); + } + return { + server, + port: address.port, + baseUrl: `http://127.0.0.1:${address.port}`, + async stop() { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }, + }; +} diff --git a/src/qa-e2e/bus-state.test.ts b/src/qa-e2e/bus-state.test.ts new file mode 100644 index 00000000000..64d0123bd33 --- /dev/null +++ b/src/qa-e2e/bus-state.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { createQaBusState } from "./bus-state.js"; + +describe("qa-bus state", () => { + it("records inbound and outbound traffic in cursor order", () => { + const state = createQaBusState(); + + const inbound = state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + text: "hello", + }); + const outbound = state.addOutboundMessage({ + to: "dm:alice", + text: "hi", + }); + + const snapshot = state.getSnapshot(); + expect(snapshot.cursor).toBe(2); + expect(snapshot.events.map((event) => event.kind)).toEqual([ + "inbound-message", + "outbound-message", + ]); + expect(snapshot.messages.map((message) => message.id)).toEqual([inbound.id, outbound.id]); + }); + + it("creates threads and mutates message state", async () => { + const state = createQaBusState(); + + const thread = state.createThread({ + conversationId: "qa-room", + title: "QA thread", + }); + const message = state.addOutboundMessage({ + to: `thread:qa-room/${thread.id}`, + text: "inside thread", + threadId: thread.id, + }); + + state.reactToMessage({ + messageId: message.id, + emoji: "white_check_mark", + }); + state.editMessage({ + messageId: message.id, + text: "inside thread (edited)", + }); + state.deleteMessage({ + messageId: message.id, + }); + + const updated = state.readMessage({ messageId: message.id }); + expect(updated.threadId).toBe(thread.id); + expect(updated.reactions).toHaveLength(1); + expect(updated.text).toContain("(edited)"); + expect(updated.deleted).toBe(true); + + const waited = await state.waitFor({ + kind: "thread-id", + threadId: thread.id, + timeoutMs: 50, + }); + expect("id" in waited && waited.id).toBe(thread.id); + }); +}); diff --git a/src/qa-e2e/bus-state.ts b/src/qa-e2e/bus-state.ts new file mode 100644 index 00000000000..76241d997fa --- /dev/null +++ b/src/qa-e2e/bus-state.ts @@ -0,0 +1,256 @@ +import { randomUUID } from "node:crypto"; +import { + type QaBusConversation, + type QaBusCreateThreadInput, + type QaBusDeleteMessageInput, + type QaBusEditMessageInput, + type QaBusEvent, + type QaBusInboundMessageInput, + type QaBusMessage, + type QaBusOutboundMessageInput, + type QaBusPollInput, + type QaBusReadMessageInput, + type QaBusReactToMessageInput, + type QaBusSearchMessagesInput, + type QaBusThread, + type QaBusWaitForInput, +} from "../../extensions/qa-channel/test-api.js"; +import { + buildQaBusSnapshot, + cloneMessage, + normalizeAccountId, + normalizeConversationFromTarget, + pollQaBusEvents, + readQaBusMessage, + searchQaBusMessages, +} from "./bus-queries.js"; +import { createQaBusWaiterStore } from "./bus-waiters.js"; + +const DEFAULT_BOT_ID = "openclaw"; +const DEFAULT_BOT_NAME = "OpenClaw QA"; + +type QaBusEventSeed = + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor">; + +export function createQaBusState() { + const conversations = new Map(); + const threads = new Map(); + const messages = new Map(); + const events: QaBusEvent[] = []; + let cursor = 0; + const waiters = createQaBusWaiterStore(() => + buildQaBusSnapshot({ + cursor, + conversations, + threads, + messages, + events, + }), + ); + + const pushEvent = (event: QaBusEventSeed | ((cursor: number) => QaBusEventSeed)): QaBusEvent => { + cursor += 1; + const next = typeof event === "function" ? event(cursor) : event; + const finalized = { cursor, ...next } as QaBusEvent; + events.push(finalized); + waiters.settle(); + return finalized; + }; + + const ensureConversation = (conversation: QaBusConversation): QaBusConversation => { + const existing = conversations.get(conversation.id); + if (existing) { + if (!existing.title && conversation.title) { + existing.title = conversation.title; + } + return existing; + } + const created = { ...conversation }; + conversations.set(created.id, created); + return created; + }; + + const createMessage = (params: { + direction: QaBusMessage["direction"]; + accountId: string; + conversation: QaBusConversation; + senderId: string; + senderName?: string; + text: string; + timestamp?: number; + threadId?: string; + threadTitle?: string; + replyToId?: string; + }): QaBusMessage => { + const conversation = ensureConversation(params.conversation); + const message: QaBusMessage = { + id: randomUUID(), + accountId: params.accountId, + direction: params.direction, + conversation, + senderId: params.senderId, + senderName: params.senderName, + text: params.text, + timestamp: params.timestamp ?? Date.now(), + threadId: params.threadId, + threadTitle: params.threadTitle, + replyToId: params.replyToId, + reactions: [], + }; + messages.set(message.id, message); + return message; + }; + + return { + reset() { + conversations.clear(); + threads.clear(); + messages.clear(); + events.length = 0; + cursor = 0; + waiters.reset(); + }, + getSnapshot() { + return buildQaBusSnapshot({ + cursor, + conversations, + threads, + messages, + events, + }); + }, + addInboundMessage(input: QaBusInboundMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = createMessage({ + direction: "inbound", + accountId, + conversation: input.conversation, + senderId: input.senderId, + senderName: input.senderName, + text: input.text, + timestamp: input.timestamp, + threadId: input.threadId, + threadTitle: input.threadTitle, + replyToId: input.replyToId, + }); + pushEvent({ + kind: "inbound-message", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + addOutboundMessage(input: QaBusOutboundMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const { conversation, threadId } = normalizeConversationFromTarget(input.to); + const message = createMessage({ + direction: "outbound", + accountId, + conversation, + senderId: input.senderId?.trim() || DEFAULT_BOT_ID, + senderName: input.senderName?.trim() || DEFAULT_BOT_NAME, + text: input.text, + timestamp: input.timestamp, + threadId: input.threadId ?? threadId, + replyToId: input.replyToId, + }); + pushEvent({ + kind: "outbound-message", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + createThread(input: QaBusCreateThreadInput) { + const accountId = normalizeAccountId(input.accountId); + const thread: QaBusThread = { + id: `thread-${randomUUID()}`, + accountId, + conversationId: input.conversationId, + title: input.title, + createdAt: input.timestamp ?? Date.now(), + createdBy: input.createdBy?.trim() || DEFAULT_BOT_ID, + }; + threads.set(thread.id, thread); + ensureConversation({ + id: input.conversationId, + kind: "channel", + }); + pushEvent({ + kind: "thread-created", + accountId, + thread: { ...thread }, + }); + return { ...thread }; + }, + reactToMessage(input: QaBusReactToMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + const reaction = { + emoji: input.emoji, + senderId: input.senderId?.trim() || DEFAULT_BOT_ID, + timestamp: input.timestamp ?? Date.now(), + }; + message.reactions.push(reaction); + pushEvent({ + kind: "reaction-added", + accountId, + message: cloneMessage(message), + emoji: reaction.emoji, + senderId: reaction.senderId, + }); + return cloneMessage(message); + }, + editMessage(input: QaBusEditMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + message.text = input.text; + message.editedAt = input.timestamp ?? Date.now(); + pushEvent({ + kind: "message-edited", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + deleteMessage(input: QaBusDeleteMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + message.deleted = true; + pushEvent({ + kind: "message-deleted", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + readMessage(input: QaBusReadMessageInput) { + return readQaBusMessage({ messages, input }); + }, + searchMessages(input: QaBusSearchMessagesInput) { + return searchQaBusMessages({ messages, input }); + }, + poll(input: QaBusPollInput = {}) { + return pollQaBusEvents({ events, cursor, input }); + }, + async waitFor(input: QaBusWaitForInput) { + return await waiters.waitFor(input); + }, + }; +} + +export type QaBusState = ReturnType; diff --git a/src/qa-e2e/bus-waiters.ts b/src/qa-e2e/bus-waiters.ts new file mode 100644 index 00000000000..07c277a8af7 --- /dev/null +++ b/src/qa-e2e/bus-waiters.ts @@ -0,0 +1,87 @@ +import type { + QaBusEvent, + QaBusMessage, + QaBusStateSnapshot, + QaBusThread, + QaBusWaitForInput, +} from "../../extensions/qa-channel/test-api.js"; + +export const DEFAULT_WAIT_TIMEOUT_MS = 5_000; + +export type QaBusWaitMatch = QaBusEvent | QaBusMessage | QaBusThread; + +type Waiter = { + resolve: (event: QaBusWaitMatch) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + matcher: (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null; +}; + +function createQaBusMatcher( + input: QaBusWaitForInput, +): (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null { + return (snapshot) => { + if (input.kind === "event-kind") { + return snapshot.events.find((event) => event.kind === input.eventKind) ?? null; + } + if (input.kind === "thread-id") { + return snapshot.threads.find((thread) => thread.id === input.threadId) ?? null; + } + return ( + snapshot.messages.find( + (message) => + (!input.direction || message.direction === input.direction) && + message.text.includes(input.textIncludes), + ) ?? null + ); + }; +} + +export function createQaBusWaiterStore(getSnapshot: () => QaBusStateSnapshot) { + const waiters = new Set(); + + return { + reset(reason = "qa-bus reset") { + for (const waiter of waiters) { + clearTimeout(waiter.timer); + waiter.reject(new Error(reason)); + } + waiters.clear(); + }, + settle() { + if (waiters.size === 0) { + return; + } + const snapshot = getSnapshot(); + for (const waiter of Array.from(waiters)) { + const match = waiter.matcher(snapshot); + if (!match) { + continue; + } + clearTimeout(waiter.timer); + waiters.delete(waiter); + waiter.resolve(match); + } + }, + async waitFor(input: QaBusWaitForInput) { + const matcher = createQaBusMatcher(input); + const immediate = matcher(getSnapshot()); + if (immediate) { + return immediate; + } + return await new Promise((resolve, reject) => { + const timeoutMs = input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const waiter: Waiter = { + resolve, + reject, + matcher, + timer: setTimeout(() => { + waiters.delete(waiter); + reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`)); + }, timeoutMs), + }; + waiters.add(waiter); + }); + }, + }; +} diff --git a/src/qa-e2e/harness-runtime.ts b/src/qa-e2e/harness-runtime.ts new file mode 100644 index 00000000000..5c59ed59d0b --- /dev/null +++ b/src/qa-e2e/harness-runtime.ts @@ -0,0 +1,75 @@ +import type { PluginRuntime } from "../plugins/runtime/types.js"; + +type SessionRecord = { + sessionKey: string; + body: string; +}; + +export function createQaRunnerRuntime(): PluginRuntime { + const sessions = new Map(); + return { + channel: { + routing: { + resolveAgentRoute({ + accountId, + peer, + }: { + accountId?: string | null; + peer?: { kind?: string; id?: string } | null; + }) { + return { + agentId: "qa-agent", + accountId: accountId ?? "default", + sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`, + mainSessionKey: "qa-agent:main", + lastRoutePolicy: "session", + matchedBy: "default", + channel: "qa-channel", + }; + }, + }, + session: { + resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) { + return agentId; + }, + readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) { + return sessions.has(sessionKey) ? Date.now() : undefined; + }, + recordInboundSession({ + sessionKey, + ctx, + }: { + sessionKey: string; + ctx: { BodyForAgent?: string; Body?: string }; + }) { + sessions.set(sessionKey, { + sessionKey, + body: String(ctx.BodyForAgent ?? ctx.Body ?? ""), + }); + }, + }, + reply: { + resolveEnvelopeFormatOptions() { + return {}; + }, + formatAgentEnvelope({ body }: { body: string }) { + return body; + }, + finalizeInboundContext(ctx: Record) { + return ctx as typeof ctx & { CommandAuthorized: boolean }; + }, + async dispatchReplyWithBufferedBlockDispatcher({ + ctx, + dispatcherOptions, + }: { + ctx: { BodyForAgent?: string; Body?: string }; + dispatcherOptions: { deliver: (payload: { text: string }) => Promise }; + }) { + await dispatcherOptions.deliver({ + text: `qa-echo: ${String(ctx.BodyForAgent ?? ctx.Body ?? "")}`, + }); + }, + }, + }, + } as unknown as PluginRuntime; +} diff --git a/src/qa-e2e/report.ts b/src/qa-e2e/report.ts new file mode 100644 index 00000000000..ff8c254d92d --- /dev/null +++ b/src/qa-e2e/report.ts @@ -0,0 +1,91 @@ +export type QaReportCheck = { + name: string; + status: "pass" | "fail" | "skip"; + details?: string; +}; + +export type QaReportScenario = { + name: string; + status: "pass" | "fail" | "skip"; + details?: string; + steps?: QaReportCheck[]; +}; + +export function renderQaMarkdownReport(params: { + title: string; + startedAt: Date; + finishedAt: Date; + checks?: QaReportCheck[]; + scenarios?: QaReportScenario[]; + timeline?: string[]; + notes?: string[]; +}) { + const checks = params.checks ?? []; + const scenarios = params.scenarios ?? []; + const passCount = + checks.filter((check) => check.status === "pass").length + + scenarios.filter((scenario) => scenario.status === "pass").length; + const failCount = + checks.filter((check) => check.status === "fail").length + + scenarios.filter((scenario) => scenario.status === "fail").length; + + const lines = [ + `# ${params.title}`, + "", + `- Started: ${params.startedAt.toISOString()}`, + `- Finished: ${params.finishedAt.toISOString()}`, + `- Duration ms: ${params.finishedAt.getTime() - params.startedAt.getTime()}`, + `- Passed: ${passCount}`, + `- Failed: ${failCount}`, + "", + ]; + + if (checks.length > 0) { + lines.push("## Checks", ""); + for (const check of checks) { + lines.push(`- [${check.status === "pass" ? "x" : " "}] ${check.name}`); + if (check.details) { + lines.push(` - ${check.details}`); + } + } + } + + if (scenarios.length > 0) { + lines.push("", "## Scenarios", ""); + for (const scenario of scenarios) { + lines.push(`### ${scenario.name}`); + lines.push(""); + lines.push(`- Status: ${scenario.status}`); + if (scenario.details) { + lines.push(`- Details: ${scenario.details}`); + } + if (scenario.steps?.length) { + lines.push("- Steps:"); + for (const step of scenario.steps) { + lines.push(` - [${step.status === "pass" ? "x" : " "}] ${step.name}`); + if (step.details) { + lines.push(` - ${step.details}`); + } + } + } + lines.push(""); + } + } + + if (params.timeline && params.timeline.length > 0) { + lines.push("## Timeline", ""); + for (const item of params.timeline) { + lines.push(`- ${item}`); + } + } + + if (params.notes && params.notes.length > 0) { + lines.push("", "## Notes", ""); + for (const note of params.notes) { + lines.push(`- ${note}`); + } + } + + lines.push(""); + return lines.join("\n"); +} diff --git a/src/qa-e2e/runner.ts b/src/qa-e2e/runner.ts new file mode 100644 index 00000000000..7903579fe79 --- /dev/null +++ b/src/qa-e2e/runner.ts @@ -0,0 +1,124 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { qaChannelPlugin, setQaChannelRuntime } from "../../extensions/qa-channel/api.js"; +import type { OpenClawConfig } from "../config/config.js"; +import { startQaBusServer } from "./bus-server.js"; +import { createQaBusState } from "./bus-state.js"; +import { createQaRunnerRuntime } from "./harness-runtime.js"; +import { renderQaMarkdownReport } from "./report.js"; +import { runQaScenario } from "./scenario.js"; +import { createQaSelfCheckScenario } from "./self-check-scenario.js"; + +export async function runQaE2eSelfCheck(params?: { outputPath?: string }) { + const startedAt = new Date(); + const state = createQaBusState(); + const bus = await startQaBusServer({ state }); + const runtime = createQaRunnerRuntime(); + setQaChannelRuntime(runtime); + + const cfg: OpenClawConfig = { + channels: { + "qa-channel": { + enabled: true, + baseUrl: bus.baseUrl, + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + allowFrom: ["*"], + }, + }, + }; + + const account = qaChannelPlugin.config.resolveAccount(cfg, "default"); + const abort = new AbortController(); + + const task = qaChannelPlugin.gateway?.startAccount?.({ + accountId: account.accountId, + account, + cfg, + runtime: { + log: () => undefined, + error: () => undefined, + exit: () => undefined, + }, + abortSignal: abort.signal, + log: { + info: () => undefined, + warn: () => undefined, + error: () => undefined, + debug: () => undefined, + }, + getStatus: () => ({ + accountId: account.accountId, + configured: true, + enabled: true, + running: true, + }), + setStatus: () => undefined, + }); + + const checks: Array<{ name: string; status: "pass" | "fail"; details?: string }> = []; + let scenarioResult: Awaited> | undefined; + + try { + scenarioResult = await runQaScenario(createQaSelfCheckScenario(cfg), { state }); + checks.push({ + name: "QA self-check scenario", + status: scenarioResult.status, + details: `${scenarioResult.steps.filter((step) => step.status === "pass").length}/${scenarioResult.steps.length} steps passed`, + }); + } catch (error) { + checks.push({ + name: "QA self-check", + status: "fail", + details: error instanceof Error ? error.message : String(error), + }); + } finally { + abort.abort(); + await task; + await bus.stop(); + } + + const finishedAt = new Date(); + const snapshot = state.getSnapshot(); + const timeline = snapshot.events.map((event) => { + switch (event.kind) { + case "thread-created": + return `${event.cursor}. ${event.kind} ${event.thread.conversationId}/${event.thread.id}`; + case "reaction-added": + return `${event.cursor}. ${event.kind} ${event.message.id} ${event.emoji}`; + default: + return `${event.cursor}. ${event.kind} ${"message" in event ? event.message.id : ""}`.trim(); + } + }); + const report = renderQaMarkdownReport({ + title: "OpenClaw QA E2E Self-Check", + startedAt, + finishedAt, + checks, + scenarios: scenarioResult + ? [ + { + name: scenarioResult.name, + status: scenarioResult.status, + details: scenarioResult.details, + steps: scenarioResult.steps, + }, + ] + : undefined, + timeline, + notes: [ + "Vertical slice only: bus + bundled qa-channel + in-process runner runtime.", + "Full Docker orchestration and model/provider matrix remain follow-up work.", + ], + }); + + const outputPath = + params?.outputPath ?? path.join(process.cwd(), ".artifacts", "qa-e2e", "self-check.md"); + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, report, "utf8"); + return { + outputPath, + report, + checks, + }; +} diff --git a/src/qa-e2e/scenario.ts b/src/qa-e2e/scenario.ts new file mode 100644 index 00000000000..6e839875aeb --- /dev/null +++ b/src/qa-e2e/scenario.ts @@ -0,0 +1,65 @@ +import type { QaBusState } from "./bus-state.js"; + +export type QaScenarioStepContext = { + state: QaBusState; +}; + +export type QaScenarioStep = { + name: string; + run: (ctx: QaScenarioStepContext) => Promise; +}; + +export type QaScenarioDefinition = { + name: string; + steps: QaScenarioStep[]; +}; + +export type QaScenarioStepResult = { + name: string; + status: "pass" | "fail"; + details?: string; +}; + +export type QaScenarioResult = { + name: string; + status: "pass" | "fail"; + steps: QaScenarioStepResult[]; + details?: string; +}; + +export async function runQaScenario( + definition: QaScenarioDefinition, + ctx: QaScenarioStepContext, +): Promise { + const steps: QaScenarioStepResult[] = []; + + for (const step of definition.steps) { + try { + const details = await step.run(ctx); + steps.push({ + name: step.name, + status: "pass", + ...(details ? { details } : {}), + }); + } catch (error) { + const details = error instanceof Error ? error.message : String(error); + steps.push({ + name: step.name, + status: "fail", + details, + }); + return { + name: definition.name, + status: "fail", + steps, + details, + }; + } + } + + return { + name: definition.name, + status: "pass", + steps, + }; +} diff --git a/src/qa-e2e/self-check-scenario.ts b/src/qa-e2e/self-check-scenario.ts new file mode 100644 index 00000000000..7aec1f81f60 --- /dev/null +++ b/src/qa-e2e/self-check-scenario.ts @@ -0,0 +1,122 @@ +import { qaChannelPlugin } from "../../extensions/qa-channel/api.js"; +import type { OpenClawConfig } from "../config/config.js"; +import { extractToolPayload } from "../infra/outbound/tool-payload.js"; +import type { QaScenarioDefinition } from "./scenario.js"; + +export function createQaSelfCheckScenario(cfg: OpenClawConfig): QaScenarioDefinition { + return { + name: "Synthetic Slack-class roundtrip", + steps: [ + { + name: "DM echo roundtrip", + async run({ state }) { + state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + senderName: "Alice", + text: "hello from qa", + }); + await state.waitFor({ + kind: "message-text", + textIncludes: "qa-echo: hello from qa", + direction: "outbound", + timeoutMs: 5_000, + }); + }, + }, + { + name: "Thread create and threaded echo", + async run({ state }) { + const threadResult = await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "thread-create", + cfg, + accountId: "default", + params: { + channelId: "qa-room", + title: "QA thread", + }, + }); + const threadPayload = (threadResult ? extractToolPayload(threadResult) : undefined) as + | { thread?: { id?: string } } + | undefined; + const threadId = threadPayload?.thread?.id; + if (!threadId) { + throw new Error("thread-create did not return thread id"); + } + + state.addInboundMessage({ + conversation: { id: "qa-room", kind: "channel", title: "QA Room" }, + senderId: "alice", + senderName: "Alice", + text: "inside thread", + threadId, + threadTitle: "QA thread", + }); + await state.waitFor({ + kind: "message-text", + textIncludes: "qa-echo: inside thread", + direction: "outbound", + timeoutMs: 5_000, + }); + return threadId; + }, + }, + { + name: "Reaction, edit, delete lifecycle", + async run({ state }) { + const outbound = state + .searchMessages({ query: "qa-echo: inside thread", conversationId: "qa-room" }) + .at(-1); + if (!outbound) { + throw new Error("threaded outbound message not found"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "react", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + emoji: "white_check_mark", + }, + }); + const reacted = state.readMessage({ messageId: outbound.id }); + if (reacted.reactions.length === 0) { + throw new Error("reaction not recorded"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "edit", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + text: "qa-echo: inside thread (edited)", + }, + }); + const edited = state.readMessage({ messageId: outbound.id }); + if (!edited.text.includes("(edited)")) { + throw new Error("edit not recorded"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "delete", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + }, + }); + const deleted = state.readMessage({ messageId: outbound.id }); + if (!deleted.deleted) { + throw new Error("delete not recorded"); + } + }, + }, + ], + }; +}