mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:20:43 +00:00
test(qa): add Slack live transport lane
This commit is contained in:
99
.github/workflows/openclaw-release-checks.yml
vendored
99
.github/workflows/openclaw-release-checks.yml
vendored
@@ -248,7 +248,7 @@ jobs:
|
||||
else
|
||||
echo "- Package Acceptance package spec: prepared release artifact"
|
||||
fi
|
||||
echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, and Telegram lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan."
|
||||
echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, Telegram, and Slack lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
prepare_release_package:
|
||||
@@ -822,6 +822,99 @@ jobs:
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
qa_live_slack_release_checks:
|
||||
name: Run QA Lab live Slack lane
|
||||
needs: [resolve_target]
|
||||
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group)
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
timeout-minutes: 60
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
environment: qa-live-shared
|
||||
env:
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1"
|
||||
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.resolve_target.outputs.revision }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
pnpm-version: ${{ env.PNPM_VERSION }}
|
||||
install-bun: "true"
|
||||
|
||||
- name: Validate required QA credential env
|
||||
env:
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
require_var() {
|
||||
local key="$1"
|
||||
if [[ -z "${!key:-}" ]]; then
|
||||
echo "Missing required ${key}." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_var OPENCLAW_QA_CONVEX_SITE_URL
|
||||
require_var OPENCLAW_QA_CONVEX_SECRET_CI
|
||||
|
||||
- name: Build private QA runtime
|
||||
run: pnpm build
|
||||
|
||||
- name: Run Slack live lane
|
||||
id: run_lane
|
||||
shell: bash
|
||||
env:
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
|
||||
OPENCLAW_QA_SLACK_CAPTURE_CONTENT: "1"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
output_dir=".artifacts/qa-e2e/slack-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
for attempt in 1 2; do
|
||||
attempt_output_dir="${output_dir}/attempt-${attempt}"
|
||||
if pnpm openclaw qa slack \
|
||||
--repo-root . \
|
||||
--output-dir "${attempt_output_dir}" \
|
||||
--provider-mode mock-openai \
|
||||
--model mock-openai/gpt-5.5 \
|
||||
--alt-model mock-openai/gpt-5.5-alt \
|
||||
--fast \
|
||||
--credential-source convex \
|
||||
--credential-role ci; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${attempt}" == "2" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "Slack live lane failed on attempt ${attempt}; retrying once..." >&2
|
||||
sleep 10
|
||||
done
|
||||
|
||||
- name: Upload Slack QA artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-qa-live-slack-${{ needs.resolve_target.outputs.revision }}
|
||||
path: .artifacts/qa-e2e/
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
summary:
|
||||
name: Verify release checks
|
||||
needs:
|
||||
@@ -835,6 +928,7 @@ jobs:
|
||||
- qa_lab_parity_report_release_checks
|
||||
- qa_live_matrix_release_checks
|
||||
- qa_live_telegram_release_checks
|
||||
- qa_live_slack_release_checks
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
permissions: {}
|
||||
@@ -855,7 +949,8 @@ jobs:
|
||||
"qa_lab_parity_lane_release_checks=${{ needs.qa_lab_parity_lane_release_checks.result }}" \
|
||||
"qa_lab_parity_report_release_checks=${{ needs.qa_lab_parity_report_release_checks.result }}" \
|
||||
"qa_live_matrix_release_checks=${{ needs.qa_live_matrix_release_checks.result }}" \
|
||||
"qa_live_telegram_release_checks=${{ needs.qa_live_telegram_release_checks.result }}"
|
||||
"qa_live_telegram_release_checks=${{ needs.qa_live_telegram_release_checks.result }}" \
|
||||
"qa_live_slack_release_checks=${{ needs.qa_live_slack_release_checks.result }}"
|
||||
do
|
||||
name="${item%%=*}"
|
||||
result="${item#*=}"
|
||||
|
||||
97
.github/workflows/qa-live-transports-convex.yml
vendored
97
.github/workflows/qa-live-transports-convex.yml
vendored
@@ -18,6 +18,10 @@ on:
|
||||
description: Optional comma-separated Discord scenario ids
|
||||
required: false
|
||||
type: string
|
||||
slack_scenario:
|
||||
description: Optional comma-separated Slack scenario ids
|
||||
required: false
|
||||
type: string
|
||||
matrix_profile:
|
||||
description: Matrix QA profile for the live Matrix lane
|
||||
required: false
|
||||
@@ -554,3 +558,96 @@ jobs:
|
||||
path: ${{ steps.run_lane.outputs.output_dir }}
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
run_live_slack:
|
||||
name: Run Slack live QA lane with Convex leases
|
||||
needs: [authorize_actor, validate_selected_ref]
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
timeout-minutes: 60
|
||||
environment: qa-live-shared
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
pnpm-version: ${{ env.PNPM_VERSION }}
|
||||
install-bun: "true"
|
||||
|
||||
- name: Validate required QA credential env
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
require_var() {
|
||||
local key="$1"
|
||||
if [[ -z "${!key:-}" ]]; then
|
||||
echo "Missing required ${key}." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_var OPENAI_API_KEY
|
||||
require_var OPENCLAW_QA_CONVEX_SITE_URL
|
||||
require_var OPENCLAW_QA_CONVEX_SECRET_CI
|
||||
|
||||
- name: Build private QA runtime
|
||||
run: pnpm build
|
||||
|
||||
- name: Run Slack live lane
|
||||
id: run_lane
|
||||
shell: bash
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
|
||||
OPENCLAW_QA_SLACK_CAPTURE_CONTENT: "1"
|
||||
INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.slack_scenario || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
output_dir=".artifacts/qa-e2e/slack-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
scenario_args=()
|
||||
|
||||
if [[ -n "${INPUT_SCENARIO// }" ]]; then
|
||||
IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}"
|
||||
for raw in "${raw_scenarios[@]}"; do
|
||||
scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
if [[ -n "${scenario}" ]]; then
|
||||
scenario_args+=(--scenario "${scenario}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
pnpm openclaw qa slack \
|
||||
--repo-root . \
|
||||
--output-dir "${output_dir}" \
|
||||
--provider-mode live-frontier \
|
||||
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
|
||||
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
|
||||
--fast \
|
||||
--credential-source convex \
|
||||
--credential-role ci \
|
||||
"${scenario_args[@]}"
|
||||
|
||||
- name: Upload Slack QA artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: qa-live-slack-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ steps.run_lane.outputs.output_dir }}
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Gateway/config: stop Gateway startup and hot reload from auto-restoring invalid config; invalid config now fails closed and `openclaw doctor --fix` owns last-known-good repair.
|
||||
- Gateway/performance: lazy-load early runtime discovery and shutdown-hook helpers, defer maintenance timers until after readiness, and trim duplicate plugin auto-enable work during Gateway startup.
|
||||
- QA/Mantis: add a `pnpm openclaw qa mantis discord-smoke` runner and manual GitHub workflow that verify the Mantis Discord bot can see the configured guild/channel, post a smoke message, add a reaction, and upload artifacts.
|
||||
- QA/Slack: add a Slack live transport QA runner with canary and mention-gating coverage for the private bot-to-bot harness. Thanks @vincentkoc.
|
||||
- Gateway/performance: lazy-load the heavy cron runtime after the rest of Gateway startup, defer restart-sentinel refresh after readiness, and let the Gateway startup benchmark write per-run V8 CPU profiles with `--cpu-prof-dir`.
|
||||
- Gateway/performance: keep raw channel-config schema parsing from discovering bundled plugin runtime metadata, and add `pnpm gateway:watch --benchmark-no-force` for profiling startup without the default port cleanup.
|
||||
- Plugins/onboarding: let Manual setup install optional official plugins, including ClawHub-backed diagnostics with npm fallback, and expose the external Codex plugin as a selectable provider setup choice. Thanks @vincentkoc.
|
||||
|
||||
@@ -47,6 +47,7 @@ script aliases; both forms are supported.
|
||||
| `qa matrix` | Live transport lane against a disposable Tuwunel homeserver. See [Matrix QA](/concepts/qa-matrix). |
|
||||
| `qa telegram` | Live transport lane against a real private Telegram group. |
|
||||
| `qa discord` | Live transport lane against a real private Discord guild channel. |
|
||||
| `qa slack` | Live transport lane against a real private Slack channel. |
|
||||
| `qa mantis` | Before and after verification runner for live transport bugs, with the first Discord status-reactions scenario. See [Mantis](/concepts/mantis). |
|
||||
|
||||
## Operator flow
|
||||
@@ -110,14 +111,15 @@ pnpm openclaw qa matrix --profile fast --fail-fast
|
||||
|
||||
The full CLI reference, profile/scenario catalog, env vars, and artifact layout for this lane live in [Matrix QA](/concepts/qa-matrix). At a glance: it provisions a disposable Tuwunel homeserver in Docker, registers temporary driver/SUT/observer users, runs the real Matrix plugin inside a child QA gateway scoped to that transport (no `qa-channel`), then writes a Markdown report, JSON summary, observed-events artifact, and combined output log under `.artifacts/qa-e2e/matrix-<timestamp>/`.
|
||||
|
||||
For transport-real Telegram and Discord smoke lanes:
|
||||
For transport-real Telegram, Discord, and Slack smoke lanes:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa telegram
|
||||
pnpm openclaw qa discord
|
||||
pnpm openclaw qa slack
|
||||
```
|
||||
|
||||
Both target a pre-existing real channel with two bots (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram and Discord QA reference](#telegram-and-discord-qa-reference) below.
|
||||
They target a pre-existing real channel with two bots (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram, Discord, and Slack QA reference](#telegram-discord-and-slack-qa-reference) below.
|
||||
|
||||
Before using pooled live credentials, run:
|
||||
|
||||
@@ -136,6 +138,7 @@ Live transport lanes share one contract instead of each inventing their own scen
|
||||
| Matrix | x | x | x | x | x | x | x | x | x | | |
|
||||
| Telegram | x | x | x | | | | | | | x | |
|
||||
| Discord | x | x | x | | | | | | | | x |
|
||||
| Slack | x | x | x | | | | | | | | |
|
||||
|
||||
This keeps `qa-channel` as the broad product-behavior suite while Matrix,
|
||||
Telegram, and future live transports share one explicit transport-contract
|
||||
@@ -162,27 +165,27 @@ guest: env-based provider keys, the QA live provider config path, and
|
||||
`CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest
|
||||
can write back through the mounted workspace.
|
||||
|
||||
## Telegram and Discord QA reference
|
||||
## Telegram, Discord, and Slack QA reference
|
||||
|
||||
Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram and Discord are smaller — a handful of scenarios each, no profile system, against pre-existing real channels — so their reference lives here.
|
||||
Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram, Discord, and Slack are smaller — a handful of scenarios each, no profile system, against pre-existing real channels — so their reference lives here.
|
||||
|
||||
### Shared CLI flags
|
||||
|
||||
Both lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags:
|
||||
These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags:
|
||||
|
||||
| Flag | Default | Description |
|
||||
| ------------------------------------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--scenario <id>` | — | Run only this scenario. Repeatable. |
|
||||
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/{telegram,discord}-<timestamp>` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. |
|
||||
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
|
||||
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
|
||||
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
|
||||
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
|
||||
| `--fast` | off | Provider fast mode where supported. |
|
||||
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
|
||||
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
|
||||
| Flag | Default | Description |
|
||||
| ------------------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--scenario <id>` | — | Run only this scenario. Repeatable. |
|
||||
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/{telegram,discord,slack}-<timestamp>` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. |
|
||||
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
|
||||
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
|
||||
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
|
||||
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
|
||||
| `--fast` | off | Provider fast mode where supported. |
|
||||
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
|
||||
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
|
||||
|
||||
Both exit non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code.
|
||||
Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code.
|
||||
|
||||
### Telegram QA
|
||||
|
||||
@@ -264,9 +267,39 @@ Output artifacts:
|
||||
- `discord-qa-observed-messages.json` — bodies redacted unless `OPENCLAW_QA_DISCORD_CAPTURE_CONTENT=1`.
|
||||
- `discord-qa-reaction-timelines.json` and `discord-status-reactions-tool-only-timeline.png` when the status-reaction scenario runs.
|
||||
|
||||
### Slack QA
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa slack
|
||||
```
|
||||
|
||||
Targets one real private Slack channel with two distinct bots: a driver bot controlled by the harness and a SUT bot started by the child OpenClaw gateway through the bundled Slack plugin.
|
||||
|
||||
Required env when `--credential-source env`:
|
||||
|
||||
- `OPENCLAW_QA_SLACK_CHANNEL_ID`
|
||||
- `OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_SLACK_SUT_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_SLACK_SUT_APP_TOKEN`
|
||||
|
||||
Optional:
|
||||
|
||||
- `OPENCLAW_QA_SLACK_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts.
|
||||
|
||||
Scenarios (`extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts:39`):
|
||||
|
||||
- `slack-canary`
|
||||
- `slack-mention-gating`
|
||||
|
||||
Output artifacts:
|
||||
|
||||
- `slack-qa-report.md`
|
||||
- `slack-qa-summary.json`
|
||||
- `slack-qa-observed-messages.json` — bodies redacted unless `OPENCLAW_QA_SLACK_CAPTURE_CONTENT=1`.
|
||||
|
||||
### Convex credential pool
|
||||
|
||||
Both Telegram and Discord lanes can lease credentials from a shared Convex pool instead of reading the env vars above. Pass `--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`); QA Lab acquires an exclusive lease, heartbeats it for the duration of the run, and releases it on shutdown. Pool kinds are `"telegram"` and `"discord"`.
|
||||
Telegram, Discord, and Slack lanes can lease credentials from a shared Convex pool instead of reading the env vars above. Pass `--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`); QA Lab acquires an exclusive lease, heartbeats it for the duration of the run, and releases it on shutdown. Pool kinds are `"telegram"`, `"discord"`, and `"slack"`.
|
||||
|
||||
Payload shapes the broker validates on `admin/add`:
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"devDependencies": {
|
||||
"@openclaw/discord": "workspace:*",
|
||||
"@openclaw/plugin-sdk": "workspace:*",
|
||||
"@openclaw/slack": "workspace:*",
|
||||
"openclaw": "workspace:*"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import { discordQaCliRegistration } from "./discord/cli.js";
|
||||
import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
|
||||
import { slackQaCliRegistration } from "./slack/cli.js";
|
||||
import { telegramQaCliRegistration } from "./telegram/cli.js";
|
||||
|
||||
function createBlockedQaRunnerCliRegistration(params: {
|
||||
@@ -38,6 +39,7 @@ function createQaRunnerCliRegistration(
|
||||
const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
|
||||
telegramQaCliRegistration,
|
||||
discordQaCliRegistration,
|
||||
slackQaCliRegistration,
|
||||
];
|
||||
|
||||
export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
|
||||
|
||||
23
extensions/qa-lab/src/live-transports/slack/cli.runtime.ts
Normal file
23
extensions/qa-lab/src/live-transports/slack/cli.runtime.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import {
|
||||
printLiveTransportQaArtifacts,
|
||||
resolveLiveTransportQaRunOptions,
|
||||
} from "../shared/live-transport-cli.runtime.js";
|
||||
import { runSlackQaLive } from "./slack-live.runtime.js";
|
||||
|
||||
export async function runQaSlackCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
const result = await runSlackQaLive(runOptions);
|
||||
printLiveTransportQaArtifacts("Slack QA", {
|
||||
report: result.reportPath,
|
||||
summary: result.summaryPath,
|
||||
"observed messages": result.observedMessagesPath,
|
||||
...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}),
|
||||
});
|
||||
if (
|
||||
!runOptions.allowFailures &&
|
||||
result.scenarios.some((scenario) => scenario.status === "fail")
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
32
extensions/qa-lab/src/live-transports/slack/cli.ts
Normal file
32
extensions/qa-lab/src/live-transports/slack/cli.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
createLiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCommandOptions,
|
||||
} from "../shared/live-transport-cli.js";
|
||||
|
||||
type SlackQaCliRuntime = typeof import("./cli.runtime.js");
|
||||
|
||||
const loadSlackQaCliRuntime = createLazyCliRuntimeLoader<SlackQaCliRuntime>(
|
||||
() => import("./cli.runtime.js"),
|
||||
);
|
||||
|
||||
async function runQaSlack(opts: LiveTransportQaCommandOptions) {
|
||||
const runtime = await loadSlackQaCliRuntime();
|
||||
await runtime.runQaSlackCommand(opts);
|
||||
}
|
||||
|
||||
export const slackQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
createLiveTransportQaCliRegistration({
|
||||
commandName: "slack",
|
||||
credentialOptions: {
|
||||
sourceDescription: "Credential source for Slack QA: env or convex (default: env)",
|
||||
roleDescription:
|
||||
"Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
|
||||
},
|
||||
description: "Run the Slack live QA lane against a private bot-to-bot channel harness",
|
||||
outputDirHelp: "Slack QA artifact directory",
|
||||
scenarioHelp: "Run only the named Slack QA scenario (repeatable)",
|
||||
sutAccountHelp: "Temporary Slack account id inside the QA gateway config",
|
||||
run: runQaSlack,
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { __testing } from "./slack-live.runtime.js";
|
||||
|
||||
describe("Slack live QA runtime helpers", () => {
|
||||
it("resolves env credential payloads", () => {
|
||||
expect(
|
||||
__testing.resolveSlackQaRuntimeEnv({
|
||||
OPENCLAW_QA_SLACK_CHANNEL_ID: "C123456789",
|
||||
OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN: "xoxb-driver",
|
||||
OPENCLAW_QA_SLACK_SUT_BOT_TOKEN: "xoxb-sut",
|
||||
OPENCLAW_QA_SLACK_SUT_APP_TOKEN: "xapp-sut",
|
||||
}),
|
||||
).toEqual({
|
||||
channelId: "C123456789",
|
||||
driverBotToken: "xoxb-driver",
|
||||
sutBotToken: "xoxb-sut",
|
||||
sutAppToken: "xapp-sut",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed Slack channel ids", () => {
|
||||
expect(() =>
|
||||
__testing.resolveSlackQaRuntimeEnv({
|
||||
OPENCLAW_QA_SLACK_CHANNEL_ID: "qa-channel",
|
||||
OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN: "xoxb-driver",
|
||||
OPENCLAW_QA_SLACK_SUT_BOT_TOKEN: "xoxb-sut",
|
||||
OPENCLAW_QA_SLACK_SUT_APP_TOKEN: "xapp-sut",
|
||||
}),
|
||||
).toThrow("OPENCLAW_QA_SLACK channelId must be a Slack id like C123 or U123.");
|
||||
});
|
||||
|
||||
it("parses Convex credential payloads", () => {
|
||||
expect(
|
||||
__testing.parseSlackQaCredentialPayload({
|
||||
channelId: "C123456789",
|
||||
driverBotToken: "xoxb-driver",
|
||||
sutBotToken: "xoxb-sut",
|
||||
sutAppToken: "xapp-sut",
|
||||
}),
|
||||
).toEqual({
|
||||
channelId: "C123456789",
|
||||
driverBotToken: "xoxb-driver",
|
||||
sutBotToken: "xoxb-sut",
|
||||
sutAppToken: "xapp-sut",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports standard live transport scenario coverage", () => {
|
||||
expect(__testing.SLACK_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]);
|
||||
});
|
||||
|
||||
it("selects Slack scenarios by id", () => {
|
||||
expect(__testing.findScenario(["slack-canary"]).map((scenario) => scenario.id)).toEqual([
|
||||
"slack-canary",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,819 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { createSlackWebClient, createSlackWriteClient } from "@openclaw/slack/api.js";
|
||||
import type { WebClient } from "@slack/web-api";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { z } from "zod";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "../../run-config.js";
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js";
|
||||
import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js";
|
||||
import {
|
||||
collectLiveTransportStandardScenarioCoverage,
|
||||
selectLiveTransportScenarios,
|
||||
type LiveTransportScenarioDefinition,
|
||||
} from "../shared/live-transport-scenarios.js";
|
||||
|
||||
type SlackQaRuntimeEnv = {
|
||||
channelId: string;
|
||||
driverBotToken: string;
|
||||
sutBotToken: string;
|
||||
sutAppToken: string;
|
||||
};
|
||||
|
||||
type SlackQaScenarioId = "slack-canary" | "slack-mention-gating";
|
||||
|
||||
type SlackQaScenarioRun = {
|
||||
expectReply: boolean;
|
||||
input: string;
|
||||
matchText: string;
|
||||
};
|
||||
|
||||
type SlackQaScenarioDefinition = LiveTransportScenarioDefinition<SlackQaScenarioId> & {
|
||||
buildRun: (sutUserId: string) => SlackQaScenarioRun;
|
||||
};
|
||||
|
||||
type SlackAuthIdentity = {
|
||||
botId?: string;
|
||||
teamId?: string;
|
||||
userId: string;
|
||||
};
|
||||
|
||||
type SlackMessage = {
|
||||
bot_id?: string;
|
||||
text?: string;
|
||||
thread_ts?: string;
|
||||
ts?: string;
|
||||
user?: string;
|
||||
};
|
||||
|
||||
type SlackObservedMessage = {
|
||||
botId?: string;
|
||||
channelId: string;
|
||||
matchedScenario?: boolean;
|
||||
scenarioId?: string;
|
||||
scenarioTitle?: string;
|
||||
text: string;
|
||||
threadTs?: string;
|
||||
ts: string;
|
||||
userId?: string;
|
||||
};
|
||||
|
||||
type SlackObservedMessageArtifact = {
|
||||
botId?: string;
|
||||
channelId?: string;
|
||||
matchedScenario?: boolean;
|
||||
scenarioId?: string;
|
||||
scenarioTitle?: string;
|
||||
text?: string;
|
||||
threadTs?: string;
|
||||
ts?: string;
|
||||
userId?: string;
|
||||
};
|
||||
|
||||
type SlackQaScenarioResult = {
|
||||
details: string;
|
||||
id: string;
|
||||
requestStartedAt?: string;
|
||||
responseObservedAt?: string;
|
||||
rttMs?: number;
|
||||
status: "fail" | "pass";
|
||||
title: string;
|
||||
};
|
||||
|
||||
export type SlackQaRunResult = {
|
||||
gatewayDebugDirPath?: string;
|
||||
observedMessagesPath: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
scenarios: SlackQaScenarioResult[];
|
||||
summaryPath: string;
|
||||
};
|
||||
|
||||
type SlackQaSummary = {
|
||||
channelId: string;
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
failed: number;
|
||||
passed: number;
|
||||
total: number;
|
||||
};
|
||||
credentials: {
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
finishedAt: string;
|
||||
scenarios: SlackQaScenarioResult[];
|
||||
startedAt: string;
|
||||
};
|
||||
|
||||
const SLACK_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_SLACK_CAPTURE_CONTENT";
|
||||
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
|
||||
const SLACK_QA_ENV_KEYS = [
|
||||
"OPENCLAW_QA_SLACK_CHANNEL_ID",
|
||||
"OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN",
|
||||
"OPENCLAW_QA_SLACK_SUT_BOT_TOKEN",
|
||||
"OPENCLAW_QA_SLACK_SUT_APP_TOKEN",
|
||||
] as const;
|
||||
|
||||
const slackQaCredentialPayloadSchema = z.object({
|
||||
channelId: z.string().trim().min(1),
|
||||
driverBotToken: z.string().trim().min(1),
|
||||
sutBotToken: z.string().trim().min(1),
|
||||
sutAppToken: z.string().trim().min(1),
|
||||
});
|
||||
|
||||
const slackAuthTestSchema = z.object({
|
||||
ok: z.boolean().optional(),
|
||||
user_id: z.string().optional(),
|
||||
bot_id: z.string().optional(),
|
||||
team_id: z.string().optional(),
|
||||
});
|
||||
|
||||
const slackPostMessageSchema = z.object({
|
||||
ok: z.boolean().optional(),
|
||||
channel: z.string().optional(),
|
||||
ts: z.string().min(1),
|
||||
});
|
||||
|
||||
const slackHistoryMessageSchema = z.object({
|
||||
bot_id: z.string().optional(),
|
||||
text: z.string().optional(),
|
||||
thread_ts: z.string().optional(),
|
||||
ts: z.string().min(1),
|
||||
user: z.string().optional(),
|
||||
});
|
||||
|
||||
const slackHistorySchema = z.object({
|
||||
ok: z.boolean().optional(),
|
||||
messages: z.array(slackHistoryMessageSchema).optional(),
|
||||
});
|
||||
|
||||
const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [
|
||||
{
|
||||
id: "slack-canary",
|
||||
standardId: "canary",
|
||||
title: "Slack canary echo",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUserId) => {
|
||||
const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
expectReply: true,
|
||||
input: `<@${sutUserId}> reply with only this exact marker: ${token}`,
|
||||
matchText: token,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "slack-mention-gating",
|
||||
standardId: "mention-gating",
|
||||
title: "Slack unmentioned bot message does not trigger",
|
||||
timeoutMs: 8_000,
|
||||
buildRun: () => {
|
||||
const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
expectReply: false,
|
||||
input: `reply with only this exact marker: ${token}`,
|
||||
matchText: token,
|
||||
};
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const SLACK_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
scenarios: SLACK_QA_SCENARIOS,
|
||||
});
|
||||
|
||||
function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof SLACK_QA_ENV_KEYS)[number]) {
|
||||
const value = env[key]?.trim();
|
||||
if (!value) {
|
||||
throw new Error(`Missing ${key}.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function isTruthyOptIn(value: string | undefined) {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true" || normalized === "yes";
|
||||
}
|
||||
|
||||
function normalizeSlackId(value: string, label: string) {
|
||||
const normalized = value.trim();
|
||||
if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) {
|
||||
throw new Error(`${label} must be a Slack id like C123 or U123.`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function validateSlackQaRuntimeEnv(runtimeEnv: SlackQaRuntimeEnv, label: string) {
|
||||
normalizeSlackId(runtimeEnv.channelId, `${label} channelId`);
|
||||
return runtimeEnv;
|
||||
}
|
||||
|
||||
function resolveSlackQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): SlackQaRuntimeEnv {
|
||||
const runtimeEnv = {
|
||||
channelId: resolveEnvValue(env, "OPENCLAW_QA_SLACK_CHANNEL_ID"),
|
||||
driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN"),
|
||||
sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN"),
|
||||
sutAppToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_APP_TOKEN"),
|
||||
};
|
||||
return validateSlackQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_SLACK");
|
||||
}
|
||||
|
||||
function parseSlackQaCredentialPayload(payload: unknown): SlackQaRuntimeEnv {
|
||||
const parsed = slackQaCredentialPayloadSchema.parse(payload);
|
||||
const runtimeEnv = {
|
||||
channelId: parsed.channelId,
|
||||
driverBotToken: parsed.driverBotToken,
|
||||
sutBotToken: parsed.sutBotToken,
|
||||
sutAppToken: parsed.sutAppToken,
|
||||
};
|
||||
return validateSlackQaRuntimeEnv(runtimeEnv, "Slack credential payload");
|
||||
}
|
||||
|
||||
function findScenario(ids?: string[]) {
|
||||
return selectLiveTransportScenarios({
|
||||
ids,
|
||||
laneLabel: "Slack",
|
||||
scenarios: SLACK_QA_SCENARIOS,
|
||||
});
|
||||
}
|
||||
|
||||
function buildSlackQaConfig(
|
||||
baseCfg: OpenClawConfig,
|
||||
params: {
|
||||
channelId: string;
|
||||
driverBotUserId: string;
|
||||
sutAccountId: string;
|
||||
sutAppToken: string;
|
||||
sutBotToken: string;
|
||||
},
|
||||
): OpenClawConfig {
|
||||
const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "slack"])];
|
||||
return {
|
||||
...baseCfg,
|
||||
plugins: {
|
||||
...baseCfg.plugins,
|
||||
allow: pluginAllow,
|
||||
entries: {
|
||||
...baseCfg.plugins?.entries,
|
||||
slack: { enabled: true },
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
...baseCfg.messages,
|
||||
groupChat: {
|
||||
...baseCfg.messages?.groupChat,
|
||||
visibleReplies: "automatic",
|
||||
},
|
||||
},
|
||||
channels: {
|
||||
...baseCfg.channels,
|
||||
slack: {
|
||||
enabled: true,
|
||||
defaultAccount: params.sutAccountId,
|
||||
accounts: {
|
||||
[params.sutAccountId]: {
|
||||
enabled: true,
|
||||
mode: "socket",
|
||||
botToken: params.sutBotToken,
|
||||
appToken: params.sutAppToken,
|
||||
groupPolicy: "allowlist",
|
||||
allowBots: true,
|
||||
channels: {
|
||||
[params.channelId]: {
|
||||
enabled: true,
|
||||
requireMention: true,
|
||||
allowBots: true,
|
||||
users: [params.driverBotUserId],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getSlackIdentity(token: string): Promise<SlackAuthIdentity> {
|
||||
const client = createSlackWebClient(token, { timeout: 15_000 });
|
||||
const auth = slackAuthTestSchema.parse(await client.auth.test());
|
||||
if (!auth.user_id) {
|
||||
throw new Error("Slack auth.test did not return user_id.");
|
||||
}
|
||||
return {
|
||||
userId: auth.user_id,
|
||||
botId: auth.bot_id,
|
||||
teamId: auth.team_id,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendSlackChannelMessage(params: {
|
||||
channelId: string;
|
||||
client: WebClient;
|
||||
text: string;
|
||||
}) {
|
||||
const sendSlackMessage = params.client.chat.postMessage.bind(params.client.chat);
|
||||
const sent = slackPostMessageSchema.parse(
|
||||
await sendSlackMessage({
|
||||
channel: params.channelId,
|
||||
text: params.text,
|
||||
unfurl_links: false,
|
||||
unfurl_media: false,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
channelId: sent.channel ?? params.channelId,
|
||||
ts: sent.ts,
|
||||
};
|
||||
}
|
||||
|
||||
async function listSlackMessages(params: {
|
||||
channelId: string;
|
||||
client: WebClient;
|
||||
oldestTs: string;
|
||||
}) {
|
||||
const history = slackHistorySchema.parse(
|
||||
await params.client.conversations.history({
|
||||
channel: params.channelId,
|
||||
inclusive: true,
|
||||
limit: 50,
|
||||
oldest: params.oldestTs,
|
||||
}),
|
||||
);
|
||||
return history.messages ?? [];
|
||||
}
|
||||
|
||||
function isSutSlackMessage(message: SlackMessage, sutIdentity: SlackAuthIdentity) {
|
||||
return (
|
||||
(message.user !== undefined && message.user === sutIdentity.userId) ||
|
||||
(message.bot_id !== undefined && message.bot_id === sutIdentity.botId)
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForSlackScenarioReply(params: {
|
||||
channelId: string;
|
||||
client: WebClient;
|
||||
matchText: string;
|
||||
observedMessages: SlackObservedMessage[];
|
||||
observationScenarioId: string;
|
||||
observationScenarioTitle: string;
|
||||
sentTs: string;
|
||||
sutIdentity: SlackAuthIdentity;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < params.timeoutMs) {
|
||||
const messages = await listSlackMessages({
|
||||
channelId: params.channelId,
|
||||
client: params.client,
|
||||
oldestTs: params.sentTs,
|
||||
});
|
||||
for (const message of messages) {
|
||||
const text = message.text ?? "";
|
||||
if (
|
||||
!message.ts ||
|
||||
message.ts === params.sentTs ||
|
||||
!isSutSlackMessage(message, params.sutIdentity)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const matchedScenario = text.includes(params.matchText);
|
||||
params.observedMessages.push({
|
||||
botId: message.bot_id,
|
||||
channelId: params.channelId,
|
||||
matchedScenario,
|
||||
scenarioId: params.observationScenarioId,
|
||||
scenarioTitle: params.observationScenarioTitle,
|
||||
text,
|
||||
threadTs: message.thread_ts,
|
||||
ts: message.ts,
|
||||
userId: message.user,
|
||||
});
|
||||
if (matchedScenario) {
|
||||
return {
|
||||
message,
|
||||
observedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack message`);
|
||||
}
|
||||
|
||||
async function waitForSlackNoReply(params: {
|
||||
channelId: string;
|
||||
client: WebClient;
|
||||
matchText: string;
|
||||
observedMessages: SlackObservedMessage[];
|
||||
observationScenarioId: string;
|
||||
observationScenarioTitle: string;
|
||||
sentTs: string;
|
||||
sutIdentity: SlackAuthIdentity;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
try {
|
||||
await waitForSlackScenarioReply(params);
|
||||
} catch (error) {
|
||||
const message = formatErrorMessage(error);
|
||||
if (message === `timed out after ${params.timeoutMs}ms waiting for Slack message`) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
throw new Error("unexpected Slack SUT reply observed");
|
||||
}
|
||||
|
||||
async function waitForSlackChannelRunning(
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
|
||||
accountId: string,
|
||||
) {
|
||||
const startedAt = Date.now();
|
||||
let lastStatus:
|
||||
| {
|
||||
connected?: boolean;
|
||||
lastConnectedAt?: number;
|
||||
lastDisconnect?: unknown;
|
||||
lastError?: string;
|
||||
restartPending?: boolean;
|
||||
running?: boolean;
|
||||
}
|
||||
| undefined;
|
||||
while (Date.now() - startedAt < 45_000) {
|
||||
try {
|
||||
const payload = (await gateway.call(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
)) as {
|
||||
channelAccounts?: Record<
|
||||
string,
|
||||
Array<{
|
||||
accountId?: string;
|
||||
connected?: boolean;
|
||||
lastConnectedAt?: number;
|
||||
lastDisconnect?: unknown;
|
||||
lastError?: string;
|
||||
restartPending?: boolean;
|
||||
running?: boolean;
|
||||
}>
|
||||
>;
|
||||
};
|
||||
const accounts = payload.channelAccounts?.slack ?? [];
|
||||
const match = accounts.find((entry) => entry.accountId === accountId);
|
||||
lastStatus = match
|
||||
? {
|
||||
connected: match.connected,
|
||||
lastConnectedAt: match.lastConnectedAt,
|
||||
lastDisconnect: match.lastDisconnect,
|
||||
lastError: match.lastError,
|
||||
restartPending: match.restartPending,
|
||||
running: match.running,
|
||||
}
|
||||
: undefined;
|
||||
if (match?.running && match.connected === true && match.restartPending !== true) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// retry
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
throw new Error(
|
||||
`slack account "${accountId}" did not become ready` +
|
||||
(lastStatus ? `; last status: ${JSON.stringify(lastStatus)}` : ""),
|
||||
);
|
||||
}
|
||||
|
||||
function toObservedSlackArtifacts(params: {
|
||||
includeContent: boolean;
|
||||
messages: SlackObservedMessage[];
|
||||
redactMetadata: boolean;
|
||||
}): SlackObservedMessageArtifact[] {
|
||||
return params.messages.map((message) => ({
|
||||
botId: params.redactMetadata ? undefined : message.botId,
|
||||
channelId: params.redactMetadata ? undefined : message.channelId,
|
||||
matchedScenario: message.matchedScenario,
|
||||
scenarioId: message.scenarioId,
|
||||
scenarioTitle: message.scenarioTitle,
|
||||
text: params.includeContent ? message.text : undefined,
|
||||
threadTs: params.redactMetadata ? undefined : message.threadTs,
|
||||
ts: params.redactMetadata ? undefined : message.ts,
|
||||
userId: params.redactMetadata ? undefined : message.userId,
|
||||
}));
|
||||
}
|
||||
|
||||
function renderSlackQaMarkdown(params: {
|
||||
channelId: string;
|
||||
cleanupIssues: string[];
|
||||
credentialSource: "convex" | "env";
|
||||
finishedAt: string;
|
||||
gatewayDebugDirPath?: string;
|
||||
redactMetadata: boolean;
|
||||
scenarios: SlackQaScenarioResult[];
|
||||
startedAt: string;
|
||||
}) {
|
||||
const lines = [
|
||||
"# Slack QA Report",
|
||||
"",
|
||||
`- Credential source: \`${params.credentialSource}\``,
|
||||
`- Channel: \`${params.redactMetadata ? "<redacted>" : params.channelId}\``,
|
||||
`- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``,
|
||||
`- Started: ${params.startedAt}`,
|
||||
`- Finished: ${params.finishedAt}`,
|
||||
];
|
||||
if (params.gatewayDebugDirPath) {
|
||||
lines.push(`- Gateway debug artifacts: \`${params.gatewayDebugDirPath}\``);
|
||||
}
|
||||
if (params.cleanupIssues.length > 0) {
|
||||
lines.push("", "## Cleanup issues", "");
|
||||
for (const issue of params.cleanupIssues) {
|
||||
lines.push(`- ${issue}`);
|
||||
}
|
||||
}
|
||||
lines.push("", "## Scenarios", "");
|
||||
for (const scenario of params.scenarios) {
|
||||
lines.push(`### ${scenario.title}`, "");
|
||||
lines.push(`- Status: ${scenario.status}`);
|
||||
lines.push(`- Details: ${scenario.details}`);
|
||||
if (scenario.rttMs !== undefined) {
|
||||
lines.push(`- RTT: ${scenario.rttMs}ms`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export async function runSlackQaLive(params: {
|
||||
alternateModel?: string;
|
||||
credentialRole?: string;
|
||||
credentialSource?: string;
|
||||
fastMode?: boolean;
|
||||
outputDir?: string;
|
||||
primaryModel?: string;
|
||||
providerMode?: QaProviderModeInput;
|
||||
repoRoot?: string;
|
||||
scenarioIds?: string[];
|
||||
sutAccountId?: string;
|
||||
}): Promise<SlackQaRunResult> {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
const outputDir =
|
||||
params.outputDir ??
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", `slack-${Date.now().toString(36)}`);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
const scenarios = findScenario(params.scenarioIds);
|
||||
|
||||
const credentialLease = await acquireQaCredentialLease({
|
||||
kind: "slack",
|
||||
source: params.credentialSource,
|
||||
role: params.credentialRole,
|
||||
resolveEnvPayload: () => resolveSlackQaRuntimeEnv(),
|
||||
parsePayload: parseSlackQaCredentialPayload,
|
||||
});
|
||||
const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease);
|
||||
const assertLeaseHealthy = () => {
|
||||
leaseHeartbeat.throwIfFailed();
|
||||
};
|
||||
|
||||
const runtimeEnv = credentialLease.payload;
|
||||
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
|
||||
const includeObservedMessageContent = isTruthyOptIn(process.env[SLACK_QA_CAPTURE_CONTENT_ENV]);
|
||||
const startedAt = new Date().toISOString();
|
||||
const observedMessages: SlackObservedMessage[] = [];
|
||||
const scenarioResults: SlackQaScenarioResult[] = [];
|
||||
const cleanupIssues: string[] = [];
|
||||
const gatewayDebugDirPath = path.join(outputDir, "gateway-debug");
|
||||
let preservedGatewayDebugArtifacts = false;
|
||||
|
||||
try {
|
||||
const [driverIdentity, sutIdentity] = await Promise.all([
|
||||
getSlackIdentity(runtimeEnv.driverBotToken),
|
||||
getSlackIdentity(runtimeEnv.sutBotToken),
|
||||
]);
|
||||
if (driverIdentity.userId === sutIdentity.userId) {
|
||||
throw new Error("Slack QA requires two distinct bots for driver and SUT.");
|
||||
}
|
||||
|
||||
const driverClient = createSlackWriteClient(runtimeEnv.driverBotToken, { timeout: 15_000 });
|
||||
const sutReadClient = createSlackWebClient(runtimeEnv.sutBotToken, { timeout: 15_000 });
|
||||
const gatewayHarness = await startQaLiveLaneGateway({
|
||||
repoRoot,
|
||||
transport: {
|
||||
requiredPluginIds: [],
|
||||
createGatewayConfig: () => ({}),
|
||||
},
|
||||
transportBaseUrl: "http://127.0.0.1:0",
|
||||
providerMode,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: params.fastMode,
|
||||
controlUiEnabled: false,
|
||||
mutateConfig: (cfg) =>
|
||||
buildSlackQaConfig(cfg, {
|
||||
channelId: runtimeEnv.channelId,
|
||||
driverBotUserId: driverIdentity.userId,
|
||||
sutAccountId,
|
||||
sutAppToken: runtimeEnv.sutAppToken,
|
||||
sutBotToken: runtimeEnv.sutBotToken,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await waitForSlackChannelRunning(gatewayHarness.gateway, sutAccountId);
|
||||
assertLeaseHealthy();
|
||||
for (const scenario of scenarios) {
|
||||
assertLeaseHealthy();
|
||||
const scenarioRun = scenario.buildRun(sutIdentity.userId);
|
||||
const requestStartedAt = new Date();
|
||||
try {
|
||||
const sent = await sendSlackChannelMessage({
|
||||
channelId: runtimeEnv.channelId,
|
||||
client: driverClient,
|
||||
text: scenarioRun.input,
|
||||
});
|
||||
if (scenarioRun.expectReply) {
|
||||
const reply = await waitForSlackScenarioReply({
|
||||
channelId: runtimeEnv.channelId,
|
||||
client: sutReadClient,
|
||||
matchText: scenarioRun.matchText,
|
||||
observedMessages,
|
||||
observationScenarioId: scenario.id,
|
||||
observationScenarioTitle: scenario.title,
|
||||
sentTs: sent.ts,
|
||||
sutIdentity,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
const responseObservedAt = new Date(reply.observedAt);
|
||||
const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime();
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: `reply matched in ${rttMs}ms`,
|
||||
rttMs,
|
||||
requestStartedAt: requestStartedAt.toISOString(),
|
||||
responseObservedAt: responseObservedAt.toISOString(),
|
||||
});
|
||||
} else {
|
||||
await waitForSlackNoReply({
|
||||
channelId: runtimeEnv.channelId,
|
||||
client: sutReadClient,
|
||||
matchText: scenarioRun.matchText,
|
||||
observedMessages,
|
||||
observationScenarioId: scenario.id,
|
||||
observationScenarioTitle: scenario.title,
|
||||
sentTs: sent.ts,
|
||||
sutIdentity,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: "no reply",
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const result = {
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "fail" as const,
|
||||
details: formatErrorMessage(error),
|
||||
};
|
||||
scenarioResults.push(result);
|
||||
preservedGatewayDebugArtifacts = true;
|
||||
await gatewayHarness.gateway
|
||||
.stop({ keepTemp: true, preserveToDir: gatewayDebugDirPath })
|
||||
.catch((stopError) => {
|
||||
appendLiveLaneIssue(cleanupIssues, "gateway debug preservation failed", stopError);
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (!preservedGatewayDebugArtifacts) {
|
||||
await gatewayHarness.stop().catch((error) => {
|
||||
appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error);
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
cleanupIssues.push(
|
||||
buildLiveLaneArtifactsError({
|
||||
heading: "Slack QA failed before scenario completion.",
|
||||
details: [formatErrorMessage(error)],
|
||||
artifacts: {
|
||||
gatewayDebug: gatewayDebugDirPath,
|
||||
},
|
||||
}),
|
||||
);
|
||||
preservedGatewayDebugArtifacts = true;
|
||||
await fs.mkdir(gatewayDebugDirPath, { recursive: true }).catch(() => {});
|
||||
scenarioResults.push({
|
||||
id: "slack-canary",
|
||||
title: "Slack canary echo",
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
} finally {
|
||||
try {
|
||||
await leaseHeartbeat.stop();
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupIssues, "credential heartbeat stop failed", error);
|
||||
}
|
||||
try {
|
||||
await credentialLease.release();
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupIssues, "credential release failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
const finishedAt = new Date().toISOString();
|
||||
const reportPath = path.join(outputDir, "slack-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "slack-qa-summary.json");
|
||||
const observedMessagesPath = path.join(outputDir, "slack-qa-observed-messages.json");
|
||||
const passed = scenarioResults.filter((entry) => entry.status === "pass").length;
|
||||
const failed = scenarioResults.filter((entry) => entry.status === "fail").length;
|
||||
const summary: SlackQaSummary = {
|
||||
credentials: {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
},
|
||||
channelId: redactPublicMetadata ? "<redacted>" : runtimeEnv.channelId,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed,
|
||||
failed,
|
||||
},
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
await fs.writeFile(
|
||||
observedMessagesPath,
|
||||
`${JSON.stringify(
|
||||
toObservedSlackArtifacts({
|
||||
messages: observedMessages,
|
||||
includeContent: includeObservedMessageContent,
|
||||
redactMetadata: redactPublicMetadata,
|
||||
}),
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderSlackQaMarkdown({
|
||||
channelId: runtimeEnv.channelId,
|
||||
cleanupIssues,
|
||||
credentialSource: credentialLease.source,
|
||||
finishedAt,
|
||||
gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined,
|
||||
redactMetadata: redactPublicMetadata,
|
||||
scenarios: scenarioResults,
|
||||
startedAt,
|
||||
})}\n`,
|
||||
);
|
||||
return {
|
||||
outputDir,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
observedMessagesPath,
|
||||
gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined,
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
findScenario,
|
||||
parseSlackQaCredentialPayload,
|
||||
resolveSlackQaRuntimeEnv,
|
||||
SLACK_QA_STANDARD_SCENARIO_IDS,
|
||||
};
|
||||
@@ -37,6 +37,7 @@
|
||||
"openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"],
|
||||
"@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"],
|
||||
"@openclaw/discord/api.js": ["../dist/plugin-sdk/extensions/discord/api.d.ts"],
|
||||
"@openclaw/slack/api.js": ["../dist/plugin-sdk/extensions/slack/api.d.ts"],
|
||||
"@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"],
|
||||
"@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"],
|
||||
"@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"]
|
||||
|
||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
@@ -1195,6 +1195,9 @@ importers:
|
||||
'@openclaw/plugin-sdk':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/plugin-sdk
|
||||
'@openclaw/slack':
|
||||
specifier: workspace:*
|
||||
version: link:../slack
|
||||
openclaw:
|
||||
specifier: workspace:*
|
||||
version: link:../..
|
||||
|
||||
@@ -47,6 +47,7 @@ export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = {
|
||||
"openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"],
|
||||
"@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"],
|
||||
"@openclaw/discord/api.js": ["../dist/plugin-sdk/extensions/discord/api.d.ts"],
|
||||
"@openclaw/slack/api.js": ["../dist/plugin-sdk/extensions/slack/api.d.ts"],
|
||||
"@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"],
|
||||
"@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"],
|
||||
"@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"],
|
||||
@@ -70,6 +71,7 @@ export const EXTENSION_PACKAGE_BOUNDARY_XAI_PATHS = {
|
||||
"openclaw/plugin-sdk/channel-secret-basic-runtime": _omitBasic,
|
||||
"openclaw/plugin-sdk/channel-secret-tts-runtime": _omitTts,
|
||||
"@openclaw/discord/api.js": _omitDiscord,
|
||||
"@openclaw/slack/api.js": _omitSlack,
|
||||
...rest
|
||||
}) => rest)(EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS),
|
||||
"../",
|
||||
|
||||
@@ -52,6 +52,13 @@ const DISCORD_DTS_INPUTS = [
|
||||
];
|
||||
const DISCORD_DTS_STAMP = "dist/plugin-sdk/extensions/discord/.boundary-dts.stamp";
|
||||
const DISCORD_DTS_REQUIRED_OUTPUTS = ["dist/plugin-sdk/extensions/discord/api.d.ts"];
|
||||
const SLACK_DTS_INPUTS = [
|
||||
"extensions/slack/api.ts",
|
||||
"extensions/slack/src/client.ts",
|
||||
"extensions/slack/tsconfig.json",
|
||||
];
|
||||
const SLACK_DTS_STAMP = "dist/plugin-sdk/extensions/slack/.boundary-dts.stamp";
|
||||
const SLACK_DTS_REQUIRED_OUTPUTS = ["dist/plugin-sdk/extensions/slack/api.d.ts"];
|
||||
const ENTRY_SHIMS_INPUTS = [
|
||||
"scripts/write-plugin-sdk-entry-dts.ts",
|
||||
"scripts/lib/plugin-sdk-entrypoints.json",
|
||||
@@ -303,6 +310,12 @@ async function main(argv = process.argv.slice(2)) {
|
||||
outputPaths: [DISCORD_DTS_STAMP, ...DISCORD_DTS_REQUIRED_OUTPUTS],
|
||||
includeFile: isRelevantTypeInput,
|
||||
}) && !hasMissingOutput(DISCORD_DTS_REQUIRED_OUTPUTS);
|
||||
const slackDtsFresh =
|
||||
isArtifactSetFresh({
|
||||
inputPaths: SLACK_DTS_INPUTS,
|
||||
outputPaths: [SLACK_DTS_STAMP, ...SLACK_DTS_REQUIRED_OUTPUTS],
|
||||
includeFile: isRelevantTypeInput,
|
||||
}) && !hasMissingOutput(SLACK_DTS_REQUIRED_OUTPUTS);
|
||||
|
||||
const prerequisiteSteps = [];
|
||||
const dependentSteps = [];
|
||||
@@ -401,6 +414,37 @@ async function main(argv = process.argv.slice(2)) {
|
||||
} else {
|
||||
process.stdout.write("[discord boundary dts] fresh; skipping\n");
|
||||
}
|
||||
if (!slackDtsFresh) {
|
||||
removeIncrementalStateForMissingOutput({
|
||||
outputPaths: SLACK_DTS_REQUIRED_OUTPUTS,
|
||||
tsBuildInfoPath: "dist/plugin-sdk/extensions/slack/.tsbuildinfo",
|
||||
});
|
||||
dependentSteps.push({
|
||||
label: "slack boundary dts",
|
||||
args: [
|
||||
runTsgoScript,
|
||||
"-p",
|
||||
"extensions/slack/tsconfig.json",
|
||||
"--declaration",
|
||||
"true",
|
||||
"--emitDeclarationOnly",
|
||||
"true",
|
||||
"--noEmit",
|
||||
"false",
|
||||
"--outDir",
|
||||
"dist/plugin-sdk/extensions/slack",
|
||||
"--rootDir",
|
||||
"extensions/slack",
|
||||
"--tsBuildInfoFile",
|
||||
"dist/plugin-sdk/extensions/slack/.tsbuildinfo",
|
||||
],
|
||||
env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" },
|
||||
timeoutMs: 300_000,
|
||||
stampPath: SLACK_DTS_STAMP,
|
||||
});
|
||||
} else {
|
||||
process.stdout.write("[slack boundary dts] fresh; skipping\n");
|
||||
}
|
||||
}
|
||||
|
||||
if (prerequisiteSteps.length > 0) {
|
||||
|
||||
@@ -143,6 +143,10 @@ export const sharedVitestConfig = {
|
||||
find: "@openclaw/discord/api.js",
|
||||
replacement: path.join(repoRoot, "extensions", "discord", "api.ts"),
|
||||
},
|
||||
{
|
||||
find: "@openclaw/slack/api.js",
|
||||
replacement: path.join(repoRoot, "extensions", "slack", "api.ts"),
|
||||
},
|
||||
...sourcePluginSdkSubpaths.map((subpath) => ({
|
||||
find: `openclaw/plugin-sdk/${subpath}`,
|
||||
replacement: path.join(repoRoot, "src", "plugin-sdk", `${subpath}.ts`),
|
||||
|
||||
Reference in New Issue
Block a user