From 31cafbb80265a006aca6d69fb6357711318d46c2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 3 May 2026 15:19:44 -0700 Subject: [PATCH] test(qa): add Slack live transport lane --- .github/workflows/openclaw-release-checks.yml | 99 ++- .../workflows/qa-live-transports-convex.yml | 97 +++ CHANGELOG.md | 1 + docs/concepts/qa-e2e-automation.md | 69 +- extensions/qa-lab/package.json | 1 + extensions/qa-lab/src/live-transports/cli.ts | 2 + .../src/live-transports/slack/cli.runtime.ts | 23 + .../qa-lab/src/live-transports/slack/cli.ts | 32 + .../slack/slack-live.runtime.test.ts | 57 ++ .../slack/slack-live.runtime.ts | 819 ++++++++++++++++++ .../tsconfig.package-boundary.paths.json | 1 + pnpm-lock.yaml | 3 + scripts/lib/extension-package-boundary.ts | 2 + ...e-extension-package-boundary-artifacts.mjs | 44 + test/vitest/vitest.shared.config.ts | 4 + 15 files changed, 1234 insertions(+), 20 deletions(-) create mode 100644 extensions/qa-lab/src/live-transports/slack/cli.runtime.ts create mode 100644 extensions/qa-lab/src/live-transports/slack/cli.ts create mode 100644 extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts create mode 100644 extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index a0e27e277a0..acd34fc497c 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -248,7 +248,7 @@ jobs: else echo "- Package Acceptance package spec: prepared release artifact" fi - echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, and Telegram lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan." + echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, Telegram, and Slack lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan." } >> "$GITHUB_STEP_SUMMARY" prepare_release_package: @@ -822,6 +822,99 @@ jobs: retention-days: 14 if-no-files-found: warn + qa_live_slack_release_checks: + name: Run QA Lab live Slack lane + needs: [resolve_target] + if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + permissions: + contents: read + pull-requests: read + environment: qa-live-shared + env: + OPENCLAW_BUILD_PRIVATE_QA: "1" + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ needs.resolve_target.outputs.revision }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Validate required QA credential env + env: + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + shell: bash + run: | + set -euo pipefail + + require_var() { + local key="$1" + if [[ -z "${!key:-}" ]]; then + echo "Missing required ${key}." >&2 + exit 1 + fi + } + + require_var OPENCLAW_QA_CONVEX_SITE_URL + require_var OPENCLAW_QA_CONVEX_SECRET_CI + + - name: Build private QA runtime + run: pnpm build + + - name: Run Slack live lane + id: run_lane + shell: bash + env: + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_SLACK_CAPTURE_CONTENT: "1" + run: | + set -euo pipefail + + output_dir=".artifacts/qa-e2e/slack-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + for attempt in 1 2; do + attempt_output_dir="${output_dir}/attempt-${attempt}" + if pnpm openclaw qa slack \ + --repo-root . \ + --output-dir "${attempt_output_dir}" \ + --provider-mode mock-openai \ + --model mock-openai/gpt-5.5 \ + --alt-model mock-openai/gpt-5.5-alt \ + --fast \ + --credential-source convex \ + --credential-role ci; then + exit 0 + fi + if [[ "${attempt}" == "2" ]]; then + exit 1 + fi + echo "Slack live lane failed on attempt ${attempt}; retrying once..." >&2 + sleep 10 + done + + - name: Upload Slack QA artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-qa-live-slack-${{ needs.resolve_target.outputs.revision }} + path: .artifacts/qa-e2e/ + retention-days: 14 + if-no-files-found: warn + summary: name: Verify release checks needs: @@ -835,6 +928,7 @@ jobs: - qa_lab_parity_report_release_checks - qa_live_matrix_release_checks - qa_live_telegram_release_checks + - qa_live_slack_release_checks if: always() runs-on: ubuntu-24.04 permissions: {} @@ -855,7 +949,8 @@ jobs: "qa_lab_parity_lane_release_checks=${{ needs.qa_lab_parity_lane_release_checks.result }}" \ "qa_lab_parity_report_release_checks=${{ needs.qa_lab_parity_report_release_checks.result }}" \ "qa_live_matrix_release_checks=${{ needs.qa_live_matrix_release_checks.result }}" \ - "qa_live_telegram_release_checks=${{ needs.qa_live_telegram_release_checks.result }}" + "qa_live_telegram_release_checks=${{ needs.qa_live_telegram_release_checks.result }}" \ + "qa_live_slack_release_checks=${{ needs.qa_live_slack_release_checks.result }}" do name="${item%%=*}" result="${item#*=}" diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index b5808910568..f2306dbfdf8 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -18,6 +18,10 @@ on: description: Optional comma-separated Discord scenario ids required: false type: string + slack_scenario: + description: Optional comma-separated Slack scenario ids + required: false + type: string matrix_profile: description: Matrix QA profile for the live Matrix lane required: false @@ -554,3 +558,96 @@ jobs: path: ${{ steps.run_lane.outputs.output_dir }} retention-days: 14 if-no-files-found: warn + + run_live_slack: + name: Run Slack live QA lane with Convex leases + needs: [authorize_actor, validate_selected_ref] + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 60 + environment: qa-live-shared + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Validate required QA credential env + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + shell: bash + run: | + set -euo pipefail + + require_var() { + local key="$1" + if [[ -z "${!key:-}" ]]; then + echo "Missing required ${key}." >&2 + exit 1 + fi + } + + require_var OPENAI_API_KEY + require_var OPENCLAW_QA_CONVEX_SITE_URL + require_var OPENCLAW_QA_CONVEX_SECRET_CI + + - name: Build private QA runtime + run: pnpm build + + - name: Run Slack live lane + id: run_lane + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_SLACK_CAPTURE_CONTENT: "1" + INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.slack_scenario || '' }} + run: | + set -euo pipefail + + output_dir=".artifacts/qa-e2e/slack-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + scenario_args=() + + if [[ -n "${INPUT_SCENARIO// }" ]]; then + IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}" + for raw in "${raw_scenarios[@]}"; do + scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [[ -n "${scenario}" ]]; then + scenario_args+=(--scenario "${scenario}") + fi + done + fi + + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + pnpm openclaw qa slack \ + --repo-root . \ + --output-dir "${output_dir}" \ + --provider-mode live-frontier \ + --model "${OPENCLAW_CI_OPENAI_MODEL}" \ + --alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \ + --fast \ + --credential-source convex \ + --credential-role ci \ + "${scenario_args[@]}" + + - name: Upload Slack QA artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: qa-live-slack-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.run_lane.outputs.output_dir }} + retention-days: 14 + if-no-files-found: warn diff --git a/CHANGELOG.md b/CHANGELOG.md index 9953fa4a21c..502f9df0b8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai - Gateway/config: stop Gateway startup and hot reload from auto-restoring invalid config; invalid config now fails closed and `openclaw doctor --fix` owns last-known-good repair. - Gateway/performance: lazy-load early runtime discovery and shutdown-hook helpers, defer maintenance timers until after readiness, and trim duplicate plugin auto-enable work during Gateway startup. - QA/Mantis: add a `pnpm openclaw qa mantis discord-smoke` runner and manual GitHub workflow that verify the Mantis Discord bot can see the configured guild/channel, post a smoke message, add a reaction, and upload artifacts. +- QA/Slack: add a Slack live transport QA runner with canary and mention-gating coverage for the private bot-to-bot harness. Thanks @vincentkoc. - Gateway/performance: lazy-load the heavy cron runtime after the rest of Gateway startup, defer restart-sentinel refresh after readiness, and let the Gateway startup benchmark write per-run V8 CPU profiles with `--cpu-prof-dir`. - Gateway/performance: keep raw channel-config schema parsing from discovering bundled plugin runtime metadata, and add `pnpm gateway:watch --benchmark-no-force` for profiling startup without the default port cleanup. - Plugins/onboarding: let Manual setup install optional official plugins, including ClawHub-backed diagnostics with npm fallback, and expose the external Codex plugin as a selectable provider setup choice. Thanks @vincentkoc. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index f238f675f82..b89c449bbec 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -47,6 +47,7 @@ script aliases; both forms are supported. | `qa matrix` | Live transport lane against a disposable Tuwunel homeserver. See [Matrix QA](/concepts/qa-matrix). | | `qa telegram` | Live transport lane against a real private Telegram group. | | `qa discord` | Live transport lane against a real private Discord guild channel. | +| `qa slack` | Live transport lane against a real private Slack channel. | | `qa mantis` | Before and after verification runner for live transport bugs, with the first Discord status-reactions scenario. See [Mantis](/concepts/mantis). | ## Operator flow @@ -110,14 +111,15 @@ pnpm openclaw qa matrix --profile fast --fail-fast The full CLI reference, profile/scenario catalog, env vars, and artifact layout for this lane live in [Matrix QA](/concepts/qa-matrix). At a glance: it provisions a disposable Tuwunel homeserver in Docker, registers temporary driver/SUT/observer users, runs the real Matrix plugin inside a child QA gateway scoped to that transport (no `qa-channel`), then writes a Markdown report, JSON summary, observed-events artifact, and combined output log under `.artifacts/qa-e2e/matrix-/`. -For transport-real Telegram and Discord smoke lanes: +For transport-real Telegram, Discord, and Slack smoke lanes: ```bash pnpm openclaw qa telegram pnpm openclaw qa discord +pnpm openclaw qa slack ``` -Both target a pre-existing real channel with two bots (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram and Discord QA reference](#telegram-and-discord-qa-reference) below. +They target a pre-existing real channel with two bots (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex credential pool are documented in [Telegram, Discord, and Slack QA reference](#telegram-discord-and-slack-qa-reference) below. Before using pooled live credentials, run: @@ -136,6 +138,7 @@ Live transport lanes share one contract instead of each inventing their own scen | Matrix | x | x | x | x | x | x | x | x | x | | | | Telegram | x | x | x | | | | | | | x | | | Discord | x | x | x | | | | | | | | x | +| Slack | x | x | x | | | | | | | | | This keeps `qa-channel` as the broad product-behavior suite while Matrix, Telegram, and future live transports share one explicit transport-contract @@ -162,27 +165,27 @@ guest: env-based provider keys, the QA live provider config path, and `CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest can write back through the mounted workspace. -## Telegram and Discord QA reference +## Telegram, Discord, and Slack QA reference -Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram and Discord are smaller — a handful of scenarios each, no profile system, against pre-existing real channels — so their reference lives here. +Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count and Docker-backed homeserver provisioning. Telegram, Discord, and Slack are smaller — a handful of scenarios each, no profile system, against pre-existing real channels — so their reference lives here. ### Shared CLI flags -Both lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags: +These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags: -| Flag | Default | Description | -| ------------------------------------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `--scenario ` | — | Run only this scenario. Repeatable. | -| `--output-dir ` | `/.artifacts/qa-e2e/{telegram,discord}-` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. | -| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | -| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | -| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | -| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | -| `--fast` | off | Provider fast mode where supported. | -| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | -| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | +| Flag | Default | Description | +| ------------------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `--scenario ` | — | Run only this scenario. Repeatable. | +| `--output-dir ` | `/.artifacts/qa-e2e/{telegram,discord,slack}-` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. | +| `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | +| `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | +| `--provider-mode ` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). | +| `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | +| `--fast` | off | Provider fast mode where supported. | +| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | +| `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | -Both exit non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code. +Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code. ### Telegram QA @@ -264,9 +267,39 @@ Output artifacts: - `discord-qa-observed-messages.json` — bodies redacted unless `OPENCLAW_QA_DISCORD_CAPTURE_CONTENT=1`. - `discord-qa-reaction-timelines.json` and `discord-status-reactions-tool-only-timeline.png` when the status-reaction scenario runs. +### Slack QA + +```bash +pnpm openclaw qa slack +``` + +Targets one real private Slack channel with two distinct bots: a driver bot controlled by the harness and a SUT bot started by the child OpenClaw gateway through the bundled Slack plugin. + +Required env when `--credential-source env`: + +- `OPENCLAW_QA_SLACK_CHANNEL_ID` +- `OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN` +- `OPENCLAW_QA_SLACK_SUT_BOT_TOKEN` +- `OPENCLAW_QA_SLACK_SUT_APP_TOKEN` + +Optional: + +- `OPENCLAW_QA_SLACK_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts. + +Scenarios (`extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts:39`): + +- `slack-canary` +- `slack-mention-gating` + +Output artifacts: + +- `slack-qa-report.md` +- `slack-qa-summary.json` +- `slack-qa-observed-messages.json` — bodies redacted unless `OPENCLAW_QA_SLACK_CAPTURE_CONTENT=1`. + ### Convex credential pool -Both Telegram and Discord lanes can lease credentials from a shared Convex pool instead of reading the env vars above. Pass `--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`); QA Lab acquires an exclusive lease, heartbeats it for the duration of the run, and releases it on shutdown. Pool kinds are `"telegram"` and `"discord"`. +Telegram, Discord, and Slack lanes can lease credentials from a shared Convex pool instead of reading the env vars above. Pass `--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`); QA Lab acquires an exclusive lease, heartbeats it for the duration of the run, and releases it on shutdown. Pool kinds are `"telegram"`, `"discord"`, and `"slack"`. Payload shapes the broker validates on `admin/add`: diff --git a/extensions/qa-lab/package.json b/extensions/qa-lab/package.json index 133dcce4d17..47591280d5e 100644 --- a/extensions/qa-lab/package.json +++ b/extensions/qa-lab/package.json @@ -14,6 +14,7 @@ "devDependencies": { "@openclaw/discord": "workspace:*", "@openclaw/plugin-sdk": "workspace:*", + "@openclaw/slack": "workspace:*", "openclaw": "workspace:*" }, "peerDependencies": { diff --git a/extensions/qa-lab/src/live-transports/cli.ts b/extensions/qa-lab/src/live-transports/cli.ts index 74f02888f31..92a0fbcf893 100644 --- a/extensions/qa-lab/src/live-transports/cli.ts +++ b/extensions/qa-lab/src/live-transports/cli.ts @@ -1,6 +1,7 @@ import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime"; import { discordQaCliRegistration } from "./discord/cli.js"; import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js"; +import { slackQaCliRegistration } from "./slack/cli.js"; import { telegramQaCliRegistration } from "./telegram/cli.js"; function createBlockedQaRunnerCliRegistration(params: { @@ -38,6 +39,7 @@ function createQaRunnerCliRegistration( const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [ telegramQaCliRegistration, discordQaCliRegistration, + slackQaCliRegistration, ]; export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] { diff --git a/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts b/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts new file mode 100644 index 00000000000..1331b880523 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts @@ -0,0 +1,23 @@ +import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; +import { + printLiveTransportQaArtifacts, + resolveLiveTransportQaRunOptions, +} from "../shared/live-transport-cli.runtime.js"; +import { runSlackQaLive } from "./slack-live.runtime.js"; + +export async function runQaSlackCommand(opts: LiveTransportQaCommandOptions) { + const runOptions = resolveLiveTransportQaRunOptions(opts); + const result = await runSlackQaLive(runOptions); + printLiveTransportQaArtifacts("Slack QA", { + report: result.reportPath, + summary: result.summaryPath, + "observed messages": result.observedMessagesPath, + ...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}), + }); + if ( + !runOptions.allowFailures && + result.scenarios.some((scenario) => scenario.status === "fail") + ) { + process.exitCode = 1; + } +} diff --git a/extensions/qa-lab/src/live-transports/slack/cli.ts b/extensions/qa-lab/src/live-transports/slack/cli.ts new file mode 100644 index 00000000000..91e9b54ce02 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/cli.ts @@ -0,0 +1,32 @@ +import { + createLazyCliRuntimeLoader, + createLiveTransportQaCliRegistration, + type LiveTransportQaCliRegistration, + type LiveTransportQaCommandOptions, +} from "../shared/live-transport-cli.js"; + +type SlackQaCliRuntime = typeof import("./cli.runtime.js"); + +const loadSlackQaCliRuntime = createLazyCliRuntimeLoader( + () => import("./cli.runtime.js"), +); + +async function runQaSlack(opts: LiveTransportQaCommandOptions) { + const runtime = await loadSlackQaCliRuntime(); + await runtime.runQaSlackCommand(opts); +} + +export const slackQaCliRegistration: LiveTransportQaCliRegistration = + createLiveTransportQaCliRegistration({ + commandName: "slack", + credentialOptions: { + sourceDescription: "Credential source for Slack QA: env or convex (default: env)", + roleDescription: + "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)", + }, + description: "Run the Slack live QA lane against a private bot-to-bot channel harness", + outputDirHelp: "Slack QA artifact directory", + scenarioHelp: "Run only the named Slack QA scenario (repeatable)", + sutAccountHelp: "Temporary Slack account id inside the QA gateway config", + run: runQaSlack, + }); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts new file mode 100644 index 00000000000..591a4d92f77 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { __testing } from "./slack-live.runtime.js"; + +describe("Slack live QA runtime helpers", () => { + it("resolves env credential payloads", () => { + expect( + __testing.resolveSlackQaRuntimeEnv({ + OPENCLAW_QA_SLACK_CHANNEL_ID: "C123456789", + OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN: "xoxb-driver", + OPENCLAW_QA_SLACK_SUT_BOT_TOKEN: "xoxb-sut", + OPENCLAW_QA_SLACK_SUT_APP_TOKEN: "xapp-sut", + }), + ).toEqual({ + channelId: "C123456789", + driverBotToken: "xoxb-driver", + sutBotToken: "xoxb-sut", + sutAppToken: "xapp-sut", + }); + }); + + it("rejects malformed Slack channel ids", () => { + expect(() => + __testing.resolveSlackQaRuntimeEnv({ + OPENCLAW_QA_SLACK_CHANNEL_ID: "qa-channel", + OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN: "xoxb-driver", + OPENCLAW_QA_SLACK_SUT_BOT_TOKEN: "xoxb-sut", + OPENCLAW_QA_SLACK_SUT_APP_TOKEN: "xapp-sut", + }), + ).toThrow("OPENCLAW_QA_SLACK channelId must be a Slack id like C123 or U123."); + }); + + it("parses Convex credential payloads", () => { + expect( + __testing.parseSlackQaCredentialPayload({ + channelId: "C123456789", + driverBotToken: "xoxb-driver", + sutBotToken: "xoxb-sut", + sutAppToken: "xapp-sut", + }), + ).toEqual({ + channelId: "C123456789", + driverBotToken: "xoxb-driver", + sutBotToken: "xoxb-sut", + sutAppToken: "xapp-sut", + }); + }); + + it("reports standard live transport scenario coverage", () => { + expect(__testing.SLACK_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]); + }); + + it("selects Slack scenarios by id", () => { + expect(__testing.findScenario(["slack-canary"]).map((scenario) => scenario.id)).toEqual([ + "slack-canary", + ]); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts new file mode 100644 index 00000000000..f9fdd8d72bd --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts @@ -0,0 +1,819 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { createSlackWebClient, createSlackWriteClient } from "@openclaw/slack/api.js"; +import type { WebClient } from "@slack/web-api"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { z } from "zod"; +import { startQaGatewayChild } from "../../gateway-child.js"; +import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; +import { + defaultQaModelForMode, + normalizeQaProviderMode, + type QaProviderModeInput, +} from "../../run-config.js"; +import { + acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat, + type QaCredentialRole, +} from "../shared/credential-lease.runtime.js"; +import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; +import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js"; +import { + collectLiveTransportStandardScenarioCoverage, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, +} from "../shared/live-transport-scenarios.js"; + +type SlackQaRuntimeEnv = { + channelId: string; + driverBotToken: string; + sutBotToken: string; + sutAppToken: string; +}; + +type SlackQaScenarioId = "slack-canary" | "slack-mention-gating"; + +type SlackQaScenarioRun = { + expectReply: boolean; + input: string; + matchText: string; +}; + +type SlackQaScenarioDefinition = LiveTransportScenarioDefinition & { + buildRun: (sutUserId: string) => SlackQaScenarioRun; +}; + +type SlackAuthIdentity = { + botId?: string; + teamId?: string; + userId: string; +}; + +type SlackMessage = { + bot_id?: string; + text?: string; + thread_ts?: string; + ts?: string; + user?: string; +}; + +type SlackObservedMessage = { + botId?: string; + channelId: string; + matchedScenario?: boolean; + scenarioId?: string; + scenarioTitle?: string; + text: string; + threadTs?: string; + ts: string; + userId?: string; +}; + +type SlackObservedMessageArtifact = { + botId?: string; + channelId?: string; + matchedScenario?: boolean; + scenarioId?: string; + scenarioTitle?: string; + text?: string; + threadTs?: string; + ts?: string; + userId?: string; +}; + +type SlackQaScenarioResult = { + details: string; + id: string; + requestStartedAt?: string; + responseObservedAt?: string; + rttMs?: number; + status: "fail" | "pass"; + title: string; +}; + +export type SlackQaRunResult = { + gatewayDebugDirPath?: string; + observedMessagesPath: string; + outputDir: string; + reportPath: string; + scenarios: SlackQaScenarioResult[]; + summaryPath: string; +}; + +type SlackQaSummary = { + channelId: string; + cleanupIssues: string[]; + counts: { + failed: number; + passed: number; + total: number; + }; + credentials: { + credentialId?: string; + kind: string; + ownerId?: string; + role?: QaCredentialRole; + source: "convex" | "env"; + }; + finishedAt: string; + scenarios: SlackQaScenarioResult[]; + startedAt: string; +}; + +const SLACK_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_SLACK_CAPTURE_CONTENT"; +const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; +const SLACK_QA_ENV_KEYS = [ + "OPENCLAW_QA_SLACK_CHANNEL_ID", + "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN", + "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN", + "OPENCLAW_QA_SLACK_SUT_APP_TOKEN", +] as const; + +const slackQaCredentialPayloadSchema = z.object({ + channelId: z.string().trim().min(1), + driverBotToken: z.string().trim().min(1), + sutBotToken: z.string().trim().min(1), + sutAppToken: z.string().trim().min(1), +}); + +const slackAuthTestSchema = z.object({ + ok: z.boolean().optional(), + user_id: z.string().optional(), + bot_id: z.string().optional(), + team_id: z.string().optional(), +}); + +const slackPostMessageSchema = z.object({ + ok: z.boolean().optional(), + channel: z.string().optional(), + ts: z.string().min(1), +}); + +const slackHistoryMessageSchema = z.object({ + bot_id: z.string().optional(), + text: z.string().optional(), + thread_ts: z.string().optional(), + ts: z.string().min(1), + user: z.string().optional(), +}); + +const slackHistorySchema = z.object({ + ok: z.boolean().optional(), + messages: z.array(slackHistoryMessageSchema).optional(), +}); + +const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ + { + id: "slack-canary", + standardId: "canary", + title: "Slack canary echo", + timeoutMs: 45_000, + buildRun: (sutUserId) => { + const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, + { + id: "slack-mention-gating", + standardId: "mention-gating", + title: "Slack unmentioned bot message does not trigger", + timeoutMs: 8_000, + buildRun: () => { + const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: false, + input: `reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, +]; + +const SLACK_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ + scenarios: SLACK_QA_SCENARIOS, +}); + +function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof SLACK_QA_ENV_KEYS)[number]) { + const value = env[key]?.trim(); + if (!value) { + throw new Error(`Missing ${key}.`); + } + return value; +} + +function isTruthyOptIn(value: string | undefined) { + const normalized = value?.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +} + +function normalizeSlackId(value: string, label: string) { + const normalized = value.trim(); + if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) { + throw new Error(`${label} must be a Slack id like C123 or U123.`); + } + return normalized; +} + +function validateSlackQaRuntimeEnv(runtimeEnv: SlackQaRuntimeEnv, label: string) { + normalizeSlackId(runtimeEnv.channelId, `${label} channelId`); + return runtimeEnv; +} + +function resolveSlackQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): SlackQaRuntimeEnv { + const runtimeEnv = { + channelId: resolveEnvValue(env, "OPENCLAW_QA_SLACK_CHANNEL_ID"), + driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN"), + sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN"), + sutAppToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_APP_TOKEN"), + }; + return validateSlackQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_SLACK"); +} + +function parseSlackQaCredentialPayload(payload: unknown): SlackQaRuntimeEnv { + const parsed = slackQaCredentialPayloadSchema.parse(payload); + const runtimeEnv = { + channelId: parsed.channelId, + driverBotToken: parsed.driverBotToken, + sutBotToken: parsed.sutBotToken, + sutAppToken: parsed.sutAppToken, + }; + return validateSlackQaRuntimeEnv(runtimeEnv, "Slack credential payload"); +} + +function findScenario(ids?: string[]) { + return selectLiveTransportScenarios({ + ids, + laneLabel: "Slack", + scenarios: SLACK_QA_SCENARIOS, + }); +} + +function buildSlackQaConfig( + baseCfg: OpenClawConfig, + params: { + channelId: string; + driverBotUserId: string; + sutAccountId: string; + sutAppToken: string; + sutBotToken: string; + }, +): OpenClawConfig { + const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "slack"])]; + return { + ...baseCfg, + plugins: { + ...baseCfg.plugins, + allow: pluginAllow, + entries: { + ...baseCfg.plugins?.entries, + slack: { enabled: true }, + }, + }, + messages: { + ...baseCfg.messages, + groupChat: { + ...baseCfg.messages?.groupChat, + visibleReplies: "automatic", + }, + }, + channels: { + ...baseCfg.channels, + slack: { + enabled: true, + defaultAccount: params.sutAccountId, + accounts: { + [params.sutAccountId]: { + enabled: true, + mode: "socket", + botToken: params.sutBotToken, + appToken: params.sutAppToken, + groupPolicy: "allowlist", + allowBots: true, + channels: { + [params.channelId]: { + enabled: true, + requireMention: true, + allowBots: true, + users: [params.driverBotUserId], + }, + }, + }, + }, + }, + }, + }; +} + +async function getSlackIdentity(token: string): Promise { + const client = createSlackWebClient(token, { timeout: 15_000 }); + const auth = slackAuthTestSchema.parse(await client.auth.test()); + if (!auth.user_id) { + throw new Error("Slack auth.test did not return user_id."); + } + return { + userId: auth.user_id, + botId: auth.bot_id, + teamId: auth.team_id, + }; +} + +async function sendSlackChannelMessage(params: { + channelId: string; + client: WebClient; + text: string; +}) { + const sendSlackMessage = params.client.chat.postMessage.bind(params.client.chat); + const sent = slackPostMessageSchema.parse( + await sendSlackMessage({ + channel: params.channelId, + text: params.text, + unfurl_links: false, + unfurl_media: false, + }), + ); + return { + channelId: sent.channel ?? params.channelId, + ts: sent.ts, + }; +} + +async function listSlackMessages(params: { + channelId: string; + client: WebClient; + oldestTs: string; +}) { + const history = slackHistorySchema.parse( + await params.client.conversations.history({ + channel: params.channelId, + inclusive: true, + limit: 50, + oldest: params.oldestTs, + }), + ); + return history.messages ?? []; +} + +function isSutSlackMessage(message: SlackMessage, sutIdentity: SlackAuthIdentity) { + return ( + (message.user !== undefined && message.user === sutIdentity.userId) || + (message.bot_id !== undefined && message.bot_id === sutIdentity.botId) + ); +} + +async function waitForSlackScenarioReply(params: { + channelId: string; + client: WebClient; + matchText: string; + observedMessages: SlackObservedMessage[]; + observationScenarioId: string; + observationScenarioTitle: string; + sentTs: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.timeoutMs) { + const messages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.sentTs, + }); + for (const message of messages) { + const text = message.text ?? ""; + if ( + !message.ts || + message.ts === params.sentTs || + !isSutSlackMessage(message, params.sutIdentity) + ) { + continue; + } + const matchedScenario = text.includes(params.matchText); + params.observedMessages.push({ + botId: message.bot_id, + channelId: params.channelId, + matchedScenario, + scenarioId: params.observationScenarioId, + scenarioTitle: params.observationScenarioTitle, + text, + threadTs: message.thread_ts, + ts: message.ts, + userId: message.user, + }); + if (matchedScenario) { + return { + message, + observedAt: new Date().toISOString(), + }; + } + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack message`); +} + +async function waitForSlackNoReply(params: { + channelId: string; + client: WebClient; + matchText: string; + observedMessages: SlackObservedMessage[]; + observationScenarioId: string; + observationScenarioTitle: string; + sentTs: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; +}) { + try { + await waitForSlackScenarioReply(params); + } catch (error) { + const message = formatErrorMessage(error); + if (message === `timed out after ${params.timeoutMs}ms waiting for Slack message`) { + return; + } + throw error; + } + throw new Error("unexpected Slack SUT reply observed"); +} + +async function waitForSlackChannelRunning( + gateway: Awaited>, + accountId: string, +) { + const startedAt = Date.now(); + let lastStatus: + | { + connected?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string; + restartPending?: boolean; + running?: boolean; + } + | undefined; + while (Date.now() - startedAt < 45_000) { + try { + const payload = (await gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { + channelAccounts?: Record< + string, + Array<{ + accountId?: string; + connected?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string; + restartPending?: boolean; + running?: boolean; + }> + >; + }; + const accounts = payload.channelAccounts?.slack ?? []; + const match = accounts.find((entry) => entry.accountId === accountId); + lastStatus = match + ? { + connected: match.connected, + lastConnectedAt: match.lastConnectedAt, + lastDisconnect: match.lastDisconnect, + lastError: match.lastError, + restartPending: match.restartPending, + running: match.running, + } + : undefined; + if (match?.running && match.connected === true && match.restartPending !== true) { + return; + } + } catch { + // retry + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error( + `slack account "${accountId}" did not become ready` + + (lastStatus ? `; last status: ${JSON.stringify(lastStatus)}` : ""), + ); +} + +function toObservedSlackArtifacts(params: { + includeContent: boolean; + messages: SlackObservedMessage[]; + redactMetadata: boolean; +}): SlackObservedMessageArtifact[] { + return params.messages.map((message) => ({ + botId: params.redactMetadata ? undefined : message.botId, + channelId: params.redactMetadata ? undefined : message.channelId, + matchedScenario: message.matchedScenario, + scenarioId: message.scenarioId, + scenarioTitle: message.scenarioTitle, + text: params.includeContent ? message.text : undefined, + threadTs: params.redactMetadata ? undefined : message.threadTs, + ts: params.redactMetadata ? undefined : message.ts, + userId: params.redactMetadata ? undefined : message.userId, + })); +} + +function renderSlackQaMarkdown(params: { + channelId: string; + cleanupIssues: string[]; + credentialSource: "convex" | "env"; + finishedAt: string; + gatewayDebugDirPath?: string; + redactMetadata: boolean; + scenarios: SlackQaScenarioResult[]; + startedAt: string; +}) { + const lines = [ + "# Slack QA Report", + "", + `- Credential source: \`${params.credentialSource}\``, + `- Channel: \`${params.redactMetadata ? "" : params.channelId}\``, + `- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``, + `- Started: ${params.startedAt}`, + `- Finished: ${params.finishedAt}`, + ]; + if (params.gatewayDebugDirPath) { + lines.push(`- Gateway debug artifacts: \`${params.gatewayDebugDirPath}\``); + } + if (params.cleanupIssues.length > 0) { + lines.push("", "## Cleanup issues", ""); + for (const issue of params.cleanupIssues) { + lines.push(`- ${issue}`); + } + } + lines.push("", "## Scenarios", ""); + for (const scenario of params.scenarios) { + lines.push(`### ${scenario.title}`, ""); + lines.push(`- Status: ${scenario.status}`); + lines.push(`- Details: ${scenario.details}`); + if (scenario.rttMs !== undefined) { + lines.push(`- RTT: ${scenario.rttMs}ms`); + } + lines.push(""); + } + return lines.join("\n"); +} + +export async function runSlackQaLive(params: { + alternateModel?: string; + credentialRole?: string; + credentialSource?: string; + fastMode?: boolean; + outputDir?: string; + primaryModel?: string; + providerMode?: QaProviderModeInput; + repoRoot?: string; + scenarioIds?: string[]; + sutAccountId?: string; +}): Promise { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const outputDir = + params.outputDir ?? + path.join(repoRoot, ".artifacts", "qa-e2e", `slack-${Date.now().toString(36)}`); + await fs.mkdir(outputDir, { recursive: true }); + + const providerMode = normalizeQaProviderMode( + params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, + ); + const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const sutAccountId = params.sutAccountId?.trim() || "sut"; + const scenarios = findScenario(params.scenarioIds); + + const credentialLease = await acquireQaCredentialLease({ + kind: "slack", + source: params.credentialSource, + role: params.credentialRole, + resolveEnvPayload: () => resolveSlackQaRuntimeEnv(), + parsePayload: parseSlackQaCredentialPayload, + }); + const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); + const assertLeaseHealthy = () => { + leaseHeartbeat.throwIfFailed(); + }; + + const runtimeEnv = credentialLease.payload; + const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); + const includeObservedMessageContent = isTruthyOptIn(process.env[SLACK_QA_CAPTURE_CONTENT_ENV]); + const startedAt = new Date().toISOString(); + const observedMessages: SlackObservedMessage[] = []; + const scenarioResults: SlackQaScenarioResult[] = []; + const cleanupIssues: string[] = []; + const gatewayDebugDirPath = path.join(outputDir, "gateway-debug"); + let preservedGatewayDebugArtifacts = false; + + try { + const [driverIdentity, sutIdentity] = await Promise.all([ + getSlackIdentity(runtimeEnv.driverBotToken), + getSlackIdentity(runtimeEnv.sutBotToken), + ]); + if (driverIdentity.userId === sutIdentity.userId) { + throw new Error("Slack QA requires two distinct bots for driver and SUT."); + } + + const driverClient = createSlackWriteClient(runtimeEnv.driverBotToken, { timeout: 15_000 }); + const sutReadClient = createSlackWebClient(runtimeEnv.sutBotToken, { timeout: 15_000 }); + const gatewayHarness = await startQaLiveLaneGateway({ + repoRoot, + transport: { + requiredPluginIds: [], + createGatewayConfig: () => ({}), + }, + transportBaseUrl: "http://127.0.0.1:0", + providerMode, + primaryModel, + alternateModel, + fastMode: params.fastMode, + controlUiEnabled: false, + mutateConfig: (cfg) => + buildSlackQaConfig(cfg, { + channelId: runtimeEnv.channelId, + driverBotUserId: driverIdentity.userId, + sutAccountId, + sutAppToken: runtimeEnv.sutAppToken, + sutBotToken: runtimeEnv.sutBotToken, + }), + }); + try { + await waitForSlackChannelRunning(gatewayHarness.gateway, sutAccountId); + assertLeaseHealthy(); + for (const scenario of scenarios) { + assertLeaseHealthy(); + const scenarioRun = scenario.buildRun(sutIdentity.userId); + const requestStartedAt = new Date(); + try { + const sent = await sendSlackChannelMessage({ + channelId: runtimeEnv.channelId, + client: driverClient, + text: scenarioRun.input, + }); + if (scenarioRun.expectReply) { + const reply = await waitForSlackScenarioReply({ + channelId: runtimeEnv.channelId, + client: sutReadClient, + matchText: scenarioRun.matchText, + observedMessages, + observationScenarioId: scenario.id, + observationScenarioTitle: scenario.title, + sentTs: sent.ts, + sutIdentity, + timeoutMs: scenario.timeoutMs, + }); + const responseObservedAt = new Date(reply.observedAt); + const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime(); + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: `reply matched in ${rttMs}ms`, + rttMs, + requestStartedAt: requestStartedAt.toISOString(), + responseObservedAt: responseObservedAt.toISOString(), + }); + } else { + await waitForSlackNoReply({ + channelId: runtimeEnv.channelId, + client: sutReadClient, + matchText: scenarioRun.matchText, + observedMessages, + observationScenarioId: scenario.id, + observationScenarioTitle: scenario.title, + sentTs: sent.ts, + sutIdentity, + timeoutMs: scenario.timeoutMs, + }); + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: "no reply", + }); + } + } catch (error) { + const result = { + id: scenario.id, + title: scenario.title, + status: "fail" as const, + details: formatErrorMessage(error), + }; + scenarioResults.push(result); + preservedGatewayDebugArtifacts = true; + await gatewayHarness.gateway + .stop({ keepTemp: true, preserveToDir: gatewayDebugDirPath }) + .catch((stopError) => { + appendLiveLaneIssue(cleanupIssues, "gateway debug preservation failed", stopError); + }); + break; + } + } + } finally { + if (!preservedGatewayDebugArtifacts) { + await gatewayHarness.stop().catch((error) => { + appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error); + }); + } + } + } catch (error) { + cleanupIssues.push( + buildLiveLaneArtifactsError({ + heading: "Slack QA failed before scenario completion.", + details: [formatErrorMessage(error)], + artifacts: { + gatewayDebug: gatewayDebugDirPath, + }, + }), + ); + preservedGatewayDebugArtifacts = true; + await fs.mkdir(gatewayDebugDirPath, { recursive: true }).catch(() => {}); + scenarioResults.push({ + id: "slack-canary", + title: "Slack canary echo", + status: "fail", + details: formatErrorMessage(error), + }); + } finally { + try { + await leaseHeartbeat.stop(); + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "credential heartbeat stop failed", error); + } + try { + await credentialLease.release(); + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "credential release failed", error); + } + } + + const finishedAt = new Date().toISOString(); + const reportPath = path.join(outputDir, "slack-qa-report.md"); + const summaryPath = path.join(outputDir, "slack-qa-summary.json"); + const observedMessagesPath = path.join(outputDir, "slack-qa-observed-messages.json"); + const passed = scenarioResults.filter((entry) => entry.status === "pass").length; + const failed = scenarioResults.filter((entry) => entry.status === "fail").length; + const summary: SlackQaSummary = { + credentials: { + source: credentialLease.source, + kind: credentialLease.kind, + role: credentialLease.role, + credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId, + ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId, + }, + channelId: redactPublicMetadata ? "" : runtimeEnv.channelId, + startedAt, + finishedAt, + cleanupIssues, + counts: { + total: scenarioResults.length, + passed, + failed, + }, + scenarios: scenarioResults, + }; + await fs.writeFile( + observedMessagesPath, + `${JSON.stringify( + toObservedSlackArtifacts({ + messages: observedMessages, + includeContent: includeObservedMessageContent, + redactMetadata: redactPublicMetadata, + }), + null, + 2, + )}\n`, + ); + await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + await fs.writeFile( + reportPath, + `${renderSlackQaMarkdown({ + channelId: runtimeEnv.channelId, + cleanupIssues, + credentialSource: credentialLease.source, + finishedAt, + gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, + redactMetadata: redactPublicMetadata, + scenarios: scenarioResults, + startedAt, + })}\n`, + ); + return { + outputDir, + reportPath, + summaryPath, + observedMessagesPath, + gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, + scenarios: scenarioResults, + }; +} + +export const __testing = { + findScenario, + parseSlackQaCredentialPayload, + resolveSlackQaRuntimeEnv, + SLACK_QA_STANDARD_SCENARIO_IDS, +}; diff --git a/extensions/tsconfig.package-boundary.paths.json b/extensions/tsconfig.package-boundary.paths.json index 1cb6134ae1b..e27f1b6b781 100644 --- a/extensions/tsconfig.package-boundary.paths.json +++ b/extensions/tsconfig.package-boundary.paths.json @@ -37,6 +37,7 @@ "openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"], "@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"], "@openclaw/discord/api.js": ["../dist/plugin-sdk/extensions/discord/api.d.ts"], + "@openclaw/slack/api.js": ["../dist/plugin-sdk/extensions/slack/api.d.ts"], "@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"], "@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"], "@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4481d9697d8..d21266936b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1195,6 +1195,9 @@ importers: '@openclaw/plugin-sdk': specifier: workspace:* version: link:../../packages/plugin-sdk + '@openclaw/slack': + specifier: workspace:* + version: link:../slack openclaw: specifier: workspace:* version: link:../.. diff --git a/scripts/lib/extension-package-boundary.ts b/scripts/lib/extension-package-boundary.ts index 0851b7f9a50..815404fe55b 100644 --- a/scripts/lib/extension-package-boundary.ts +++ b/scripts/lib/extension-package-boundary.ts @@ -47,6 +47,7 @@ export const EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS = { "openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"], "@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"], "@openclaw/discord/api.js": ["../dist/plugin-sdk/extensions/discord/api.d.ts"], + "@openclaw/slack/api.js": ["../dist/plugin-sdk/extensions/slack/api.d.ts"], "@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"], "@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"], "@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"], @@ -70,6 +71,7 @@ export const EXTENSION_PACKAGE_BOUNDARY_XAI_PATHS = { "openclaw/plugin-sdk/channel-secret-basic-runtime": _omitBasic, "openclaw/plugin-sdk/channel-secret-tts-runtime": _omitTts, "@openclaw/discord/api.js": _omitDiscord, + "@openclaw/slack/api.js": _omitSlack, ...rest }) => rest)(EXTENSION_PACKAGE_BOUNDARY_BASE_PATHS), "../", diff --git a/scripts/prepare-extension-package-boundary-artifacts.mjs b/scripts/prepare-extension-package-boundary-artifacts.mjs index 24583a654d5..b36aaa11ddd 100644 --- a/scripts/prepare-extension-package-boundary-artifacts.mjs +++ b/scripts/prepare-extension-package-boundary-artifacts.mjs @@ -52,6 +52,13 @@ const DISCORD_DTS_INPUTS = [ ]; const DISCORD_DTS_STAMP = "dist/plugin-sdk/extensions/discord/.boundary-dts.stamp"; const DISCORD_DTS_REQUIRED_OUTPUTS = ["dist/plugin-sdk/extensions/discord/api.d.ts"]; +const SLACK_DTS_INPUTS = [ + "extensions/slack/api.ts", + "extensions/slack/src/client.ts", + "extensions/slack/tsconfig.json", +]; +const SLACK_DTS_STAMP = "dist/plugin-sdk/extensions/slack/.boundary-dts.stamp"; +const SLACK_DTS_REQUIRED_OUTPUTS = ["dist/plugin-sdk/extensions/slack/api.d.ts"]; const ENTRY_SHIMS_INPUTS = [ "scripts/write-plugin-sdk-entry-dts.ts", "scripts/lib/plugin-sdk-entrypoints.json", @@ -303,6 +310,12 @@ async function main(argv = process.argv.slice(2)) { outputPaths: [DISCORD_DTS_STAMP, ...DISCORD_DTS_REQUIRED_OUTPUTS], includeFile: isRelevantTypeInput, }) && !hasMissingOutput(DISCORD_DTS_REQUIRED_OUTPUTS); + const slackDtsFresh = + isArtifactSetFresh({ + inputPaths: SLACK_DTS_INPUTS, + outputPaths: [SLACK_DTS_STAMP, ...SLACK_DTS_REQUIRED_OUTPUTS], + includeFile: isRelevantTypeInput, + }) && !hasMissingOutput(SLACK_DTS_REQUIRED_OUTPUTS); const prerequisiteSteps = []; const dependentSteps = []; @@ -401,6 +414,37 @@ async function main(argv = process.argv.slice(2)) { } else { process.stdout.write("[discord boundary dts] fresh; skipping\n"); } + if (!slackDtsFresh) { + removeIncrementalStateForMissingOutput({ + outputPaths: SLACK_DTS_REQUIRED_OUTPUTS, + tsBuildInfoPath: "dist/plugin-sdk/extensions/slack/.tsbuildinfo", + }); + dependentSteps.push({ + label: "slack boundary dts", + args: [ + runTsgoScript, + "-p", + "extensions/slack/tsconfig.json", + "--declaration", + "true", + "--emitDeclarationOnly", + "true", + "--noEmit", + "false", + "--outDir", + "dist/plugin-sdk/extensions/slack", + "--rootDir", + "extensions/slack", + "--tsBuildInfoFile", + "dist/plugin-sdk/extensions/slack/.tsbuildinfo", + ], + env: { OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1" }, + timeoutMs: 300_000, + stampPath: SLACK_DTS_STAMP, + }); + } else { + process.stdout.write("[slack boundary dts] fresh; skipping\n"); + } } if (prerequisiteSteps.length > 0) { diff --git a/test/vitest/vitest.shared.config.ts b/test/vitest/vitest.shared.config.ts index a1c1804bbaa..2490008bc30 100644 --- a/test/vitest/vitest.shared.config.ts +++ b/test/vitest/vitest.shared.config.ts @@ -143,6 +143,10 @@ export const sharedVitestConfig = { find: "@openclaw/discord/api.js", replacement: path.join(repoRoot, "extensions", "discord", "api.ts"), }, + { + find: "@openclaw/slack/api.js", + replacement: path.join(repoRoot, "extensions", "slack", "api.ts"), + }, ...sourcePluginSdkSubpaths.map((subpath) => ({ find: `openclaw/plugin-sdk/${subpath}`, replacement: path.join(repoRoot, "src", "plugin-sdk", `${subpath}.ts`),