mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:31:40 +00:00
refactor(qa): migrate Telegram scenarios into QA Lab (#108430)
* refactor(qa): migrate Telegram scenarios into QA Lab * refactor(qa): remove retired Telegram runner exports
This commit is contained in:
4
.github/workflows/mantis-telegram-live.yml
vendored
4
.github/workflows/mantis-telegram-live.yml
vendored
@@ -444,7 +444,7 @@ jobs:
|
||||
telegram_exit=$?
|
||||
set -e
|
||||
|
||||
if [[ ! -f "$root/qa-evidence.json" && ! -f "$root/telegram-qa-summary.json" ]]; then
|
||||
if [[ ! -f "$root/qa-evidence.json" ]]; then
|
||||
echo "Telegram live QA did not produce an evidence summary." >&2
|
||||
exit "$telegram_exit"
|
||||
fi
|
||||
@@ -496,7 +496,7 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
cat "$root/telegram-qa-report.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
cat "$root/qa-suite-report.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload Mantis Telegram artifacts
|
||||
id: upload_artifact
|
||||
|
||||
2
.github/workflows/npm-telegram-beta-e2e.yml
vendored
2
.github/workflows/npm-telegram-beta-e2e.yml
vendored
@@ -423,7 +423,7 @@ jobs:
|
||||
|
||||
append_telegram_summary() {
|
||||
local status=$?
|
||||
local report="${output_dir}/telegram-qa-report.md"
|
||||
local report="${output_dir}/qa-suite-report.md"
|
||||
if [[ -n "${GITHUB_STEP_SUMMARY:-}" && -f "${report}" ]]; then
|
||||
{
|
||||
echo "## Package Telegram E2E"
|
||||
|
||||
@@ -249,8 +249,6 @@ extensions/qa-lab/src/lab-server.ts
|
||||
extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts
|
||||
extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-e2ee-destructive.ts
|
||||
extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
|
||||
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts
|
||||
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts
|
||||
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
|
||||
extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts
|
||||
extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts
|
||||
|
||||
@@ -325,7 +325,7 @@ Comment triggers, from a PR with write/maintain/admin access:
|
||||
@openclaw-mantis discord status reactions baseline=origin/main candidate=HEAD
|
||||
@openclaw-mantis telegram
|
||||
@openclaw-mantis telegram scenario=telegram-status-command
|
||||
@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply
|
||||
@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary
|
||||
@openclaw-mantis web ui chat
|
||||
@openclaw-mantis web-ui-chat candidate=HEAD
|
||||
```
|
||||
|
||||
@@ -491,11 +491,12 @@ Required env when `--credential-source env`:
|
||||
- `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN`
|
||||
|
||||
Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`):
|
||||
The `release` profile selects the maintained Telegram YAML scenarios; `all`
|
||||
adds opt-in session, usage, reply-chain, and streaming stress checks. Explicit
|
||||
`--scenario` values override the profile.
|
||||
|
||||
- `telegram-canary`
|
||||
- `telegram-mention-gating`
|
||||
- `telegram-mentioned-message-reply`
|
||||
- `channel-canary`
|
||||
- `channel-mention-gating`
|
||||
- `telegram-help-command`
|
||||
- `telegram-commands-command`
|
||||
- `telegram-tools-compact-command`
|
||||
@@ -511,19 +512,21 @@ Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime
|
||||
- `telegram-long-final-reuses-preview`
|
||||
- `telegram-long-final-three-chunks`
|
||||
|
||||
The implicit default set always covers canary, mention gating, native command
|
||||
The `release` profile always covers canary, mention gating, native command
|
||||
replies, command addressing, and bot-to-bot group replies. `mock-openai`
|
||||
defaults also include deterministic reply-chain and final-message streaming
|
||||
checks. `telegram-current-session-status-tool` and
|
||||
also includes the deterministic long-final preview check.
|
||||
`telegram-current-session-status-tool` and
|
||||
`telegram-tool-only-usage-footer` remain opt-in: the former is only stable
|
||||
when threaded directly after canary, and the latter is a real-Telegram proof
|
||||
of the `/usage` footer on tool-only replies. Use `pnpm openclaw qa telegram
|
||||
--list-scenarios --provider-mode mock-openai` to print the current
|
||||
default/optional split with regression refs.
|
||||
default/optional split with regression refs. Use `--profile all` for every
|
||||
Telegram scenario and `--fail-fast` to stop after the first failed scenario.
|
||||
|
||||
Output artifacts:
|
||||
|
||||
- `telegram-qa-report.md`
|
||||
- `qa-suite-report.md`
|
||||
- `qa-suite-summary.json`
|
||||
- `qa-evidence.json` - evidence entries for the live transport checks,
|
||||
including profile, coverage, provider, channel, artifacts, result, and RTT
|
||||
fields.
|
||||
@@ -542,7 +545,7 @@ When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper
|
||||
leases a `kind: "telegram"` credential, exports the leased group/driver/SUT
|
||||
bot env into the installed-package run, heartbeats the lease, and releases it
|
||||
on shutdown. The package wrapper defaults to 20 RTT checks of
|
||||
`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role
|
||||
`channel-canary`, a 30s RTT timeout, and Convex role
|
||||
`maintainer` outside CI when Convex is selected. Override
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`,
|
||||
or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without
|
||||
|
||||
@@ -276,9 +276,8 @@ inside every shard.
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`,
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the run.
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of
|
||||
Telegram QA check IDs to sample; when unset, the default RTT-capable
|
||||
check is `telegram-mentioned-message-reply`.
|
||||
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` selects the Telegram QA scenario to
|
||||
sample; the supported RTT target is `channel-canary`.
|
||||
- Uses the same Telegram env credentials or Convex credential source as
|
||||
`pnpm openclaw qa telegram`. For CI/release automation, set
|
||||
`OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus
|
||||
@@ -441,7 +440,7 @@ set. Maintainers can start it from the Actions UI through `Mantis Scenario`
|
||||
```text
|
||||
@openclaw-mantis telegram
|
||||
@openclaw-mantis telegram scenario=telegram-status-command
|
||||
@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply
|
||||
@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary
|
||||
```
|
||||
|
||||
`Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { QaScenarioPack } from "./scenario-catalog.js";
|
||||
|
||||
const {
|
||||
@@ -11,10 +11,7 @@ const {
|
||||
runQaSuite,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
runCanonicalLiveScenarios,
|
||||
listLiveTransportQaAdapterFactories,
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
startQaLabServer,
|
||||
writeQaDockerHarnessFiles,
|
||||
buildQaDockerHarnessImage,
|
||||
@@ -27,10 +24,7 @@ const {
|
||||
runQaSuite: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
runCanonicalLiveScenarios: vi.fn(),
|
||||
listLiveTransportQaAdapterFactories: vi.fn(),
|
||||
listTelegramQaScenarioCatalog: vi.fn(),
|
||||
runTelegramQaLive: vi.fn(),
|
||||
startQaLabServer: vi.fn(),
|
||||
writeQaDockerHarnessFiles: vi.fn(),
|
||||
buildQaDockerHarnessImage: vi.fn(),
|
||||
@@ -61,20 +55,10 @@ vi.mock("./live-transports/cli.js", () => ({
|
||||
listLiveTransportQaAdapterFactories,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/shared/canonical-scenarios.js", async (importOriginal) => ({
|
||||
...(await importOriginal()),
|
||||
runCanonicalLiveScenarios,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/adapter.runtime.js", () => ({
|
||||
createTelegramQaTransportAdapter: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
}));
|
||||
|
||||
vi.mock("./lab-server.js", () => ({
|
||||
startQaLabServer,
|
||||
}));
|
||||
@@ -118,7 +102,6 @@ import {
|
||||
} from "./cli.runtime.js";
|
||||
import { QaSuiteInfraError } from "./errors.js";
|
||||
import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
|
||||
import { loadNonYamlScenarioRefs } from "./live-transports/shared/live-transport-scenarios.js";
|
||||
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
@@ -253,10 +236,7 @@ describe("qa cli runtime", () => {
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
runCanonicalLiveScenarios.mockReset();
|
||||
listLiveTransportQaAdapterFactories.mockReset();
|
||||
listTelegramQaScenarioCatalog.mockReset();
|
||||
runTelegramQaLive.mockReset();
|
||||
startQaLabServer.mockReset();
|
||||
writeQaDockerHarnessFiles.mockReset();
|
||||
buildQaDockerHarnessImage.mockReset();
|
||||
@@ -300,28 +280,6 @@ describe("qa cli runtime", () => {
|
||||
vmName: "openclaw-qa-test",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
runTelegramQaLive.mockResolvedValue({
|
||||
outputDir: telegramArtifactsDir,
|
||||
reportPath: path.join(telegramArtifactsDir, "report.md"),
|
||||
summaryPath: telegramSummaryPath,
|
||||
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
|
||||
scenarios: [],
|
||||
});
|
||||
runCanonicalLiveScenarios.mockResolvedValue({
|
||||
outputDir: telegramArtifactsDir,
|
||||
reportPath: path.join(telegramArtifactsDir, "canonical-report.md"),
|
||||
summaryPath: telegramSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
listTelegramQaScenarioCatalog.mockReturnValue([
|
||||
{
|
||||
id: "telegram-stream-final-single-message",
|
||||
title: "Telegram streamed final reply",
|
||||
defaultEnabled: false,
|
||||
rationale: "stream rationale",
|
||||
regressionRefs: [],
|
||||
},
|
||||
]);
|
||||
listLiveTransportQaAdapterFactories.mockReturnValue([
|
||||
{
|
||||
id: "telegram",
|
||||
@@ -998,22 +956,20 @@ describe("qa cli runtime", () => {
|
||||
sutAccountId: "sut-live",
|
||||
});
|
||||
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
|
||||
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
fastMode: true,
|
||||
channelDriver: "live",
|
||||
channelId: "telegram",
|
||||
options: expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
alternateModel: "openai/gpt-5.6-luna",
|
||||
fastMode: true,
|
||||
sutAccountId: "sut-live",
|
||||
}),
|
||||
adapterOptions: expect.objectContaining({ sutAccountId: "sut-live" }),
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
}),
|
||||
);
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects output dirs that escape the repo root", () => {
|
||||
@@ -1031,31 +987,33 @@ describe("qa cli runtime", () => {
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
});
|
||||
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
|
||||
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
allowFailures: undefined,
|
||||
}),
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
}),
|
||||
);
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps remaining Telegram defaults when Commander supplies an empty scenario list", async () => {
|
||||
it("resolves the Telegram release profile when Commander supplies an empty scenario list", async () => {
|
||||
await runQaTelegramCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
scenarioIds: [],
|
||||
});
|
||||
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ scenarioIds: undefined }),
|
||||
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
scenarioIds: expect.arrayContaining([
|
||||
"channel-canary",
|
||||
"channel-mention-gating",
|
||||
"telegram-other-bot-command-gating",
|
||||
]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the trusted Telegram launcher for canonical and legacy gateways", async () => {
|
||||
it("uses the trusted Telegram launcher for the shared suite gateway", async () => {
|
||||
const candidateRoot = path.join(telegramArtifactsDir, "candidate");
|
||||
const boundaryDir = path.join(telegramArtifactsDir, "boundary");
|
||||
const launcherPath = path.join(telegramArtifactsDir, "openclaw-telegram-sut-launcher");
|
||||
@@ -1097,12 +1055,9 @@ describe("qa cli runtime", () => {
|
||||
terminationRetryTimeoutMs: 60_000,
|
||||
},
|
||||
};
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
options: expect.objectContaining({ sutOpenClawCommand }),
|
||||
}),
|
||||
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ sutOpenClawCommand }),
|
||||
);
|
||||
expect(runTelegramQaLive).toHaveBeenCalledWith(expect.objectContaining({ sutOpenClawCommand }));
|
||||
});
|
||||
|
||||
it("rejects relative Telegram launcher paths before starting a gateway", async () => {
|
||||
@@ -1114,8 +1069,7 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
).rejects.toThrow("OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must be an absolute file path.");
|
||||
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects non-executable Telegram launcher files before starting a gateway", async () => {
|
||||
@@ -1131,8 +1085,7 @@ describe("qa cli runtime", () => {
|
||||
`OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must point to an executable regular file: ${launcherPath}`,
|
||||
);
|
||||
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects unknown mixed Telegram selections before resolving the SUT launcher", async () => {
|
||||
@@ -1144,8 +1097,7 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario");
|
||||
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("prints telegram scenario catalog without resolving the SUT launcher", async () => {
|
||||
@@ -1156,9 +1108,7 @@ describe("qa cli runtime", () => {
|
||||
listScenarios: true,
|
||||
});
|
||||
|
||||
expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai");
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
|
||||
expectWriteContains(
|
||||
stdoutWrite,
|
||||
"telegram-status-command\tdefault\tTelegram status command reply\tVerify Telegram status returns model, session, and activation details. refs=openclaw/openclaw#74698",
|
||||
@@ -1176,11 +1126,10 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runTelegramQaLive.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: telegramArtifactsDir,
|
||||
reportPath: path.join(telegramArtifactsDir, "report.md"),
|
||||
summaryPath: telegramSummaryPath,
|
||||
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
|
||||
scenarios: [],
|
||||
});
|
||||
|
||||
@@ -1205,11 +1154,10 @@ describe("qa cli runtime", () => {
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
runTelegramQaLive.mockResolvedValueOnce({
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
|
||||
outputDir: telegramArtifactsDir,
|
||||
reportPath: path.join(telegramArtifactsDir, "report.md"),
|
||||
summaryPath: telegramSummaryPath,
|
||||
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
|
||||
scenarios: [
|
||||
{
|
||||
id: "telegram-help-command",
|
||||
@@ -1999,11 +1947,6 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
|
||||
describe("coverage inventory command", () => {
|
||||
beforeAll(async () => {
|
||||
listTelegramQaScenarioCatalog.mockReturnValue([]);
|
||||
await loadNonYamlScenarioRefs();
|
||||
});
|
||||
|
||||
it("prints a markdown report from scenario metadata", async () => {
|
||||
await runQaCoverageReportCommand({ repoRoot: process.cwd() });
|
||||
|
||||
|
||||
@@ -277,12 +277,12 @@ describe("qa coverage report", () => {
|
||||
nonYamlScenarios: [
|
||||
{
|
||||
id: scenario.id,
|
||||
sourcePath: "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts",
|
||||
sourcePath: "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts",
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow(
|
||||
"duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts)",
|
||||
"duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts)",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -308,7 +308,7 @@ describe("qa coverage report", () => {
|
||||
expect(report).toContain("personal-share-safe-diagnostics-artifact");
|
||||
expect(report).toContain("## Live Transport Lanes");
|
||||
expect(report).toContain(
|
||||
"- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
|
||||
"- telegram (telegram): canary: channel-canary, help-command: telegram-help-command, mention-gating: channel-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
|
||||
);
|
||||
expect(report).toContain("## Scorecard Taxonomy");
|
||||
expect(report).toContain("- Taxonomy: taxonomy.yaml");
|
||||
|
||||
@@ -152,7 +152,7 @@ describe("evidence summary", () => {
|
||||
const evidence = buildLiveTransportEvidenceSummary({
|
||||
artifactPaths: [
|
||||
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
|
||||
{ kind: "report", path: "telegram-qa-report.md" },
|
||||
{ kind: "report", path: "qa-suite-report.md" },
|
||||
],
|
||||
env: {
|
||||
OPENCLAW_QA_RUNNER: "crabbox",
|
||||
@@ -218,7 +218,7 @@ describe("evidence summary", () => {
|
||||
},
|
||||
{
|
||||
kind: "report",
|
||||
path: "telegram-qa-report.md",
|
||||
path: "qa-suite-report.md",
|
||||
source: "telegram-live-transport",
|
||||
},
|
||||
],
|
||||
@@ -244,9 +244,9 @@ describe("evidence summary", () => {
|
||||
providerMode: "live-frontier",
|
||||
checks: [
|
||||
{
|
||||
id: "telegram-mentioned-message-reply",
|
||||
id: "channel-canary",
|
||||
coverageIds: ["channels.telegram.mention-gating"],
|
||||
title: "Telegram mentioned message gets a reply",
|
||||
title: "Channel transport canary",
|
||||
status: "pass",
|
||||
details: "5 samples collected.",
|
||||
rttMs: 2000,
|
||||
|
||||
@@ -8,14 +8,12 @@ const { runQaFlowSuiteFromRuntime } = vi.hoisted(() => ({
|
||||
vi.mock("../../suite-launch.runtime.js", () => ({
|
||||
runQaFlowSuiteFromRuntime,
|
||||
}));
|
||||
import { readQaScenarioPack } from "../../scenario-catalog.js";
|
||||
import {
|
||||
assertKnownScenarioIds,
|
||||
partitionCanonicalScenarioIds,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
whatsappDefaultCanonicalScenarioIds,
|
||||
listCanonicalScenarios,
|
||||
runCanonicalLiveScenarios,
|
||||
} from "./canonical-scenarios.js";
|
||||
import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js";
|
||||
@@ -23,42 +21,27 @@ import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js";
|
||||
describe("canonical live-transport scenarios", () => {
|
||||
it("loads every migrated routing, command, and session-context scenario from YAML", () => {
|
||||
const whatsAppMockDefaultIds = whatsappDefaultCanonicalScenarioIds("mock-openai");
|
||||
const telegram = listCanonicalScenarios({
|
||||
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
});
|
||||
const whatsapp = listCanonicalScenarios({
|
||||
ids: WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: whatsAppMockDefaultIds,
|
||||
});
|
||||
const expectedIds = new Set<string>(WHATSAPP_CANONICAL_SCENARIO_IDS);
|
||||
const whatsappIds = new Set(
|
||||
readQaScenarioPack()
|
||||
.scenarios.filter((scenario) => expectedIds.has(scenario.id))
|
||||
.map((scenario) => scenario.id),
|
||||
);
|
||||
|
||||
expect(telegram.map(({ id }) => id).toSorted()).toEqual(
|
||||
[...TELEGRAM_CANONICAL_SCENARIO_IDS].toSorted(),
|
||||
);
|
||||
expect(whatsapp.map(({ id }) => id).toSorted()).toEqual(
|
||||
[...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted(),
|
||||
);
|
||||
expect(telegram.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
|
||||
expect.arrayContaining([...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]),
|
||||
);
|
||||
expect(whatsapp.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
|
||||
expect.arrayContaining([...whatsAppMockDefaultIds]),
|
||||
);
|
||||
expect([...whatsappIds].toSorted()).toEqual([...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted());
|
||||
expect([...whatsAppMockDefaultIds].every((id) => whatsappIds.has(id))).toBe(true);
|
||||
expect(whatsappDefaultCanonicalScenarioIds("live-frontier")).toEqual(["whatsapp-help-command"]);
|
||||
expect(telegram.find(({ id }) => id === "telegram-status-command")?.regressionRefs).toEqual([
|
||||
"openclaw/openclaw#74698",
|
||||
]);
|
||||
});
|
||||
|
||||
it("partitions canonical aliases from remaining imperative scenarios", () => {
|
||||
expect(
|
||||
partitionCanonicalScenarioIds(
|
||||
["telegram-help-command", "telegram-mentioned-message-reply"],
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
["whatsapp-help-command", "whatsapp-canary"],
|
||||
WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
),
|
||||
).toEqual({
|
||||
canonical: ["telegram-help-command"],
|
||||
legacy: ["telegram-mentioned-message-reply"],
|
||||
canonical: ["whatsapp-help-command"],
|
||||
legacy: ["whatsapp-canary"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -107,10 +90,7 @@ describe("canonical live-transport scenarios", () => {
|
||||
it("removes migrated ids from non-YAML scenario ownership", async () => {
|
||||
const nonYamlIds = new Set((await loadNonYamlScenarioRefs()).map(({ id }) => id));
|
||||
|
||||
for (const scenarioId of [
|
||||
...TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
...WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
]) {
|
||||
for (const scenarioId of WHATSAPP_CANONICAL_SCENARIO_IDS) {
|
||||
expect(nonYamlIds.has(scenarioId), scenarioId).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -2,37 +2,9 @@
|
||||
import path from "node:path";
|
||||
import type { QaGatewayChildCommand } from "../../gateway-child.js";
|
||||
import type { QaTransportAdapterFactory } from "../../qa-transport-registry.js";
|
||||
import { readQaScenarioPack } from "../../scenario-catalog.js";
|
||||
import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js";
|
||||
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
|
||||
|
||||
export const TELEGRAM_CANONICAL_SCENARIO_IDS = [
|
||||
"channel-canary",
|
||||
"channel-mention-gating",
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-context-command",
|
||||
"telegram-current-session-status-tool",
|
||||
"telegram-tool-only-usage-footer",
|
||||
"telegram-reply-chain-exact-marker",
|
||||
] as const;
|
||||
|
||||
export const TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS = [
|
||||
"channel-canary",
|
||||
"channel-mention-gating",
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-context-command",
|
||||
] as const;
|
||||
|
||||
export const WHATSAPP_ROUTING_CANONICAL_SCENARIO_IDS = [
|
||||
"channel-canary",
|
||||
"channel-dm-group-routing",
|
||||
@@ -100,23 +72,6 @@ export function partitionCanonicalScenarioIds(
|
||||
return { canonical, legacy };
|
||||
}
|
||||
|
||||
export function listCanonicalScenarios(params: {
|
||||
ids: readonly string[];
|
||||
defaultIds: readonly string[];
|
||||
}) {
|
||||
const requestedIds = new Set(params.ids);
|
||||
const defaultIds = new Set(params.defaultIds);
|
||||
return readQaScenarioPack()
|
||||
.scenarios.filter((scenario) => requestedIds.has(scenario.id))
|
||||
.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
rationale: scenario.objective,
|
||||
regressionRefs: scenario.regressionRefs ?? [],
|
||||
defaultEnabled: defaultIds.has(scenario.id),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function runCanonicalLiveScenarios(params: {
|
||||
channelId: string;
|
||||
factory: QaTransportAdapterFactory;
|
||||
|
||||
@@ -1,63 +1,11 @@
|
||||
// Qa Lab plugin module implements shared live-transport RTT behavior.
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import type { QaEvidenceTiming } from "../../evidence-summary.js";
|
||||
|
||||
export type LiveTransportRttOptions<CheckId extends string = string> = {
|
||||
count: number;
|
||||
timeoutMs: number;
|
||||
maxFailures: number;
|
||||
checkIds: Set<CheckId>;
|
||||
};
|
||||
|
||||
export type LiveTransportRttSample = {
|
||||
rttMs?: number;
|
||||
status: "pass" | "fail";
|
||||
};
|
||||
|
||||
function normalizePositiveRttInteger(value: number | undefined) {
|
||||
if (
|
||||
typeof value !== "number" ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value <= 0 ||
|
||||
value > MAX_TIMER_TIMEOUT_MS
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeLiveTransportRttOptions<CheckId extends string>(params: {
|
||||
count?: number;
|
||||
defaultCheckIds: readonly CheckId[];
|
||||
knownCheckIds: ReadonlySet<CheckId>;
|
||||
maxFailures?: number;
|
||||
rawCheckIds?: readonly string[];
|
||||
timeoutMs?: number;
|
||||
unknownCheckMessage: (checkId: string) => string;
|
||||
}): LiveTransportRttOptions<CheckId> | undefined {
|
||||
const count = normalizePositiveRttInteger(params.count);
|
||||
if (count === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const rawCheckIds =
|
||||
params.rawCheckIds && params.rawCheckIds.length > 0
|
||||
? params.rawCheckIds
|
||||
: params.defaultCheckIds;
|
||||
const checkIds = new Set<CheckId>();
|
||||
for (const checkId of rawCheckIds) {
|
||||
if (!params.knownCheckIds.has(checkId as CheckId)) {
|
||||
throw new Error(params.unknownCheckMessage(checkId));
|
||||
}
|
||||
checkIds.add(checkId as CheckId);
|
||||
}
|
||||
return {
|
||||
count,
|
||||
maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count,
|
||||
checkIds,
|
||||
timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000,
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(sortedValues: readonly number[], percentileValue: number) {
|
||||
if (sortedValues.length === 0) {
|
||||
return undefined;
|
||||
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { discordQaLiveRuntime } from "../discord/discord-live.runtime.js";
|
||||
import { __testing as slackTesting } from "../slack/slack-live.runtime.js";
|
||||
import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js";
|
||||
import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js";
|
||||
import {
|
||||
buildLiveTransportCoverageLaneSummaries,
|
||||
@@ -115,6 +114,7 @@ describe("live transport scenario helpers", () => {
|
||||
]);
|
||||
expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({
|
||||
standardId: "canary",
|
||||
scenarioId: "channel-canary",
|
||||
});
|
||||
expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({
|
||||
standardId: "restart-resume",
|
||||
@@ -151,10 +151,7 @@ describe("live transport scenario helpers", () => {
|
||||
expect.arrayContaining(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS),
|
||||
);
|
||||
expect(lanes.get("telegram")).toEqual(
|
||||
expect.arrayContaining([
|
||||
"help-command",
|
||||
...telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS,
|
||||
]),
|
||||
expect.arrayContaining(["canary", "help-command", "mention-gating"]),
|
||||
);
|
||||
expect(lanes.get("whatsapp")).toEqual(
|
||||
expect.arrayContaining([
|
||||
|
||||
@@ -57,9 +57,9 @@ const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [
|
||||
transportId: "telegram",
|
||||
commandName: "telegram",
|
||||
members: [
|
||||
{ standardId: "canary" },
|
||||
{ standardId: "canary", scenarioId: "channel-canary" },
|
||||
{ standardId: "help-command", scenarioId: "telegram-help-command" },
|
||||
{ standardId: "mention-gating", scenarioId: "telegram-mention-gating" },
|
||||
{ standardId: "mention-gating", scenarioId: "channel-mention-gating" },
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -107,10 +107,9 @@ export function buildLiveTransportCoverageLaneSummaries(
|
||||
}
|
||||
|
||||
export async function loadNonYamlScenarioRefs() {
|
||||
const [discord, slack, telegram, whatsapp] = await Promise.all([
|
||||
const [discord, slack, whatsapp] = await Promise.all([
|
||||
import("../discord/discord-live.runtime.js"),
|
||||
import("../slack/slack-live.runtime.js"),
|
||||
import("../telegram/telegram-live.runtime.js"),
|
||||
import("../whatsapp/whatsapp-live.runtime.js"),
|
||||
]);
|
||||
const refs = (sourcePath: string, scenarios: readonly { id: string }[]) =>
|
||||
@@ -124,10 +123,6 @@ export async function loadNonYamlScenarioRefs() {
|
||||
"extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts",
|
||||
slack.listSlackQaScenarioCatalog(),
|
||||
),
|
||||
...refs(
|
||||
"extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts",
|
||||
telegram.listTelegramQaScenarioCatalog(),
|
||||
),
|
||||
...refs(
|
||||
"extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts",
|
||||
whatsapp.listWhatsAppQaScenarioCatalog(),
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
acquireQaCredentialLease: vi.fn(),
|
||||
assertQaGatewayCredentialLeaseQuarantine: vi.fn(),
|
||||
callTelegramApi: vi.fn(),
|
||||
flushTelegramUpdates: vi.fn(),
|
||||
heartbeatStop: vi.fn(),
|
||||
heartbeatThrowIfFailed: vi.fn(),
|
||||
leaseHeartbeat: vi.fn(),
|
||||
leaseRelease: vi.fn(),
|
||||
shouldRetainQaGatewayCredentialLease: vi.fn(),
|
||||
waitForTelegramChannelRunning: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../shared/credential-lease.runtime.js", () => ({
|
||||
acquireQaCredentialLease: mocks.acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat: () => ({
|
||||
stop: mocks.heartbeatStop,
|
||||
throwIfFailed: mocks.heartbeatThrowIfFailed,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../../gateway-process-boundary.js", () => ({
|
||||
assertQaGatewayCredentialLeaseQuarantine: mocks.assertQaGatewayCredentialLeaseQuarantine,
|
||||
shouldRetainQaGatewayCredentialLease: mocks.shouldRetainQaGatewayCredentialLease,
|
||||
}));
|
||||
|
||||
vi.mock("./telegram-api.runtime.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./telegram-api.runtime.js")>()),
|
||||
callTelegramApi: mocks.callTelegramApi,
|
||||
flushTelegramUpdates: mocks.flushTelegramUpdates,
|
||||
waitForTelegramChannelRunning: mocks.waitForTelegramChannelRunning,
|
||||
}));
|
||||
|
||||
import { createTelegramQaTransportAdapter } from "./adapter.runtime.js";
|
||||
|
||||
describe("Telegram QA transport adapter", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.acquireQaCredentialLease.mockResolvedValue({
|
||||
payload: {
|
||||
groupId: "-100123",
|
||||
driverToken: "placeholder",
|
||||
sutToken: "placeholder",
|
||||
},
|
||||
source: "env",
|
||||
heartbeat: mocks.leaseHeartbeat,
|
||||
release: mocks.leaseRelease,
|
||||
});
|
||||
mocks.flushTelegramUpdates.mockResolvedValue(0);
|
||||
mocks.shouldRetainQaGatewayCredentialLease.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it("rejects credentials that do not identify two distinct bots", async () => {
|
||||
mocks.callTelegramApi.mockResolvedValue({
|
||||
id: 1,
|
||||
is_bot: true,
|
||||
first_name: "bot",
|
||||
username: "same_bot",
|
||||
});
|
||||
|
||||
await expect(
|
||||
createTelegramQaTransportAdapter({ adapterOptions: {}, messages: {} } as never),
|
||||
).rejects.toThrow("requires two distinct bots");
|
||||
expect(mocks.heartbeatStop).toHaveBeenCalledOnce();
|
||||
expect(mocks.leaseRelease).toHaveBeenCalledOnce();
|
||||
expect(mocks.flushTelegramUpdates).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("maps native sends, replies, edits, and cleanup inside the adapter", async () => {
|
||||
const pollResolvers: Array<(updates: unknown[]) => void> = [];
|
||||
let getMeCalls = 0;
|
||||
let sendMessageCalls = 0;
|
||||
mocks.callTelegramApi.mockImplementation(
|
||||
async (_token: string, method: string): Promise<unknown> => {
|
||||
if (method === "getMe") {
|
||||
getMeCalls += 1;
|
||||
return getMeCalls === 1
|
||||
? { id: 1, is_bot: true, first_name: "driver", username: "driver_bot" }
|
||||
: { id: 2, is_bot: true, first_name: "sut", username: "sut_bot" };
|
||||
}
|
||||
if (method === "sendMessage") {
|
||||
sendMessageCalls += 1;
|
||||
return { message_id: sendMessageCalls === 1 ? 10 : 12 };
|
||||
}
|
||||
if (method === "getUpdates") {
|
||||
return await new Promise<unknown[]>((resolve) => {
|
||||
pollResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected Telegram API method: ${method}`);
|
||||
},
|
||||
);
|
||||
const addInboundMessage = vi.fn().mockResolvedValue({ id: "in-1" });
|
||||
const addOutboundMessage = vi.fn().mockResolvedValue({ id: "out-1" });
|
||||
const editMessage = vi.fn().mockResolvedValue({ id: "out-1" });
|
||||
const adapter = await createTelegramQaTransportAdapter({
|
||||
adapterOptions: { sutAccountId: "sut" },
|
||||
messages: { addInboundMessage, addOutboundMessage, editMessage },
|
||||
} as never);
|
||||
|
||||
await vi.waitFor(() => expect(pollResolvers).toHaveLength(1));
|
||||
await adapter.sendInbound?.({
|
||||
conversation: { id: "logical-room", kind: "group" },
|
||||
senderId: "driver",
|
||||
text: "@openclaw reply exactly: QA-MARKER",
|
||||
});
|
||||
expect(mocks.callTelegramApi).toHaveBeenCalledWith(
|
||||
"placeholder",
|
||||
"sendMessage",
|
||||
expect.objectContaining({
|
||||
chat_id: "-100123",
|
||||
text: "@sut_bot reply exactly: QA-MARKER",
|
||||
}),
|
||||
);
|
||||
|
||||
pollResolvers[0]?.([
|
||||
{
|
||||
update_id: 1,
|
||||
message: {
|
||||
message_id: 11,
|
||||
date: 100,
|
||||
chat: { id: -100123 },
|
||||
from: { id: 2, is_bot: true, username: "sut_bot" },
|
||||
text: "preview",
|
||||
reply_to_message: { message_id: 10 },
|
||||
},
|
||||
},
|
||||
]);
|
||||
await vi.waitFor(() => expect(addOutboundMessage).toHaveBeenCalledOnce());
|
||||
expect(addOutboundMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
to: "group:logical-room",
|
||||
text: "preview",
|
||||
replyToId: "in-1",
|
||||
}),
|
||||
);
|
||||
await adapter.sendInbound?.({
|
||||
conversation: { id: "logical-room", kind: "group" },
|
||||
senderId: "driver",
|
||||
text: "follow-up",
|
||||
replyToId: "out-1",
|
||||
});
|
||||
expect(mocks.callTelegramApi).toHaveBeenCalledWith(
|
||||
"placeholder",
|
||||
"sendMessage",
|
||||
expect.objectContaining({
|
||||
reply_parameters: {
|
||||
message_id: 11,
|
||||
allow_sending_without_reply: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await vi.waitFor(() => expect(pollResolvers).toHaveLength(2));
|
||||
pollResolvers[1]?.([
|
||||
{
|
||||
update_id: 2,
|
||||
edited_message: {
|
||||
message_id: 11,
|
||||
date: 101,
|
||||
chat: { id: -100123 },
|
||||
from: { id: 2, is_bot: true, username: "sut_bot" },
|
||||
text: "final",
|
||||
},
|
||||
},
|
||||
]);
|
||||
await vi.waitFor(() => expect(editMessage).toHaveBeenCalledOnce());
|
||||
expect(editMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ messageId: "out-1", text: "final", timestamp: 101_000 }),
|
||||
);
|
||||
|
||||
await vi.waitFor(() => expect(pollResolvers).toHaveLength(3));
|
||||
const cleanup = adapter.cleanup?.();
|
||||
pollResolvers[2]?.([]);
|
||||
await cleanup;
|
||||
expect(mocks.heartbeatStop).toHaveBeenCalledOnce();
|
||||
expect(mocks.leaseRelease).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
// Qa Lab plugin module implements Telegram live transport adapter behavior.
|
||||
import type { TelegramBotUpdate } from "@openclaw/telegram/api.js";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import {
|
||||
@@ -10,23 +9,34 @@ import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import { __testing as telegramLive } from "./telegram-live.runtime.js";
|
||||
import {
|
||||
buildTelegramQaConfig,
|
||||
callTelegramApi,
|
||||
flushTelegramUpdates,
|
||||
isRecoverableTelegramQaPollError,
|
||||
normalizeTelegramObservedMessage,
|
||||
parseTelegramQaCredentialPayload,
|
||||
resolveTelegramQaRuntimeEnv,
|
||||
waitForTelegramChannelRunning,
|
||||
waitForTelegramPollRetryDelay,
|
||||
type TelegramBotIdentity,
|
||||
type TelegramQaRuntimeEnv,
|
||||
type TelegramUpdate,
|
||||
} from "./telegram-api.runtime.js";
|
||||
|
||||
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
|
||||
type FactoryContext = Parameters<AdapterFactory["create"]>[0];
|
||||
type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>>;
|
||||
type TelegramRuntimeEnv = ReturnType<typeof telegramLive.resolveTelegramQaRuntimeEnv>;
|
||||
|
||||
export async function createTelegramQaTransportAdapter(
|
||||
context: FactoryContext,
|
||||
): Promise<AdapterDefinition> {
|
||||
const options = context.adapterOptions ?? {};
|
||||
const credentialLease = await acquireQaCredentialLease<TelegramRuntimeEnv>({
|
||||
const credentialLease = await acquireQaCredentialLease<TelegramQaRuntimeEnv>({
|
||||
kind: "telegram",
|
||||
source: options.credentialSource,
|
||||
role: options.credentialRole,
|
||||
resolveEnvPayload: () => telegramLive.resolveTelegramQaRuntimeEnv(),
|
||||
parsePayload: telegramLive.parseTelegramQaCredentialPayload,
|
||||
resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(),
|
||||
parsePayload: parseTelegramQaCredentialPayload,
|
||||
});
|
||||
try {
|
||||
assertQaGatewayCredentialLeaseQuarantine(credentialLease);
|
||||
@@ -36,17 +46,26 @@ export async function createTelegramQaTransportAdapter(
|
||||
}
|
||||
const heartbeat = startQaCredentialLeaseHeartbeat(credentialLease);
|
||||
const runtimeEnv = credentialLease.payload;
|
||||
let driverIdentity: { id: number; username?: string };
|
||||
let sutIdentity: { id: number; username?: string };
|
||||
let driverIdentity: TelegramBotIdentity;
|
||||
let sutIdentity: TelegramBotIdentity;
|
||||
let offset: number;
|
||||
try {
|
||||
[driverIdentity, sutIdentity, offset] = await Promise.all([
|
||||
telegramLive.callTelegramApi<{ id: number; username?: string }>(
|
||||
runtimeEnv.driverToken,
|
||||
"getMe",
|
||||
),
|
||||
telegramLive.callTelegramApi<{ id: number; username?: string }>(runtimeEnv.sutToken, "getMe"),
|
||||
telegramLive.flushTelegramUpdates(runtimeEnv.driverToken),
|
||||
[driverIdentity, sutIdentity] = await Promise.all([
|
||||
callTelegramApi<TelegramBotIdentity>(runtimeEnv.driverToken, "getMe"),
|
||||
callTelegramApi<TelegramBotIdentity>(runtimeEnv.sutToken, "getMe"),
|
||||
]);
|
||||
if (!driverIdentity.is_bot || !sutIdentity.is_bot) {
|
||||
throw new Error("Telegram QA credentials must belong to bots.");
|
||||
}
|
||||
if (driverIdentity.id === sutIdentity.id) {
|
||||
throw new Error("Telegram QA requires two distinct bots for driver and SUT.");
|
||||
}
|
||||
if (!sutIdentity.username?.trim()) {
|
||||
throw new Error("Telegram QA requires the SUT bot to have a Telegram username.");
|
||||
}
|
||||
[offset] = await Promise.all([
|
||||
flushTelegramUpdates(runtimeEnv.driverToken),
|
||||
flushTelegramUpdates(runtimeEnv.sutToken),
|
||||
]);
|
||||
} catch (error) {
|
||||
await heartbeat.stop();
|
||||
@@ -64,39 +83,56 @@ export async function createTelegramQaTransportAdapter(
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
const updates = await telegramLive.callTelegramApi<TelegramBotUpdate[]>(
|
||||
runtimeEnv.driverToken,
|
||||
"getUpdates",
|
||||
{ offset, timeout: 1, allowed_updates: ["message", "edited_message"] },
|
||||
6_000,
|
||||
);
|
||||
let updates: TelegramUpdate[];
|
||||
try {
|
||||
updates = await callTelegramApi<TelegramUpdate[]>(
|
||||
runtimeEnv.driverToken,
|
||||
"getUpdates",
|
||||
{ offset, timeout: 1, allowed_updates: ["message", "edited_message"] },
|
||||
6_000,
|
||||
);
|
||||
} catch (error) {
|
||||
if (!isRecoverableTelegramQaPollError(error)) {
|
||||
throw error;
|
||||
}
|
||||
await waitForTelegramPollRetryDelay();
|
||||
continue;
|
||||
}
|
||||
for (const update of updates) {
|
||||
offset = Math.max(offset, update.update_id + 1);
|
||||
const message = update.edited_message ?? update.message;
|
||||
if (!message?.from?.id || message.from.id !== sutIdentity.id) {
|
||||
const message = normalizeTelegramObservedMessage(update);
|
||||
if (
|
||||
!message ||
|
||||
message.chatId !== Number(runtimeEnv.groupId) ||
|
||||
message.senderId !== sutIdentity.id
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const existingMessageId = busMessageIds.get(message.message_id);
|
||||
if (update.edited_message && existingMessageId) {
|
||||
await context.messages.editMessage({
|
||||
accountId: options.sutAccountId?.trim() || "sut",
|
||||
messageId: existingMessageId,
|
||||
text: message.text ?? message.caption ?? "",
|
||||
});
|
||||
const existingMessageId = busMessageIds.get(message.messageId);
|
||||
if (update.edited_message) {
|
||||
if (existingMessageId) {
|
||||
await context.messages.editMessage({
|
||||
accountId: options.sutAccountId?.trim() || "sut",
|
||||
messageId: existingMessageId,
|
||||
text: message.text,
|
||||
timestamp: message.timestamp,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const outbound = await context.messages.addOutboundMessage({
|
||||
accountId: options.sutAccountId?.trim() || "sut",
|
||||
to: `${logicalConversationKind}:${logicalConversationId}`,
|
||||
senderId: String(message.from.id),
|
||||
senderName: message.from.username,
|
||||
text: message.text ?? message.caption ?? "",
|
||||
timestamp: message.date * 1_000,
|
||||
replyToId: message.reply_to_message?.message_id
|
||||
? busMessageIds.get(message.reply_to_message.message_id)
|
||||
senderId: String(message.senderId),
|
||||
senderName: message.senderUsername,
|
||||
text: message.text,
|
||||
timestamp: message.timestamp,
|
||||
replyToId: message.replyToMessageId
|
||||
? busMessageIds.get(message.replyToMessageId)
|
||||
: undefined,
|
||||
});
|
||||
busMessageIds.set(message.message_id, outbound.id);
|
||||
nativeMessageIds.set(outbound.id, message.messageId);
|
||||
busMessageIds.set(message.messageId, outbound.id);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -127,7 +163,7 @@ export async function createTelegramQaTransportAdapter(
|
||||
? input.text.replaceAll("@openclaw", `@${sutIdentity.username}`)
|
||||
: input.text;
|
||||
const nativeReplyToId = input.replyToId ? nativeMessageIds.get(input.replyToId) : undefined;
|
||||
const sent = await telegramLive.callTelegramApi<{ message_id: number }>(
|
||||
const sent = await callTelegramApi<{ message_id: number }>(
|
||||
runtimeEnv.driverToken,
|
||||
"sendMessage",
|
||||
{
|
||||
@@ -161,14 +197,14 @@ export async function createTelegramQaTransportAdapter(
|
||||
busMessageIds.clear();
|
||||
},
|
||||
createGatewayConfig: () =>
|
||||
telegramLive.buildTelegramQaConfig({} as OpenClawConfig, {
|
||||
buildTelegramQaConfig({} as OpenClawConfig, {
|
||||
groupId: runtimeEnv.groupId,
|
||||
sutToken: runtimeEnv.sutToken,
|
||||
driverBotId: driverIdentity.id,
|
||||
sutAccountId: accountId,
|
||||
}),
|
||||
waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) =>
|
||||
await telegramLive.waitForTelegramChannelRunning(gateway as never, accountId, {
|
||||
await waitForTelegramChannelRunning(gateway, accountId, {
|
||||
timeoutMs,
|
||||
pollMs: pollIntervalMs,
|
||||
}),
|
||||
|
||||
@@ -2,22 +2,15 @@ import { constants as fsConstants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { QaGatewayChildCommand } from "../../gateway-child.js";
|
||||
import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js";
|
||||
import type { QaSuiteRoundTripProbe } from "../../suite-round-trip.js";
|
||||
import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js";
|
||||
import {
|
||||
assertKnownScenarioIds,
|
||||
canonicalScenarioOutputDir,
|
||||
listCanonicalScenarios,
|
||||
partitionCanonicalScenarioIds,
|
||||
runCanonicalLiveScenarios,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
} from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js";
|
||||
import { createTelegramQaTransportAdapter } from "./adapter.runtime.js";
|
||||
import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js";
|
||||
import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js";
|
||||
|
||||
const TELEGRAM_QA_SUT_OPENCLAW_COMMAND_ENV = "OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND";
|
||||
const TELEGRAM_QA_SUT_UID_ENV = "OPENCLAW_QA_TELEGRAM_SUT_UID";
|
||||
@@ -142,17 +135,15 @@ async function resolveTelegramQaSutOpenClawCommand(
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
|
||||
type TelegramQaSuiteOptions = LiveTransportQaCommandOptions & {
|
||||
roundTripProbe?: QaSuiteRoundTripProbe;
|
||||
sutOpenClawCommand?: QaGatewayChildCommand;
|
||||
};
|
||||
|
||||
export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
if (runOptions.listScenarios) {
|
||||
const scenarios = [
|
||||
...listCanonicalScenarios({
|
||||
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
}),
|
||||
...listTelegramQaScenarioCatalog(runOptions.providerMode),
|
||||
];
|
||||
for (const scenario of scenarios) {
|
||||
for (const scenario of listTelegramQaScenarios(runOptions.providerMode)) {
|
||||
const defaultLabel = scenario.defaultEnabled ? "default" : "optional";
|
||||
const refs =
|
||||
scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : "";
|
||||
@@ -160,62 +151,40 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
|
||||
`${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
return undefined;
|
||||
}
|
||||
const selected = partitionCanonicalScenarioIds(
|
||||
runOptions.scenarioIds,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
);
|
||||
const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0;
|
||||
if (hasExplicitScenarioIds) {
|
||||
assertKnownScenarioIds({
|
||||
ids: selected.legacy,
|
||||
knownIds: listTelegramQaScenarioCatalog(runOptions.providerMode).map(({ id }) => id),
|
||||
laneLabel: "Telegram",
|
||||
});
|
||||
}
|
||||
const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot);
|
||||
const executionOptions = {
|
||||
...runOptions,
|
||||
...(sutOpenClawCommand ? { sutOpenClawCommand } : {}),
|
||||
};
|
||||
const canonicalScenarioIds = hasExplicitScenarioIds
|
||||
? selected.canonical
|
||||
: [...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS];
|
||||
const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0;
|
||||
if (canonicalScenarioIds.length > 0) {
|
||||
const canonical = await runCanonicalLiveScenarios({
|
||||
channelId: "telegram",
|
||||
factory: {
|
||||
const scenarioIds = resolveTelegramQaScenarioIds({
|
||||
profile: opts.profile,
|
||||
providerMode: runOptions.providerMode,
|
||||
scenarioIds: runOptions.scenarioIds,
|
||||
});
|
||||
const result = await runQaFlowSuiteFromRuntime({
|
||||
adapterFactories: [
|
||||
{
|
||||
id: "telegram",
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
|
||||
create: createTelegramQaTransportAdapter,
|
||||
},
|
||||
options: {
|
||||
...executionOptions,
|
||||
outputDir: canonicalScenarioOutputDir(executionOptions, runsLegacyScenarios),
|
||||
},
|
||||
scenarioIds: canonicalScenarioIds,
|
||||
});
|
||||
printLiveTransportQaArtifacts("Telegram canonical QA", {
|
||||
report: canonical.reportPath,
|
||||
summary: canonical.summaryPath,
|
||||
});
|
||||
if (!runOptions.allowFailures) {
|
||||
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(
|
||||
canonical.summaryPath,
|
||||
);
|
||||
if (failedScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!runsLegacyScenarios) {
|
||||
return;
|
||||
}
|
||||
const result = await runTelegramQaLive({
|
||||
...executionOptions,
|
||||
scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined,
|
||||
],
|
||||
adapterOptions: {
|
||||
repoRoot: runOptions.repoRoot,
|
||||
...(runOptions.credentialRole ? { credentialRole: runOptions.credentialRole } : {}),
|
||||
...(runOptions.credentialSource ? { credentialSource: runOptions.credentialSource } : {}),
|
||||
...(runOptions.sutAccountId ? { sutAccountId: runOptions.sutAccountId } : {}),
|
||||
},
|
||||
channelDriver: "live",
|
||||
channelId: "telegram",
|
||||
concurrency: 1,
|
||||
...(runOptions.alternateModel ? { alternateModel: runOptions.alternateModel } : {}),
|
||||
...(runOptions.fastMode !== undefined ? { fastMode: runOptions.fastMode } : {}),
|
||||
...(runOptions.failFast !== undefined ? { failFast: runOptions.failFast } : {}),
|
||||
...(runOptions.outputDir ? { outputDir: runOptions.outputDir } : {}),
|
||||
...(runOptions.primaryModel ? { primaryModel: runOptions.primaryModel } : {}),
|
||||
providerMode: runOptions.providerMode,
|
||||
repoRoot: runOptions.repoRoot,
|
||||
roundTripProbe: opts.roundTripProbe,
|
||||
scenarioIds,
|
||||
sutOpenClawCommand: opts.sutOpenClawCommand,
|
||||
});
|
||||
printLiveTransportQaArtifacts("Telegram QA", {
|
||||
report: result.reportPath,
|
||||
@@ -227,4 +196,22 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
if (runOptions.listScenarios) {
|
||||
return await runQaTelegramSuite(opts);
|
||||
}
|
||||
resolveTelegramQaScenarioIds({
|
||||
profile: opts.profile,
|
||||
providerMode: runOptions.providerMode,
|
||||
scenarioIds: runOptions.scenarioIds,
|
||||
});
|
||||
const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot);
|
||||
return await runQaTelegramSuite({
|
||||
...opts,
|
||||
...(sutOpenClawCommand ? { sutOpenClawCommand } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
13
extensions/qa-lab/src/live-transports/telegram/cli.test.ts
Normal file
13
extensions/qa-lab/src/live-transports/telegram/cli.test.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { telegramQaCliRegistration } from "./cli.js";
|
||||
import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js";
|
||||
|
||||
describe("Telegram QA CLI registration", () => {
|
||||
it("keeps the generic live baseline outside command profile selection", () => {
|
||||
expect(telegramQaCliRegistration.adapterFactory?.scenarioIds).toEqual([
|
||||
"channel-chat-baseline",
|
||||
...TELEGRAM_QA_ALL_SCENARIO_IDS,
|
||||
]);
|
||||
expect(TELEGRAM_QA_ALL_SCENARIO_IDS).not.toContain("channel-chat-baseline");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,3 @@
|
||||
import { TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
@@ -6,6 +5,7 @@ import {
|
||||
type LiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCommandOptions,
|
||||
} from "../shared/live-transport-cli.js";
|
||||
import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js";
|
||||
|
||||
type TelegramQaAdapterRuntime = typeof import("./adapter.runtime.js");
|
||||
type TelegramQaCliRuntime = typeof import("./cli.runtime.js");
|
||||
@@ -23,7 +23,7 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
|
||||
|
||||
const telegramQaAdapterFactory: NonNullable<LiveTransportQaCliRegistration["adapterFactory"]> = {
|
||||
id: "telegram",
|
||||
scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS],
|
||||
scenarioIds: ["channel-chat-baseline", ...TELEGRAM_QA_ALL_SCENARIO_IDS],
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
|
||||
async create(context) {
|
||||
return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context);
|
||||
@@ -42,6 +42,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
|
||||
listScenariosHelp: "Print available Telegram scenario ids and exit",
|
||||
outputDirHelp: "Telegram QA artifact directory",
|
||||
profileHelp: "QA Lab Telegram profile: release or all (default: release)",
|
||||
run: runQaTelegram,
|
||||
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
|
||||
sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import fs from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readQaScenarioById } from "../../scenario-catalog.js";
|
||||
|
||||
const LEGACY_TELEGRAM_BEHAVIOR_LEDGER = [
|
||||
{
|
||||
id: "telegram-other-bot-command-gating",
|
||||
owner: "qa/scenarios/channels/telegram-other-bot-command-gating.yaml",
|
||||
scenarioId: "telegram-other-bot-command-gating",
|
||||
},
|
||||
{
|
||||
id: "telegram-mentioned-message-reply",
|
||||
owner: "qa/scenarios/channels/channel-canary.yaml",
|
||||
scenarioId: "channel-canary",
|
||||
},
|
||||
{
|
||||
id: "telegram-stream-final-single-message",
|
||||
owner: "qa/scenarios/channels/telegram-stream-final-single-message.yaml",
|
||||
scenarioId: "telegram-stream-final-single-message",
|
||||
},
|
||||
{
|
||||
id: "telegram-long-final-reuses-preview",
|
||||
owner: "qa/scenarios/channels/telegram-long-final-reuses-preview.yaml",
|
||||
scenarioId: "telegram-long-final-reuses-preview",
|
||||
},
|
||||
{
|
||||
id: "telegram-long-final-three-chunks",
|
||||
owner: "qa/scenarios/channels/telegram-long-final-three-chunks.yaml",
|
||||
scenarioId: "telegram-long-final-three-chunks",
|
||||
},
|
||||
{
|
||||
id: "telegram-mention-gating",
|
||||
owner: "qa/scenarios/channels/channel-mention-gating.yaml",
|
||||
scenarioId: "channel-mention-gating",
|
||||
},
|
||||
] as const;
|
||||
|
||||
const LEGACY_TELEGRAM_CAPABILITY_LEDGER = [
|
||||
{
|
||||
id: "credentials-and-lease-cleanup",
|
||||
owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts",
|
||||
proof: "acquireQaCredentialLease",
|
||||
},
|
||||
{
|
||||
id: "bot-api-and-rich-message-observation",
|
||||
owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts",
|
||||
proof: "normalizeTelegramObservedMessage",
|
||||
},
|
||||
{
|
||||
id: "native-send-reply-edit-mapping",
|
||||
owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts",
|
||||
proof: "reply_parameters",
|
||||
},
|
||||
{
|
||||
id: "gateway-readiness-and-config",
|
||||
owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts",
|
||||
proof: "waitForTelegramChannelRunning",
|
||||
},
|
||||
{
|
||||
id: "selection-and-standard-artifacts",
|
||||
owner: "extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts",
|
||||
proof: "runQaFlowSuiteFromRuntime",
|
||||
},
|
||||
{
|
||||
id: "package-rtt-sampling",
|
||||
owner: "scripts/e2e/npm-telegram-live-runner.ts",
|
||||
proof: "createRoundTripProbe",
|
||||
},
|
||||
] as const;
|
||||
|
||||
describe("legacy Telegram runner parity", () => {
|
||||
it("maps every retired scenario to one maintained YAML owner", () => {
|
||||
expect(LEGACY_TELEGRAM_BEHAVIOR_LEDGER).toHaveLength(6);
|
||||
expect(new Set(LEGACY_TELEGRAM_BEHAVIOR_LEDGER.map(({ id }) => id)).size).toBe(6);
|
||||
|
||||
for (const entry of LEGACY_TELEGRAM_BEHAVIOR_LEDGER) {
|
||||
const scenario = readQaScenarioById(entry.scenarioId);
|
||||
expect(scenario.sourcePath, entry.id).toBe(entry.owner);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps every runner capability at its intended owner boundary", () => {
|
||||
for (const entry of LEGACY_TELEGRAM_CAPABILITY_LEDGER) {
|
||||
expect(fs.readFileSync(entry.owner, "utf8"), entry.id).toContain(entry.proof);
|
||||
}
|
||||
});
|
||||
|
||||
it("has no unresolved legacy runner owner", () => {
|
||||
expect(
|
||||
fs.existsSync("extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts"),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
listTelegramQaScenarios,
|
||||
resolveTelegramQaScenarioIds,
|
||||
TELEGRAM_QA_ALL_SCENARIO_IDS,
|
||||
} from "./profiles.js";
|
||||
|
||||
describe("Telegram QA profiles", () => {
|
||||
it("keeps release focused and adds the scripted long-final check for mock runs", () => {
|
||||
const live = resolveTelegramQaScenarioIds({ providerMode: "live-frontier" });
|
||||
const mock = resolveTelegramQaScenarioIds({ providerMode: "mock-openai" });
|
||||
|
||||
expect(live).toContain("telegram-other-bot-command-gating");
|
||||
expect(live).not.toContain("telegram-long-final-reuses-preview");
|
||||
expect(mock).toEqual([...live, "telegram-long-final-reuses-preview"]);
|
||||
});
|
||||
|
||||
it("selects every migrated Telegram scenario through all", () => {
|
||||
expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual([
|
||||
...TELEGRAM_QA_ALL_SCENARIO_IDS,
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets explicit scenarios override profile selection", () => {
|
||||
expect(
|
||||
resolveTelegramQaScenarioIds({
|
||||
profile: "all",
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["telegram-status-command"],
|
||||
}),
|
||||
).toEqual(["telegram-status-command"]);
|
||||
});
|
||||
|
||||
it("rejects unknown profiles and scenarios before gateway startup", () => {
|
||||
expect(() =>
|
||||
resolveTelegramQaScenarioIds({ providerMode: "live-frontier", profile: "transport" }),
|
||||
).toThrow('Unknown QA Lab Telegram profile "transport"');
|
||||
expect(() =>
|
||||
resolveTelegramQaScenarioIds({
|
||||
providerMode: "live-frontier",
|
||||
scenarioIds: ["telegram-missing"],
|
||||
}),
|
||||
).toThrow("unknown Telegram QA scenario id(s): telegram-missing");
|
||||
});
|
||||
|
||||
it("lists the YAML catalog with provider-specific release defaults", () => {
|
||||
const scenarios = listTelegramQaScenarios("mock-openai");
|
||||
|
||||
expect(scenarios.map(({ id }) => id).toSorted()).toEqual(
|
||||
[...TELEGRAM_QA_ALL_SCENARIO_IDS].toSorted(),
|
||||
);
|
||||
expect(
|
||||
scenarios.find(({ id }) => id === "telegram-long-final-reuses-preview")?.defaultEnabled,
|
||||
).toBe(true);
|
||||
expect(
|
||||
scenarios.find(({ id }) => id === "telegram-long-final-three-chunks")?.defaultEnabled,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
77
extensions/qa-lab/src/live-transports/telegram/profiles.ts
Normal file
77
extensions/qa-lab/src/live-transports/telegram/profiles.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { readQaScenarioPack } from "../../scenario-catalog.js";
|
||||
|
||||
const TELEGRAM_QA_RELEASE_SCENARIO_IDS = [
|
||||
"channel-canary",
|
||||
"channel-mention-gating",
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-context-command",
|
||||
"telegram-other-bot-command-gating",
|
||||
] as const;
|
||||
|
||||
const TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS = [
|
||||
...TELEGRAM_QA_RELEASE_SCENARIO_IDS,
|
||||
"telegram-long-final-reuses-preview",
|
||||
] as const;
|
||||
|
||||
export const TELEGRAM_QA_ALL_SCENARIO_IDS = [
|
||||
...TELEGRAM_QA_RELEASE_SCENARIO_IDS,
|
||||
"telegram-current-session-status-tool",
|
||||
"telegram-tool-only-usage-footer",
|
||||
"telegram-reply-chain-exact-marker",
|
||||
"telegram-stream-final-single-message",
|
||||
"telegram-long-final-reuses-preview",
|
||||
"telegram-long-final-three-chunks",
|
||||
] as const;
|
||||
|
||||
type TelegramQaProfile = "all" | "release";
|
||||
|
||||
function resolveTelegramQaProfile(profile: string | undefined): TelegramQaProfile {
|
||||
const normalized = profile?.trim() || "release";
|
||||
if (normalized === "all" || normalized === "release") {
|
||||
return normalized;
|
||||
}
|
||||
throw new Error(
|
||||
`Unknown QA Lab Telegram profile "${normalized}". Expected one of: all, release.`,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveTelegramQaScenarioIds(params: {
|
||||
profile?: string;
|
||||
providerMode: string;
|
||||
scenarioIds?: readonly string[];
|
||||
}): string[] {
|
||||
const knownIds = new Set<string>(TELEGRAM_QA_ALL_SCENARIO_IDS);
|
||||
if (params.scenarioIds && params.scenarioIds.length > 0) {
|
||||
const unknownIds = params.scenarioIds.filter((id) => !knownIds.has(id));
|
||||
if (unknownIds.length > 0) {
|
||||
throw new Error(`unknown Telegram QA scenario id(s): ${unknownIds.join(", ")}`);
|
||||
}
|
||||
return [...params.scenarioIds];
|
||||
}
|
||||
const profile = resolveTelegramQaProfile(params.profile);
|
||||
if (profile === "all") {
|
||||
return [...TELEGRAM_QA_ALL_SCENARIO_IDS];
|
||||
}
|
||||
return params.providerMode === "mock-openai"
|
||||
? [...TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS]
|
||||
: [...TELEGRAM_QA_RELEASE_SCENARIO_IDS];
|
||||
}
|
||||
|
||||
export function listTelegramQaScenarios(providerMode: string) {
|
||||
const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode, profile: "release" }));
|
||||
const allIds = new Set<string>(TELEGRAM_QA_ALL_SCENARIO_IDS);
|
||||
return readQaScenarioPack()
|
||||
.scenarios.filter((scenario) => allIds.has(scenario.id))
|
||||
.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
rationale: scenario.objective,
|
||||
regressionRefs: scenario.regressionRefs ?? [],
|
||||
defaultEnabled: defaultIds.has(scenario.id),
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildTelegramQaConfig,
|
||||
isRecoverableTelegramQaPollError,
|
||||
normalizeTelegramObservedMessage,
|
||||
parseTelegramQaCredentialPayload,
|
||||
resolveTelegramQaRuntimeEnv,
|
||||
waitForTelegramChannelRunning,
|
||||
} from "./telegram-api.runtime.js";
|
||||
|
||||
describe("Telegram QA API boundary", () => {
|
||||
it("parses env and leased credential payloads", () => {
|
||||
expect(
|
||||
resolveTelegramQaRuntimeEnv({
|
||||
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
|
||||
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "placeholder",
|
||||
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "placeholder",
|
||||
}),
|
||||
).toEqual({
|
||||
groupId: "-100123",
|
||||
driverToken: "placeholder",
|
||||
sutToken: "placeholder",
|
||||
});
|
||||
expect(
|
||||
parseTelegramQaCredentialPayload({
|
||||
groupId: "-100456",
|
||||
driverToken: "placeholder",
|
||||
sutToken: "placeholder",
|
||||
}),
|
||||
).toEqual({
|
||||
groupId: "-100456",
|
||||
driverToken: "placeholder",
|
||||
sutToken: "placeholder",
|
||||
});
|
||||
expect(() =>
|
||||
parseTelegramQaCredentialPayload({
|
||||
groupId: "group-name",
|
||||
driverToken: "placeholder",
|
||||
sutToken: "placeholder",
|
||||
}),
|
||||
).toThrow("numeric Telegram chat id");
|
||||
});
|
||||
|
||||
it("normalizes rich edited messages and native reply metadata", () => {
|
||||
expect(
|
||||
normalizeTelegramObservedMessage({
|
||||
update_id: 9,
|
||||
edited_message: {
|
||||
message_id: 42,
|
||||
date: 123,
|
||||
chat: { id: -100123 },
|
||||
from: { id: 2, is_bot: true, username: "sut_bot" },
|
||||
rich_message: {
|
||||
blocks: [{ text: "final " }, { text: [{ text: "reply" }] }],
|
||||
},
|
||||
reply_to_message: { message_id: 41 },
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
updateId: 9,
|
||||
messageId: 42,
|
||||
chatId: -100123,
|
||||
senderId: 2,
|
||||
senderIsBot: true,
|
||||
text: "final \nreply",
|
||||
replyToMessageId: 41,
|
||||
timestamp: 123_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("builds the isolated Telegram gateway config", () => {
|
||||
const config = buildTelegramQaConfig(
|
||||
{ plugins: { allow: ["qa-lab"] } },
|
||||
{
|
||||
groupId: "-100123",
|
||||
sutToken: "placeholder",
|
||||
driverBotId: 1,
|
||||
sutAccountId: "sut",
|
||||
},
|
||||
);
|
||||
|
||||
expect(config.plugins?.allow).toEqual(["qa-lab", "telegram"]);
|
||||
expect(config.channels?.telegram).toMatchObject({
|
||||
enabled: true,
|
||||
defaultAccount: "sut",
|
||||
accounts: {
|
||||
sut: {
|
||||
botToken: "placeholder",
|
||||
dmPolicy: "disabled",
|
||||
replyToMode: "first",
|
||||
groups: {
|
||||
"-100123": {
|
||||
groupPolicy: "allowlist",
|
||||
allowFrom: ["1"],
|
||||
requireMention: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("waits for the selected Telegram account to become connected", async () => {
|
||||
const call = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
telegram: [{ accountId: "sut", running: true, connected: false }],
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
telegram: [{ accountId: "sut", running: true, connected: true }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitForTelegramChannelRunning({ call }, "sut", { timeoutMs: 100, pollMs: 1 });
|
||||
expect(call).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("classifies transient polling failures", () => {
|
||||
expect(isRecoverableTelegramQaPollError(new Error("socket hang up"))).toBe(true);
|
||||
expect(isRecoverableTelegramQaPollError(new Error("Telegram unauthorized"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,433 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
// Qa Lab plugin module owns Telegram live adapter API and credential behavior.
|
||||
import type { TelegramBotMessage, TelegramBotUpdate } from "@openclaw/telegram/api.js";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
parseStrictPositiveInteger,
|
||||
resolveTimerTimeoutMs,
|
||||
} from "openclaw/plugin-sdk/number-runtime";
|
||||
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { z } from "zod";
|
||||
|
||||
export type TelegramQaRuntimeEnv = {
|
||||
groupId: string;
|
||||
driverToken: string;
|
||||
sutToken: string;
|
||||
};
|
||||
|
||||
export type TelegramBotIdentity = {
|
||||
id: number;
|
||||
is_bot: boolean;
|
||||
first_name: string;
|
||||
username?: string;
|
||||
};
|
||||
|
||||
type TelegramApiEnvelope<T> = {
|
||||
ok: boolean;
|
||||
result?: T;
|
||||
description?: string;
|
||||
};
|
||||
|
||||
type TelegramReplyMarkup = {
|
||||
inline_keyboard?: Array<Array<{ text?: string }>>;
|
||||
};
|
||||
|
||||
type TelegramRichMessage = {
|
||||
markdown?: string;
|
||||
html?: string;
|
||||
blocks?: unknown[];
|
||||
};
|
||||
|
||||
type TelegramMessage = Pick<TelegramBotMessage, "date" | "message_id"> &
|
||||
Partial<Pick<TelegramBotMessage, "caption" | "text">> & {
|
||||
audio?: unknown;
|
||||
chat: { id: number };
|
||||
document?: unknown;
|
||||
from?: Pick<NonNullable<TelegramBotMessage["from"]>, "id" | "is_bot" | "username">;
|
||||
photo?: unknown[];
|
||||
rich_message?: TelegramRichMessage;
|
||||
reply_markup?: TelegramReplyMarkup;
|
||||
reply_to_message?: { message_id?: number };
|
||||
sticker?: unknown;
|
||||
video?: unknown;
|
||||
voice?: unknown;
|
||||
};
|
||||
|
||||
export type TelegramUpdate = Pick<TelegramBotUpdate, "update_id"> & {
|
||||
edited_message?: TelegramMessage;
|
||||
message?: TelegramMessage;
|
||||
};
|
||||
|
||||
type TelegramObservedMessage = {
|
||||
updateId: number;
|
||||
messageId: number;
|
||||
chatId: number;
|
||||
senderId: number;
|
||||
senderIsBot: boolean;
|
||||
senderUsername?: string;
|
||||
text: string;
|
||||
caption?: string;
|
||||
replyToMessageId?: number;
|
||||
timestamp: number;
|
||||
inlineButtons: string[];
|
||||
mediaKinds: string[];
|
||||
};
|
||||
|
||||
type TelegramChannelStatus = {
|
||||
accountId?: string;
|
||||
connected?: boolean;
|
||||
lastConnectedAt?: number;
|
||||
lastDisconnect?: unknown;
|
||||
lastError?: string | null;
|
||||
restartPending?: boolean;
|
||||
running?: boolean;
|
||||
};
|
||||
|
||||
type TelegramGatewayClient = {
|
||||
call: (method: string, params?: unknown, options?: { timeoutMs?: number }) => Promise<unknown>;
|
||||
};
|
||||
|
||||
const TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS = 45_000;
|
||||
const TELEGRAM_QA_ENV_FIELDS = [
|
||||
{ field: "groupId", envKey: "OPENCLAW_QA_TELEGRAM_GROUP_ID" },
|
||||
{ field: "driverToken", envKey: "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN" },
|
||||
{ field: "sutToken", envKey: "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN" },
|
||||
] as const;
|
||||
|
||||
const telegramQaCredentialPayloadSchema = z.object({
|
||||
groupId: z.string().trim().min(1),
|
||||
driverToken: z.string().trim().min(1),
|
||||
sutToken: z.string().trim().min(1),
|
||||
});
|
||||
|
||||
function resolveEnvValue(
|
||||
env: NodeJS.ProcessEnv,
|
||||
key: (typeof TELEGRAM_QA_ENV_FIELDS)[number]["envKey"],
|
||||
) {
|
||||
const value = env[key]?.trim();
|
||||
if (!value) {
|
||||
throw new Error(`Missing ${key}.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function resolveTelegramQaRuntimeEnv(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): TelegramQaRuntimeEnv {
|
||||
return parseTelegramQaCredentialPayload(
|
||||
Object.fromEntries(
|
||||
TELEGRAM_QA_ENV_FIELDS.map(({ envKey, field }) => [field, resolveEnvValue(env, envKey)]),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function parseTelegramQaCredentialPayload(payload: unknown): TelegramQaRuntimeEnv {
|
||||
const parsed = telegramQaCredentialPayloadSchema.parse(payload);
|
||||
if (!/^-?\d+$/u.test(parsed.groupId)) {
|
||||
throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id.");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function flattenInlineButtons(replyMarkup?: TelegramReplyMarkup) {
|
||||
return (replyMarkup?.inline_keyboard ?? [])
|
||||
.flat()
|
||||
.map((button) => button.text?.trim())
|
||||
.filter((text): text is string => Boolean(text));
|
||||
}
|
||||
|
||||
function detectMediaKinds(message: TelegramMessage) {
|
||||
const kinds: string[] = [];
|
||||
if (Array.isArray(message.photo) && message.photo.length > 0) {
|
||||
kinds.push("photo");
|
||||
}
|
||||
for (const kind of ["document", "audio", "video", "voice", "sticker"] as const) {
|
||||
if (message[kind]) {
|
||||
kinds.push(kind);
|
||||
}
|
||||
}
|
||||
return kinds;
|
||||
}
|
||||
|
||||
function flattenTelegramRichText(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((part) => flattenTelegramRichText(part)).join("");
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return "";
|
||||
}
|
||||
if ("text" in value) {
|
||||
return flattenTelegramRichText(value.text);
|
||||
}
|
||||
if (typeof value.alternative_text === "string") {
|
||||
return value.alternative_text;
|
||||
}
|
||||
return typeof value.expression === "string" ? value.expression : "";
|
||||
}
|
||||
|
||||
function flattenTelegramRichTableCells(value: unknown): string {
|
||||
if (!Array.isArray(value)) {
|
||||
return flattenTelegramRichBlock(value);
|
||||
}
|
||||
return value
|
||||
.map((row) => {
|
||||
const cells = Array.isArray(row) ? row : [row];
|
||||
return cells
|
||||
.map((cell) => flattenTelegramRichBlock(cell))
|
||||
.filter((cell) => cell.trim())
|
||||
.join("\t");
|
||||
})
|
||||
.filter((row) => row.trim())
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function flattenTelegramRichBlocks(value: unknown): string {
|
||||
const blocks = Array.isArray(value) ? value : [value];
|
||||
return blocks
|
||||
.map((block) => flattenTelegramRichBlock(block))
|
||||
.filter((part) => part.trim())
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function flattenTelegramRichBlock(value: unknown): string {
|
||||
if (typeof value === "string" || Array.isArray(value)) {
|
||||
return flattenTelegramRichText(value);
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return "";
|
||||
}
|
||||
const parts = [
|
||||
"text" in value ? flattenTelegramRichText(value.text) : "",
|
||||
"summary" in value ? flattenTelegramRichText(value.summary) : "",
|
||||
typeof value.label === "string" ? value.label : "",
|
||||
typeof value.expression === "string" ? value.expression : "",
|
||||
"blocks" in value ? flattenTelegramRichBlocks(value.blocks) : "",
|
||||
"items" in value ? flattenTelegramRichBlocks(value.items) : "",
|
||||
"cells" in value ? flattenTelegramRichTableCells(value.cells) : "",
|
||||
"caption" in value ? flattenTelegramRichBlock(value.caption) : "",
|
||||
"credit" in value ? flattenTelegramRichText(value.credit) : "",
|
||||
];
|
||||
return parts.filter((part) => part.trim()).join("\n");
|
||||
}
|
||||
|
||||
function selectTelegramObservedText(message: TelegramMessage) {
|
||||
return (
|
||||
message.text ||
|
||||
message.caption ||
|
||||
message.rich_message?.markdown ||
|
||||
message.rich_message?.html ||
|
||||
flattenTelegramRichBlocks(message.rich_message?.blocks) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
export function normalizeTelegramObservedMessage(
|
||||
update: TelegramUpdate,
|
||||
): TelegramObservedMessage | null {
|
||||
const message = update.edited_message ?? update.message;
|
||||
if (!message?.from?.id) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
updateId: update.update_id,
|
||||
messageId: message.message_id,
|
||||
chatId: message.chat.id,
|
||||
senderId: message.from.id,
|
||||
senderIsBot: message.from.is_bot,
|
||||
senderUsername: message.from.username,
|
||||
text: selectTelegramObservedText(message),
|
||||
caption: message.caption,
|
||||
replyToMessageId: message.reply_to_message?.message_id,
|
||||
timestamp: message.date * 1_000,
|
||||
inlineButtons: flattenInlineButtons(message.reply_markup),
|
||||
mediaKinds: detectMediaKinds(message),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildTelegramQaConfig(
|
||||
baseCfg: OpenClawConfig,
|
||||
params: {
|
||||
groupId: string;
|
||||
sutToken: string;
|
||||
driverBotId: number;
|
||||
sutAccountId: string;
|
||||
},
|
||||
): OpenClawConfig {
|
||||
return {
|
||||
...baseCfg,
|
||||
agents: {
|
||||
...baseCfg.agents,
|
||||
defaults: {
|
||||
...baseCfg.agents?.defaults,
|
||||
models: {
|
||||
...baseCfg.agents?.defaults?.models,
|
||||
"openai/gpt-5.6-luna": {
|
||||
...baseCfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"],
|
||||
agentRuntime: { id: "openclaw" },
|
||||
},
|
||||
},
|
||||
skipBootstrap: true,
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
...baseCfg.plugins,
|
||||
allow: uniqueStrings([...(baseCfg.plugins?.allow ?? []), "telegram"]),
|
||||
entries: {
|
||||
...baseCfg.plugins?.entries,
|
||||
telegram: { enabled: true },
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
...baseCfg.messages,
|
||||
groupChat: {
|
||||
...baseCfg.messages?.groupChat,
|
||||
visibleReplies: "automatic",
|
||||
},
|
||||
},
|
||||
channels: {
|
||||
...baseCfg.channels,
|
||||
telegram: {
|
||||
enabled: true,
|
||||
defaultAccount: params.sutAccountId,
|
||||
accounts: {
|
||||
[params.sutAccountId]: {
|
||||
enabled: true,
|
||||
botToken: params.sutToken,
|
||||
dmPolicy: "disabled",
|
||||
replyToMode: "first",
|
||||
groups: {
|
||||
[params.groupId]: {
|
||||
groupPolicy: "allowlist",
|
||||
allowFrom: [String(params.driverBotId)],
|
||||
requireMention: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function callTelegramApi<T>(
|
||||
token: string,
|
||||
method: string,
|
||||
body?: Record<string, unknown>,
|
||||
timeoutMs = 15_000,
|
||||
): Promise<T> {
|
||||
const requestTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 15_000);
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: `https://api.telegram.org/bot${token}/${method}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(body ?? {}),
|
||||
},
|
||||
timeoutMs: requestTimeoutMs,
|
||||
policy: { hostnameAllowlist: ["api.telegram.org"] },
|
||||
auditContext: "qa-lab-telegram-live",
|
||||
capture: false,
|
||||
});
|
||||
try {
|
||||
const payload = await readProviderJsonResponse<TelegramApiEnvelope<T>>(
|
||||
response,
|
||||
`qa-lab-telegram-live.${method}`,
|
||||
);
|
||||
if (!response.ok || !payload.ok || payload.result === undefined) {
|
||||
throw new Error(
|
||||
payload.description?.trim() || `${method} failed with status ${response.status}`,
|
||||
);
|
||||
}
|
||||
return payload.result;
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
export function isRecoverableTelegramQaPollError(error: unknown): boolean {
|
||||
const message = formatErrorMessage(error).toLowerCase();
|
||||
return [
|
||||
"fetch failed",
|
||||
"aborted due to timeout",
|
||||
"operation was aborted",
|
||||
"request timed out",
|
||||
"aborterror",
|
||||
"econnreset",
|
||||
"etimedout",
|
||||
"socket hang up",
|
||||
"terminated",
|
||||
].some((fragment) => message.includes(fragment));
|
||||
}
|
||||
|
||||
export async function waitForTelegramPollRetryDelay(remainingMs = 250) {
|
||||
await sleep(Math.min(250, Math.max(100, remainingMs)));
|
||||
}
|
||||
|
||||
export async function flushTelegramUpdates(token: string) {
|
||||
const startedAt = Date.now();
|
||||
let offset = 0;
|
||||
while (Date.now() - startedAt < 15_000) {
|
||||
const updates = await callTelegramApi<TelegramUpdate[]>(
|
||||
token,
|
||||
"getUpdates",
|
||||
{ offset, timeout: 0, allowed_updates: ["message", "edited_message"] },
|
||||
15_000,
|
||||
);
|
||||
if (updates.length === 0) {
|
||||
return offset;
|
||||
}
|
||||
offset = (updates.at(-1)?.update_id ?? offset) + 1;
|
||||
}
|
||||
throw new Error("timed out after 15000ms draining Telegram updates");
|
||||
}
|
||||
|
||||
function resolveTelegramQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) {
|
||||
const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS;
|
||||
return raw
|
||||
? (parseStrictPositiveInteger(raw) ?? TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS)
|
||||
: TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
export async function waitForTelegramChannelRunning(
|
||||
gateway: TelegramGatewayClient,
|
||||
accountId: string,
|
||||
options?: { env?: NodeJS.ProcessEnv; pollMs?: number; timeoutMs?: number },
|
||||
) {
|
||||
const startedAt = Date.now();
|
||||
const timeoutMs = options?.timeoutMs ?? resolveTelegramQaReadyTimeoutMs(options?.env);
|
||||
const pollMs = options?.pollMs ?? 500;
|
||||
let lastProbeError: string | undefined;
|
||||
let lastStatus: TelegramChannelStatus | undefined;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
try {
|
||||
const payload = (await gateway.call(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
)) as { channelAccounts?: Record<string, TelegramChannelStatus[]> };
|
||||
const match = (payload.channelAccounts?.telegram ?? []).find(
|
||||
(entry) => entry.accountId === accountId,
|
||||
);
|
||||
lastProbeError = undefined;
|
||||
lastStatus = match;
|
||||
if (match?.running && match.connected === true && match.restartPending !== true) {
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
lastProbeError = formatErrorMessage(error);
|
||||
}
|
||||
await sleep(pollMs);
|
||||
}
|
||||
const details = lastStatus
|
||||
? `; last status: ${JSON.stringify(lastStatus)}`
|
||||
: lastProbeError
|
||||
? `; last probe error: ${lastProbeError}`
|
||||
: "";
|
||||
throw new Error(`telegram account "${accountId}" did not become ready${details}`);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
66
extensions/qa-lab/src/suite-round-trip.test.ts
Normal file
66
extensions/qa-lab/src/suite-round-trip.test.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { runQaSuiteRoundTripProbe } from "./suite-round-trip.js";
|
||||
|
||||
describe("QA suite round-trip probe", () => {
|
||||
it("collects requested samples and chains native replies", async () => {
|
||||
const messages: Array<{ direction: "outbound"; id: string }> = [];
|
||||
const sendInbound = vi.fn().mockResolvedValue({ id: "inbound" });
|
||||
const waitForOutbound = vi.fn().mockImplementation(async () => {
|
||||
const reply = { id: `out-${messages.length + 1}` };
|
||||
messages.push({ direction: "outbound", id: reply.id });
|
||||
return reply;
|
||||
});
|
||||
|
||||
const result = await runQaSuiteRoundTripProbe({
|
||||
probe: {
|
||||
scenarioId: "channel-canary",
|
||||
count: 2,
|
||||
maxFailures: 2,
|
||||
timeoutMs: 1_000,
|
||||
markerPrefix: "QA-RTT",
|
||||
input: {
|
||||
conversation: { id: "room", kind: "group" },
|
||||
senderId: "driver",
|
||||
},
|
||||
textPrefix: "Reply exactly: ",
|
||||
chainReplies: true,
|
||||
},
|
||||
transport: {
|
||||
state: {
|
||||
getSnapshot: () => ({ messages }),
|
||||
},
|
||||
sendInbound,
|
||||
waitForOutbound,
|
||||
} as never,
|
||||
});
|
||||
|
||||
expect(result.passed).toBe(2);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.timing.samples).toBe(2);
|
||||
expect(sendInbound.mock.calls[1]?.[0]).toMatchObject({ replyToId: "out-1" });
|
||||
});
|
||||
|
||||
it("stops when the failure budget is exhausted", async () => {
|
||||
const result = await runQaSuiteRoundTripProbe({
|
||||
probe: {
|
||||
scenarioId: "channel-canary",
|
||||
count: 3,
|
||||
maxFailures: 1,
|
||||
timeoutMs: 10,
|
||||
markerPrefix: "QA-RTT",
|
||||
input: {
|
||||
conversation: { id: "room", kind: "group" },
|
||||
senderId: "driver",
|
||||
},
|
||||
textPrefix: "Reply exactly: ",
|
||||
},
|
||||
transport: {
|
||||
state: { getSnapshot: () => ({ messages: [] }) },
|
||||
sendInbound: vi.fn(),
|
||||
waitForOutbound: vi.fn().mockRejectedValue(new Error("timeout")),
|
||||
} as never,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ passed: 0, failed: 1 });
|
||||
});
|
||||
});
|
||||
66
extensions/qa-lab/src/suite-round-trip.ts
Normal file
66
extensions/qa-lab/src/suite-round-trip.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
summarizeLiveTransportRttSamples,
|
||||
type LiveTransportRttSample,
|
||||
} from "./live-transports/shared/live-transport-rtt.js";
|
||||
import type { QaTransportAdapter } from "./qa-transport.js";
|
||||
import type { QaBusInboundMessageInput } from "./runtime-api.js";
|
||||
|
||||
export type QaSuiteRoundTripProbe = {
|
||||
scenarioId: string;
|
||||
count: number;
|
||||
maxFailures: number;
|
||||
timeoutMs: number;
|
||||
markerPrefix: string;
|
||||
input: Omit<QaBusInboundMessageInput, "replyToId" | "text">;
|
||||
textPrefix: string;
|
||||
chainReplies?: boolean;
|
||||
};
|
||||
|
||||
export async function runQaSuiteRoundTripProbe(params: {
|
||||
probe: QaSuiteRoundTripProbe;
|
||||
transport: QaTransportAdapter;
|
||||
}) {
|
||||
const samples: LiveTransportRttSample[] = [];
|
||||
let failures = 0;
|
||||
let passed = 0;
|
||||
let latestReplyId = params.transport.state
|
||||
.getSnapshot()
|
||||
.messages.findLast((message) => message.direction === "outbound")?.id;
|
||||
|
||||
for (let index = 1; passed < params.probe.count; index += 1) {
|
||||
const marker = `${params.probe.markerPrefix}-${index}-${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
const outboundStartIndex = params.transport.state
|
||||
.getSnapshot()
|
||||
.messages.filter((message) => message.direction === "outbound").length;
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
await params.transport.sendInbound({
|
||||
...params.probe.input,
|
||||
text: `${params.probe.textPrefix}${marker}`,
|
||||
...(params.probe.chainReplies && latestReplyId ? { replyToId: latestReplyId } : {}),
|
||||
});
|
||||
const reply = await params.transport.waitForOutbound({
|
||||
conversation: params.probe.input.conversation,
|
||||
sinceIndex: outboundStartIndex,
|
||||
textIncludes: marker,
|
||||
timeoutMs: params.probe.timeoutMs,
|
||||
});
|
||||
latestReplyId = reply.id;
|
||||
samples.push({ status: "pass", rttMs: Math.max(1, Date.now() - startedAt) });
|
||||
passed += 1;
|
||||
} catch {
|
||||
samples.push({ status: "fail" });
|
||||
failures += 1;
|
||||
}
|
||||
if (failures >= params.probe.maxFailures) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const summary = summarizeLiveTransportRttSamples(samples);
|
||||
return {
|
||||
...summary,
|
||||
details: `${summary.passed}/${samples.length} RTT checks passed`,
|
||||
};
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { QaSuiteArtifactError } from "./errors.js";
|
||||
import type { QaEvidenceSummaryJson } from "./evidence-summary.js";
|
||||
import type { QaEvidenceSummaryJson, QaEvidenceTiming } from "./evidence-summary.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import type { RuntimeId, RuntimeParityResult } from "./runtime-parity.js";
|
||||
import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js";
|
||||
@@ -12,6 +12,7 @@ type QaSuiteSummaryScenario = {
|
||||
status: "pass" | "fail" | "skip" | "skipped";
|
||||
steps: unknown[];
|
||||
details?: string;
|
||||
timing?: QaEvidenceTiming;
|
||||
runtimeParity?: RuntimeParityResult;
|
||||
};
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import {
|
||||
buildQaSuiteEvidenceSummary,
|
||||
QA_EVIDENCE_FILENAME,
|
||||
type QaEvidenceTiming,
|
||||
type QaEvidenceSummaryJson,
|
||||
} from "./evidence-summary.js";
|
||||
import {
|
||||
@@ -88,6 +89,7 @@ import {
|
||||
shouldUseIsolatedQaSuiteScenarioWorkers,
|
||||
splitModelRef,
|
||||
} from "./suite-planning.js";
|
||||
import { runQaSuiteRoundTripProbe, type QaSuiteRoundTripProbe } from "./suite-round-trip.js";
|
||||
import {
|
||||
createQaSuiteScenarioStepRunner,
|
||||
runQaSuiteScenarioDefinition,
|
||||
@@ -115,6 +117,7 @@ export type QaSuiteScenarioResult = {
|
||||
status: "pass" | "fail";
|
||||
steps: QaReportCheck[];
|
||||
details?: string;
|
||||
timing?: QaEvidenceTiming;
|
||||
runtimeParity?: RuntimeParityResult;
|
||||
};
|
||||
|
||||
@@ -201,6 +204,7 @@ export type QaSuiteRunParams = {
|
||||
forcedRuntime?: RuntimeId;
|
||||
runtimePair?: [RuntimeId, RuntimeId];
|
||||
captureRuntimeParityCell?: boolean;
|
||||
roundTripProbe?: QaSuiteRoundTripProbe;
|
||||
// Unified suite partitions consume child evidence in memory; only the
|
||||
// parent should write the aggregate qa-evidence.json artifact.
|
||||
writeEvidenceFile?: boolean;
|
||||
@@ -545,6 +549,10 @@ function buildQaIsolatedScenarioWorkerParams(params: {
|
||||
transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs,
|
||||
workerStartStaggerMs: params.input?.workerStartStaggerMs,
|
||||
forcedRuntime: params.input?.forcedRuntime,
|
||||
roundTripProbe:
|
||||
params.input?.roundTripProbe?.scenarioId === params.scenario.id
|
||||
? params.input.roundTripProbe
|
||||
: undefined,
|
||||
writeEvidenceFile: params.input?.writeEvidenceFile,
|
||||
};
|
||||
}
|
||||
@@ -1276,6 +1284,17 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
||||
requested: requestedModels,
|
||||
scenarios: selectedScenarios,
|
||||
});
|
||||
if (
|
||||
params?.roundTripProbe &&
|
||||
!selectedScenarios.some((scenario) => scenario.id === params.roundTripProbe?.scenarioId)
|
||||
) {
|
||||
throw new Error(
|
||||
`QA round-trip probe scenario is not selected: ${params.roundTripProbe.scenarioId}`,
|
||||
);
|
||||
}
|
||||
if (params?.roundTripProbe && params.runtimePair) {
|
||||
throw new Error("QA round-trip probes are not supported with runtime-pair runs.");
|
||||
}
|
||||
const enabledPluginIds = [
|
||||
...new Set([
|
||||
...collectQaSuitePluginIds(selectedScenarios),
|
||||
@@ -1443,7 +1462,14 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
||||
|
||||
let isolatedRunFailed = false;
|
||||
let isolatedRunError: unknown;
|
||||
let parentTransportCleaned = false;
|
||||
try {
|
||||
if (params?.channelDriver === "live") {
|
||||
// The parent only renders aggregate artifacts. Release its live credentials
|
||||
// before child workers acquire the same exclusive transport lease.
|
||||
parentTransportCleaned = true;
|
||||
await transportFactoryResult.cleanup();
|
||||
}
|
||||
updateScenarioRun();
|
||||
const workerStartStaggerMs =
|
||||
params?.workerStartStaggerMs ?? resolveQaSuiteWorkerStartStaggerMs(concurrency);
|
||||
@@ -1616,7 +1642,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
||||
throw error;
|
||||
} finally {
|
||||
const cleanupSteps: Array<() => Promise<void>> = [
|
||||
() => transportFactoryResult.cleanup(),
|
||||
...(!parentTransportCleaned ? [() => transportFactoryResult.cleanup()] : []),
|
||||
() => disposeRegisteredAgentHarnesses(),
|
||||
];
|
||||
if (ownsLab) {
|
||||
@@ -1810,7 +1836,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
||||
const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario);
|
||||
const scenarioRetryCount =
|
||||
scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined;
|
||||
const result =
|
||||
let result: QaSuiteScenarioResult =
|
||||
scenarioRetryCount === 0
|
||||
? await runSelectedScenario()
|
||||
: await runQaScenarioWithFlakeRetry(runSelectedScenario, () =>
|
||||
@@ -1819,6 +1845,27 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
|
||||
`scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`,
|
||||
),
|
||||
);
|
||||
if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) {
|
||||
const probeResult = await runQaSuiteRoundTripProbe({
|
||||
probe: params.roundTripProbe,
|
||||
transport,
|
||||
});
|
||||
const probePassed = probeResult.passed >= params.roundTripProbe.count;
|
||||
result = {
|
||||
...result,
|
||||
status: probePassed ? "pass" : "fail",
|
||||
details: [result.details, probeResult.details].filter(Boolean).join(" | "),
|
||||
timing: probeResult.timing,
|
||||
steps: [
|
||||
...result.steps,
|
||||
{
|
||||
name: "Round-trip samples",
|
||||
status: probePassed ? "pass" : "fail",
|
||||
details: probeResult.details,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
sampleGatewayProcessRss(`scenario:${scenario.id}:finish`);
|
||||
scenarios.push(result);
|
||||
writeQaSuiteProgress(
|
||||
|
||||
@@ -60,4 +60,7 @@ flow:
|
||||
expr: matchingOutbound.length === 1
|
||||
message:
|
||||
expr: "`expected one canary reply, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: config.conversationId })}`"
|
||||
- assert:
|
||||
expr: "transport.id !== 'telegram' || Boolean(outbound.replyToId)"
|
||||
message: Telegram canary reply was not threaded to the driver message
|
||||
detailsExpr: outbound.text
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
title: Telegram long final reuses its preview
|
||||
|
||||
scenario:
|
||||
id: telegram-long-final-reuses-preview
|
||||
surface: channel-framework
|
||||
category: channel-framework.conversation-routing-and-delivery
|
||||
coverage:
|
||||
primary: [runtime.delivery]
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#39905
|
||||
objective: Verify a long Telegram final replaces its preview instead of leaving stale text.
|
||||
successCriteria:
|
||||
- The joined final contains both scripted boundary markers.
|
||||
- Telegram leaves one or two final chunks after streaming settles.
|
||||
codeRefs:
|
||||
- extensions/telegram/src/send.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run the scripted long final and verify preview reuse and chunk count.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: reuses the long-final preview
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: outboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
senderId: qa-stream-operator
|
||||
senderName: QA Stream Operator
|
||||
text: "@openclaw Telegram long final QA check. Use the scripted long final response."
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
sinceIndex: { ref: outboundStartIndex }
|
||||
textIncludes: TELEGRAM-LONG-FINAL-BEGIN
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [4000]
|
||||
- set: finalMessages
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
|
||||
- set: joinedFinal
|
||||
value:
|
||||
expr: "finalMessages.map((message) => message.text).join('')"
|
||||
- assert:
|
||||
expr: "finalMessages.length >= 1 && finalMessages.length <= 2 && joinedFinal.includes('TELEGRAM-LONG-FINAL-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-END')"
|
||||
message:
|
||||
expr: "`expected 1-2 complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`"
|
||||
detailsExpr: "`${finalMessages.length} final chunk(s)`"
|
||||
56
qa/scenarios/channels/telegram-long-final-three-chunks.yaml
Normal file
56
qa/scenarios/channels/telegram-long-final-three-chunks.yaml
Normal file
@@ -0,0 +1,56 @@
|
||||
title: Telegram long final keeps three final chunks
|
||||
|
||||
scenario:
|
||||
id: telegram-long-final-three-chunks
|
||||
surface: channel-framework
|
||||
category: channel-framework.conversation-routing-and-delivery
|
||||
coverage:
|
||||
primary: [runtime.delivery]
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#39905
|
||||
objective: Verify a three-chunk Telegram final removes previews and keeps only final chunks.
|
||||
successCriteria:
|
||||
- The joined final contains both scripted three-chunk boundary markers.
|
||||
- Exactly three final Telegram messages remain after streaming settles.
|
||||
codeRefs:
|
||||
- extensions/telegram/src/send.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run the scripted three-chunk final and verify final chunk accounting.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: keeps exactly three final chunks
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: outboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
senderId: qa-stream-operator
|
||||
senderName: QA Stream Operator
|
||||
text: "@openclaw Telegram long final three chunk QA check. Use the scripted three chunk final response."
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
sinceIndex: { ref: outboundStartIndex }
|
||||
textIncludes: TELEGRAM-LONG-FINAL-3CHUNK-BEGIN
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [4000]
|
||||
- set: finalMessages
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
|
||||
- set: joinedFinal
|
||||
value:
|
||||
expr: "finalMessages.map((message) => message.text).join('')"
|
||||
- assert:
|
||||
expr: "finalMessages.length === 3 && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-END')"
|
||||
message:
|
||||
expr: "`expected three complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`"
|
||||
detailsExpr: "`${finalMessages.length} final chunks`"
|
||||
39
qa/scenarios/channels/telegram-other-bot-command-gating.yaml
Normal file
39
qa/scenarios/channels/telegram-other-bot-command-gating.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
title: Telegram command addressed to another bot
|
||||
|
||||
scenario:
|
||||
id: telegram-other-bot-command-gating
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify a command addressed to another Telegram bot does not wake OpenClaw.
|
||||
successCriteria:
|
||||
- The driver sends a native status command addressed to another bot.
|
||||
- The Telegram group remains quiet for the observation window.
|
||||
codeRefs:
|
||||
- extensions/telegram/src/bot-message-context.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Address a command to another bot and verify OpenClaw ignores it.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: ignores another bot's command
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: outboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /status@OpenClawQaOtherBot
|
||||
- waitForNoOutbound:
|
||||
quietMs: 8000
|
||||
sinceIndex: { ref: outboundStartIndex }
|
||||
detailsExpr: "'command addressed to another bot stayed quiet'"
|
||||
@@ -0,0 +1,55 @@
|
||||
title: Telegram streamed final stays one message
|
||||
|
||||
scenario:
|
||||
id: telegram-stream-final-single-message
|
||||
surface: channel-framework
|
||||
category: channel-framework.conversation-routing-and-delivery
|
||||
coverage:
|
||||
primary: [runtime.delivery]
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#39905
|
||||
objective: Verify Telegram quiet streaming finalizes one preview without a duplicate reply.
|
||||
successCriteria:
|
||||
- The scripted streaming reply contains the exact marker.
|
||||
- One final outbound Telegram message remains after streaming settles.
|
||||
codeRefs:
|
||||
- extensions/telegram/src/send.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run the scripted quiet-streaming reply and verify one final message.
|
||||
config:
|
||||
marker: QA-TELEGRAM-STREAM-SINGLE-OK
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: finalizes one streaming message
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: outboundStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
senderId: qa-stream-operator
|
||||
senderName: QA Stream Operator
|
||||
text:
|
||||
expr: "`@openclaw Quiet streaming QA check. Reply exactly: ${config.marker}`"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-stream-room, kind: channel }
|
||||
sinceIndex: { ref: outboundStartIndex }
|
||||
textIncludes: { ref: config.marker }
|
||||
timeoutMs: 75000
|
||||
- call: sleep
|
||||
args: [4000]
|
||||
- set: finalMessages
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
|
||||
- assert:
|
||||
expr: "finalMessages.length === 1 && finalMessages[0].text.includes(config.marker)"
|
||||
message:
|
||||
expr: "`expected one final Telegram message; saw ${finalMessages.length}: ${finalMessages.map((message) => message.text).join(' | ')}`"
|
||||
detailsExpr: finalMessages[0].text
|
||||
@@ -6,6 +6,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { QaProviderMode } from "../../extensions/qa-lab/src/run-config.ts";
|
||||
import type { QaSuiteRoundTripProbe } from "../../extensions/qa-lab/src/suite-round-trip.ts";
|
||||
|
||||
function parseBoolean(value: string | undefined) {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
@@ -53,24 +54,47 @@ function resolvePackageTelegramOutputDir(env: NodeJS.ProcessEnv, repoRoot: strin
|
||||
);
|
||||
}
|
||||
|
||||
const DEFAULT_RTT_CHECK_ID = "telegram-mentioned-message-reply";
|
||||
const DEFAULT_RTT_CHECK_ID = "channel-canary";
|
||||
|
||||
function resolveRttOptions(env: NodeJS.ProcessEnv, selectedScenarioIds: readonly string[] = []) {
|
||||
const explicitCheckIds = splitCsv(env.OPENCLAW_NPM_TELEGRAM_RTT_CHECKS);
|
||||
const checkIds = explicitCheckIds.length > 0 ? explicitCheckIds : [DEFAULT_RTT_CHECK_ID];
|
||||
const unknownCheckIds = checkIds.filter((checkId) => checkId !== DEFAULT_RTT_CHECK_ID);
|
||||
if (unknownCheckIds.length > 0) {
|
||||
throw new Error(`unknown Telegram QA RTT check: ${unknownCheckIds[0]}`);
|
||||
}
|
||||
if (
|
||||
explicitCheckIds.length === 0 &&
|
||||
selectedScenarioIds.length > 0 &&
|
||||
!selectedScenarioIds.includes(DEFAULT_RTT_CHECK_ID)
|
||||
) {
|
||||
return {};
|
||||
return undefined;
|
||||
}
|
||||
const rttCount = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20;
|
||||
const count = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20;
|
||||
return {
|
||||
rttCount,
|
||||
rttTimeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"),
|
||||
maxRttFailures:
|
||||
parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? rttCount,
|
||||
rttCheckIds: explicitCheckIds,
|
||||
scenarioId: DEFAULT_RTT_CHECK_ID,
|
||||
count,
|
||||
timeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS") ?? 30_000,
|
||||
maxFailures: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? count,
|
||||
};
|
||||
}
|
||||
|
||||
function createRoundTripProbe(
|
||||
options: ReturnType<typeof resolveRttOptions>,
|
||||
): QaSuiteRoundTripProbe | undefined {
|
||||
if (!options) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...options,
|
||||
markerPrefix: "QA-TELEGRAM-RTT",
|
||||
input: {
|
||||
conversation: { id: "telegram-rtt-room", kind: "group" },
|
||||
senderId: "qa-rtt-driver",
|
||||
senderName: "QA RTT Driver",
|
||||
},
|
||||
textPrefix: "@openclaw Telegram RTT check. Reply exactly: ",
|
||||
chainReplies: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -117,8 +141,8 @@ async function resolveTrustedOpenClawCommand(
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { runTelegramQaLive } =
|
||||
await import("../../extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts");
|
||||
const { runQaTelegramSuite } =
|
||||
await import("../../extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts");
|
||||
const rawSutOpenClawCommand = process.env.OPENCLAW_NPM_TELEGRAM_SUT_COMMAND?.trim();
|
||||
if (!rawSutOpenClawCommand) {
|
||||
throw new Error("Missing OPENCLAW_NPM_TELEGRAM_SUT_COMMAND.");
|
||||
@@ -128,8 +152,8 @@ async function main() {
|
||||
const repoRoot = path.resolve(process.env.OPENCLAW_NPM_TELEGRAM_REPO_ROOT ?? process.cwd());
|
||||
const outputDir = resolvePackageTelegramOutputDir(process.env, repoRoot);
|
||||
const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS);
|
||||
const result = await runTelegramQaLive({
|
||||
env: process.env,
|
||||
const result = await runQaTelegramSuite({
|
||||
allowFailures: true,
|
||||
repoRoot,
|
||||
outputDir,
|
||||
sutOpenClawCommand,
|
||||
@@ -138,11 +162,14 @@ async function main() {
|
||||
alternateModel: process.env.OPENCLAW_NPM_TELEGRAM_ALT_MODEL,
|
||||
fastMode: parseBoolean(process.env.OPENCLAW_NPM_TELEGRAM_FAST),
|
||||
scenarioIds,
|
||||
...resolveRttOptions(process.env, scenarioIds),
|
||||
roundTripProbe: createRoundTripProbe(resolveRttOptions(process.env, scenarioIds)),
|
||||
sutAccountId: process.env.OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT,
|
||||
credentialSource: resolveCredentialSource(process.env),
|
||||
credentialRole: resolveCredentialRole(process.env),
|
||||
});
|
||||
if (!result) {
|
||||
throw new Error("Package Telegram QA did not produce suite artifacts.");
|
||||
}
|
||||
|
||||
process.stdout.write(`Package Telegram QA report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`Package Telegram QA summary: ${result.summaryPath}\n`);
|
||||
@@ -180,6 +207,7 @@ export const testing = {
|
||||
resolvePackageTelegramOutputDir,
|
||||
resolveCredentialRole,
|
||||
resolveCredentialSource,
|
||||
createRoundTripProbe,
|
||||
resolveRttOptions,
|
||||
resolveTrustedOpenClawCommand,
|
||||
shouldFailPackageTelegramRun,
|
||||
|
||||
@@ -70,62 +70,6 @@ function evidenceCredentialSource(summary) {
|
||||
);
|
||||
}
|
||||
|
||||
// Historical Telegram summary artifacts can still appear in old Mantis uploads.
|
||||
// Current QA producers write qa-evidence.json for gate inputs.
|
||||
function legacyTelegramSummaryToEvidenceSummary(summary) {
|
||||
const scenarios = Array.isArray(summary.scenarios) ? summary.scenarios : [];
|
||||
return {
|
||||
kind: "openclaw.qa.evidence-summary",
|
||||
schemaVersion: 2,
|
||||
generatedAt: new Date().toISOString(),
|
||||
entries: scenarios.map((scenario) => ({
|
||||
test: {
|
||||
kind: "legacy-telegram-qa-scenario",
|
||||
id: scenario?.id ?? "legacy-telegram-scenario",
|
||||
title: scenario?.title ?? scenario?.id ?? "Legacy Telegram scenario",
|
||||
},
|
||||
mapping: {
|
||||
profile: "release",
|
||||
coverage: [],
|
||||
},
|
||||
execution: {
|
||||
runner: "legacy-telegram-qa",
|
||||
environment: {
|
||||
ref: null,
|
||||
os: "unknown",
|
||||
nodeVersion: "unknown",
|
||||
},
|
||||
provider: {
|
||||
id: "unknown",
|
||||
live: true,
|
||||
model: { name: null, ref: null },
|
||||
auth: summary.credentials?.source ?? "unknown",
|
||||
},
|
||||
channel: {
|
||||
id: "telegram",
|
||||
live: true,
|
||||
driver: "native",
|
||||
},
|
||||
packageSource: { kind: "unknown" },
|
||||
artifacts: [
|
||||
{
|
||||
kind: "summary",
|
||||
path: "telegram-qa-summary.json",
|
||||
source: "legacy-telegram-qa",
|
||||
},
|
||||
],
|
||||
},
|
||||
result: {
|
||||
status: scenario?.status === "skip" ? "skipped" : (scenario?.status ?? "fail"),
|
||||
...(scenario?.status === "pass"
|
||||
? {}
|
||||
: { failure: { reason: scenario?.details ?? "legacy scenario did not pass" } }),
|
||||
...(typeof scenario?.rttMs === "number" ? { timing: { rttMs: scenario.rttMs } } : {}),
|
||||
},
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function renderScenarioList(summary) {
|
||||
const entries = evidenceEntries(summary);
|
||||
if (entries.length === 0) {
|
||||
@@ -420,7 +364,7 @@ export function buildTelegramEvidenceManifest({
|
||||
kind: "report",
|
||||
lane: "run",
|
||||
label: "Telegram QA report",
|
||||
path: "telegram-qa-report.md",
|
||||
path: "qa-suite-report.md",
|
||||
targetPath: "report.md",
|
||||
},
|
||||
];
|
||||
@@ -453,17 +397,11 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
|
||||
const outputDir = path.resolve(args.output_dir);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const evidenceSummaryPath = path.join(outputDir, "qa-evidence.json");
|
||||
const legacySummaryPath = path.join(outputDir, "telegram-qa-summary.json");
|
||||
const usesCurrentEvidenceSummary = existsSync(evidenceSummaryPath);
|
||||
const summaryPath = usesCurrentEvidenceSummary ? evidenceSummaryPath : legacySummaryPath;
|
||||
const observedPath = path.join(outputDir, "telegram-qa-observed-messages.json");
|
||||
const reportPath = path.join(outputDir, "telegram-qa-report.md");
|
||||
if (!existsSync(summaryPath)) {
|
||||
const reportPath = path.join(outputDir, "qa-suite-report.md");
|
||||
if (!existsSync(evidenceSummaryPath)) {
|
||||
throw new Error(`Missing Telegram QA evidence summary: ${evidenceSummaryPath}`);
|
||||
}
|
||||
const summary = usesCurrentEvidenceSummary
|
||||
? readJson(evidenceSummaryPath)
|
||||
: legacyTelegramSummaryToEvidenceSummary(readJson(legacySummaryPath));
|
||||
const summary = readJson(evidenceSummaryPath);
|
||||
const counts = evidenceCounts(summary);
|
||||
const pass = counts.failed === 0 && Number(counts.total ?? 0) > 0;
|
||||
if (!existsSync(reportPath)) {
|
||||
@@ -472,17 +410,15 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
|
||||
}
|
||||
writeFileSync(reportPath, "# Mantis Telegram Live QA\n\nTelegram QA report was unavailable.\n");
|
||||
}
|
||||
const hasLegacyObservedMessages = !usesCurrentEvidenceSummary && existsSync(observedPath);
|
||||
const observedMessages = hasLegacyObservedMessages ? readJson(observedPath) : [];
|
||||
const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages, summary });
|
||||
const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages: [], summary });
|
||||
writeFileSync(path.join(outputDir, "telegram-live-transcript.html"), transcriptHtml, "utf8");
|
||||
const manifest = buildTelegramEvidenceManifest({
|
||||
candidateRef: args.candidate_ref,
|
||||
candidateSha: args.candidate_sha,
|
||||
hasObservedMessages: hasLegacyObservedMessages,
|
||||
hasObservedMessages: false,
|
||||
scenarioLabel: args.scenario_label,
|
||||
summary,
|
||||
summaryArtifactPath: path.basename(summaryPath),
|
||||
summaryArtifactPath: path.basename(evidenceSummaryPath),
|
||||
});
|
||||
writeFileSync(
|
||||
path.join(outputDir, "mantis-evidence.json"),
|
||||
|
||||
@@ -449,7 +449,7 @@ async function main(): Promise<void> {
|
||||
if (telegramRunId) {
|
||||
await pollRun(options.repo, telegramRunId);
|
||||
const artifactDir = downloadTelegramArtifact(options.repo, telegramRunId);
|
||||
const report = findFile(artifactDir, "telegram-qa-report.md");
|
||||
const report = findFile(artifactDir, "qa-suite-report.md");
|
||||
if (report && existsSync(report)) {
|
||||
console.log(`\nTelegram report: ${report}\n`);
|
||||
console.log(readFileSync(report, "utf8"));
|
||||
|
||||
@@ -80,35 +80,11 @@ function makeTelegramOutput({ includeReport = true, summary = {} } = {}) {
|
||||
]),
|
||||
);
|
||||
if (includeReport) {
|
||||
writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n");
|
||||
writeFileSync(path.join(dir, "qa-suite-report.md"), "# Telegram QA\n\npass\n");
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function makeLegacyTelegramOutput() {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "mantis-telegram-evidence-test-"));
|
||||
tempDirs.push(dir);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(dir, "telegram-qa-summary.json"),
|
||||
JSON.stringify({
|
||||
credentials: { source: "convex" },
|
||||
counts: { total: 1, passed: 1, failed: 0 },
|
||||
scenarios: [
|
||||
{
|
||||
id: "telegram-status-command",
|
||||
title: "Telegram status command reply",
|
||||
status: "pass",
|
||||
details: "Observed expected status response.",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
writeFileSync(path.join(dir, "telegram-qa-observed-messages.json"), JSON.stringify([]));
|
||||
writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n");
|
||||
return dir;
|
||||
}
|
||||
|
||||
describe("scripts/mantis/build-telegram-evidence", () => {
|
||||
it("renders redacted Telegram observed messages as a transcript HTML page", () => {
|
||||
const html = renderTelegramEvidenceHtml({
|
||||
@@ -202,21 +178,6 @@ describe("scripts/mantis/build-telegram-evidence", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("renders historical Telegram summaries when evidence summaries are absent", () => {
|
||||
const dir = makeLegacyTelegramOutput();
|
||||
|
||||
const result = writeTelegramEvidence(["--output-dir", dir]);
|
||||
|
||||
expect(readFileSync(result.transcriptPath, "utf8")).toContain("Telegram status command reply");
|
||||
expect(result.manifest.comparison.pass).toBe(true);
|
||||
expect(
|
||||
result.manifest.artifacts.find((artifact) => artifact.targetPath === "summary.json"),
|
||||
).toMatchObject({ path: "telegram-qa-summary.json" });
|
||||
expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).toContain(
|
||||
"observed-messages.json",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not fabricate a required report artifact for passing Telegram summaries", () => {
|
||||
const dir = makeTelegramOutput({ includeReport: false });
|
||||
|
||||
@@ -250,7 +211,7 @@ describe("scripts/mantis/build-telegram-evidence", () => {
|
||||
const result = writeTelegramEvidence(["--output-dir", dir]);
|
||||
|
||||
expect(result.manifest.comparison.pass).toBe(false);
|
||||
expect(readFileSync(path.join(dir, "telegram-qa-report.md"), "utf8")).toContain(
|
||||
expect(readFileSync(path.join(dir, "qa-suite-report.md"), "utf8")).toContain(
|
||||
"Telegram QA report was unavailable",
|
||||
);
|
||||
expect(loadEvidenceManifest(result.manifestPath).comparison.pass).toBe(false);
|
||||
|
||||
@@ -284,15 +284,15 @@ describe("package Telegram live Docker E2E", () => {
|
||||
|
||||
it("defaults package Telegram RTT for the normal package live lane", () => {
|
||||
expect(testing.resolveRttOptions({})).toEqual({
|
||||
rttCount: 20,
|
||||
rttTimeoutMs: undefined,
|
||||
maxRttFailures: 20,
|
||||
rttCheckIds: [],
|
||||
scenarioId: "channel-canary",
|
||||
count: 20,
|
||||
timeoutMs: 30_000,
|
||||
maxFailures: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not force default RTT onto focused non-RTT scenario runs", () => {
|
||||
expect(testing.resolveRttOptions({}, ["telegram-canary"])).toEqual({});
|
||||
expect(testing.resolveRttOptions({}, ["telegram-status-command"])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("maps repeated RTT env onto package Telegram live options", () => {
|
||||
@@ -301,16 +301,40 @@ describe("package Telegram live Docker E2E", () => {
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7",
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS: "45000",
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES: "2",
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply",
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "channel-canary",
|
||||
}),
|
||||
).toEqual({
|
||||
rttCount: 7,
|
||||
rttTimeoutMs: 45_000,
|
||||
maxRttFailures: 2,
|
||||
rttCheckIds: ["telegram-mentioned-message-reply"],
|
||||
scenarioId: "channel-canary",
|
||||
count: 7,
|
||||
timeoutMs: 45_000,
|
||||
maxFailures: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("builds a generic suite probe for the Telegram RTT lane", () => {
|
||||
const probe = testing.createRoundTripProbe(testing.resolveRttOptions({}));
|
||||
|
||||
expect(probe).toMatchObject({
|
||||
scenarioId: "channel-canary",
|
||||
count: 20,
|
||||
timeoutMs: 30_000,
|
||||
markerPrefix: "QA-TELEGRAM-RTT",
|
||||
textPrefix: "@openclaw Telegram RTT check. Reply exactly: ",
|
||||
chainReplies: true,
|
||||
input: {
|
||||
conversation: { id: "telegram-rtt-room", kind: "group" },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects retired RTT scenario ids", () => {
|
||||
expect(() =>
|
||||
testing.resolveRttOptions({
|
||||
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply",
|
||||
}),
|
||||
).toThrow("unknown Telegram QA RTT check: telegram-mentioned-message-reply");
|
||||
});
|
||||
|
||||
it("rejects invalid repeated RTT env", () => {
|
||||
expect(() =>
|
||||
testing.resolveRttOptions({
|
||||
|
||||
Reference in New Issue
Block a user