refactor(qa): migrate Telegram scenarios into QA Lab (#108430)

* refactor(qa): migrate Telegram scenarios into QA Lab

* refactor(qa): remove retired Telegram runner exports
This commit is contained in:
Dallin Romney
2026-07-15 12:55:00 -07:00
committed by GitHub
parent d201acd6e8
commit 77d07dc3e9
40 changed files with 1673 additions and 3955 deletions

View File

@@ -444,7 +444,7 @@ jobs:
telegram_exit=$?
set -e
if [[ ! -f "$root/qa-evidence.json" && ! -f "$root/telegram-qa-summary.json" ]]; then
if [[ ! -f "$root/qa-evidence.json" ]]; then
echo "Telegram live QA did not produce an evidence summary." >&2
exit "$telegram_exit"
fi
@@ -496,7 +496,7 @@ jobs:
fi
fi
cat "$root/telegram-qa-report.md" >> "$GITHUB_STEP_SUMMARY"
cat "$root/qa-suite-report.md" >> "$GITHUB_STEP_SUMMARY"
- name: Upload Mantis Telegram artifacts
id: upload_artifact

View File

@@ -423,7 +423,7 @@ jobs:
append_telegram_summary() {
local status=$?
local report="${output_dir}/telegram-qa-report.md"
local report="${output_dir}/qa-suite-report.md"
if [[ -n "${GITHUB_STEP_SUMMARY:-}" && -f "${report}" ]]; then
{
echo "## Package Telegram E2E"

View File

@@ -249,8 +249,6 @@ extensions/qa-lab/src/lab-server.ts
extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts
extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-e2ee-destructive.ts
extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts
extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts
extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts
extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts

View File

@@ -325,7 +325,7 @@ Comment triggers, from a PR with write/maintain/admin access:
@openclaw-mantis discord status reactions baseline=origin/main candidate=HEAD
@openclaw-mantis telegram
@openclaw-mantis telegram scenario=telegram-status-command
@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply
@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary
@openclaw-mantis web ui chat
@openclaw-mantis web-ui-chat candidate=HEAD
```

View File

@@ -491,11 +491,12 @@ Required env when `--credential-source env`:
- `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN`
- `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN`
Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`):
The `release` profile selects the maintained Telegram YAML scenarios; `all`
adds opt-in session, usage, reply-chain, and streaming stress checks. Explicit
`--scenario` values override the profile.
- `telegram-canary`
- `telegram-mention-gating`
- `telegram-mentioned-message-reply`
- `channel-canary`
- `channel-mention-gating`
- `telegram-help-command`
- `telegram-commands-command`
- `telegram-tools-compact-command`
@@ -511,19 +512,21 @@ Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime
- `telegram-long-final-reuses-preview`
- `telegram-long-final-three-chunks`
The implicit default set always covers canary, mention gating, native command
The `release` profile always covers canary, mention gating, native command
replies, command addressing, and bot-to-bot group replies. `mock-openai`
defaults also include deterministic reply-chain and final-message streaming
checks. `telegram-current-session-status-tool` and
also includes the deterministic long-final preview check.
`telegram-current-session-status-tool` and
`telegram-tool-only-usage-footer` remain opt-in: the former is only stable
when threaded directly after canary, and the latter is a real-Telegram proof
of the `/usage` footer on tool-only replies. Use `pnpm openclaw qa telegram
--list-scenarios --provider-mode mock-openai` to print the current
default/optional split with regression refs.
default/optional split with regression refs. Use `--profile all` for every
Telegram scenario and `--fail-fast` to stop after the first failed scenario.
Output artifacts:
- `telegram-qa-report.md`
- `qa-suite-report.md`
- `qa-suite-summary.json`
- `qa-evidence.json` - evidence entries for the live transport checks,
including profile, coverage, provider, channel, artifacts, result, and RTT
fields.
@@ -542,7 +545,7 @@ When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper
leases a `kind: "telegram"` credential, exports the leased group/driver/SUT
bot env into the installed-package run, heartbeats the lease, and releases it
on shutdown. The package wrapper defaults to 20 RTT checks of
`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role
`channel-canary`, a 30s RTT timeout, and Convex role
`maintainer` outside CI when Convex is selected. Override
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`,
or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without

View File

@@ -276,9 +276,8 @@ inside every shard.
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`,
`OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or
`OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the run.
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of
Telegram QA check IDs to sample; when unset, the default RTT-capable
check is `telegram-mentioned-message-reply`.
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` selects the Telegram QA scenario to
sample; the supported RTT target is `channel-canary`.
- Uses the same Telegram env credentials or Convex credential source as
`pnpm openclaw qa telegram`. For CI/release automation, set
`OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus
@@ -441,7 +440,7 @@ set. Maintainers can start it from the Actions UI through `Mantis Scenario`
```text
@openclaw-mantis telegram
@openclaw-mantis telegram scenario=telegram-status-command
@openclaw-mantis telegram scenarios=telegram-status-command,telegram-mentioned-message-reply
@openclaw-mantis telegram scenarios=telegram-status-command,channel-canary
```
`Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop

View File

@@ -2,7 +2,7 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { QaScenarioPack } from "./scenario-catalog.js";
const {
@@ -11,10 +11,7 @@ const {
runQaSuite,
runQaCharacterEval,
runQaMultipass,
runCanonicalLiveScenarios,
listLiveTransportQaAdapterFactories,
listTelegramQaScenarioCatalog,
runTelegramQaLive,
startQaLabServer,
writeQaDockerHarnessFiles,
buildQaDockerHarnessImage,
@@ -27,10 +24,7 @@ const {
runQaSuite: vi.fn(),
runQaCharacterEval: vi.fn(),
runQaMultipass: vi.fn(),
runCanonicalLiveScenarios: vi.fn(),
listLiveTransportQaAdapterFactories: vi.fn(),
listTelegramQaScenarioCatalog: vi.fn(),
runTelegramQaLive: vi.fn(),
startQaLabServer: vi.fn(),
writeQaDockerHarnessFiles: vi.fn(),
buildQaDockerHarnessImage: vi.fn(),
@@ -61,20 +55,10 @@ vi.mock("./live-transports/cli.js", () => ({
listLiveTransportQaAdapterFactories,
}));
vi.mock("./live-transports/shared/canonical-scenarios.js", async (importOriginal) => ({
...(await importOriginal()),
runCanonicalLiveScenarios,
}));
vi.mock("./live-transports/telegram/adapter.runtime.js", () => ({
createTelegramQaTransportAdapter: vi.fn(),
}));
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
listTelegramQaScenarioCatalog,
runTelegramQaLive,
}));
vi.mock("./lab-server.js", () => ({
startQaLabServer,
}));
@@ -118,7 +102,6 @@ import {
} from "./cli.runtime.js";
import { QaSuiteInfraError } from "./errors.js";
import { QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
import { loadNonYamlScenarioRefs } from "./live-transports/shared/live-transport-scenarios.js";
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import type { QaProviderModeInput } from "./run-config.js";
@@ -253,10 +236,7 @@ describe("qa cli runtime", () => {
runQaCharacterEval.mockReset();
runQaManualLane.mockReset();
runQaMultipass.mockReset();
runCanonicalLiveScenarios.mockReset();
listLiveTransportQaAdapterFactories.mockReset();
listTelegramQaScenarioCatalog.mockReset();
runTelegramQaLive.mockReset();
startQaLabServer.mockReset();
writeQaDockerHarnessFiles.mockReset();
buildQaDockerHarnessImage.mockReset();
@@ -300,28 +280,6 @@ describe("qa cli runtime", () => {
vmName: "openclaw-qa-test",
scenarioIds: ["channel-chat-baseline"],
});
runTelegramQaLive.mockResolvedValue({
outputDir: telegramArtifactsDir,
reportPath: path.join(telegramArtifactsDir, "report.md"),
summaryPath: telegramSummaryPath,
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
scenarios: [],
});
runCanonicalLiveScenarios.mockResolvedValue({
outputDir: telegramArtifactsDir,
reportPath: path.join(telegramArtifactsDir, "canonical-report.md"),
summaryPath: telegramSummaryPath,
scenarios: [],
});
listTelegramQaScenarioCatalog.mockReturnValue([
{
id: "telegram-stream-final-single-message",
title: "Telegram streamed final reply",
defaultEnabled: false,
rationale: "stream rationale",
regressionRefs: [],
},
]);
listLiveTransportQaAdapterFactories.mockReturnValue([
{
id: "telegram",
@@ -998,22 +956,20 @@ describe("qa cli runtime", () => {
sutAccountId: "sut-live",
});
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
fastMode: true,
channelDriver: "live",
channelId: "telegram",
options: expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
fastMode: true,
sutAccountId: "sut-live",
}),
adapterOptions: expect.objectContaining({ sutAccountId: "sut-live" }),
scenarioIds: ["telegram-help-command"],
}),
);
expect(runTelegramQaLive).not.toHaveBeenCalled();
});
it("rejects output dirs that escape the repo root", () => {
@@ -1031,31 +987,33 @@ describe("qa cli runtime", () => {
scenarioIds: ["telegram-help-command"],
});
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
expect.objectContaining({
options: expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
providerMode: "live-frontier",
allowFailures: undefined,
}),
repoRoot: path.resolve("/tmp/openclaw-repo"),
providerMode: "live-frontier",
scenarioIds: ["telegram-help-command"],
}),
);
expect(runTelegramQaLive).not.toHaveBeenCalled();
});
it("keeps remaining Telegram defaults when Commander supplies an empty scenario list", async () => {
it("resolves the Telegram release profile when Commander supplies an empty scenario list", async () => {
await runQaTelegramCommand({
repoRoot: "/tmp/openclaw-repo",
scenarioIds: [],
});
expect(runCanonicalLiveScenarios).toHaveBeenCalled();
expect(runTelegramQaLive).toHaveBeenCalledWith(
expect.objectContaining({ scenarioIds: undefined }),
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
expect.objectContaining({
scenarioIds: expect.arrayContaining([
"channel-canary",
"channel-mention-gating",
"telegram-other-bot-command-gating",
]),
}),
);
});
it("uses the trusted Telegram launcher for canonical and legacy gateways", async () => {
it("uses the trusted Telegram launcher for the shared suite gateway", async () => {
const candidateRoot = path.join(telegramArtifactsDir, "candidate");
const boundaryDir = path.join(telegramArtifactsDir, "boundary");
const launcherPath = path.join(telegramArtifactsDir, "openclaw-telegram-sut-launcher");
@@ -1097,12 +1055,9 @@ describe("qa cli runtime", () => {
terminationRetryTimeoutMs: 60_000,
},
};
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
expect.objectContaining({
options: expect.objectContaining({ sutOpenClawCommand }),
}),
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
expect.objectContaining({ sutOpenClawCommand }),
);
expect(runTelegramQaLive).toHaveBeenCalledWith(expect.objectContaining({ sutOpenClawCommand }));
});
it("rejects relative Telegram launcher paths before starting a gateway", async () => {
@@ -1114,8 +1069,7 @@ describe("qa cli runtime", () => {
}),
).rejects.toThrow("OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must be an absolute file path.");
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
expect(runTelegramQaLive).not.toHaveBeenCalled();
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
});
it("rejects non-executable Telegram launcher files before starting a gateway", async () => {
@@ -1131,8 +1085,7 @@ describe("qa cli runtime", () => {
`OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND must point to an executable regular file: ${launcherPath}`,
);
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
expect(runTelegramQaLive).not.toHaveBeenCalled();
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
});
it("rejects unknown mixed Telegram selections before resolving the SUT launcher", async () => {
@@ -1144,8 +1097,7 @@ describe("qa cli runtime", () => {
}),
).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario");
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
expect(runTelegramQaLive).not.toHaveBeenCalled();
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
});
it("prints telegram scenario catalog without resolving the SUT launcher", async () => {
@@ -1156,9 +1108,7 @@ describe("qa cli runtime", () => {
listScenarios: true,
});
expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai");
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
expect(runTelegramQaLive).not.toHaveBeenCalled();
expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled();
expectWriteContains(
stdoutWrite,
"telegram-status-command\tdefault\tTelegram status command reply\tVerify Telegram status returns model, session, and activation details. refs=openclaw/openclaw#74698",
@@ -1176,11 +1126,10 @@ describe("qa cli runtime", () => {
}),
"utf8",
);
runTelegramQaLive.mockResolvedValueOnce({
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
outputDir: telegramArtifactsDir,
reportPath: path.join(telegramArtifactsDir, "report.md"),
summaryPath: telegramSummaryPath,
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
scenarios: [],
});
@@ -1205,11 +1154,10 @@ describe("qa cli runtime", () => {
}),
"utf8",
);
runTelegramQaLive.mockResolvedValueOnce({
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({
outputDir: telegramArtifactsDir,
reportPath: path.join(telegramArtifactsDir, "report.md"),
summaryPath: telegramSummaryPath,
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
scenarios: [
{
id: "telegram-help-command",
@@ -1999,11 +1947,6 @@ describe("qa cli runtime", () => {
});
describe("coverage inventory command", () => {
beforeAll(async () => {
listTelegramQaScenarioCatalog.mockReturnValue([]);
await loadNonYamlScenarioRefs();
});
it("prints a markdown report from scenario metadata", async () => {
await runQaCoverageReportCommand({ repoRoot: process.cwd() });

View File

@@ -277,12 +277,12 @@ describe("qa coverage report", () => {
nonYamlScenarios: [
{
id: scenario.id,
sourcePath: "extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts",
sourcePath: "extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts",
},
],
}),
).toThrow(
"duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts)",
"duplicate qa scenario id(s): test-scenario (qa/scenarios/test/test-scenario.yaml, extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts)",
);
});
@@ -308,7 +308,7 @@ describe("qa coverage report", () => {
expect(report).toContain("personal-share-safe-diagnostics-artifact");
expect(report).toContain("## Live Transport Lanes");
expect(report).toContain(
"- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
"- telegram (telegram): canary: channel-canary, help-command: telegram-help-command, mention-gating: channel-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
);
expect(report).toContain("## Scorecard Taxonomy");
expect(report).toContain("- Taxonomy: taxonomy.yaml");

View File

@@ -152,7 +152,7 @@ describe("evidence summary", () => {
const evidence = buildLiveTransportEvidenceSummary({
artifactPaths: [
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
{ kind: "report", path: "telegram-qa-report.md" },
{ kind: "report", path: "qa-suite-report.md" },
],
env: {
OPENCLAW_QA_RUNNER: "crabbox",
@@ -218,7 +218,7 @@ describe("evidence summary", () => {
},
{
kind: "report",
path: "telegram-qa-report.md",
path: "qa-suite-report.md",
source: "telegram-live-transport",
},
],
@@ -244,9 +244,9 @@ describe("evidence summary", () => {
providerMode: "live-frontier",
checks: [
{
id: "telegram-mentioned-message-reply",
id: "channel-canary",
coverageIds: ["channels.telegram.mention-gating"],
title: "Telegram mentioned message gets a reply",
title: "Channel transport canary",
status: "pass",
details: "5 samples collected.",
rttMs: 2000,

View File

@@ -8,14 +8,12 @@ const { runQaFlowSuiteFromRuntime } = vi.hoisted(() => ({
vi.mock("../../suite-launch.runtime.js", () => ({
runQaFlowSuiteFromRuntime,
}));
import { readQaScenarioPack } from "../../scenario-catalog.js";
import {
assertKnownScenarioIds,
partitionCanonicalScenarioIds,
TELEGRAM_CANONICAL_SCENARIO_IDS,
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
WHATSAPP_CANONICAL_SCENARIO_IDS,
whatsappDefaultCanonicalScenarioIds,
listCanonicalScenarios,
runCanonicalLiveScenarios,
} from "./canonical-scenarios.js";
import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js";
@@ -23,42 +21,27 @@ import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js";
describe("canonical live-transport scenarios", () => {
it("loads every migrated routing, command, and session-context scenario from YAML", () => {
const whatsAppMockDefaultIds = whatsappDefaultCanonicalScenarioIds("mock-openai");
const telegram = listCanonicalScenarios({
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
});
const whatsapp = listCanonicalScenarios({
ids: WHATSAPP_CANONICAL_SCENARIO_IDS,
defaultIds: whatsAppMockDefaultIds,
});
const expectedIds = new Set<string>(WHATSAPP_CANONICAL_SCENARIO_IDS);
const whatsappIds = new Set(
readQaScenarioPack()
.scenarios.filter((scenario) => expectedIds.has(scenario.id))
.map((scenario) => scenario.id),
);
expect(telegram.map(({ id }) => id).toSorted()).toEqual(
[...TELEGRAM_CANONICAL_SCENARIO_IDS].toSorted(),
);
expect(whatsapp.map(({ id }) => id).toSorted()).toEqual(
[...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted(),
);
expect(telegram.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
expect.arrayContaining([...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]),
);
expect(whatsapp.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
expect.arrayContaining([...whatsAppMockDefaultIds]),
);
expect([...whatsappIds].toSorted()).toEqual([...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted());
expect([...whatsAppMockDefaultIds].every((id) => whatsappIds.has(id))).toBe(true);
expect(whatsappDefaultCanonicalScenarioIds("live-frontier")).toEqual(["whatsapp-help-command"]);
expect(telegram.find(({ id }) => id === "telegram-status-command")?.regressionRefs).toEqual([
"openclaw/openclaw#74698",
]);
});
it("partitions canonical aliases from remaining imperative scenarios", () => {
expect(
partitionCanonicalScenarioIds(
["telegram-help-command", "telegram-mentioned-message-reply"],
TELEGRAM_CANONICAL_SCENARIO_IDS,
["whatsapp-help-command", "whatsapp-canary"],
WHATSAPP_CANONICAL_SCENARIO_IDS,
),
).toEqual({
canonical: ["telegram-help-command"],
legacy: ["telegram-mentioned-message-reply"],
canonical: ["whatsapp-help-command"],
legacy: ["whatsapp-canary"],
});
});
@@ -107,10 +90,7 @@ describe("canonical live-transport scenarios", () => {
it("removes migrated ids from non-YAML scenario ownership", async () => {
const nonYamlIds = new Set((await loadNonYamlScenarioRefs()).map(({ id }) => id));
for (const scenarioId of [
...TELEGRAM_CANONICAL_SCENARIO_IDS,
...WHATSAPP_CANONICAL_SCENARIO_IDS,
]) {
for (const scenarioId of WHATSAPP_CANONICAL_SCENARIO_IDS) {
expect(nonYamlIds.has(scenarioId), scenarioId).toBe(false);
}
});

View File

@@ -2,37 +2,9 @@
import path from "node:path";
import type { QaGatewayChildCommand } from "../../gateway-child.js";
import type { QaTransportAdapterFactory } from "../../qa-transport-registry.js";
import { readQaScenarioPack } from "../../scenario-catalog.js";
import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js";
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
export const TELEGRAM_CANONICAL_SCENARIO_IDS = [
"channel-canary",
"channel-mention-gating",
"telegram-help-command",
"telegram-commands-command",
"telegram-tools-compact-command",
"telegram-whoami-command",
"telegram-status-command",
"telegram-repeated-command-authorization",
"telegram-context-command",
"telegram-current-session-status-tool",
"telegram-tool-only-usage-footer",
"telegram-reply-chain-exact-marker",
] as const;
export const TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS = [
"channel-canary",
"channel-mention-gating",
"telegram-help-command",
"telegram-commands-command",
"telegram-tools-compact-command",
"telegram-whoami-command",
"telegram-status-command",
"telegram-repeated-command-authorization",
"telegram-context-command",
] as const;
export const WHATSAPP_ROUTING_CANONICAL_SCENARIO_IDS = [
"channel-canary",
"channel-dm-group-routing",
@@ -100,23 +72,6 @@ export function partitionCanonicalScenarioIds(
return { canonical, legacy };
}
export function listCanonicalScenarios(params: {
ids: readonly string[];
defaultIds: readonly string[];
}) {
const requestedIds = new Set(params.ids);
const defaultIds = new Set(params.defaultIds);
return readQaScenarioPack()
.scenarios.filter((scenario) => requestedIds.has(scenario.id))
.map((scenario) => ({
id: scenario.id,
title: scenario.title,
rationale: scenario.objective,
regressionRefs: scenario.regressionRefs ?? [],
defaultEnabled: defaultIds.has(scenario.id),
}));
}
export async function runCanonicalLiveScenarios(params: {
channelId: string;
factory: QaTransportAdapterFactory;

View File

@@ -1,63 +1,11 @@
// Qa Lab plugin module implements shared live-transport RTT behavior.
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import type { QaEvidenceTiming } from "../../evidence-summary.js";
export type LiveTransportRttOptions<CheckId extends string = string> = {
count: number;
timeoutMs: number;
maxFailures: number;
checkIds: Set<CheckId>;
};
export type LiveTransportRttSample = {
rttMs?: number;
status: "pass" | "fail";
};
function normalizePositiveRttInteger(value: number | undefined) {
if (
typeof value !== "number" ||
!Number.isSafeInteger(value) ||
value <= 0 ||
value > MAX_TIMER_TIMEOUT_MS
) {
return undefined;
}
return value;
}
export function normalizeLiveTransportRttOptions<CheckId extends string>(params: {
count?: number;
defaultCheckIds: readonly CheckId[];
knownCheckIds: ReadonlySet<CheckId>;
maxFailures?: number;
rawCheckIds?: readonly string[];
timeoutMs?: number;
unknownCheckMessage: (checkId: string) => string;
}): LiveTransportRttOptions<CheckId> | undefined {
const count = normalizePositiveRttInteger(params.count);
if (count === undefined) {
return undefined;
}
const rawCheckIds =
params.rawCheckIds && params.rawCheckIds.length > 0
? params.rawCheckIds
: params.defaultCheckIds;
const checkIds = new Set<CheckId>();
for (const checkId of rawCheckIds) {
if (!params.knownCheckIds.has(checkId as CheckId)) {
throw new Error(params.unknownCheckMessage(checkId));
}
checkIds.add(checkId as CheckId);
}
return {
count,
maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count,
checkIds,
timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000,
};
}
function percentile(sortedValues: readonly number[], percentileValue: number) {
if (sortedValues.length === 0) {
return undefined;

View File

@@ -8,7 +8,6 @@ import {
import { describe, expect, it } from "vitest";
import { discordQaLiveRuntime } from "../discord/discord-live.runtime.js";
import { __testing as slackTesting } from "../slack/slack-live.runtime.js";
import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js";
import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js";
import {
buildLiveTransportCoverageLaneSummaries,
@@ -115,6 +114,7 @@ describe("live transport scenario helpers", () => {
]);
expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({
standardId: "canary",
scenarioId: "channel-canary",
});
expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({
standardId: "restart-resume",
@@ -151,10 +151,7 @@ describe("live transport scenario helpers", () => {
expect.arrayContaining(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS),
);
expect(lanes.get("telegram")).toEqual(
expect.arrayContaining([
"help-command",
...telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS,
]),
expect.arrayContaining(["canary", "help-command", "mention-gating"]),
);
expect(lanes.get("whatsapp")).toEqual(
expect.arrayContaining([

View File

@@ -57,9 +57,9 @@ const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [
transportId: "telegram",
commandName: "telegram",
members: [
{ standardId: "canary" },
{ standardId: "canary", scenarioId: "channel-canary" },
{ standardId: "help-command", scenarioId: "telegram-help-command" },
{ standardId: "mention-gating", scenarioId: "telegram-mention-gating" },
{ standardId: "mention-gating", scenarioId: "channel-mention-gating" },
],
},
{
@@ -107,10 +107,9 @@ export function buildLiveTransportCoverageLaneSummaries(
}
export async function loadNonYamlScenarioRefs() {
const [discord, slack, telegram, whatsapp] = await Promise.all([
const [discord, slack, whatsapp] = await Promise.all([
import("../discord/discord-live.runtime.js"),
import("../slack/slack-live.runtime.js"),
import("../telegram/telegram-live.runtime.js"),
import("../whatsapp/whatsapp-live.runtime.js"),
]);
const refs = (sourcePath: string, scenarios: readonly { id: string }[]) =>
@@ -124,10 +123,6 @@ export async function loadNonYamlScenarioRefs() {
"extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts",
slack.listSlackQaScenarioCatalog(),
),
...refs(
"extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts",
telegram.listTelegramQaScenarioCatalog(),
),
...refs(
"extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts",
whatsapp.listWhatsAppQaScenarioCatalog(),

View File

@@ -0,0 +1,181 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
acquireQaCredentialLease: vi.fn(),
assertQaGatewayCredentialLeaseQuarantine: vi.fn(),
callTelegramApi: vi.fn(),
flushTelegramUpdates: vi.fn(),
heartbeatStop: vi.fn(),
heartbeatThrowIfFailed: vi.fn(),
leaseHeartbeat: vi.fn(),
leaseRelease: vi.fn(),
shouldRetainQaGatewayCredentialLease: vi.fn(),
waitForTelegramChannelRunning: vi.fn(),
}));
vi.mock("../shared/credential-lease.runtime.js", () => ({
acquireQaCredentialLease: mocks.acquireQaCredentialLease,
startQaCredentialLeaseHeartbeat: () => ({
stop: mocks.heartbeatStop,
throwIfFailed: mocks.heartbeatThrowIfFailed,
}),
}));
vi.mock("../../gateway-process-boundary.js", () => ({
assertQaGatewayCredentialLeaseQuarantine: mocks.assertQaGatewayCredentialLeaseQuarantine,
shouldRetainQaGatewayCredentialLease: mocks.shouldRetainQaGatewayCredentialLease,
}));
vi.mock("./telegram-api.runtime.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./telegram-api.runtime.js")>()),
callTelegramApi: mocks.callTelegramApi,
flushTelegramUpdates: mocks.flushTelegramUpdates,
waitForTelegramChannelRunning: mocks.waitForTelegramChannelRunning,
}));
import { createTelegramQaTransportAdapter } from "./adapter.runtime.js";
describe("Telegram QA transport adapter", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.acquireQaCredentialLease.mockResolvedValue({
payload: {
groupId: "-100123",
driverToken: "placeholder",
sutToken: "placeholder",
},
source: "env",
heartbeat: mocks.leaseHeartbeat,
release: mocks.leaseRelease,
});
mocks.flushTelegramUpdates.mockResolvedValue(0);
mocks.shouldRetainQaGatewayCredentialLease.mockResolvedValue(false);
});
it("rejects credentials that do not identify two distinct bots", async () => {
mocks.callTelegramApi.mockResolvedValue({
id: 1,
is_bot: true,
first_name: "bot",
username: "same_bot",
});
await expect(
createTelegramQaTransportAdapter({ adapterOptions: {}, messages: {} } as never),
).rejects.toThrow("requires two distinct bots");
expect(mocks.heartbeatStop).toHaveBeenCalledOnce();
expect(mocks.leaseRelease).toHaveBeenCalledOnce();
expect(mocks.flushTelegramUpdates).not.toHaveBeenCalled();
});
it("maps native sends, replies, edits, and cleanup inside the adapter", async () => {
const pollResolvers: Array<(updates: unknown[]) => void> = [];
let getMeCalls = 0;
let sendMessageCalls = 0;
mocks.callTelegramApi.mockImplementation(
async (_token: string, method: string): Promise<unknown> => {
if (method === "getMe") {
getMeCalls += 1;
return getMeCalls === 1
? { id: 1, is_bot: true, first_name: "driver", username: "driver_bot" }
: { id: 2, is_bot: true, first_name: "sut", username: "sut_bot" };
}
if (method === "sendMessage") {
sendMessageCalls += 1;
return { message_id: sendMessageCalls === 1 ? 10 : 12 };
}
if (method === "getUpdates") {
return await new Promise<unknown[]>((resolve) => {
pollResolvers.push(resolve);
});
}
throw new Error(`unexpected Telegram API method: ${method}`);
},
);
const addInboundMessage = vi.fn().mockResolvedValue({ id: "in-1" });
const addOutboundMessage = vi.fn().mockResolvedValue({ id: "out-1" });
const editMessage = vi.fn().mockResolvedValue({ id: "out-1" });
const adapter = await createTelegramQaTransportAdapter({
adapterOptions: { sutAccountId: "sut" },
messages: { addInboundMessage, addOutboundMessage, editMessage },
} as never);
await vi.waitFor(() => expect(pollResolvers).toHaveLength(1));
await adapter.sendInbound?.({
conversation: { id: "logical-room", kind: "group" },
senderId: "driver",
text: "@openclaw reply exactly: QA-MARKER",
});
expect(mocks.callTelegramApi).toHaveBeenCalledWith(
"placeholder",
"sendMessage",
expect.objectContaining({
chat_id: "-100123",
text: "@sut_bot reply exactly: QA-MARKER",
}),
);
pollResolvers[0]?.([
{
update_id: 1,
message: {
message_id: 11,
date: 100,
chat: { id: -100123 },
from: { id: 2, is_bot: true, username: "sut_bot" },
text: "preview",
reply_to_message: { message_id: 10 },
},
},
]);
await vi.waitFor(() => expect(addOutboundMessage).toHaveBeenCalledOnce());
expect(addOutboundMessage).toHaveBeenCalledWith(
expect.objectContaining({
to: "group:logical-room",
text: "preview",
replyToId: "in-1",
}),
);
await adapter.sendInbound?.({
conversation: { id: "logical-room", kind: "group" },
senderId: "driver",
text: "follow-up",
replyToId: "out-1",
});
expect(mocks.callTelegramApi).toHaveBeenCalledWith(
"placeholder",
"sendMessage",
expect.objectContaining({
reply_parameters: {
message_id: 11,
allow_sending_without_reply: true,
},
}),
);
await vi.waitFor(() => expect(pollResolvers).toHaveLength(2));
pollResolvers[1]?.([
{
update_id: 2,
edited_message: {
message_id: 11,
date: 101,
chat: { id: -100123 },
from: { id: 2, is_bot: true, username: "sut_bot" },
text: "final",
},
},
]);
await vi.waitFor(() => expect(editMessage).toHaveBeenCalledOnce());
expect(editMessage).toHaveBeenCalledWith(
expect.objectContaining({ messageId: "out-1", text: "final", timestamp: 101_000 }),
);
await vi.waitFor(() => expect(pollResolvers).toHaveLength(3));
const cleanup = adapter.cleanup?.();
pollResolvers[2]?.([]);
await cleanup;
expect(mocks.heartbeatStop).toHaveBeenCalledOnce();
expect(mocks.leaseRelease).toHaveBeenCalledOnce();
});
});

View File

@@ -1,5 +1,4 @@
// Qa Lab plugin module implements Telegram live transport adapter behavior.
import type { TelegramBotUpdate } from "@openclaw/telegram/api.js";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
import {
@@ -10,23 +9,34 @@ import {
acquireQaCredentialLease,
startQaCredentialLeaseHeartbeat,
} from "../shared/credential-lease.runtime.js";
import { __testing as telegramLive } from "./telegram-live.runtime.js";
import {
buildTelegramQaConfig,
callTelegramApi,
flushTelegramUpdates,
isRecoverableTelegramQaPollError,
normalizeTelegramObservedMessage,
parseTelegramQaCredentialPayload,
resolveTelegramQaRuntimeEnv,
waitForTelegramChannelRunning,
waitForTelegramPollRetryDelay,
type TelegramBotIdentity,
type TelegramQaRuntimeEnv,
type TelegramUpdate,
} from "./telegram-api.runtime.js";
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
type FactoryContext = Parameters<AdapterFactory["create"]>[0];
type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>>;
type TelegramRuntimeEnv = ReturnType<typeof telegramLive.resolveTelegramQaRuntimeEnv>;
export async function createTelegramQaTransportAdapter(
context: FactoryContext,
): Promise<AdapterDefinition> {
const options = context.adapterOptions ?? {};
const credentialLease = await acquireQaCredentialLease<TelegramRuntimeEnv>({
const credentialLease = await acquireQaCredentialLease<TelegramQaRuntimeEnv>({
kind: "telegram",
source: options.credentialSource,
role: options.credentialRole,
resolveEnvPayload: () => telegramLive.resolveTelegramQaRuntimeEnv(),
parsePayload: telegramLive.parseTelegramQaCredentialPayload,
resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(),
parsePayload: parseTelegramQaCredentialPayload,
});
try {
assertQaGatewayCredentialLeaseQuarantine(credentialLease);
@@ -36,17 +46,26 @@ export async function createTelegramQaTransportAdapter(
}
const heartbeat = startQaCredentialLeaseHeartbeat(credentialLease);
const runtimeEnv = credentialLease.payload;
let driverIdentity: { id: number; username?: string };
let sutIdentity: { id: number; username?: string };
let driverIdentity: TelegramBotIdentity;
let sutIdentity: TelegramBotIdentity;
let offset: number;
try {
[driverIdentity, sutIdentity, offset] = await Promise.all([
telegramLive.callTelegramApi<{ id: number; username?: string }>(
runtimeEnv.driverToken,
"getMe",
),
telegramLive.callTelegramApi<{ id: number; username?: string }>(runtimeEnv.sutToken, "getMe"),
telegramLive.flushTelegramUpdates(runtimeEnv.driverToken),
[driverIdentity, sutIdentity] = await Promise.all([
callTelegramApi<TelegramBotIdentity>(runtimeEnv.driverToken, "getMe"),
callTelegramApi<TelegramBotIdentity>(runtimeEnv.sutToken, "getMe"),
]);
if (!driverIdentity.is_bot || !sutIdentity.is_bot) {
throw new Error("Telegram QA credentials must belong to bots.");
}
if (driverIdentity.id === sutIdentity.id) {
throw new Error("Telegram QA requires two distinct bots for driver and SUT.");
}
if (!sutIdentity.username?.trim()) {
throw new Error("Telegram QA requires the SUT bot to have a Telegram username.");
}
[offset] = await Promise.all([
flushTelegramUpdates(runtimeEnv.driverToken),
flushTelegramUpdates(runtimeEnv.sutToken),
]);
} catch (error) {
await heartbeat.stop();
@@ -64,39 +83,56 @@ export async function createTelegramQaTransportAdapter(
if (stopped) {
return;
}
const updates = await telegramLive.callTelegramApi<TelegramBotUpdate[]>(
runtimeEnv.driverToken,
"getUpdates",
{ offset, timeout: 1, allowed_updates: ["message", "edited_message"] },
6_000,
);
let updates: TelegramUpdate[];
try {
updates = await callTelegramApi<TelegramUpdate[]>(
runtimeEnv.driverToken,
"getUpdates",
{ offset, timeout: 1, allowed_updates: ["message", "edited_message"] },
6_000,
);
} catch (error) {
if (!isRecoverableTelegramQaPollError(error)) {
throw error;
}
await waitForTelegramPollRetryDelay();
continue;
}
for (const update of updates) {
offset = Math.max(offset, update.update_id + 1);
const message = update.edited_message ?? update.message;
if (!message?.from?.id || message.from.id !== sutIdentity.id) {
const message = normalizeTelegramObservedMessage(update);
if (
!message ||
message.chatId !== Number(runtimeEnv.groupId) ||
message.senderId !== sutIdentity.id
) {
continue;
}
const existingMessageId = busMessageIds.get(message.message_id);
if (update.edited_message && existingMessageId) {
await context.messages.editMessage({
accountId: options.sutAccountId?.trim() || "sut",
messageId: existingMessageId,
text: message.text ?? message.caption ?? "",
});
const existingMessageId = busMessageIds.get(message.messageId);
if (update.edited_message) {
if (existingMessageId) {
await context.messages.editMessage({
accountId: options.sutAccountId?.trim() || "sut",
messageId: existingMessageId,
text: message.text,
timestamp: message.timestamp,
});
}
continue;
}
const outbound = await context.messages.addOutboundMessage({
accountId: options.sutAccountId?.trim() || "sut",
to: `${logicalConversationKind}:${logicalConversationId}`,
senderId: String(message.from.id),
senderName: message.from.username,
text: message.text ?? message.caption ?? "",
timestamp: message.date * 1_000,
replyToId: message.reply_to_message?.message_id
? busMessageIds.get(message.reply_to_message.message_id)
senderId: String(message.senderId),
senderName: message.senderUsername,
text: message.text,
timestamp: message.timestamp,
replyToId: message.replyToMessageId
? busMessageIds.get(message.replyToMessageId)
: undefined,
});
busMessageIds.set(message.message_id, outbound.id);
nativeMessageIds.set(outbound.id, message.messageId);
busMessageIds.set(message.messageId, outbound.id);
}
}
};
@@ -127,7 +163,7 @@ export async function createTelegramQaTransportAdapter(
? input.text.replaceAll("@openclaw", `@${sutIdentity.username}`)
: input.text;
const nativeReplyToId = input.replyToId ? nativeMessageIds.get(input.replyToId) : undefined;
const sent = await telegramLive.callTelegramApi<{ message_id: number }>(
const sent = await callTelegramApi<{ message_id: number }>(
runtimeEnv.driverToken,
"sendMessage",
{
@@ -161,14 +197,14 @@ export async function createTelegramQaTransportAdapter(
busMessageIds.clear();
},
createGatewayConfig: () =>
telegramLive.buildTelegramQaConfig({} as OpenClawConfig, {
buildTelegramQaConfig({} as OpenClawConfig, {
groupId: runtimeEnv.groupId,
sutToken: runtimeEnv.sutToken,
driverBotId: driverIdentity.id,
sutAccountId: accountId,
}),
waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) =>
await telegramLive.waitForTelegramChannelRunning(gateway as never, accountId, {
await waitForTelegramChannelRunning(gateway, accountId, {
timeoutMs,
pollMs: pollIntervalMs,
}),

View File

@@ -2,22 +2,15 @@ import { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import type { QaGatewayChildCommand } from "../../gateway-child.js";
import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js";
import type { QaSuiteRoundTripProbe } from "../../suite-round-trip.js";
import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js";
import {
assertKnownScenarioIds,
canonicalScenarioOutputDir,
listCanonicalScenarios,
partitionCanonicalScenarioIds,
runCanonicalLiveScenarios,
TELEGRAM_CANONICAL_SCENARIO_IDS,
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
} from "../shared/canonical-scenarios.js";
// Qa Lab plugin module implements cli behavior.
import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js";
import { createTelegramQaTransportAdapter } from "./adapter.runtime.js";
import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js";
import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js";
const TELEGRAM_QA_SUT_OPENCLAW_COMMAND_ENV = "OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND";
const TELEGRAM_QA_SUT_UID_ENV = "OPENCLAW_QA_TELEGRAM_SUT_UID";
@@ -142,17 +135,15 @@ async function resolveTelegramQaSutOpenClawCommand(
};
}
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
type TelegramQaSuiteOptions = LiveTransportQaCommandOptions & {
roundTripProbe?: QaSuiteRoundTripProbe;
sutOpenClawCommand?: QaGatewayChildCommand;
};
export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) {
const runOptions = resolveLiveTransportQaRunOptions(opts);
if (runOptions.listScenarios) {
const scenarios = [
...listCanonicalScenarios({
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
}),
...listTelegramQaScenarioCatalog(runOptions.providerMode),
];
for (const scenario of scenarios) {
for (const scenario of listTelegramQaScenarios(runOptions.providerMode)) {
const defaultLabel = scenario.defaultEnabled ? "default" : "optional";
const refs =
scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : "";
@@ -160,62 +151,40 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
`${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`,
);
}
return;
return undefined;
}
const selected = partitionCanonicalScenarioIds(
runOptions.scenarioIds,
TELEGRAM_CANONICAL_SCENARIO_IDS,
);
const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0;
if (hasExplicitScenarioIds) {
assertKnownScenarioIds({
ids: selected.legacy,
knownIds: listTelegramQaScenarioCatalog(runOptions.providerMode).map(({ id }) => id),
laneLabel: "Telegram",
});
}
const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot);
const executionOptions = {
...runOptions,
...(sutOpenClawCommand ? { sutOpenClawCommand } : {}),
};
const canonicalScenarioIds = hasExplicitScenarioIds
? selected.canonical
: [...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS];
const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0;
if (canonicalScenarioIds.length > 0) {
const canonical = await runCanonicalLiveScenarios({
channelId: "telegram",
factory: {
const scenarioIds = resolveTelegramQaScenarioIds({
profile: opts.profile,
providerMode: runOptions.providerMode,
scenarioIds: runOptions.scenarioIds,
});
const result = await runQaFlowSuiteFromRuntime({
adapterFactories: [
{
id: "telegram",
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
create: createTelegramQaTransportAdapter,
},
options: {
...executionOptions,
outputDir: canonicalScenarioOutputDir(executionOptions, runsLegacyScenarios),
},
scenarioIds: canonicalScenarioIds,
});
printLiveTransportQaArtifacts("Telegram canonical QA", {
report: canonical.reportPath,
summary: canonical.summaryPath,
});
if (!runOptions.allowFailures) {
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(
canonical.summaryPath,
);
if (failedScenarioCount > 0) {
process.exitCode = 1;
}
}
}
if (!runsLegacyScenarios) {
return;
}
const result = await runTelegramQaLive({
...executionOptions,
scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined,
],
adapterOptions: {
repoRoot: runOptions.repoRoot,
...(runOptions.credentialRole ? { credentialRole: runOptions.credentialRole } : {}),
...(runOptions.credentialSource ? { credentialSource: runOptions.credentialSource } : {}),
...(runOptions.sutAccountId ? { sutAccountId: runOptions.sutAccountId } : {}),
},
channelDriver: "live",
channelId: "telegram",
concurrency: 1,
...(runOptions.alternateModel ? { alternateModel: runOptions.alternateModel } : {}),
...(runOptions.fastMode !== undefined ? { fastMode: runOptions.fastMode } : {}),
...(runOptions.failFast !== undefined ? { failFast: runOptions.failFast } : {}),
...(runOptions.outputDir ? { outputDir: runOptions.outputDir } : {}),
...(runOptions.primaryModel ? { primaryModel: runOptions.primaryModel } : {}),
providerMode: runOptions.providerMode,
repoRoot: runOptions.repoRoot,
roundTripProbe: opts.roundTripProbe,
scenarioIds,
sutOpenClawCommand: opts.sutOpenClawCommand,
});
printLiveTransportQaArtifacts("Telegram QA", {
report: result.reportPath,
@@ -227,4 +196,22 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
process.exitCode = 1;
}
}
return result;
}
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
const runOptions = resolveLiveTransportQaRunOptions(opts);
if (runOptions.listScenarios) {
return await runQaTelegramSuite(opts);
}
resolveTelegramQaScenarioIds({
profile: opts.profile,
providerMode: runOptions.providerMode,
scenarioIds: runOptions.scenarioIds,
});
const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot);
return await runQaTelegramSuite({
...opts,
...(sutOpenClawCommand ? { sutOpenClawCommand } : {}),
});
}

View File

@@ -0,0 +1,13 @@
import { describe, expect, it } from "vitest";
import { telegramQaCliRegistration } from "./cli.js";
import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js";
describe("Telegram QA CLI registration", () => {
it("keeps the generic live baseline outside command profile selection", () => {
expect(telegramQaCliRegistration.adapterFactory?.scenarioIds).toEqual([
"channel-chat-baseline",
...TELEGRAM_QA_ALL_SCENARIO_IDS,
]);
expect(TELEGRAM_QA_ALL_SCENARIO_IDS).not.toContain("channel-chat-baseline");
});
});

View File

@@ -1,4 +1,3 @@
import { TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js";
// Qa Lab plugin module implements cli behavior.
import {
createLazyCliRuntimeLoader,
@@ -6,6 +5,7 @@ import {
type LiveTransportQaCliRegistration,
type LiveTransportQaCommandOptions,
} from "../shared/live-transport-cli.js";
import { TELEGRAM_QA_ALL_SCENARIO_IDS } from "./profiles.js";
type TelegramQaAdapterRuntime = typeof import("./adapter.runtime.js");
type TelegramQaCliRuntime = typeof import("./cli.runtime.js");
@@ -23,7 +23,7 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
const telegramQaAdapterFactory: NonNullable<LiveTransportQaCliRegistration["adapterFactory"]> = {
id: "telegram",
scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS],
scenarioIds: ["channel-chat-baseline", ...TELEGRAM_QA_ALL_SCENARIO_IDS],
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
async create(context) {
return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context);
@@ -42,6 +42,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
listScenariosHelp: "Print available Telegram scenario ids and exit",
outputDirHelp: "Telegram QA artifact directory",
profileHelp: "QA Lab Telegram profile: release or all (default: release)",
run: runQaTelegram,
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",

View File

@@ -0,0 +1,93 @@
import fs from "node:fs";
import { describe, expect, it } from "vitest";
import { readQaScenarioById } from "../../scenario-catalog.js";
const LEGACY_TELEGRAM_BEHAVIOR_LEDGER = [
{
id: "telegram-other-bot-command-gating",
owner: "qa/scenarios/channels/telegram-other-bot-command-gating.yaml",
scenarioId: "telegram-other-bot-command-gating",
},
{
id: "telegram-mentioned-message-reply",
owner: "qa/scenarios/channels/channel-canary.yaml",
scenarioId: "channel-canary",
},
{
id: "telegram-stream-final-single-message",
owner: "qa/scenarios/channels/telegram-stream-final-single-message.yaml",
scenarioId: "telegram-stream-final-single-message",
},
{
id: "telegram-long-final-reuses-preview",
owner: "qa/scenarios/channels/telegram-long-final-reuses-preview.yaml",
scenarioId: "telegram-long-final-reuses-preview",
},
{
id: "telegram-long-final-three-chunks",
owner: "qa/scenarios/channels/telegram-long-final-three-chunks.yaml",
scenarioId: "telegram-long-final-three-chunks",
},
{
id: "telegram-mention-gating",
owner: "qa/scenarios/channels/channel-mention-gating.yaml",
scenarioId: "channel-mention-gating",
},
] as const;
const LEGACY_TELEGRAM_CAPABILITY_LEDGER = [
{
id: "credentials-and-lease-cleanup",
owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts",
proof: "acquireQaCredentialLease",
},
{
id: "bot-api-and-rich-message-observation",
owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts",
proof: "normalizeTelegramObservedMessage",
},
{
id: "native-send-reply-edit-mapping",
owner: "extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts",
proof: "reply_parameters",
},
{
id: "gateway-readiness-and-config",
owner: "extensions/qa-lab/src/live-transports/telegram/telegram-api.runtime.ts",
proof: "waitForTelegramChannelRunning",
},
{
id: "selection-and-standard-artifacts",
owner: "extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts",
proof: "runQaFlowSuiteFromRuntime",
},
{
id: "package-rtt-sampling",
owner: "scripts/e2e/npm-telegram-live-runner.ts",
proof: "createRoundTripProbe",
},
] as const;
describe("legacy Telegram runner parity", () => {
it("maps every retired scenario to one maintained YAML owner", () => {
expect(LEGACY_TELEGRAM_BEHAVIOR_LEDGER).toHaveLength(6);
expect(new Set(LEGACY_TELEGRAM_BEHAVIOR_LEDGER.map(({ id }) => id)).size).toBe(6);
for (const entry of LEGACY_TELEGRAM_BEHAVIOR_LEDGER) {
const scenario = readQaScenarioById(entry.scenarioId);
expect(scenario.sourcePath, entry.id).toBe(entry.owner);
}
});
it("keeps every runner capability at its intended owner boundary", () => {
for (const entry of LEGACY_TELEGRAM_CAPABILITY_LEDGER) {
expect(fs.readFileSync(entry.owner, "utf8"), entry.id).toContain(entry.proof);
}
});
it("has no unresolved legacy runner owner", () => {
expect(
fs.existsSync("extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts"),
).toBe(false);
});
});

View File

@@ -0,0 +1,59 @@
import { describe, expect, it } from "vitest";
import {
listTelegramQaScenarios,
resolveTelegramQaScenarioIds,
TELEGRAM_QA_ALL_SCENARIO_IDS,
} from "./profiles.js";
describe("Telegram QA profiles", () => {
it("keeps release focused and adds the scripted long-final check for mock runs", () => {
const live = resolveTelegramQaScenarioIds({ providerMode: "live-frontier" });
const mock = resolveTelegramQaScenarioIds({ providerMode: "mock-openai" });
expect(live).toContain("telegram-other-bot-command-gating");
expect(live).not.toContain("telegram-long-final-reuses-preview");
expect(mock).toEqual([...live, "telegram-long-final-reuses-preview"]);
});
it("selects every migrated Telegram scenario through all", () => {
expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual([
...TELEGRAM_QA_ALL_SCENARIO_IDS,
]);
});
it("lets explicit scenarios override profile selection", () => {
expect(
resolveTelegramQaScenarioIds({
profile: "all",
providerMode: "live-frontier",
scenarioIds: ["telegram-status-command"],
}),
).toEqual(["telegram-status-command"]);
});
it("rejects unknown profiles and scenarios before gateway startup", () => {
expect(() =>
resolveTelegramQaScenarioIds({ providerMode: "live-frontier", profile: "transport" }),
).toThrow('Unknown QA Lab Telegram profile "transport"');
expect(() =>
resolveTelegramQaScenarioIds({
providerMode: "live-frontier",
scenarioIds: ["telegram-missing"],
}),
).toThrow("unknown Telegram QA scenario id(s): telegram-missing");
});
it("lists the YAML catalog with provider-specific release defaults", () => {
const scenarios = listTelegramQaScenarios("mock-openai");
expect(scenarios.map(({ id }) => id).toSorted()).toEqual(
[...TELEGRAM_QA_ALL_SCENARIO_IDS].toSorted(),
);
expect(
scenarios.find(({ id }) => id === "telegram-long-final-reuses-preview")?.defaultEnabled,
).toBe(true);
expect(
scenarios.find(({ id }) => id === "telegram-long-final-three-chunks")?.defaultEnabled,
).toBe(false);
});
});

View File

@@ -0,0 +1,77 @@
import { readQaScenarioPack } from "../../scenario-catalog.js";
const TELEGRAM_QA_RELEASE_SCENARIO_IDS = [
"channel-canary",
"channel-mention-gating",
"telegram-help-command",
"telegram-commands-command",
"telegram-tools-compact-command",
"telegram-whoami-command",
"telegram-status-command",
"telegram-repeated-command-authorization",
"telegram-context-command",
"telegram-other-bot-command-gating",
] as const;
const TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS = [
...TELEGRAM_QA_RELEASE_SCENARIO_IDS,
"telegram-long-final-reuses-preview",
] as const;
export const TELEGRAM_QA_ALL_SCENARIO_IDS = [
...TELEGRAM_QA_RELEASE_SCENARIO_IDS,
"telegram-current-session-status-tool",
"telegram-tool-only-usage-footer",
"telegram-reply-chain-exact-marker",
"telegram-stream-final-single-message",
"telegram-long-final-reuses-preview",
"telegram-long-final-three-chunks",
] as const;
type TelegramQaProfile = "all" | "release";
function resolveTelegramQaProfile(profile: string | undefined): TelegramQaProfile {
const normalized = profile?.trim() || "release";
if (normalized === "all" || normalized === "release") {
return normalized;
}
throw new Error(
`Unknown QA Lab Telegram profile "${normalized}". Expected one of: all, release.`,
);
}
export function resolveTelegramQaScenarioIds(params: {
profile?: string;
providerMode: string;
scenarioIds?: readonly string[];
}): string[] {
const knownIds = new Set<string>(TELEGRAM_QA_ALL_SCENARIO_IDS);
if (params.scenarioIds && params.scenarioIds.length > 0) {
const unknownIds = params.scenarioIds.filter((id) => !knownIds.has(id));
if (unknownIds.length > 0) {
throw new Error(`unknown Telegram QA scenario id(s): ${unknownIds.join(", ")}`);
}
return [...params.scenarioIds];
}
const profile = resolveTelegramQaProfile(params.profile);
if (profile === "all") {
return [...TELEGRAM_QA_ALL_SCENARIO_IDS];
}
return params.providerMode === "mock-openai"
? [...TELEGRAM_QA_MOCK_RELEASE_SCENARIO_IDS]
: [...TELEGRAM_QA_RELEASE_SCENARIO_IDS];
}
export function listTelegramQaScenarios(providerMode: string) {
const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode, profile: "release" }));
const allIds = new Set<string>(TELEGRAM_QA_ALL_SCENARIO_IDS);
return readQaScenarioPack()
.scenarios.filter((scenario) => allIds.has(scenario.id))
.map((scenario) => ({
id: scenario.id,
title: scenario.title,
rationale: scenario.objective,
regressionRefs: scenario.regressionRefs ?? [],
defaultEnabled: defaultIds.has(scenario.id),
}));
}

View File

@@ -0,0 +1,125 @@
import { describe, expect, it, vi } from "vitest";
import {
buildTelegramQaConfig,
isRecoverableTelegramQaPollError,
normalizeTelegramObservedMessage,
parseTelegramQaCredentialPayload,
resolveTelegramQaRuntimeEnv,
waitForTelegramChannelRunning,
} from "./telegram-api.runtime.js";
describe("Telegram QA API boundary", () => {
it("parses env and leased credential payloads", () => {
expect(
resolveTelegramQaRuntimeEnv({
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "placeholder",
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "placeholder",
}),
).toEqual({
groupId: "-100123",
driverToken: "placeholder",
sutToken: "placeholder",
});
expect(
parseTelegramQaCredentialPayload({
groupId: "-100456",
driverToken: "placeholder",
sutToken: "placeholder",
}),
).toEqual({
groupId: "-100456",
driverToken: "placeholder",
sutToken: "placeholder",
});
expect(() =>
parseTelegramQaCredentialPayload({
groupId: "group-name",
driverToken: "placeholder",
sutToken: "placeholder",
}),
).toThrow("numeric Telegram chat id");
});
it("normalizes rich edited messages and native reply metadata", () => {
expect(
normalizeTelegramObservedMessage({
update_id: 9,
edited_message: {
message_id: 42,
date: 123,
chat: { id: -100123 },
from: { id: 2, is_bot: true, username: "sut_bot" },
rich_message: {
blocks: [{ text: "final " }, { text: [{ text: "reply" }] }],
},
reply_to_message: { message_id: 41 },
},
}),
).toMatchObject({
updateId: 9,
messageId: 42,
chatId: -100123,
senderId: 2,
senderIsBot: true,
text: "final \nreply",
replyToMessageId: 41,
timestamp: 123_000,
});
});
it("builds the isolated Telegram gateway config", () => {
const config = buildTelegramQaConfig(
{ plugins: { allow: ["qa-lab"] } },
{
groupId: "-100123",
sutToken: "placeholder",
driverBotId: 1,
sutAccountId: "sut",
},
);
expect(config.plugins?.allow).toEqual(["qa-lab", "telegram"]);
expect(config.channels?.telegram).toMatchObject({
enabled: true,
defaultAccount: "sut",
accounts: {
sut: {
botToken: "placeholder",
dmPolicy: "disabled",
replyToMode: "first",
groups: {
"-100123": {
groupPolicy: "allowlist",
allowFrom: ["1"],
requireMention: true,
},
},
},
},
});
});
it("waits for the selected Telegram account to become connected", async () => {
const call = vi
.fn()
.mockResolvedValueOnce({
channelAccounts: {
telegram: [{ accountId: "sut", running: true, connected: false }],
},
})
.mockResolvedValueOnce({
channelAccounts: {
telegram: [{ accountId: "sut", running: true, connected: true }],
},
});
await waitForTelegramChannelRunning({ call }, "sut", { timeoutMs: 100, pollMs: 1 });
expect(call).toHaveBeenCalledTimes(2);
});
it("classifies transient polling failures", () => {
expect(isRecoverableTelegramQaPollError(new Error("socket hang up"))).toBe(true);
expect(isRecoverableTelegramQaPollError(new Error("Telegram unauthorized"))).toBe(false);
});
});

View File

@@ -0,0 +1,433 @@
import { setTimeout as sleep } from "node:timers/promises";
// Qa Lab plugin module owns Telegram live adapter API and credential behavior.
import type { TelegramBotMessage, TelegramBotUpdate } from "@openclaw/telegram/api.js";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
parseStrictPositiveInteger,
resolveTimerTimeoutMs,
} from "openclaw/plugin-sdk/number-runtime";
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import { z } from "zod";
export type TelegramQaRuntimeEnv = {
groupId: string;
driverToken: string;
sutToken: string;
};
export type TelegramBotIdentity = {
id: number;
is_bot: boolean;
first_name: string;
username?: string;
};
type TelegramApiEnvelope<T> = {
ok: boolean;
result?: T;
description?: string;
};
type TelegramReplyMarkup = {
inline_keyboard?: Array<Array<{ text?: string }>>;
};
type TelegramRichMessage = {
markdown?: string;
html?: string;
blocks?: unknown[];
};
type TelegramMessage = Pick<TelegramBotMessage, "date" | "message_id"> &
Partial<Pick<TelegramBotMessage, "caption" | "text">> & {
audio?: unknown;
chat: { id: number };
document?: unknown;
from?: Pick<NonNullable<TelegramBotMessage["from"]>, "id" | "is_bot" | "username">;
photo?: unknown[];
rich_message?: TelegramRichMessage;
reply_markup?: TelegramReplyMarkup;
reply_to_message?: { message_id?: number };
sticker?: unknown;
video?: unknown;
voice?: unknown;
};
export type TelegramUpdate = Pick<TelegramBotUpdate, "update_id"> & {
edited_message?: TelegramMessage;
message?: TelegramMessage;
};
type TelegramObservedMessage = {
updateId: number;
messageId: number;
chatId: number;
senderId: number;
senderIsBot: boolean;
senderUsername?: string;
text: string;
caption?: string;
replyToMessageId?: number;
timestamp: number;
inlineButtons: string[];
mediaKinds: string[];
};
type TelegramChannelStatus = {
accountId?: string;
connected?: boolean;
lastConnectedAt?: number;
lastDisconnect?: unknown;
lastError?: string | null;
restartPending?: boolean;
running?: boolean;
};
type TelegramGatewayClient = {
call: (method: string, params?: unknown, options?: { timeoutMs?: number }) => Promise<unknown>;
};
const TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS = 45_000;
const TELEGRAM_QA_ENV_FIELDS = [
{ field: "groupId", envKey: "OPENCLAW_QA_TELEGRAM_GROUP_ID" },
{ field: "driverToken", envKey: "OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN" },
{ field: "sutToken", envKey: "OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN" },
] as const;
const telegramQaCredentialPayloadSchema = z.object({
groupId: z.string().trim().min(1),
driverToken: z.string().trim().min(1),
sutToken: z.string().trim().min(1),
});
function resolveEnvValue(
env: NodeJS.ProcessEnv,
key: (typeof TELEGRAM_QA_ENV_FIELDS)[number]["envKey"],
) {
const value = env[key]?.trim();
if (!value) {
throw new Error(`Missing ${key}.`);
}
return value;
}
export function resolveTelegramQaRuntimeEnv(
env: NodeJS.ProcessEnv = process.env,
): TelegramQaRuntimeEnv {
return parseTelegramQaCredentialPayload(
Object.fromEntries(
TELEGRAM_QA_ENV_FIELDS.map(({ envKey, field }) => [field, resolveEnvValue(env, envKey)]),
),
);
}
export function parseTelegramQaCredentialPayload(payload: unknown): TelegramQaRuntimeEnv {
const parsed = telegramQaCredentialPayloadSchema.parse(payload);
if (!/^-?\d+$/u.test(parsed.groupId)) {
throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id.");
}
return parsed;
}
function flattenInlineButtons(replyMarkup?: TelegramReplyMarkup) {
return (replyMarkup?.inline_keyboard ?? [])
.flat()
.map((button) => button.text?.trim())
.filter((text): text is string => Boolean(text));
}
function detectMediaKinds(message: TelegramMessage) {
const kinds: string[] = [];
if (Array.isArray(message.photo) && message.photo.length > 0) {
kinds.push("photo");
}
for (const kind of ["document", "audio", "video", "voice", "sticker"] as const) {
if (message[kind]) {
kinds.push(kind);
}
}
return kinds;
}
function flattenTelegramRichText(value: unknown): string {
if (typeof value === "string") {
return value;
}
if (Array.isArray(value)) {
return value.map((part) => flattenTelegramRichText(part)).join("");
}
if (!isRecord(value)) {
return "";
}
if ("text" in value) {
return flattenTelegramRichText(value.text);
}
if (typeof value.alternative_text === "string") {
return value.alternative_text;
}
return typeof value.expression === "string" ? value.expression : "";
}
function flattenTelegramRichTableCells(value: unknown): string {
if (!Array.isArray(value)) {
return flattenTelegramRichBlock(value);
}
return value
.map((row) => {
const cells = Array.isArray(row) ? row : [row];
return cells
.map((cell) => flattenTelegramRichBlock(cell))
.filter((cell) => cell.trim())
.join("\t");
})
.filter((row) => row.trim())
.join("\n");
}
function flattenTelegramRichBlocks(value: unknown): string {
const blocks = Array.isArray(value) ? value : [value];
return blocks
.map((block) => flattenTelegramRichBlock(block))
.filter((part) => part.trim())
.join("\n");
}
function flattenTelegramRichBlock(value: unknown): string {
if (typeof value === "string" || Array.isArray(value)) {
return flattenTelegramRichText(value);
}
if (!isRecord(value)) {
return "";
}
const parts = [
"text" in value ? flattenTelegramRichText(value.text) : "",
"summary" in value ? flattenTelegramRichText(value.summary) : "",
typeof value.label === "string" ? value.label : "",
typeof value.expression === "string" ? value.expression : "",
"blocks" in value ? flattenTelegramRichBlocks(value.blocks) : "",
"items" in value ? flattenTelegramRichBlocks(value.items) : "",
"cells" in value ? flattenTelegramRichTableCells(value.cells) : "",
"caption" in value ? flattenTelegramRichBlock(value.caption) : "",
"credit" in value ? flattenTelegramRichText(value.credit) : "",
];
return parts.filter((part) => part.trim()).join("\n");
}
function selectTelegramObservedText(message: TelegramMessage) {
return (
message.text ||
message.caption ||
message.rich_message?.markdown ||
message.rich_message?.html ||
flattenTelegramRichBlocks(message.rich_message?.blocks) ||
""
);
}
export function normalizeTelegramObservedMessage(
update: TelegramUpdate,
): TelegramObservedMessage | null {
const message = update.edited_message ?? update.message;
if (!message?.from?.id) {
return null;
}
return {
updateId: update.update_id,
messageId: message.message_id,
chatId: message.chat.id,
senderId: message.from.id,
senderIsBot: message.from.is_bot,
senderUsername: message.from.username,
text: selectTelegramObservedText(message),
caption: message.caption,
replyToMessageId: message.reply_to_message?.message_id,
timestamp: message.date * 1_000,
inlineButtons: flattenInlineButtons(message.reply_markup),
mediaKinds: detectMediaKinds(message),
};
}
export function buildTelegramQaConfig(
baseCfg: OpenClawConfig,
params: {
groupId: string;
sutToken: string;
driverBotId: number;
sutAccountId: string;
},
): OpenClawConfig {
return {
...baseCfg,
agents: {
...baseCfg.agents,
defaults: {
...baseCfg.agents?.defaults,
models: {
...baseCfg.agents?.defaults?.models,
"openai/gpt-5.6-luna": {
...baseCfg.agents?.defaults?.models?.["openai/gpt-5.6-luna"],
agentRuntime: { id: "openclaw" },
},
},
skipBootstrap: true,
},
},
plugins: {
...baseCfg.plugins,
allow: uniqueStrings([...(baseCfg.plugins?.allow ?? []), "telegram"]),
entries: {
...baseCfg.plugins?.entries,
telegram: { enabled: true },
},
},
messages: {
...baseCfg.messages,
groupChat: {
...baseCfg.messages?.groupChat,
visibleReplies: "automatic",
},
},
channels: {
...baseCfg.channels,
telegram: {
enabled: true,
defaultAccount: params.sutAccountId,
accounts: {
[params.sutAccountId]: {
enabled: true,
botToken: params.sutToken,
dmPolicy: "disabled",
replyToMode: "first",
groups: {
[params.groupId]: {
groupPolicy: "allowlist",
allowFrom: [String(params.driverBotId)],
requireMention: true,
},
},
},
},
},
},
};
}
export async function callTelegramApi<T>(
token: string,
method: string,
body?: Record<string, unknown>,
timeoutMs = 15_000,
): Promise<T> {
const requestTimeoutMs = resolveTimerTimeoutMs(timeoutMs, 15_000);
const { response, release } = await fetchWithSsrFGuard({
url: `https://api.telegram.org/bot${token}/${method}`,
init: {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body ?? {}),
},
timeoutMs: requestTimeoutMs,
policy: { hostnameAllowlist: ["api.telegram.org"] },
auditContext: "qa-lab-telegram-live",
capture: false,
});
try {
const payload = await readProviderJsonResponse<TelegramApiEnvelope<T>>(
response,
`qa-lab-telegram-live.${method}`,
);
if (!response.ok || !payload.ok || payload.result === undefined) {
throw new Error(
payload.description?.trim() || `${method} failed with status ${response.status}`,
);
}
return payload.result;
} finally {
await release();
}
}
export function isRecoverableTelegramQaPollError(error: unknown): boolean {
const message = formatErrorMessage(error).toLowerCase();
return [
"fetch failed",
"aborted due to timeout",
"operation was aborted",
"request timed out",
"aborterror",
"econnreset",
"etimedout",
"socket hang up",
"terminated",
].some((fragment) => message.includes(fragment));
}
export async function waitForTelegramPollRetryDelay(remainingMs = 250) {
await sleep(Math.min(250, Math.max(100, remainingMs)));
}
export async function flushTelegramUpdates(token: string) {
const startedAt = Date.now();
let offset = 0;
while (Date.now() - startedAt < 15_000) {
const updates = await callTelegramApi<TelegramUpdate[]>(
token,
"getUpdates",
{ offset, timeout: 0, allowed_updates: ["message", "edited_message"] },
15_000,
);
if (updates.length === 0) {
return offset;
}
offset = (updates.at(-1)?.update_id ?? offset) + 1;
}
throw new Error("timed out after 15000ms draining Telegram updates");
}
function resolveTelegramQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) {
const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS;
return raw
? (parseStrictPositiveInteger(raw) ?? TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS)
: TELEGRAM_QA_DEFAULT_READY_TIMEOUT_MS;
}
export async function waitForTelegramChannelRunning(
gateway: TelegramGatewayClient,
accountId: string,
options?: { env?: NodeJS.ProcessEnv; pollMs?: number; timeoutMs?: number },
) {
const startedAt = Date.now();
const timeoutMs = options?.timeoutMs ?? resolveTelegramQaReadyTimeoutMs(options?.env);
const pollMs = options?.pollMs ?? 500;
let lastProbeError: string | undefined;
let lastStatus: TelegramChannelStatus | undefined;
while (Date.now() - startedAt < timeoutMs) {
try {
const payload = (await gateway.call(
"channels.status",
{ probe: false, timeoutMs: 2_000 },
{ timeoutMs: 5_000 },
)) as { channelAccounts?: Record<string, TelegramChannelStatus[]> };
const match = (payload.channelAccounts?.telegram ?? []).find(
(entry) => entry.accountId === accountId,
);
lastProbeError = undefined;
lastStatus = match;
if (match?.running && match.connected === true && match.restartPending !== true) {
return;
}
} catch (error) {
lastProbeError = formatErrorMessage(error);
}
await sleep(pollMs);
}
const details = lastStatus
? `; last status: ${JSON.stringify(lastStatus)}`
: lastProbeError
? `; last probe error: ${lastProbeError}`
: "";
throw new Error(`telegram account "${accountId}" did not become ready${details}`);
}

View File

@@ -0,0 +1,66 @@
import { describe, expect, it, vi } from "vitest";
import { runQaSuiteRoundTripProbe } from "./suite-round-trip.js";
describe("QA suite round-trip probe", () => {
it("collects requested samples and chains native replies", async () => {
const messages: Array<{ direction: "outbound"; id: string }> = [];
const sendInbound = vi.fn().mockResolvedValue({ id: "inbound" });
const waitForOutbound = vi.fn().mockImplementation(async () => {
const reply = { id: `out-${messages.length + 1}` };
messages.push({ direction: "outbound", id: reply.id });
return reply;
});
const result = await runQaSuiteRoundTripProbe({
probe: {
scenarioId: "channel-canary",
count: 2,
maxFailures: 2,
timeoutMs: 1_000,
markerPrefix: "QA-RTT",
input: {
conversation: { id: "room", kind: "group" },
senderId: "driver",
},
textPrefix: "Reply exactly: ",
chainReplies: true,
},
transport: {
state: {
getSnapshot: () => ({ messages }),
},
sendInbound,
waitForOutbound,
} as never,
});
expect(result.passed).toBe(2);
expect(result.failed).toBe(0);
expect(result.timing.samples).toBe(2);
expect(sendInbound.mock.calls[1]?.[0]).toMatchObject({ replyToId: "out-1" });
});
it("stops when the failure budget is exhausted", async () => {
const result = await runQaSuiteRoundTripProbe({
probe: {
scenarioId: "channel-canary",
count: 3,
maxFailures: 1,
timeoutMs: 10,
markerPrefix: "QA-RTT",
input: {
conversation: { id: "room", kind: "group" },
senderId: "driver",
},
textPrefix: "Reply exactly: ",
},
transport: {
state: { getSnapshot: () => ({ messages: [] }) },
sendInbound: vi.fn(),
waitForOutbound: vi.fn().mockRejectedValue(new Error("timeout")),
} as never,
});
expect(result).toMatchObject({ passed: 0, failed: 1 });
});
});

View File

@@ -0,0 +1,66 @@
import { randomUUID } from "node:crypto";
import {
summarizeLiveTransportRttSamples,
type LiveTransportRttSample,
} from "./live-transports/shared/live-transport-rtt.js";
import type { QaTransportAdapter } from "./qa-transport.js";
import type { QaBusInboundMessageInput } from "./runtime-api.js";
export type QaSuiteRoundTripProbe = {
scenarioId: string;
count: number;
maxFailures: number;
timeoutMs: number;
markerPrefix: string;
input: Omit<QaBusInboundMessageInput, "replyToId" | "text">;
textPrefix: string;
chainReplies?: boolean;
};
export async function runQaSuiteRoundTripProbe(params: {
probe: QaSuiteRoundTripProbe;
transport: QaTransportAdapter;
}) {
const samples: LiveTransportRttSample[] = [];
let failures = 0;
let passed = 0;
let latestReplyId = params.transport.state
.getSnapshot()
.messages.findLast((message) => message.direction === "outbound")?.id;
for (let index = 1; passed < params.probe.count; index += 1) {
const marker = `${params.probe.markerPrefix}-${index}-${randomUUID().slice(0, 8).toUpperCase()}`;
const outboundStartIndex = params.transport.state
.getSnapshot()
.messages.filter((message) => message.direction === "outbound").length;
const startedAt = Date.now();
try {
await params.transport.sendInbound({
...params.probe.input,
text: `${params.probe.textPrefix}${marker}`,
...(params.probe.chainReplies && latestReplyId ? { replyToId: latestReplyId } : {}),
});
const reply = await params.transport.waitForOutbound({
conversation: params.probe.input.conversation,
sinceIndex: outboundStartIndex,
textIncludes: marker,
timeoutMs: params.probe.timeoutMs,
});
latestReplyId = reply.id;
samples.push({ status: "pass", rttMs: Math.max(1, Date.now() - startedAt) });
passed += 1;
} catch {
samples.push({ status: "fail" });
failures += 1;
}
if (failures >= params.probe.maxFailures) {
break;
}
}
const summary = summarizeLiveTransportRttSamples(samples);
return {
...summary,
details: `${summary.passed}/${samples.length} RTT checks passed`,
};
}

View File

@@ -2,7 +2,7 @@
import fs from "node:fs/promises";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { QaSuiteArtifactError } from "./errors.js";
import type { QaEvidenceSummaryJson } from "./evidence-summary.js";
import type { QaEvidenceSummaryJson, QaEvidenceTiming } from "./evidence-summary.js";
import type { QaProviderMode } from "./model-selection.js";
import type { RuntimeId, RuntimeParityResult } from "./runtime-parity.js";
import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js";
@@ -12,6 +12,7 @@ type QaSuiteSummaryScenario = {
status: "pass" | "fail" | "skip" | "skipped";
steps: unknown[];
details?: string;
timing?: QaEvidenceTiming;
runtimeParity?: RuntimeParityResult;
};

View File

@@ -26,6 +26,7 @@ import {
import {
buildQaSuiteEvidenceSummary,
QA_EVIDENCE_FILENAME,
type QaEvidenceTiming,
type QaEvidenceSummaryJson,
} from "./evidence-summary.js";
import {
@@ -88,6 +89,7 @@ import {
shouldUseIsolatedQaSuiteScenarioWorkers,
splitModelRef,
} from "./suite-planning.js";
import { runQaSuiteRoundTripProbe, type QaSuiteRoundTripProbe } from "./suite-round-trip.js";
import {
createQaSuiteScenarioStepRunner,
runQaSuiteScenarioDefinition,
@@ -115,6 +117,7 @@ export type QaSuiteScenarioResult = {
status: "pass" | "fail";
steps: QaReportCheck[];
details?: string;
timing?: QaEvidenceTiming;
runtimeParity?: RuntimeParityResult;
};
@@ -201,6 +204,7 @@ export type QaSuiteRunParams = {
forcedRuntime?: RuntimeId;
runtimePair?: [RuntimeId, RuntimeId];
captureRuntimeParityCell?: boolean;
roundTripProbe?: QaSuiteRoundTripProbe;
// Unified suite partitions consume child evidence in memory; only the
// parent should write the aggregate qa-evidence.json artifact.
writeEvidenceFile?: boolean;
@@ -545,6 +549,10 @@ function buildQaIsolatedScenarioWorkerParams(params: {
transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs,
workerStartStaggerMs: params.input?.workerStartStaggerMs,
forcedRuntime: params.input?.forcedRuntime,
roundTripProbe:
params.input?.roundTripProbe?.scenarioId === params.scenario.id
? params.input.roundTripProbe
: undefined,
writeEvidenceFile: params.input?.writeEvidenceFile,
};
}
@@ -1276,6 +1284,17 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
requested: requestedModels,
scenarios: selectedScenarios,
});
if (
params?.roundTripProbe &&
!selectedScenarios.some((scenario) => scenario.id === params.roundTripProbe?.scenarioId)
) {
throw new Error(
`QA round-trip probe scenario is not selected: ${params.roundTripProbe.scenarioId}`,
);
}
if (params?.roundTripProbe && params.runtimePair) {
throw new Error("QA round-trip probes are not supported with runtime-pair runs.");
}
const enabledPluginIds = [
...new Set([
...collectQaSuitePluginIds(selectedScenarios),
@@ -1443,7 +1462,14 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
let isolatedRunFailed = false;
let isolatedRunError: unknown;
let parentTransportCleaned = false;
try {
if (params?.channelDriver === "live") {
// The parent only renders aggregate artifacts. Release its live credentials
// before child workers acquire the same exclusive transport lease.
parentTransportCleaned = true;
await transportFactoryResult.cleanup();
}
updateScenarioRun();
const workerStartStaggerMs =
params?.workerStartStaggerMs ?? resolveQaSuiteWorkerStartStaggerMs(concurrency);
@@ -1616,7 +1642,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
throw error;
} finally {
const cleanupSteps: Array<() => Promise<void>> = [
() => transportFactoryResult.cleanup(),
...(!parentTransportCleaned ? [() => transportFactoryResult.cleanup()] : []),
() => disposeRegisteredAgentHarnesses(),
];
if (ownsLab) {
@@ -1810,7 +1836,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario);
const scenarioRetryCount =
scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined;
const result =
let result: QaSuiteScenarioResult =
scenarioRetryCount === 0
? await runSelectedScenario()
: await runQaScenarioWithFlakeRetry(runSelectedScenario, () =>
@@ -1819,6 +1845,27 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
`scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`,
),
);
if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) {
const probeResult = await runQaSuiteRoundTripProbe({
probe: params.roundTripProbe,
transport,
});
const probePassed = probeResult.passed >= params.roundTripProbe.count;
result = {
...result,
status: probePassed ? "pass" : "fail",
details: [result.details, probeResult.details].filter(Boolean).join(" | "),
timing: probeResult.timing,
steps: [
...result.steps,
{
name: "Round-trip samples",
status: probePassed ? "pass" : "fail",
details: probeResult.details,
},
],
};
}
sampleGatewayProcessRss(`scenario:${scenario.id}:finish`);
scenarios.push(result);
writeQaSuiteProgress(

View File

@@ -60,4 +60,7 @@ flow:
expr: matchingOutbound.length === 1
message:
expr: "`expected one canary reply, saw ${matchingOutbound.length}; transcript=${formatTransportTranscript(state, { conversationId: config.conversationId })}`"
- assert:
expr: "transport.id !== 'telegram' || Boolean(outbound.replyToId)"
message: Telegram canary reply was not threaded to the driver message
detailsExpr: outbound.text

View File

@@ -0,0 +1,56 @@
title: Telegram long final reuses its preview
scenario:
id: telegram-long-final-reuses-preview
surface: channel-framework
category: channel-framework.conversation-routing-and-delivery
coverage:
primary: [runtime.delivery]
regressionRefs:
- openclaw/openclaw#39905
objective: Verify a long Telegram final replaces its preview instead of leaving stale text.
successCriteria:
- The joined final contains both scripted boundary markers.
- Telegram leaves one or two final chunks after streaming settles.
codeRefs:
- extensions/telegram/src/send.ts
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
execution:
kind: flow
channel: telegram
summary: Run the scripted long final and verify preview reuse and chunk count.
config:
requiredChannelDriver: live
requiredProviderMode: mock-openai
flow:
steps:
- name: reuses the long-final preview
actions:
- resetTransport: true
- set: outboundStartIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
- sendInbound:
conversation: { id: telegram-stream-room, kind: channel }
senderId: qa-stream-operator
senderName: QA Stream Operator
text: "@openclaw Telegram long final QA check. Use the scripted long final response."
- waitForOutbound:
conversation: { id: telegram-stream-room, kind: channel }
sinceIndex: { ref: outboundStartIndex }
textIncludes: TELEGRAM-LONG-FINAL-BEGIN
timeoutMs: 60000
- call: sleep
args: [4000]
- set: finalMessages
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
- set: joinedFinal
value:
expr: "finalMessages.map((message) => message.text).join('')"
- assert:
expr: "finalMessages.length >= 1 && finalMessages.length <= 2 && joinedFinal.includes('TELEGRAM-LONG-FINAL-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-END')"
message:
expr: "`expected 1-2 complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`"
detailsExpr: "`${finalMessages.length} final chunk(s)`"

View File

@@ -0,0 +1,56 @@
title: Telegram long final keeps three final chunks
scenario:
id: telegram-long-final-three-chunks
surface: channel-framework
category: channel-framework.conversation-routing-and-delivery
coverage:
primary: [runtime.delivery]
regressionRefs:
- openclaw/openclaw#39905
objective: Verify a three-chunk Telegram final removes previews and keeps only final chunks.
successCriteria:
- The joined final contains both scripted three-chunk boundary markers.
- Exactly three final Telegram messages remain after streaming settles.
codeRefs:
- extensions/telegram/src/send.ts
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
execution:
kind: flow
channel: telegram
summary: Run the scripted three-chunk final and verify final chunk accounting.
config:
requiredChannelDriver: live
requiredProviderMode: mock-openai
flow:
steps:
- name: keeps exactly three final chunks
actions:
- resetTransport: true
- set: outboundStartIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
- sendInbound:
conversation: { id: telegram-stream-room, kind: channel }
senderId: qa-stream-operator
senderName: QA Stream Operator
text: "@openclaw Telegram long final three chunk QA check. Use the scripted three chunk final response."
- waitForOutbound:
conversation: { id: telegram-stream-room, kind: channel }
sinceIndex: { ref: outboundStartIndex }
textIncludes: TELEGRAM-LONG-FINAL-3CHUNK-BEGIN
timeoutMs: 60000
- call: sleep
args: [4000]
- set: finalMessages
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
- set: joinedFinal
value:
expr: "finalMessages.map((message) => message.text).join('')"
- assert:
expr: "finalMessages.length === 3 && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-BEGIN') && joinedFinal.includes('TELEGRAM-LONG-FINAL-3CHUNK-END')"
message:
expr: "`expected three complete final chunks; saw ${finalMessages.length}: ${joinedFinal}`"
detailsExpr: "`${finalMessages.length} final chunks`"

View File

@@ -0,0 +1,39 @@
title: Telegram command addressed to another bot
scenario:
id: telegram-other-bot-command-gating
surface: channel-framework
category: channel-framework.channel-actions-commands-and-approvals
coverage:
primary: [channels.native-commands]
objective: Verify a command addressed to another Telegram bot does not wake OpenClaw.
successCriteria:
- The driver sends a native status command addressed to another bot.
- The Telegram group remains quiet for the observation window.
codeRefs:
- extensions/telegram/src/bot-message-context.ts
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
execution:
kind: flow
channel: telegram
summary: Address a command to another bot and verify OpenClaw ignores it.
config:
requiredChannelDriver: live
flow:
steps:
- name: ignores another bot's command
actions:
- resetTransport: true
- set: outboundStartIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
- sendInbound:
conversation: { id: telegram-command-room, kind: channel }
senderId: qa-command-operator
senderName: QA Command Operator
text: /status@OpenClawQaOtherBot
- waitForNoOutbound:
quietMs: 8000
sinceIndex: { ref: outboundStartIndex }
detailsExpr: "'command addressed to another bot stayed quiet'"

View File

@@ -0,0 +1,55 @@
title: Telegram streamed final stays one message
scenario:
id: telegram-stream-final-single-message
surface: channel-framework
category: channel-framework.conversation-routing-and-delivery
coverage:
primary: [runtime.delivery]
regressionRefs:
- openclaw/openclaw#39905
objective: Verify Telegram quiet streaming finalizes one preview without a duplicate reply.
successCriteria:
- The scripted streaming reply contains the exact marker.
- One final outbound Telegram message remains after streaming settles.
codeRefs:
- extensions/telegram/src/send.ts
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
execution:
kind: flow
channel: telegram
summary: Run the scripted quiet-streaming reply and verify one final message.
config:
marker: QA-TELEGRAM-STREAM-SINGLE-OK
requiredChannelDriver: live
requiredProviderMode: mock-openai
flow:
steps:
- name: finalizes one streaming message
actions:
- resetTransport: true
- set: outboundStartIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
- sendInbound:
conversation: { id: telegram-stream-room, kind: channel }
senderId: qa-stream-operator
senderName: QA Stream Operator
text:
expr: "`@openclaw Quiet streaming QA check. Reply exactly: ${config.marker}`"
- waitForOutbound:
conversation: { id: telegram-stream-room, kind: channel }
sinceIndex: { ref: outboundStartIndex }
textIncludes: { ref: config.marker }
timeoutMs: 75000
- call: sleep
args: [4000]
- set: finalMessages
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
- assert:
expr: "finalMessages.length === 1 && finalMessages[0].text.includes(config.marker)"
message:
expr: "`expected one final Telegram message; saw ${finalMessages.length}: ${finalMessages.map((message) => message.text).join(' | ')}`"
detailsExpr: finalMessages[0].text

View File

@@ -6,6 +6,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import type { QaProviderMode } from "../../extensions/qa-lab/src/run-config.ts";
import type { QaSuiteRoundTripProbe } from "../../extensions/qa-lab/src/suite-round-trip.ts";
function parseBoolean(value: string | undefined) {
const normalized = value?.trim().toLowerCase();
@@ -53,24 +54,47 @@ function resolvePackageTelegramOutputDir(env: NodeJS.ProcessEnv, repoRoot: strin
);
}
const DEFAULT_RTT_CHECK_ID = "telegram-mentioned-message-reply";
const DEFAULT_RTT_CHECK_ID = "channel-canary";
function resolveRttOptions(env: NodeJS.ProcessEnv, selectedScenarioIds: readonly string[] = []) {
const explicitCheckIds = splitCsv(env.OPENCLAW_NPM_TELEGRAM_RTT_CHECKS);
const checkIds = explicitCheckIds.length > 0 ? explicitCheckIds : [DEFAULT_RTT_CHECK_ID];
const unknownCheckIds = checkIds.filter((checkId) => checkId !== DEFAULT_RTT_CHECK_ID);
if (unknownCheckIds.length > 0) {
throw new Error(`unknown Telegram QA RTT check: ${unknownCheckIds[0]}`);
}
if (
explicitCheckIds.length === 0 &&
selectedScenarioIds.length > 0 &&
!selectedScenarioIds.includes(DEFAULT_RTT_CHECK_ID)
) {
return {};
return undefined;
}
const rttCount = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20;
const count = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20;
return {
rttCount,
rttTimeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"),
maxRttFailures:
parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? rttCount,
rttCheckIds: explicitCheckIds,
scenarioId: DEFAULT_RTT_CHECK_ID,
count,
timeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS") ?? 30_000,
maxFailures: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? count,
};
}
function createRoundTripProbe(
options: ReturnType<typeof resolveRttOptions>,
): QaSuiteRoundTripProbe | undefined {
if (!options) {
return undefined;
}
return {
...options,
markerPrefix: "QA-TELEGRAM-RTT",
input: {
conversation: { id: "telegram-rtt-room", kind: "group" },
senderId: "qa-rtt-driver",
senderName: "QA RTT Driver",
},
textPrefix: "@openclaw Telegram RTT check. Reply exactly: ",
chainReplies: true,
};
}
@@ -117,8 +141,8 @@ async function resolveTrustedOpenClawCommand(
}
async function main() {
const { runTelegramQaLive } =
await import("../../extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts");
const { runQaTelegramSuite } =
await import("../../extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts");
const rawSutOpenClawCommand = process.env.OPENCLAW_NPM_TELEGRAM_SUT_COMMAND?.trim();
if (!rawSutOpenClawCommand) {
throw new Error("Missing OPENCLAW_NPM_TELEGRAM_SUT_COMMAND.");
@@ -128,8 +152,8 @@ async function main() {
const repoRoot = path.resolve(process.env.OPENCLAW_NPM_TELEGRAM_REPO_ROOT ?? process.cwd());
const outputDir = resolvePackageTelegramOutputDir(process.env, repoRoot);
const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS);
const result = await runTelegramQaLive({
env: process.env,
const result = await runQaTelegramSuite({
allowFailures: true,
repoRoot,
outputDir,
sutOpenClawCommand,
@@ -138,11 +162,14 @@ async function main() {
alternateModel: process.env.OPENCLAW_NPM_TELEGRAM_ALT_MODEL,
fastMode: parseBoolean(process.env.OPENCLAW_NPM_TELEGRAM_FAST),
scenarioIds,
...resolveRttOptions(process.env, scenarioIds),
roundTripProbe: createRoundTripProbe(resolveRttOptions(process.env, scenarioIds)),
sutAccountId: process.env.OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT,
credentialSource: resolveCredentialSource(process.env),
credentialRole: resolveCredentialRole(process.env),
});
if (!result) {
throw new Error("Package Telegram QA did not produce suite artifacts.");
}
process.stdout.write(`Package Telegram QA report: ${result.reportPath}\n`);
process.stdout.write(`Package Telegram QA summary: ${result.summaryPath}\n`);
@@ -180,6 +207,7 @@ export const testing = {
resolvePackageTelegramOutputDir,
resolveCredentialRole,
resolveCredentialSource,
createRoundTripProbe,
resolveRttOptions,
resolveTrustedOpenClawCommand,
shouldFailPackageTelegramRun,

View File

@@ -70,62 +70,6 @@ function evidenceCredentialSource(summary) {
);
}
// Historical Telegram summary artifacts can still appear in old Mantis uploads.
// Current QA producers write qa-evidence.json for gate inputs.
function legacyTelegramSummaryToEvidenceSummary(summary) {
const scenarios = Array.isArray(summary.scenarios) ? summary.scenarios : [];
return {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: new Date().toISOString(),
entries: scenarios.map((scenario) => ({
test: {
kind: "legacy-telegram-qa-scenario",
id: scenario?.id ?? "legacy-telegram-scenario",
title: scenario?.title ?? scenario?.id ?? "Legacy Telegram scenario",
},
mapping: {
profile: "release",
coverage: [],
},
execution: {
runner: "legacy-telegram-qa",
environment: {
ref: null,
os: "unknown",
nodeVersion: "unknown",
},
provider: {
id: "unknown",
live: true,
model: { name: null, ref: null },
auth: summary.credentials?.source ?? "unknown",
},
channel: {
id: "telegram",
live: true,
driver: "native",
},
packageSource: { kind: "unknown" },
artifacts: [
{
kind: "summary",
path: "telegram-qa-summary.json",
source: "legacy-telegram-qa",
},
],
},
result: {
status: scenario?.status === "skip" ? "skipped" : (scenario?.status ?? "fail"),
...(scenario?.status === "pass"
? {}
: { failure: { reason: scenario?.details ?? "legacy scenario did not pass" } }),
...(typeof scenario?.rttMs === "number" ? { timing: { rttMs: scenario.rttMs } } : {}),
},
})),
};
}
function renderScenarioList(summary) {
const entries = evidenceEntries(summary);
if (entries.length === 0) {
@@ -420,7 +364,7 @@ export function buildTelegramEvidenceManifest({
kind: "report",
lane: "run",
label: "Telegram QA report",
path: "telegram-qa-report.md",
path: "qa-suite-report.md",
targetPath: "report.md",
},
];
@@ -453,17 +397,11 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
const outputDir = path.resolve(args.output_dir);
mkdirSync(outputDir, { recursive: true });
const evidenceSummaryPath = path.join(outputDir, "qa-evidence.json");
const legacySummaryPath = path.join(outputDir, "telegram-qa-summary.json");
const usesCurrentEvidenceSummary = existsSync(evidenceSummaryPath);
const summaryPath = usesCurrentEvidenceSummary ? evidenceSummaryPath : legacySummaryPath;
const observedPath = path.join(outputDir, "telegram-qa-observed-messages.json");
const reportPath = path.join(outputDir, "telegram-qa-report.md");
if (!existsSync(summaryPath)) {
const reportPath = path.join(outputDir, "qa-suite-report.md");
if (!existsSync(evidenceSummaryPath)) {
throw new Error(`Missing Telegram QA evidence summary: ${evidenceSummaryPath}`);
}
const summary = usesCurrentEvidenceSummary
? readJson(evidenceSummaryPath)
: legacyTelegramSummaryToEvidenceSummary(readJson(legacySummaryPath));
const summary = readJson(evidenceSummaryPath);
const counts = evidenceCounts(summary);
const pass = counts.failed === 0 && Number(counts.total ?? 0) > 0;
if (!existsSync(reportPath)) {
@@ -472,17 +410,15 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
}
writeFileSync(reportPath, "# Mantis Telegram Live QA\n\nTelegram QA report was unavailable.\n");
}
const hasLegacyObservedMessages = !usesCurrentEvidenceSummary && existsSync(observedPath);
const observedMessages = hasLegacyObservedMessages ? readJson(observedPath) : [];
const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages, summary });
const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages: [], summary });
writeFileSync(path.join(outputDir, "telegram-live-transcript.html"), transcriptHtml, "utf8");
const manifest = buildTelegramEvidenceManifest({
candidateRef: args.candidate_ref,
candidateSha: args.candidate_sha,
hasObservedMessages: hasLegacyObservedMessages,
hasObservedMessages: false,
scenarioLabel: args.scenario_label,
summary,
summaryArtifactPath: path.basename(summaryPath),
summaryArtifactPath: path.basename(evidenceSummaryPath),
});
writeFileSync(
path.join(outputDir, "mantis-evidence.json"),

View File

@@ -449,7 +449,7 @@ async function main(): Promise<void> {
if (telegramRunId) {
await pollRun(options.repo, telegramRunId);
const artifactDir = downloadTelegramArtifact(options.repo, telegramRunId);
const report = findFile(artifactDir, "telegram-qa-report.md");
const report = findFile(artifactDir, "qa-suite-report.md");
if (report && existsSync(report)) {
console.log(`\nTelegram report: ${report}\n`);
console.log(readFileSync(report, "utf8"));

View File

@@ -80,35 +80,11 @@ function makeTelegramOutput({ includeReport = true, summary = {} } = {}) {
]),
);
if (includeReport) {
writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n");
writeFileSync(path.join(dir, "qa-suite-report.md"), "# Telegram QA\n\npass\n");
}
return dir;
}
function makeLegacyTelegramOutput() {
const dir = mkdtempSync(path.join(tmpdir(), "mantis-telegram-evidence-test-"));
tempDirs.push(dir);
mkdirSync(dir, { recursive: true });
writeFileSync(
path.join(dir, "telegram-qa-summary.json"),
JSON.stringify({
credentials: { source: "convex" },
counts: { total: 1, passed: 1, failed: 0 },
scenarios: [
{
id: "telegram-status-command",
title: "Telegram status command reply",
status: "pass",
details: "Observed expected status response.",
},
],
}),
);
writeFileSync(path.join(dir, "telegram-qa-observed-messages.json"), JSON.stringify([]));
writeFileSync(path.join(dir, "telegram-qa-report.md"), "# Telegram QA\n\npass\n");
return dir;
}
describe("scripts/mantis/build-telegram-evidence", () => {
it("renders redacted Telegram observed messages as a transcript HTML page", () => {
const html = renderTelegramEvidenceHtml({
@@ -202,21 +178,6 @@ describe("scripts/mantis/build-telegram-evidence", () => {
);
});
it("renders historical Telegram summaries when evidence summaries are absent", () => {
const dir = makeLegacyTelegramOutput();
const result = writeTelegramEvidence(["--output-dir", dir]);
expect(readFileSync(result.transcriptPath, "utf8")).toContain("Telegram status command reply");
expect(result.manifest.comparison.pass).toBe(true);
expect(
result.manifest.artifacts.find((artifact) => artifact.targetPath === "summary.json"),
).toMatchObject({ path: "telegram-qa-summary.json" });
expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).toContain(
"observed-messages.json",
);
});
it("does not fabricate a required report artifact for passing Telegram summaries", () => {
const dir = makeTelegramOutput({ includeReport: false });
@@ -250,7 +211,7 @@ describe("scripts/mantis/build-telegram-evidence", () => {
const result = writeTelegramEvidence(["--output-dir", dir]);
expect(result.manifest.comparison.pass).toBe(false);
expect(readFileSync(path.join(dir, "telegram-qa-report.md"), "utf8")).toContain(
expect(readFileSync(path.join(dir, "qa-suite-report.md"), "utf8")).toContain(
"Telegram QA report was unavailable",
);
expect(loadEvidenceManifest(result.manifestPath).comparison.pass).toBe(false);

View File

@@ -284,15 +284,15 @@ describe("package Telegram live Docker E2E", () => {
it("defaults package Telegram RTT for the normal package live lane", () => {
expect(testing.resolveRttOptions({})).toEqual({
rttCount: 20,
rttTimeoutMs: undefined,
maxRttFailures: 20,
rttCheckIds: [],
scenarioId: "channel-canary",
count: 20,
timeoutMs: 30_000,
maxFailures: 20,
});
});
it("does not force default RTT onto focused non-RTT scenario runs", () => {
expect(testing.resolveRttOptions({}, ["telegram-canary"])).toEqual({});
expect(testing.resolveRttOptions({}, ["telegram-status-command"])).toBeUndefined();
});
it("maps repeated RTT env onto package Telegram live options", () => {
@@ -301,16 +301,40 @@ describe("package Telegram live Docker E2E", () => {
OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7",
OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS: "45000",
OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES: "2",
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply",
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "channel-canary",
}),
).toEqual({
rttCount: 7,
rttTimeoutMs: 45_000,
maxRttFailures: 2,
rttCheckIds: ["telegram-mentioned-message-reply"],
scenarioId: "channel-canary",
count: 7,
timeoutMs: 45_000,
maxFailures: 2,
});
});
it("builds a generic suite probe for the Telegram RTT lane", () => {
const probe = testing.createRoundTripProbe(testing.resolveRttOptions({}));
expect(probe).toMatchObject({
scenarioId: "channel-canary",
count: 20,
timeoutMs: 30_000,
markerPrefix: "QA-TELEGRAM-RTT",
textPrefix: "@openclaw Telegram RTT check. Reply exactly: ",
chainReplies: true,
input: {
conversation: { id: "telegram-rtt-room", kind: "group" },
},
});
});
it("rejects retired RTT scenario ids", () => {
expect(() =>
testing.resolveRttOptions({
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply",
}),
).toThrow("unknown Telegram QA RTT check: telegram-mentioned-message-reply");
});
it("rejects invalid repeated RTT env", () => {
expect(() =>
testing.resolveRttOptions({