vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 08:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

qa-live: stream telegram scenario progress logs in realtime

Browse Source
This commit is contained in:
joshavant
2026-04-23 00:19:43 -05:00
parent c78562d8a2
commit e6d0342629
2 changed files with 169 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts
View File

@@ -66,6 +66,51 @@ describe("telegram live qa runtime", () => {
).toThrow("OPENCLAW_QA_TELEGRAM_GROUP_ID must be a numeric Telegram chat id.");
});
it("parses Telegram live progress env booleans", () => {
expect(__testing.parseTelegramQaProgressBooleanEnv("true")).toBe(true);
expect(__testing.parseTelegramQaProgressBooleanEnv("on")).toBe(true);
expect(__testing.parseTelegramQaProgressBooleanEnv("false")).toBe(false);
expect(__testing.parseTelegramQaProgressBooleanEnv("off")).toBe(false);
expect(__testing.parseTelegramQaProgressBooleanEnv("maybe")).toBeUndefined();
});
it("defaults Telegram live progress logging from CI when no override is set", () => {
expect(__testing.shouldLogTelegramQaLiveProgress({ CI: "true" })).toBe(true);
expect(__testing.shouldLogTelegramQaLiveProgress({ CI: "false" })).toBe(false);
});
it("applies OPENCLAW_QA_SUITE_PROGRESS override to Telegram live logging", () => {
expect(
__testing.shouldLogTelegramQaLiveProgress({
CI: "false",
OPENCLAW_QA_SUITE_PROGRESS: "true",
}),
).toBe(true);
expect(
__testing.shouldLogTelegramQaLiveProgress({
CI: "true",
OPENCLAW_QA_SUITE_PROGRESS: "false",
}),
).toBe(false);
expect(
__testing.shouldLogTelegramQaLiveProgress({
CI: "true",
OPENCLAW_QA_SUITE_PROGRESS: "definitely",
}),
).toBe(true);
});
it("sanitizes and truncates Telegram live progress details", () => {
expect(__testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe(
"scenario id value",
);
expect(__testing.sanitizeTelegramQaProgressValue("\u0000\u0001")).toBe("<empty>");
const details = __testing.formatTelegramQaProgressDetails(`header\n${"x".repeat(500)}`);
expect(details.startsWith("header ")).toBe(true);
expect(details.length).toBeLessThanOrEqual(240);
expect(details.endsWith("...")).toBe(true);
});
it("parses Telegram pooled credential payloads", () => {
expect(
__testing.parseTelegramQaCredentialPayload({

140
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts
View File

@@ -298,6 +298,9 @@ const TELEGRAM_QA_ENV_KEYS = [
] as const;
const TELEGRAM_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT";
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
const QA_SUITE_PROGRESS_ENV = "OPENCLAW_QA_SUITE_PROGRESS";
const TELEGRAM_QA_PROGRESS_DETAIL_LIMIT = 240;
const TELEGRAM_QA_PROGRESS_PREFIX = "[qa-telegram-live]";
const telegramQaCredentialPayloadSchema = z.object({
groupId: z.string().trim().min(1),
@@ -318,6 +321,57 @@ function isTruthyOptIn(value: string | undefined) {
return normalized === "1" || normalized === "true" || normalized === "yes";
}
function parseTelegramQaProgressBooleanEnv(value: string | undefined): boolean | undefined {
const normalized = value?.trim().toLowerCase();
if (!normalized) {
return undefined;
}
if (normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on") {
return true;
}
if (normalized === "0" || normalized === "false" || normalized === "no" || normalized === "off") {
return false;
}
return undefined;
}
function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) {
const override = parseTelegramQaProgressBooleanEnv(env[QA_SUITE_PROGRESS_ENV]);
if (override !== undefined) {
return override;
}
return parseTelegramQaProgressBooleanEnv(env.CI) === true;
}
function writeTelegramQaProgress(enabled: boolean, message: string) {
if (!enabled) {
return;
}
process.stderr.write(`${TELEGRAM_QA_PROGRESS_PREFIX} ${message}\n`);
}
function sanitizeTelegramQaProgressValue(value: string): string {
let normalized = "";
for (const char of value) {
const code = char.codePointAt(0);
if (code === undefined) {
continue;
}
const isControl = code <= 0x1f || (code >= 0x7f && code <= 0x9f);
normalized += isControl ? " " : char;
}
normalized = normalized.replace(/\s+/gu, " ").trim();
return normalized.length > 0 ? normalized : "<empty>";
}
function formatTelegramQaProgressDetails(details: string): string {
const sanitized = sanitizeTelegramQaProgressValue(details);
if (sanitized.length <= TELEGRAM_QA_PROGRESS_DETAIL_LIMIT) {
return sanitized;
}
return `${sanitized.slice(0, TELEGRAM_QA_PROGRESS_DETAIL_LIMIT - 3).trimEnd()}...`;
}
export function resolveTelegramQaRuntimeEnv(
env: NodeJS.ProcessEnv = process.env,
): TelegramQaRuntimeEnv {
@@ -908,6 +962,19 @@ export async function runTelegramQaLive(params: {
path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${Date.now().toString(36)}`);
await fs.mkdir(outputDir, { recursive: true });
const providerMode = normalizeQaProviderMode(
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findScenario(params.scenarioIds);
const progressEnabled = shouldLogTelegramQaLiveProgress();
writeTelegramQaProgress(
progressEnabled,
`run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"}`,
);
const credentialLease = await acquireQaCredentialLease({
kind: "telegram",
source: params.credentialSource,
@@ -919,18 +986,19 @@ export async function runTelegramQaLive(params: {
const assertLeaseHealthy = () => {
leaseHeartbeat.throwIfFailed();
};
writeTelegramQaProgress(
progressEnabled,
`credentials ready: source=${credentialLease.source} role=${credentialLease.role ?? "<none>"}`,
);
const runtimeEnv = credentialLease.payload;
const providerMode = normalizeQaProviderMode(
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findScenario(params.scenarioIds);
const observedMessages: TelegramObservedMessage[] = [];
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
const includeObservedMessageContent = isTruthyOptIn(process.env[TELEGRAM_QA_CAPTURE_CONTENT_ENV]);
writeTelegramQaProgress(
progressEnabled,
`runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"} captureContent=${includeObservedMessageContent ? "on" : "off"}`,
);
const startedAt = new Date().toISOString();
const scenarioResults: TelegramQaScenarioResult[] = [];
const cleanupIssues: string[] = [];
@@ -976,6 +1044,7 @@ export async function runTelegramQaLive(params: {
await waitForTelegramChannelRunning(gatewayHarness.gateway, sutAccountId);
assertLeaseHealthy();
try {
writeTelegramQaProgress(progressEnabled, "canary start");
await runCanary({
driverToken: runtimeEnv.driverToken,
groupId: runtimeEnv.groupId,
@@ -983,6 +1052,7 @@ export async function runTelegramQaLive(params: {
sutBotId: sutIdentity.id,
observedMessages,
});
writeTelegramQaProgress(progressEnabled, "canary pass");
} catch (error) {
canaryFailure = canaryFailureMessage({
error,
@@ -999,11 +1069,21 @@ export async function runTelegramQaLive(params: {
status: "fail",
details: canaryFailure,
});
writeTelegramQaProgress(
progressEnabled,
`canary fail: details=${formatTelegramQaProgressDetails(canaryFailure)}`,
);
}
assertLeaseHealthy();
if (!canaryFailure) {
let driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken);
for (const scenario of scenarios) {
for (const [scenarioIndex, scenario] of scenarios.entries()) {
const scenarioIndexLabel = `${scenarioIndex + 1}/${scenarios.length}`;
const scenarioIdForLog = sanitizeTelegramQaProgressValue(scenario.id);
writeTelegramQaProgress(
progressEnabled,
`scenario start (${scenarioIndexLabel}): ${scenarioIdForLog}`,
);
assertLeaseHealthy();
const scenarioRun = scenario.buildRun(sutUsername);
try {
@@ -1036,35 +1116,50 @@ export async function runTelegramQaLive(params: {
expectedTextIncludes: scenarioRun.expectedTextIncludes,
message: matched.message,
});
scenarioResults.push({
const result = {
id: scenario.id,
title: scenario.title,
status: "pass",
details: redactPublicMetadata
? "reply matched"
: `reply message ${matched.message.messageId} matched`,
});
} satisfies TelegramQaScenarioResult;
scenarioResults.push(result);
writeTelegramQaProgress(
progressEnabled,
`scenario pass (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`,
);
} catch (error) {
if (!scenarioRun.expectReply) {
const details = formatErrorMessage(error);
if (
details === `timed out after ${scenario.timeoutMs}ms waiting for Telegram message`
) {
scenarioResults.push({
const result = {
id: scenario.id,
title: scenario.title,
status: "pass",
details: "no reply",
});
} satisfies TelegramQaScenarioResult;
scenarioResults.push(result);
writeTelegramQaProgress(
progressEnabled,
`scenario pass (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`,
);
continue;
}
}
scenarioResults.push({
const result = {
id: scenario.id,
title: scenario.title,
status: "fail",
details: formatErrorMessage(error),
});
} satisfies TelegramQaScenarioResult;
scenarioResults.push(result);
writeTelegramQaProgress(
progressEnabled,
`scenario fail (${scenarioIndexLabel}): ${scenarioIdForLog} details=${formatTelegramQaProgressDetails(result.details)}`,
);
}
assertLeaseHealthy();
}
@@ -1089,6 +1184,15 @@ export async function runTelegramQaLive(params: {
const publishedCleanupIssues = redactPublicMetadata
? cleanupIssues.map(() => "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)")
: cleanupIssues;
const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length;
const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length;
writeTelegramQaProgress(
progressEnabled,
`run complete: passed=${passedCount} failed=${failedCount} total=${scenarioResults.length}`,
);
if (cleanupIssues.length > 0) {
writeTelegramQaProgress(progressEnabled, `cleanup issues: count=${cleanupIssues.length}`);
}
const summary: TelegramQaSummary = {
credentials: {
source: credentialLease.source,
@@ -1103,8 +1207,8 @@ export async function runTelegramQaLive(params: {
cleanupIssues: publishedCleanupIssues,
counts: {
total: scenarioResults.length,
passed: scenarioResults.filter((entry) => entry.status === "pass").length,
failed: scenarioResults.filter((entry) => entry.status === "fail").length,
passed: passedCount,
failed: failedCount,
},
scenarios: scenarioResults,
};
@@ -1185,6 +1289,10 @@ export const __testing = {
findScenario,
matchesTelegramScenarioReply,
normalizeTelegramObservedMessage,
parseTelegramQaProgressBooleanEnv,
parseTelegramQaCredentialPayload,
resolveTelegramQaRuntimeEnv,
sanitizeTelegramQaProgressValue,
shouldLogTelegramQaLiveProgress,
formatTelegramQaProgressDetails,
};