From fcc0f4996c7acb877ff11a32535323aeb7a624a1 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Fri, 1 May 2026 18:09:45 +0530 Subject: [PATCH] test(e2e): measure telegram normal reply rtt --- scripts/e2e/mock-openai-server.mjs | 21 +++++---- scripts/e2e/npm-telegram-rtt-config.mjs | 1 + scripts/e2e/npm-telegram-rtt-driver.mjs | 57 ++++++++++++++++--------- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/scripts/e2e/mock-openai-server.mjs b/scripts/e2e/mock-openai-server.mjs index 814d7b9766c..3185a72802b 100644 --- a/scripts/e2e/mock-openai-server.mjs +++ b/scripts/e2e/mock-openai-server.mjs @@ -76,13 +76,13 @@ function writeSse(res, events) { res.end(); } -function writeChatCompletion(res, stream) { +function writeChatCompletion(res, stream, text = successMarker) { if (stream) { writeSse(res, [ { id: "chatcmpl_e2e", object: "chat.completion.chunk", - choices: [{ index: 0, delta: { role: "assistant", content: successMarker } }], + choices: [{ index: 0, delta: { role: "assistant", content: text } }], }, { id: "chatcmpl_e2e", @@ -95,13 +95,16 @@ function writeChatCompletion(res, stream) { writeJson(res, 200, { id: "chatcmpl_e2e", object: "chat.completion", - choices: [ - { index: 0, message: { role: "assistant", content: successMarker }, finish_reason: "stop" }, - ], + choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }], usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 }, }); } +function resolveResponseText(bodyText) { + const matches = Array.from(bodyText.matchAll(/\bOPENCLAW_E2E_OK(?:_\d+)?\b/gu)); + return matches.at(-1)?.[0] ?? successMarker; +} + const server = http.createServer(async (req, res) => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); if (req.method === "GET" && url.pathname === "/health") { @@ -131,6 +134,7 @@ const server = http.createServer(async (req, res) => { } if (req.method === "POST" && url.pathname === "/v1/responses") { + const responseText = resolveResponseText(bodyText); if (body.stream === false) { writeJson(res, 200, { id: "resp_e2e", @@ -142,19 +146,20 @@ const server = http.createServer(async (req, res) => { id: "msg_e2e_1", role: "assistant", status: "completed", - content: [{ type: "output_text", text: successMarker, annotations: [] }], + content: [{ type: "output_text", text: responseText, annotations: [] }], }, ], usage: { input_tokens: 11, output_tokens: 7, total_tokens: 18 }, }); return; } - writeSse(res, responseEvents(successMarker)); + writeSse(res, responseEvents(responseText)); return; } if (req.method === "POST" && url.pathname === "/v1/chat/completions") { - writeChatCompletion(res, body.stream !== false); + const responseText = resolveResponseText(bodyText); + writeChatCompletion(res, body.stream !== false, responseText); return; } diff --git a/scripts/e2e/npm-telegram-rtt-config.mjs b/scripts/e2e/npm-telegram-rtt-config.mjs index ff898e828f1..440eee4a409 100755 --- a/scripts/e2e/npm-telegram-rtt-config.mjs +++ b/scripts/e2e/npm-telegram-rtt-config.mjs @@ -39,6 +39,7 @@ config.models.providers.openai = { id: "OPENAI_API_KEY", }, baseUrl: `http://127.0.0.1:${mockPort}/v1`, + request: { allowPrivateNetwork: true }, models: [ { id: "gpt-5.5", diff --git a/scripts/e2e/npm-telegram-rtt-driver.mjs b/scripts/e2e/npm-telegram-rtt-driver.mjs index 88f96fe7077..c115508f418 100755 --- a/scripts/e2e/npm-telegram-rtt-driver.mjs +++ b/scripts/e2e/npm-telegram-rtt-driver.mjs @@ -13,6 +13,7 @@ const canaryTimeoutMs = Number( const warmSampleCount = Number(process.env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES ?? "20"); const sampleTimeoutMs = Number(process.env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS ?? "30000"); const maxWarmFailures = Number(process.env.OPENCLAW_NPM_TELEGRAM_MAX_FAILURES ?? "3"); +const successMarker = process.env.OPENCLAW_NPM_TELEGRAM_SUCCESS_MARKER ?? "OPENCLAW_E2E_OK"; const scenarioIds = ( process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS ?? "telegram-mentioned-message-reply" ) @@ -82,7 +83,10 @@ function messageText(message) { } async function flushUpdates(bot) { - let updates = await bot.getUpdates({ timeout: 0, allowed_updates: ["message"] }); + let updates = await bot.getUpdates({ + timeout: 0, + allowed_updates: ["message", "edited_message"], + }); let nextOffset; while (updates.length > 0) { const lastUpdateId = updates.at(-1).update_id; @@ -90,7 +94,7 @@ async function flushUpdates(bot) { updates = await bot.getUpdates({ offset: nextOffset, timeout: 0, - allowed_updates: ["message"], + allowed_updates: ["message", "edited_message"], }); } return nextOffset; @@ -102,15 +106,16 @@ async function waitForSutReply(params) { const updates = await driver.getUpdates({ offset: driverUpdateOffset, timeout: 5, - allowed_updates: ["message"], + allowed_updates: ["message", "edited_message"], }); for (const update of updates) { driverUpdateOffset = Math.max(driverUpdateOffset, update.update_id + 1); - const message = update.message; + const message = update.message ?? update.edited_message; if (!message || String(message.chat?.id) !== String(groupId)) { continue; } observedMessages.push({ + updateType: update.edited_message ? "edited_message" : "message", updateId: update.update_id, messageId: message.message_id, fromId: message.from?.id, @@ -128,10 +133,12 @@ async function waitForSutReply(params) { continue; } const text = messageText(message); + if (params.matchText && !text.includes(params.matchText)) { + continue; + } const replyMatches = message.reply_to_message?.message_id === params.requestMessageId; - const markerMatches = params.matchText ? text.includes(params.matchText) : false; const anySutReplyMatches = params.allowAnySutReply; - if (replyMatches || markerMatches || anySutReplyMatches) { + if (replyMatches || anySutReplyMatches || params.matchText) { return message; } } @@ -143,11 +150,15 @@ async function waitForSutReply(params) { async function runScenario(params) { const startedAt = new Date(); const startedUnixSeconds = Math.floor(startedAt.getTime() / 1000); - const request = await driver.sendMessage({ + const sendParams = { chat_id: groupId, text: params.input, disable_notification: true, - }); + }; + if (params.replyToMessageId) { + sendParams.reply_parameters = { message_id: params.replyToMessageId }; + } + const request = await driver.sendMessage(sendParams); try { const reply = await waitForSutReply({ @@ -167,6 +178,7 @@ async function runScenario(params) { title: params.title, status: "pass", details: `observed SUT message ${reply.message_id}`, + messageId: reply.message_id, rttMs, }; } catch (error) { @@ -207,10 +219,13 @@ async function runWarmScenario(params) { let failures = 0; let passed = 0; for (let index = 0; passed < params.sampleCount; index += 1) { + const sampleMarker = `${successMarker}_${index + 1}`; const sample = await runScenario({ - allowAnySutReply: true, + allowAnySutReply: false, id: params.id, - input: `/status@${params.sutUsername}`, + input: `@${params.sutUsername} RTT sample ${index + 1}. Reply with exactly ${sampleMarker}.`, + matchText: sampleMarker, + replyToMessageId: params.replyToMessageId, sampleIndex: index + 1, sutId: params.sutId, timeoutMs: params.sampleTimeoutMs, @@ -282,27 +297,27 @@ async function main() { driverUpdateOffset = (await flushUpdates(driver)) ?? driverUpdateOffset; const scenarios = []; - scenarios.push( - await runScenario({ - allowAnySutReply: true, - id: "telegram-canary", - input: `/status@${sutMe.username}`, - sutId: sutMe.id, - timeoutMs: canaryTimeoutMs, - title: "Telegram canary", - }), - ); + const canary = await runScenario({ + allowAnySutReply: true, + id: "telegram-canary", + input: `/status@${sutMe.username}`, + sutId: sutMe.id, + timeoutMs: canaryTimeoutMs, + title: "Telegram canary", + }); + scenarios.push(canary); if (scenarioIds.includes("telegram-mentioned-message-reply")) { scenarios.push( await runWarmScenario({ id: "telegram-mentioned-message-reply", maxFailures: maxWarmFailures, + replyToMessageId: canary.messageId, sampleCount: warmSampleCount, sampleTimeoutMs, sutId: sutMe.id, sutUsername: sutMe.username, - title: "Telegram status command reply", + title: "Telegram normal reply", }), ); }