From 56ffa2bca48bf9ae6c82e491dcd9009d0e61183e Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sat, 1 Aug 2026 21:31:41 +0800 Subject: [PATCH] fix(qa): preserve Slack fallback evidence (#117420) --- .../slack/slack-live.config.ts | 4 +- .../slack/slack-live.observations.ts | 36 +++++++++--- .../slack/slack-live.runtime.test.ts | 57 +++++++++++++++++-- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts index ce02ca09d137..32dcc7b780a2 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts @@ -62,6 +62,7 @@ type SlackQaPostMessageAttempt = { formattingDisabled: boolean; nativeDataBlockCount: number; status: "failed" | "sent"; + text: string; }; export function countSlackNativeDataBlocks(value: unknown) { @@ -85,10 +86,11 @@ export function instrumentSlackPostMessage(client: WebClient) { const originalPostMessage = client.chat.postMessage; const attempts: SlackQaPostMessageAttempt[] = []; client.chat.postMessage = (async (payload) => { - const payloadRecord = payload as { blocks?: unknown; mrkdwn?: boolean }; + const payloadRecord = payload as { blocks?: unknown; mrkdwn?: boolean; text?: unknown }; const attempt = { formattingDisabled: payloadRecord.mrkdwn === false, nativeDataBlockCount: countSlackNativeDataBlocks(payloadRecord.blocks), + text: typeof payloadRecord.text === "string" ? payloadRecord.text : "", }; try { const response = await originalPostMessage.call(client.chat, payload); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts index c96ee933fdab..afb1b295dada 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts @@ -220,6 +220,15 @@ function normalizeSlackAccessibleText(value: string) { return value.trim().replace(/\s+/gu, " "); } +const SLACK_QA_TEXT_DIAGNOSTIC_MAX_CHARS = 500; + +function describeSlackObservedText(value: string) { + const preview = value.slice(0, SLACK_QA_TEXT_DIAGNOSTIC_MAX_CHARS); + return `${value.length} characters; actual=${JSON.stringify(preview)}${ + preview.length < value.length ? "…" : "" + }`; +} + export function isExpectedSlackNativeChartMessage( message: SlackMessage, expectedAccessibleText: string, @@ -365,6 +374,11 @@ export async function runSlackTableInvalidBlocksFallbackScenario( ) { throw new Error("Slack fallback did not use one formatting-disabled blockless API request"); } + if (fallbackAttempt.text !== probe.fallbackText) { + throw new Error( + `Slack fallback API request was incomplete: expected ${probe.fallbackText.length} characters, observed ${describeSlackObservedText(fallbackAttempt.text)}`, + ); + } const message = await waitForSlackStoredMessage({ channelId: context.channelId, @@ -379,16 +393,20 @@ export async function runSlackTableInvalidBlocksFallbackScenario( if (countSlackNativeDataBlocks(message.blocks) !== 0) { throw new Error("stored Slack fallback retained a native data block"); } - const storedLines = storedText.split("\n"); - if (!storedLines.includes(probe.firstRowText)) { - throw new Error("stored Slack fallback omitted the exact first data row"); - } - if (!storedLines.includes(probe.finalRowText)) { - throw new Error("stored Slack fallback omitted the exact final data row"); - } - if (storedText !== probe.fallbackText) { + const normalizedStoredText = normalizeSlackAccessibleText(storedText); + if (!normalizedStoredText.includes(normalizeSlackAccessibleText(probe.firstRowText))) { throw new Error( - `stored Slack fallback was incomplete: expected ${probe.fallbackText.length} characters, observed ${storedText.length}`, + `stored Slack fallback omitted the first data row: ${describeSlackObservedText(storedText)}`, + ); + } + if (!normalizedStoredText.includes(normalizeSlackAccessibleText(probe.finalRowText))) { + throw new Error( + `stored Slack fallback omitted the final data row: ${describeSlackObservedText(storedText)}`, + ); + } + if (normalizedStoredText !== normalizeSlackAccessibleText(probe.fallbackText)) { + throw new Error( + `stored Slack fallback was incomplete: expected ${probe.fallbackText.length} characters, observed ${describeSlackObservedText(storedText)}`, ); } return { diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 6db0979f8bad..e8255c982ed0 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -1167,7 +1167,10 @@ describe("Slack live QA runtime helpers", () => { ? [ { blocks: storedPayload.blocks, - text: storedPayload.text, + text: + typeof storedPayload.text === "string" + ? storedPayload.text.replace(/\s+/gu, " ") + : storedPayload.text, ts: "2.000000", user: "U999999999", }, @@ -1203,17 +1206,59 @@ describe("Slack live QA runtime helpers", () => { const fallbackText = typeof fallbackRequest?.text === "string" ? fallbackRequest.text : ""; expect(fallbackText.split("\n")).toContain(probe.firstRowText); expect(fallbackText.split("\n")).toContain(probe.finalRowText); - expect(result.message).toMatchObject({ - text: fallbackText, - ts: "2.000000", - user: "U999999999", - }); + expect(result.message).toMatchObject({ ts: "2.000000", user: "U999999999" }); + expect(result.message.text).toBe(fallbackText.replace(/\s+/gu, " ")); expect(result.details).toContain("first API failure=invalid_blocks"); expect(result.details).toContain("fallback formatting disabled=true"); expect(result.details).toContain("complete delivery=true"); expect(sutWriteClient.chat.postMessage).toBe(postMessage); }); + it("bounds invalid_blocks readback diagnostics while showing the observed text", async () => { + const malformedReadback = `BROKEN-${"x".repeat(2_000)}`; + const postMessage = vi.fn(async () => ({ + channel: "C123456789", + ok: true, + ts: "2.000000", + })); + const sutWriteClient = { chat: { postMessage } }; + const cfg = testing.buildSlackQaConfig( + {}, + { + channelId: "C123456789", + driverBotUserId: "U111111111", + sutAccountId: "sut", + sutAppToken: "xapp-sut", + sutBotToken: "xoxb-sut", + }, + ); + + const error = await testing + .runSlackTableInvalidBlocksFallbackScenario({ + cfg, + channelId: "C123456789", + sutAccountId: "sut", + sutIdentity: { userId: "U999999999" }, + sutReadClient: { + conversations: { + history: vi.fn(async () => ({ + messages: [{ text: malformedReadback, ts: "2.000000", user: "U999999999" }], + })), + }, + } as never, + sutWriteClient: sutWriteClient as never, + timeoutMs: 0, + }) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain(`${malformedReadback.length} characters`); + expect(message).toContain('actual="BROKEN-'); + expect(message).toContain("…"); + expect(message.length).toBeLessThan(700); + }); + it("reports the real Slack error when the fallback request fails", async () => { const postMessage = vi.fn(async () => { throw Object.assign(new Error("do not persist this raw platform detail"), {