From a027ac0195f4edfe774b5a5614d49768096341e5 Mon Sep 17 00:00:00 2001
From: Vincent Koc <vincentkoc@ieee.org>
Date: Sun, 3 May 2026 15:31:28 -0700
Subject: [PATCH] fix(qa): fail slack no-reply on any reply

---
 CHANGELOG.md                                  |  1 +
 .../slack/slack-live.runtime.test.ts          | 37 ++++++++++++++++++
 .../slack/slack-live.runtime.ts               | 39 +++++++++++++++----
 3 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a948d499c6..b4f29c7d520 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,6 +36,7 @@ Docs: https://docs.openclaw.ai
 - Google Meet: refresh realtime browser state during status and retry delayed speech after Meet finishes joining, so a just-opened in-call tab no longer leaves speech stuck behind stale `not-in-call` health.
 - Plugins/install: recover the install ledger from the managed npm root when `plugins/installs.json` is empty or partial, so reinstalling Discord and Codex no longer makes the other installed plugin disappear.
 - Google Meet: grant Meet media permissions through the Playwright browser context when CDP grants do not affect the attached Chrome page, and report in-call microphone/speaker permission problems instead of marking realtime speech ready.
+- QA/Slack: fail the live mention-gating scenario on any unexpected SUT reply, even when the reply does not echo the expected marker. Thanks @vincentkoc.
 - Channel docs: keep JSON5 channel config examples parseable and schema-valid, fixing BlueBubbles, QQ Bot, and Slack snippets that could not be copied into config or app manifests as shown. Thanks @vincentkoc.
 - Tlon: expose `groupInviteAllowlist` in the channel config schema and clarify that group invite auto-accept fails closed without an invite allowlist. Thanks @vincentkoc.
 - Google Chat: update the setup example to use the accepted `groups.<space>.enabled` key instead of the legacy `allow` alias, with a schema regression for the documented group shape. Thanks @vincentkoc.
diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
index 591a4d92f77..083a231cbf7 100644
--- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
+++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts
@@ -54,4 +54,41 @@ describe("Slack live QA runtime helpers", () => {
       "slack-canary",
     ]);
   });
+
+  it("fails mention-gating when the SUT replies without the marker", async () => {
+    const observedMessages: Array<unknown> = [];
+    await expect(
+      __testing.waitForSlackNoReply({
+        channelId: "C123456789",
+        client: {
+          conversations: {
+            history: async () => ({
+              messages: [
+                {
+                  text: "I should not have replied",
+                  ts: "2.000000",
+                  user: "U999999999",
+                },
+              ],
+            }),
+          },
+        } as never,
+        matchText: "SLACK_QA_NOMENTION_MARKER",
+        observedMessages: observedMessages as never,
+        observationScenarioId: "slack-mention-gating",
+        observationScenarioTitle: "Slack unmentioned bot message does not trigger",
+        sentTs: "1.000000",
+        sutIdentity: { userId: "U999999999" },
+        timeoutMs: 1_000,
+      }),
+    ).rejects.toThrow("unexpected Slack SUT reply observed");
+    expect(observedMessages).toMatchObject([
+      {
+        matchedScenario: false,
+        text: "I should not have replied",
+        ts: "2.000000",
+        userId: "U999999999",
+      },
+    ]);
+  });
 });
diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts
index f9fdd8d72bd..b76aa4c776d 100644
--- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts
+++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts
@@ -428,16 +428,38 @@ async function waitForSlackNoReply(params: {
   sutIdentity: SlackAuthIdentity;
   timeoutMs: number;
 }) {
-  try {
-    await waitForSlackScenarioReply(params);
-  } catch (error) {
-    const message = formatErrorMessage(error);
-    if (message === `timed out after ${params.timeoutMs}ms waiting for Slack message`) {
-      return;
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < params.timeoutMs) {
+    const messages = await listSlackMessages({
+      channelId: params.channelId,
+      client: params.client,
+      oldestTs: params.sentTs,
+    });
+    for (const message of messages) {
+      const text = message.text ?? "";
+      if (
+        !message.ts ||
+        message.ts === params.sentTs ||
+        !isSutSlackMessage(message, params.sutIdentity)
+      ) {
+        continue;
+      }
+      const matchedScenario = text.includes(params.matchText);
+      params.observedMessages.push({
+        botId: message.bot_id,
+        channelId: params.channelId,
+        matchedScenario,
+        scenarioId: params.observationScenarioId,
+        scenarioTitle: params.observationScenarioTitle,
+        text,
+        threadTs: message.thread_ts,
+        ts: message.ts,
+        userId: message.user,
+      });
+      throw new Error("unexpected Slack SUT reply observed");
     }
-    throw error;
+    await new Promise((resolve) => setTimeout(resolve, 1_000));
   }
-  throw new Error("unexpected Slack SUT reply observed");
 }
 
 async function waitForSlackChannelRunning(
@@ -816,4 +838,5 @@ export const __testing = {
   parseSlackQaCredentialPayload,
   resolveSlackQaRuntimeEnv,
   SLACK_QA_STANDARD_SCENARIO_IDS,
+  waitForSlackNoReply,
 };