diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb2c03c44b..a0dd402899d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Docs: https://docs.openclaw.ai - Google Meet: grant Meet media permissions through the Playwright browser context when CDP grants do not affect the attached Chrome page, and report in-call microphone/speaker permission problems instead of marking realtime speech ready. - QA/Slack: fail the live mention-gating scenario on any unexpected SUT reply, even when the reply does not echo the expected marker. Thanks @vincentkoc. - QA/Matrix: steer the live tool-progress preview check away from `HEARTBEAT.md` and report final preview candidates when the live marker reply misses the exact token. Thanks @vincentkoc. +- QA/Matrix: let the live tool-progress preview check verify progress replacement events without depending on the preview saying `Working`. Thanks @vincentkoc. - Tlon: expose `groupInviteAllowlist` in the channel config schema and clarify that group invite auto-accept fails closed without an invite allowlist. Thanks @vincentkoc. - Control UI/WebChat: collapse duplicate in-flight internal text sends onto the active Gateway run so rapid repeat submits do not start fresh `agent:main:main` dispatches. Fixes #75737. Thanks @dsdsddd1 and @BunsDev. - Mattermost: accept the documented `channels.mattermost.streaming` config and honor `streaming: "off"` by disabling draft preview posts. Thanks @vincentkoc. diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index 541653fba31..5754ba44a0f 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -738,7 +738,7 @@ function buildMatrixQaToolProgressTimeoutMessage(params: { return ( event.eventId === params.previewEventId || event.relatesTo?.eventId === params.previewEventId || - /\bWorking\b/i.test(event.body ?? "") + event.body !== undefined ); }) .slice(-8); @@ -812,18 +812,31 @@ async function runMatrixToolProgressScenario( mentionUserIds: [context.sutUserId], roomId: context.roomId, }); - const preview = await client.waitForRoomEvent({ - observedEvents: context.observedEvents, - predicate: (event) => - event.roomId === context.roomId && - event.sender === context.sutUserId && - event.kind === params.expectedPreviewKind && - event.relatesTo === undefined && - /\bWorking\b/i.test(event.body ?? ""), - roomId: context.roomId, - since: startSince, - timeoutMs: context.timeoutMs, - }); + const preview = await client + .waitForRoomEvent({ + observedEvents: context.observedEvents, + predicate: (event) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + event.kind === params.expectedPreviewKind && + event.relatesTo === undefined, + roomId: context.roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }) + .catch((err: unknown) => { + throw new Error( + buildMatrixQaToolProgressTimeoutMessage({ + cause: err, + events: context.observedEvents, + expectedPreviewKind: params.expectedPreviewKind, + previewEventId: "", + roomId: context.roomId, + startIndex: startObservedIndex, + sutUserId: context.sutUserId, + }), + ); + }); const matchesExpectedProgress = (body: string | undefined) => params.progressPattern.test(body ?? "") || (params.allowGenericProgressLine === true && hasMatrixQaToolProgressPreviewLine(body)); @@ -838,7 +851,6 @@ async function runMatrixToolProgressScenario( event.kind === params.expectedPreviewKind && event.relatesTo?.relType === "m.replace" && event.relatesTo.eventId === preview.event.eventId && - /\bWorking\b/i.test(event.body ?? "") && matchesExpectedProgress(event.body), roomId: context.roomId, since: preview.since, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index e2bf2ab9e0a..ddd5908e4c5 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -2567,7 +2567,7 @@ describe("matrix live qa scenarios", () => { event: matrixQaMessageEvent({ kind: "notice", eventId: previewEventId, - body: "Working...", + body: "One moment.", }), since: "driver-sync-preview", }, @@ -2575,7 +2575,7 @@ describe("matrix live qa scenarios", () => { event: matrixQaMessageEvent({ kind: "notice", eventId: "$tool-progress-generic-update", - body: "Working...\n- `tool: exec_command`", + body: "- `tool: exec_command`", relatesTo: { relType: "m.replace", eventId: previewEventId, @@ -2610,7 +2610,7 @@ describe("matrix live qa scenarios", () => { await expect(runMatrixQaScenario(scenario!, matrixQaScenarioContext())).resolves.toMatchObject({ artifacts: { driverEventId: "$tool-progress-generic-trigger", - previewBodyPreview: "Working...\n- `tool: exec_command`", + previewBodyPreview: "- `tool: exec_command`", previewEventId: "$tool-progress-generic-preview", reply: { eventId: "$tool-progress-generic-final",