From 8e79392dccf4e064ef4d559e2cbe85b13a3041c2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 3 May 2026 18:11:17 -0700 Subject: [PATCH] test(qa): accept Matrix progress edits without draft root --- CHANGELOG.md | 2 +- .../runners/contract/scenario-runtime-room.ts | 26 ++++++++++++------- .../src/runners/contract/scenarios.test.ts | 18 ++++++++----- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd1539b6ca..f85973ba9ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,7 @@ Docs: https://docs.openclaw.ai - Google Meet: keep Chrome realtime transport tests hermetic on Linux prerelease shards while preserving the macOS-only runtime guard. Thanks @vincentkoc. - QA/Slack: fail the live mention-gating scenario on any unexpected SUT reply, even when the reply does not echo the expected marker. Thanks @vincentkoc. - QA/Matrix: steer the live tool-progress preview check away from `HEARTBEAT.md` and report final preview candidates when the live marker reply misses the exact token. Thanks @vincentkoc. -- QA/Matrix: let the live tool-progress preview and error checks verify progress replacement events without depending on the preview saying `Working`, `tool: read`, or an unlabelled/pathless `read from`. Thanks @vincentkoc. +- QA/Matrix: let the live tool-progress preview and error checks verify progress replacement events without depending on the preview saying `Working`, `tool: read`, an unlabelled/pathless `read from`, or the original draft root being observed. Thanks @vincentkoc. - QA/Matrix: wait for live approval reactions to echo before starting the threaded approval decision timeout. Thanks @vincentkoc. - QA/Matrix: reuse the primed driver sync stream when confirming approval reaction echoes, avoiding missed self-reactions in live release runs. Thanks @vincentkoc. - Tlon: expose `groupInviteAllowlist` in the channel config schema and clarify that group invite auto-accept fails closed without an invite allowlist. Thanks @vincentkoc. diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index b7ca1e4e60a..58b6289de9d 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -812,6 +812,13 @@ async function runMatrixToolProgressScenario( mentionUserIds: [context.sutUserId], roomId: context.roomId, }); + const matchesExpectedProgress = (body: string | undefined) => + params.progressPattern.test(body ?? "") || + (params.allowGenericProgressLine === true && hasMatrixQaToolProgressPreviewLine(body)); + const getPreviewRootEventId = (event: MatrixQaObservedEvent) => + event.relatesTo?.relType === "m.replace" && event.relatesTo.eventId + ? event.relatesTo.eventId + : event.eventId; const preview = await client .waitForRoomEvent({ observedEvents: context.observedEvents, @@ -819,7 +826,8 @@ async function runMatrixToolProgressScenario( event.roomId === context.roomId && event.sender === context.sutUserId && event.kind === params.expectedPreviewKind && - event.relatesTo === undefined, + (event.relatesTo === undefined || + (event.relatesTo.relType === "m.replace" && matchesExpectedProgress(event.body))), roomId: context.roomId, since: startSince, timeoutMs: context.timeoutMs, @@ -837,9 +845,7 @@ async function runMatrixToolProgressScenario( }), ); }); - const matchesExpectedProgress = (body: string | undefined) => - params.progressPattern.test(body ?? "") || - (params.allowGenericProgressLine === true && hasMatrixQaToolProgressPreviewLine(body)); + const previewRootEventId = getPreviewRootEventId(preview.event); const progress = matchesExpectedProgress(preview.event.body) ? preview : await client @@ -850,7 +856,7 @@ async function runMatrixToolProgressScenario( event.sender === context.sutUserId && event.kind === params.expectedPreviewKind && event.relatesTo?.relType === "m.replace" && - event.relatesTo.eventId === preview.event.eventId && + event.relatesTo.eventId === previewRootEventId && matchesExpectedProgress(event.body), roomId: context.roomId, since: preview.since, @@ -862,7 +868,7 @@ async function runMatrixToolProgressScenario( cause: err, events: context.observedEvents, expectedPreviewKind: params.expectedPreviewKind, - previewEventId: preview.event.eventId, + previewEventId: previewRootEventId, roomId: context.roomId, startIndex: startObservedIndex, sutUserId: context.sutUserId, @@ -882,7 +888,7 @@ async function runMatrixToolProgressScenario( event.sender === context.sutUserId && isMatrixQaMessageLikeKind(event.kind) && event.relatesTo?.relType === "m.replace" && - event.relatesTo.eventId === preview.event.eventId && + event.relatesTo.eventId === previewRootEventId && doesMatrixQaReplyBodyMatchToken(event, params.finalText), roomId: context.roomId, since: progress.since, @@ -893,7 +899,7 @@ async function runMatrixToolProgressScenario( buildMatrixQaToolProgressFinalTimeoutMessage({ cause: err, events: context.observedEvents, - previewEventId: preview.event.eventId, + previewEventId: previewRootEventId, roomId: context.roomId, startIndex: startObservedIndex, sutUserId: context.sutUserId, @@ -904,7 +910,7 @@ async function runMatrixToolProgressScenario( const unexpectedWorkingEvents = findMatrixQaUnexpectedWorkingEvents({ events: context.observedEvents, finalEventId: finalized.event.eventId, - previewEventId: preview.event.eventId, + previewEventId: previewRootEventId, startIndex: startObservedIndex, sutUserId: context.sutUserId, }); @@ -924,7 +930,7 @@ async function runMatrixToolProgressScenario( artifacts: { driverEventId, previewBodyPreview: progress.event.body?.slice(0, 200), - previewEventId: preview.event.eventId, + previewEventId: previewRootEventId, previewFormattedBodyPreview: progress.event.formattedBody?.slice(0, 200), previewMentions: progress.event.mentions, reply: finalReply, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index 0892065e9ce..67562725e7d 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -2858,15 +2858,20 @@ describe("matrix live qa scenarios", () => { it("finalizes Matrix tool progress previews after tool errors", async () => { const previewEventId = "$tool-progress-error-preview"; - const { sendTextMessage } = mockMatrixQaRoomClient({ + const progressEvent = matrixQaMessageEvent({ + kind: "notice", + eventId: "$tool-progress-error-progress", + body: "Pearling...\n`📖 Read: from /tmp/qa/workspace/missing-matrix-tool-progress-target.txt`", + relatesTo: { + relType: "m.replace", + eventId: previewEventId, + }, + }); + const { sendTextMessage, waitForRoomEvent } = mockMatrixQaRoomClient({ driverEventId: "$tool-progress-error-trigger", events: [ { - event: matrixQaMessageEvent({ - kind: "notice", - eventId: previewEventId, - body: "Pearling...\n`📖 Read: from /tmp/qa/workspace/missing-matrix-tool-progress-target.txt`", - }), + event: progressEvent, since: "driver-sync-preview", }, { @@ -2909,6 +2914,7 @@ describe("matrix live qa scenarios", () => { }, }); + expect(waitForRoomEvent.mock.calls[0]?.[0].predicate(progressEvent)).toBe(true); expect(sendTextMessage).toHaveBeenCalledWith({ body: expect.stringContaining("Tool progress error QA check"), mentionUserIds: ["@sut:matrix-qa.test"],