From 8bd78de5daa02edeb951ecd9ff9f9284fccd25d8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 13:35:54 +0100 Subject: [PATCH] test(macos): fix timestamp-anchoring flakes in chat haptics and stream replay tests (#109954) Both flakes (CI run 29573675642, macos-swift) were the same mechanism: fixture timestamps anchored to wall clock at test start, raced against the optimistic user-echo timestamp captured later at send time. The pending-run drain (clearPendingRunIfAssistantMessagePresent -> assistantHapticEvent(after:)) only matches assistant rows timestamped at or after the user echo, and the fake transports' default waitForRunCompletion returns .unavailable, so this comparison is the only drain path in both tests. ChatHapticsTests "durable assistant failure fires run failed": the durable failure row was stamped test start + 1000ms. Under CI load a >1s stall between fixture creation and send() left the row permanently older than the user echo, so .runFailed never fired for either parameterized case. Fix: HapticsTestTransport now takes a history closure and the test stamps the failure row at requestHistory time, which is always post-send by event order. ChatStreamReplayTests "reconnect mid-run converges via history refetch": the completed transcript rows were stamped test start + 100/900ms; the scripted payload has no inFlightRun/sessionInfo, so the foreground refetch cannot drain and only the pending-run owner's timestamp comparison can. Bootstrap plus send taking >900ms made the assistant row older than the echo and the run never drained. Fix: anchor the reconnect transcript to a timestamp captured after the send completed. Proof: injecting a 1.5s/1.2s stall before send reproduced both CI failures exactly (timeout after ~16.5s/16.2s, both haptics cases failing); with the fixes the tests pass even with the stalls injected. Full ChatHapticsTests + ChatStreamReplayTests suites green; both fixed tests stressed 10x with zero failures. --- .../OpenClawKitTests/ChatHapticsTests.swift | 38 ++++++++++++------- .../ChatStreamReplayTests.swift | 11 ++++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatHapticsTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatHapticsTests.swift index 495f76b8f7d8..61c54ce8a43c 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatHapticsTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatHapticsTests.swift @@ -22,11 +22,15 @@ private final class HapticRecorder: @unchecked Sendable { private final class HapticsTestTransport: @unchecked Sendable, OpenClawChatTransport { private let response: OpenClawChatSendResponse - private let historyMessages: [AnyCodable] + // History rows are built per fetch so fixtures can stamp timestamps at + // request time; every fetch in the send flow happens after the optimistic + // user echo exists, which keeps fixture rows ordered after the user turn + // regardless of how long the test was starved before sending. + private let historyMessages: @Sendable () -> [AnyCodable] private let stream: AsyncStream private let continuation: AsyncStream.Continuation - init(status: String, historyMessages: [AnyCodable] = []) { + init(status: String, historyMessages: @escaping @Sendable () -> [AnyCodable] = { [] }) { self.response = OpenClawChatSendResponse(runId: "run-1", status: status) self.historyMessages = historyMessages var continuation: AsyncStream.Continuation! @@ -38,7 +42,7 @@ private final class HapticsTestTransport: @unchecked Sendable, OpenClawChatTrans OpenClawChatHistoryPayload( sessionKey: sessionKey, sessionId: "session-1", - messages: self.historyMessages, + messages: self.historyMessages(), thinkingLevel: "off") } @@ -65,7 +69,9 @@ private final class HapticsTestTransport: @unchecked Sendable, OpenClawChatTrans } } -private func makeHapticsViewModel(status: String, historyMessages: [AnyCodable] = []) async -> ( +private func makeHapticsViewModel( + status: String, + historyMessages: @escaping @Sendable () -> [AnyCodable] = { [] }) async -> ( HapticsTestTransport, OpenClawChatViewModel, HapticRecorder) @@ -117,16 +123,20 @@ struct ChatHapticsTests { @Test(arguments: ["error", "aborted"]) func `durable assistant failure fires run failed`(stopReason: String) async throws { - let error = AnyCodable([ - "role": "assistant", - "content": [], - "timestamp": Date().timeIntervalSince1970 * 1000 + 1000, - "stopReason": stopReason, - "errorMessage": "provider failed", - ] as [String: Any]) - let (_, viewModel, recorder) = await makeHapticsViewModel( - status: "started", - historyMessages: [error]) + // The run-failed drain only fires for assistant rows timestamped at or + // after the optimistic user echo. Stamp the durable failure when history + // is fetched (always post-send) instead of at test start, where a >1s + // scheduling stall before send() left the row permanently "older" than + // the user turn and the wait timed out on loaded CI runners. + let (_, viewModel, recorder) = await makeHapticsViewModel(status: "started") { + [AnyCodable([ + "role": "assistant", + "content": [], + "timestamp": Date().timeIntervalSince1970 * 1000, + "stopReason": stopReason, + "errorMessage": "provider failed", + ] as [String: Any])] + } await sendHapticsTestMessage(viewModel) try await waitUntil("run failed haptic") { recorder.events == [.messageSent, .runFailed] diff --git a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatStreamReplayTests.swift b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatStreamReplayTests.swift index 3cd12d68675d..7a155b9aca97 100644 --- a/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatStreamReplayTests.swift +++ b/apps/shared/OpenClawKit/Tests/OpenClawKitTests/ChatStreamReplayTests.swift @@ -500,7 +500,6 @@ struct ChatStreamReplayTests { } @Test func `reconnect mid-run converges via history refetch and drains pending run`() async throws { - let now = Date().timeIntervalSince1970 * 1000 let harness = try await StreamReplayHarness.bootstrapped() let runId = try await harness.send("please finish") @@ -509,17 +508,23 @@ struct ChatStreamReplayTests { // Stream stops here (no final, no lifecycle end). The gateway finished the // run while the client was away, so the next history fetch returns the // completed transcript keyed to this run. + // With no terminal event, the pending run drains only via the history + // poller, which requires the durable assistant row to be timestamped at + // or after the optimistic user echo. Anchor the transcript after the + // send instead of at test start, where slow bootstrap (>900ms on loaded + // CI runners) left the row "older" than the echo and the run never drained. + let reconnectNow = Date().timeIntervalSince1970 * 1000 await harness.transport.setHistory( replayHistory(messages: [ replayRawMessage( role: "user", text: "please finish", - timestamp: now + 100, + timestamp: reconnectNow + 100, idempotencyKey: "\(runId):user"), replayRawMessage( role: "assistant", text: "Finished while you were away.", - timestamp: now + 900, + timestamp: reconnectNow + 900, idempotencyKey: runId), ])) await MainActor.run { harness.vm.resumeFromForeground() }