From b7f99d0ffd2aee29baea352175367eb2f4962c2d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 18 Jul 2026 19:08:23 +0100 Subject: [PATCH] fix(macos): harden Quick Chat capture (area-send visibility, AX text quality, overlay race) (#110830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(macos): harden Quick Chat capture — drop AX-role text, restore bar after area capture, guard overlays post-permission * chore(i18n): refresh native inventory line numbers after capture hardening --- apps/.i18n/native-source.json | 12 ++++++------ .../OpenClaw/QuickChatFocusedTextCapture.swift | 9 +++++---- .../Sources/OpenClaw/QuickChatWindowPicker.swift | 10 ++++++++++ .../QuickChatFocusedTextCollectorTests.swift | 13 +++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 39863472654a..7b7b289af154 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -34219,7 +34219,7 @@ }, { "kind": "ui-localized-call", - "line": 215, + "line": 221, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Allow OpenClaw to capture windows", "surface": "apple", @@ -34227,7 +34227,7 @@ }, { "kind": "ui-localized-call", - "line": 216, + "line": 222, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Allow OpenClaw to capture a screen area", "surface": "apple", @@ -34235,7 +34235,7 @@ }, { "kind": "ui-localized-call", - "line": 217, + "line": 223, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Sending a screenshot uses macOS Screen Recording access.", "surface": "apple", @@ -34243,7 +34243,7 @@ }, { "kind": "ui-localized-call", - "line": 218, + "line": 224, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Grant Access", "surface": "apple", @@ -34251,7 +34251,7 @@ }, { "kind": "ui-localized-call", - "line": 219, + "line": 225, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Cancel", "surface": "apple", @@ -34259,7 +34259,7 @@ }, { "kind": "ui-localized-call", - "line": 493, + "line": 503, "path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift", "source": "Selected area", "surface": "apple", diff --git a/apps/macos/Sources/OpenClaw/QuickChatFocusedTextCapture.swift b/apps/macos/Sources/OpenClaw/QuickChatFocusedTextCapture.swift index 8054a8ae5981..0472caf8909a 100644 --- a/apps/macos/Sources/OpenClaw/QuickChatFocusedTextCapture.swift +++ b/apps/macos/Sources/OpenClaw/QuickChatFocusedTextCapture.swift @@ -353,10 +353,11 @@ private struct QuickChatAXTextTreeNode: QuickChatTextTreeNode, Sendable { return value } } - return self.stringAttribute(kAXRoleAttribute)? - .replacingOccurrences(of: "AX", with: "") - .trimmingCharacters(in: .whitespacesAndNewlines) - .nonEmpty + // Deliberately no AX-role fallback: a role name like "Window"/"Group" is structure, + // not readable content. Emitting it would count toward textEntryCount and let a + // canvas/image-only window send a chip of role words instead of failing with + // "No readable text". + return nil } func children(limit: Int) -> QuickChatTextTreeChildren { diff --git a/apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift b/apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift index 2e73e98021a4..ff60308805bc 100644 --- a/apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift +++ b/apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift @@ -161,6 +161,12 @@ final class QuickChatWindowPicker { self.finishInteraction() return } + // The permission check above suspends; a dismissal/reopen during it may have + // taken over the interaction. Re-validate before installing any picker chrome + // so a cancelled operation cannot strand full-screen area overlays. + guard self.operationID == operationID, self.isInteractionActive, !Task.isCancelled else { + return + } if mode == .area { guard self.showAreaOverlays() else { @@ -487,6 +493,10 @@ final class QuickChatWindowPicker { self.clearCaptureTask(for: operationID) return } + // Pixels are captured; restore the bar now (kept hidden only so it could + // not appear inside the selected region) so image processing and the + // chat.send round-trip show progress instead of a vanished panel. + self.finishInteraction(invalidateOperation: false) let accepted = await self.model.sendCapturedImage( pipelineID: pipelineID, data: result.imageData, diff --git a/apps/macos/Tests/OpenClawIPCTests/QuickChatFocusedTextCollectorTests.swift b/apps/macos/Tests/OpenClawIPCTests/QuickChatFocusedTextCollectorTests.swift index 125942b4b4be..5acc9fb95aa7 100644 --- a/apps/macos/Tests/OpenClawIPCTests/QuickChatFocusedTextCollectorTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/QuickChatFocusedTextCollectorTests.swift @@ -118,6 +118,19 @@ struct QuickChatFocusedTextCollectorTests { #expect(result.text.hasSuffix(QuickChatFocusedTextCollector.truncationMarker)) #expect(result.wasTruncated) } + + @Test func `collector reports no text when every node is structure-only`() { + // Mirrors a canvas/image window: nodes expose no textual attributes (the AX + // adapter now returns nil instead of a role name). textEntryCount must stay 0 + // so the caller surfaces "No readable text" rather than a chip of role words. + let child = FakeTextNode(id: 2) + let root = FakeTextNode(id: 1, children: [child]) + + let result = QuickChatFocusedTextCollector.collect(root: root) + + #expect(result.text.isEmpty) + #expect(result.textEntryCount == 0) + } } private final class FakeTextNode: QuickChatTextTreeNode, Sendable {