fix(macos): harden Quick Chat capture (area-send visibility, AX text quality, overlay race) (#110830)

* fix(macos): harden Quick Chat capture — drop AX-role text, restore bar after area capture, guard overlays post-permission

* chore(i18n): refresh native inventory line numbers after capture hardening
This commit is contained in:
Peter Steinberger
2026-07-18 19:08:23 +01:00
committed by GitHub
parent e04ea7916c
commit b7f99d0ffd
4 changed files with 34 additions and 10 deletions

View File

@@ -34219,7 +34219,7 @@
},
{
"kind": "ui-localized-call",
"line": 215,
"line": 221,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Allow OpenClaw to capture windows",
"surface": "apple",
@@ -34227,7 +34227,7 @@
},
{
"kind": "ui-localized-call",
"line": 216,
"line": 222,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Allow OpenClaw to capture a screen area",
"surface": "apple",
@@ -34235,7 +34235,7 @@
},
{
"kind": "ui-localized-call",
"line": 217,
"line": 223,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Sending a screenshot uses macOS Screen Recording access.",
"surface": "apple",
@@ -34243,7 +34243,7 @@
},
{
"kind": "ui-localized-call",
"line": 218,
"line": 224,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Grant Access",
"surface": "apple",
@@ -34251,7 +34251,7 @@
},
{
"kind": "ui-localized-call",
"line": 219,
"line": 225,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Cancel",
"surface": "apple",
@@ -34259,7 +34259,7 @@
},
{
"kind": "ui-localized-call",
"line": 493,
"line": 503,
"path": "apps/macos/Sources/OpenClaw/QuickChatWindowPicker.swift",
"source": "Selected area",
"surface": "apple",

View File

@@ -353,10 +353,11 @@ private struct QuickChatAXTextTreeNode: QuickChatTextTreeNode, Sendable {
return value
}
}
return self.stringAttribute(kAXRoleAttribute)?
.replacingOccurrences(of: "AX", with: "")
.trimmingCharacters(in: .whitespacesAndNewlines)
.nonEmpty
// Deliberately no AX-role fallback: a role name like "Window"/"Group" is structure,
// not readable content. Emitting it would count toward textEntryCount and let a
// canvas/image-only window send a chip of role words instead of failing with
// "No readable text".
return nil
}
func children(limit: Int) -> QuickChatTextTreeChildren {

View File

@@ -161,6 +161,12 @@ final class QuickChatWindowPicker {
self.finishInteraction()
return
}
// The permission check above suspends; a dismissal/reopen during it may have
// taken over the interaction. Re-validate before installing any picker chrome
// so a cancelled operation cannot strand full-screen area overlays.
guard self.operationID == operationID, self.isInteractionActive, !Task.isCancelled else {
return
}
if mode == .area {
guard self.showAreaOverlays() else {
@@ -487,6 +493,10 @@ final class QuickChatWindowPicker {
self.clearCaptureTask(for: operationID)
return
}
// Pixels are captured; restore the bar now (kept hidden only so it could
// not appear inside the selected region) so image processing and the
// chat.send round-trip show progress instead of a vanished panel.
self.finishInteraction(invalidateOperation: false)
let accepted = await self.model.sendCapturedImage(
pipelineID: pipelineID,
data: result.imageData,

View File

@@ -118,6 +118,19 @@ struct QuickChatFocusedTextCollectorTests {
#expect(result.text.hasSuffix(QuickChatFocusedTextCollector.truncationMarker))
#expect(result.wasTruncated)
}
@Test func `collector reports no text when every node is structure-only`() {
// Mirrors a canvas/image window: nodes expose no textual attributes (the AX
// adapter now returns nil instead of a role name). textEntryCount must stay 0
// so the caller surfaces "No readable text" rather than a chip of role words.
let child = FakeTextNode(id: 2)
let root = FakeTextNode(id: 1, children: [child])
let result = QuickChatFocusedTextCollector.collect(root: root)
#expect(result.text.isEmpty)
#expect(result.textEntryCount == 0)
}
}
private final class FakeTextNode: QuickChatTextTreeNode, Sendable {