feat: add macOS screen snapshots for monitor preview (#67954) thanks @BunsDev

Co-authored-by: Val Alexander <68980965+BunsDev@users.noreply.github.com>
This commit is contained in:
Val Alexander
2026-04-17 02:58:21 -05:00
committed by GitHub
parent 0b6c39be18
commit f377db1015
12 changed files with 321 additions and 1 deletions

View File

@@ -146,6 +146,7 @@ final class MacNodeModeCoordinator {
OpenClawCanvasA2UICommand.push.rawValue,
OpenClawCanvasA2UICommand.pushJSONL.rawValue,
OpenClawCanvasA2UICommand.reset.rawValue,
MacNodeScreenCommand.snapshot.rawValue,
MacNodeScreenCommand.record.rawValue,
OpenClawSystemCommand.notify.rawValue,
OpenClawSystemCommand.which.rawValue,

View File

@@ -63,6 +63,8 @@ actor MacNodeRuntime {
return try await self.handleCameraInvoke(req)
case OpenClawLocationCommand.get.rawValue:
return try await self.handleLocationInvoke(req)
case MacNodeScreenCommand.snapshot.rawValue:
return try await self.handleScreenSnapshotInvoke(req)
case MacNodeScreenCommand.record.rawValue:
return try await self.handleScreenRecordInvoke(req)
case OpenClawSystemCommand.run.rawValue:
@@ -352,6 +354,34 @@ actor MacNodeRuntime {
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
}
private func handleScreenSnapshotInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let params = (try? Self.decodeParams(MacNodeScreenSnapshotParams.self, from: req.paramsJSON)) ??
MacNodeScreenSnapshotParams()
let services = await self.mainActorServices()
let capturedAtMs = Int64(Date().timeIntervalSince1970 * 1000)
let res = try await services.snapshotScreen(
screenIndex: params.screenIndex,
maxWidth: params.maxWidth,
quality: params.quality,
format: params.format)
struct ScreenSnapshotPayload: Encodable {
var format: String
var base64: String
var width: Int
var height: Int
var screenIndex: Int?
var capturedAtMs: Int64
}
let payload = try Self.encodePayload(ScreenSnapshotPayload(
format: res.format.rawValue,
base64: res.data.base64EncodedString(),
width: res.width,
height: res.height,
screenIndex: params.screenIndex,
capturedAtMs: capturedAtMs))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
}
private func mainActorServices() async -> any MacNodeRuntimeMainActorServices {
if let cachedMainActorServices { return cachedMainActorServices }
let services = await self.makeMainActorServices()

View File

@@ -4,6 +4,13 @@ import OpenClawKit
@MainActor
protocol MacNodeRuntimeMainActorServices: Sendable {
func snapshotScreen(
screenIndex: Int?,
maxWidth: Int?,
quality: Double?,
format: OpenClawScreenSnapshotFormat?) async throws
-> (data: Data, format: OpenClawScreenSnapshotFormat, width: Int, height: Int)
func recordScreen(
screenIndex: Int?,
durationMs: Int?,
@@ -21,9 +28,24 @@ protocol MacNodeRuntimeMainActorServices: Sendable {
@MainActor
final class LiveMacNodeRuntimeMainActorServices: MacNodeRuntimeMainActorServices, @unchecked Sendable {
private let screenSnapshotter = ScreenSnapshotService()
private let screenRecorder = ScreenRecordService()
private let locationService = MacNodeLocationService()
func snapshotScreen(
screenIndex: Int?,
maxWidth: Int?,
quality: Double?,
format: OpenClawScreenSnapshotFormat?) async throws
-> (data: Data, format: OpenClawScreenSnapshotFormat, width: Int, height: Int)
{
try await self.screenSnapshotter.snapshot(
screenIndex: screenIndex,
maxWidth: maxWidth,
quality: quality,
format: format)
}
func recordScreen(
screenIndex: Int?,
durationMs: Int?,

View File

@@ -1,9 +1,18 @@
import Foundation
import OpenClawKit
enum MacNodeScreenCommand: String, Codable {
case snapshot = "screen.snapshot"
case record = "screen.record"
}
struct MacNodeScreenSnapshotParams: Codable, Equatable {
var screenIndex: Int?
var maxWidth: Int?
var quality: Double?
var format: OpenClawScreenSnapshotFormat?
}
struct MacNodeScreenRecordParams: Codable, Equatable {
var screenIndex: Int?
var durationMs: Int?