feat: QR code scanning for gateway onboarding

iOS:
- QR scanner view using DataScannerViewController
- Photo library QR detection via CIDetector for saved QR images
- Deep link parser for openclaw://gateway URLs and base64url setup codes
- Onboarding wizard: full-screen welcome with "Scan QR Code" button,
  auto-connect on scan, back navigation, step indicators for manual flow

Backend:
- Add /pair qr action to device-pair extension for QR code generation
- TUI/WebUI differentiation: ASCII QR for TUI, markdown image for WebUI
- Telegram: send QR as media attachment via sendMessageTelegram
- Add data URI support to loadWebMedia for generic base64 media handling
- Export renderQrPngBase64 from plugin SDK for extension use

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
(cherry picked from commit d79ed65be0)
This commit is contained in:
Hongwei Ma
2026-02-14 21:47:33 +08:00
committed by Mariano Belinky
parent df6d0ee92b
commit 06adadc759
6 changed files with 238 additions and 1 deletions

View File

@@ -577,6 +577,8 @@ final class NodeAppModel {
switch route {
case let .agent(link):
await self.handleAgentDeepLink(link, originalURL: url)
case .gateway:
break
}
}

View File

@@ -0,0 +1,65 @@
import OpenClawKit
import SwiftUI
import VisionKit
struct QRScannerView: UIViewControllerRepresentable {
let onGatewayLink: (GatewayConnectDeepLink) -> Void
let onError: (String) -> Void
let onDismiss: () -> Void
func makeUIViewController(context: Context) -> DataScannerViewController {
let scanner = DataScannerViewController(
recognizedDataTypes: [.barcode(symbologies: [.qr])],
isHighlightingEnabled: true)
scanner.delegate = context.coordinator
try? scanner.startScanning()
return scanner
}
func updateUIViewController(_: DataScannerViewController, context _: Context) {}
func makeCoordinator() -> Coordinator {
Coordinator(parent: self)
}
final class Coordinator: NSObject, DataScannerViewControllerDelegate {
let parent: QRScannerView
private var handled = false
init(parent: QRScannerView) {
self.parent = parent
}
func dataScanner(_: DataScannerViewController, didAdd items: [RecognizedItem], allItems _: [RecognizedItem]) {
guard !self.handled else { return }
for item in items {
guard case let .barcode(barcode) = item,
let payload = barcode.payloadStringValue
else { continue }
// Try setup code format first (base64url JSON from /pair qr).
if let link = GatewayConnectDeepLink.fromSetupCode(payload) {
self.handled = true
self.parent.onGatewayLink(link)
return
}
// Fall back to deep link URL format (openclaw://gateway?...).
if let url = URL(string: payload),
let route = DeepLinkParser.parse(url),
case let .gateway(link) = route
{
self.handled = true
self.parent.onGatewayLink(link)
return
}
}
}
func dataScanner(_: DataScannerViewController, didRemove _: [RecognizedItem], allItems _: [RecognizedItem]) {}
func dataScanner(_: DataScannerViewController, becameUnavailableWithError error: DataScannerViewController.ScanningUnavailable) {
self.parent.onError("Camera is not available on this device.")
}
}
}

View File

@@ -2,6 +2,56 @@ import Foundation
public enum DeepLinkRoute: Sendable, Equatable {
case agent(AgentDeepLink)
case gateway(GatewayConnectDeepLink)
}
public struct GatewayConnectDeepLink: Codable, Sendable, Equatable {
public let host: String
public let port: Int
public let tls: Bool
public let token: String?
public let password: String?
public init(host: String, port: Int, tls: Bool, token: String?, password: String?) {
self.host = host
self.port = port
self.tls = tls
self.token = token
self.password = password
}
public var websocketURL: URL? {
let scheme = self.tls ? "wss" : "ws"
return URL(string: "\(scheme)://\(self.host):\(self.port)")
}
/// Parse a device-pair setup code (base64url-encoded JSON: `{url, token?, password?}`).
public static func fromSetupCode(_ code: String) -> GatewayConnectDeepLink? {
guard let data = Self.decodeBase64Url(code) else { return nil }
guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil }
guard let urlString = json["url"] as? String,
let parsed = URLComponents(string: urlString),
let hostname = parsed.host, !hostname.isEmpty
else { return nil }
let scheme = parsed.scheme ?? "ws"
let tls = scheme == "wss"
let port = parsed.port ?? 18789
let token = json["token"] as? String
let password = json["password"] as? String
return GatewayConnectDeepLink(host: hostname, port: port, tls: tls, token: token, password: password)
}
private static func decodeBase64Url(_ input: String) -> Data? {
var base64 = input
.replacingOccurrences(of: "-", with: "+")
.replacingOccurrences(of: "_", with: "/")
let remainder = base64.count % 4
if remainder > 0 {
base64.append(contentsOf: String(repeating: "=", count: 4 - remainder))
}
return Data(base64Encoded: base64)
}
}
public struct AgentDeepLink: Codable, Sendable, Equatable {
@@ -69,6 +119,23 @@ public enum DeepLinkParser {
channel: query["channel"],
timeoutSeconds: timeoutSeconds,
key: query["key"]))
case "gateway":
guard let hostParam = query["host"],
!hostParam.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
else {
return nil
}
let port = query["port"].flatMap { Int($0) } ?? 18789
let tls = (query["tls"] as NSString?)?.boolValue ?? false
return .gateway(
.init(
host: hostParam,
port: port,
tls: tls,
token: query["token"],
password: query["password"]))
default:
return nil
}

View File

@@ -1,6 +1,15 @@
import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
import os from "node:os";
import { approveDevicePairing, listDevicePairing } from "openclaw/plugin-sdk";
import { approveDevicePairing, listDevicePairing, renderQrPngBase64 } from "openclaw/plugin-sdk";
import qrcode from "qrcode-terminal";
function renderQrAscii(data: string): Promise<string> {
return new Promise((resolve) => {
qrcode.generate(data, { small: true }, (output: string) => {
resolve(output);
});
});
}
const DEFAULT_GATEWAY_PORT = 18789;
@@ -434,6 +443,84 @@ export default function register(api: OpenClawPluginApi) {
password: auth.password,
};
if (action === "qr") {
const setupCode = encodeSetupCode(payload);
const [qrBase64, qrAscii] = await Promise.all([
renderQrPngBase64(setupCode),
renderQrAscii(setupCode),
]);
const authLabel = auth.label ?? "auth";
const dataUrl = `data:image/png;base64,${qrBase64}`;
const channel = ctx.channel;
const target = ctx.senderId?.trim() || ctx.from?.trim() || ctx.to?.trim() || "";
if (channel === "telegram" && target) {
try {
const send = api.runtime?.channel?.telegram?.sendMessageTelegram;
if (send) {
await send(target, "Scan this QR code with the OpenClaw iOS app:", {
...(ctx.messageThreadId != null ? { messageThreadId: ctx.messageThreadId } : {}),
...(ctx.accountId ? { accountId: ctx.accountId } : {}),
mediaUrl: dataUrl,
});
return {
text: [
`Gateway: ${payload.url}`,
`Auth: ${authLabel}`,
"",
"After scanning, come back here and run `/pair approve` to complete pairing.",
].join("\n"),
};
}
} catch (err) {
api.logger.warn?.(
`device-pair: telegram QR send failed, falling back (${String(
(err as Error)?.message ?? err,
)})`,
);
}
}
// Render based on channel capability
api.logger.info?.(`device-pair: QR fallback channel=${channel} target=${target}`);
const infoLines = [
`Gateway: ${payload.url}`,
`Auth: ${authLabel}`,
"",
"After scanning, run `/pair approve` to complete pairing.",
];
// TUI (gateway-client) needs ASCII, WebUI can render markdown images
const isTui = target === "gateway-client" || channel !== "webchat";
if (!isTui) {
// WebUI: markdown image only
return {
text: [
"Scan this QR code with the OpenClaw iOS app:",
"",
`![Pairing QR](${dataUrl})`,
"",
...infoLines,
].join("\n"),
};
}
// CLI/TUI: ASCII QR only
return {
text: [
"Scan this QR code with the OpenClaw iOS app:",
"",
"```",
qrAscii,
"```",
"",
...infoLines,
].join("\n"),
};
}
const channel = ctx.channel;
const target = ctx.senderId?.trim() || ctx.from?.trim() || ctx.to?.trim() || "";
const authLabel = auth.label ?? "auth";

View File

@@ -451,3 +451,6 @@ export type { ProcessedLineMessage } from "../line/markdown-to-line.js";
// Media utilities
export { loadWebMedia, type WebMediaResult } from "../web/media.js";
// QR code utilities
export { renderQrPngBase64 } from "../web/qr-image.js";

View File

@@ -273,6 +273,19 @@ async function loadWebMediaInternal(
};
};
// Handle data: URLs (base64-encoded inline data)
if (mediaUrl.startsWith("data:")) {
const match = mediaUrl.match(/^data:([^;,]+)?(?:;base64)?,(.*)$/);
if (!match) {
throw new Error("Invalid data: URL format");
}
const contentType = match[1] || "application/octet-stream";
const base64Data = match[2];
const buffer = Buffer.from(base64Data, "base64");
const kind = mediaKindFromMime(contentType);
return await clampAndFinalize({ buffer, contentType, kind });
}
if (/^https?:\/\//i.test(mediaUrl)) {
// Enforce a download cap during fetch to avoid unbounded memory usage.
// For optimized images, allow fetching larger payloads before compression.