From 052dc59526a2dfdab64b3b16d8fe31650d902856 Mon Sep 17 00:00:00 2001 From: Josh Avant <830519+joshavant@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:17:21 -0500 Subject: [PATCH] fix: /pair qr fails to show QR code in chat surfaces (#97933) * fix(pair): render qr per chat surface * test(plugin-sdk): update reply helper surface budget * fix(pair): keep qr metadata internal --- extensions/device-pair/index.test.ts | 13 ++++- extensions/device-pair/index.ts | 9 ++- .../device-pair/pairing-qr-channel-data.ts | 19 +++++++ src/auto-reply/reply-payload.test.ts | 30 ++++++++++ src/auto-reply/reply-payload.ts | 41 ++++++++++++++ .../chat.directive-tags.test.ts | 55 ++++++++++++++++++- src/gateway/server-methods/chat.ts | 46 +++++++++++++++- src/tui/tui-formatters.ts | 30 +++++++++- src/tui/tui-stream-assembler.test.ts | 18 ++++++ ui/src/ui/chat/grouped-render.test.ts | 21 +++++++ ui/src/ui/chat/grouped-render.ts | 8 +++ 11 files changed, 281 insertions(+), 9 deletions(-) create mode 100644 extensions/device-pair/pairing-qr-channel-data.ts create mode 100644 src/auto-reply/reply-payload.test.ts diff --git a/extensions/device-pair/index.test.ts b/extensions/device-pair/index.test.ts index 4f700041313b..e4773d38f24c 100644 --- a/extensions/device-pair/index.test.ts +++ b/extensions/device-pair/index.test.ts @@ -290,7 +290,12 @@ describe("device-pair /pair qr", () => { gatewayClientScopes: INTERNAL_SETUP_SCOPES, }), ); - const payload = result as { text?: string; mediaUrl?: string; sensitiveMedia?: boolean }; + const payload = result as { + text?: string; + mediaUrl?: string; + channelData?: Record; + sensitiveMedia?: boolean; + }; const text = requireText(result); expect(pluginApiMocks.renderQrPngDataUrl).toHaveBeenCalledTimes(1); @@ -301,7 +306,11 @@ describe("device-pair /pair qr", () => { }, }); expect(text).toContain("Scan this QR code with the OpenClaw iOS app:"); - expect(payload.mediaUrl).toBe("data:image/png;base64,ZmFrZXBuZw=="); + expect(payload.mediaUrl).toBeUndefined(); + expect(payload.channelData?.openclawPairingQr).toEqual({ + setupCode: expect.any(String), + expiresAtMs: expect.any(Number), + }); expect(payload.sensitiveMedia).toBe(true); expect(text).toContain("- Security: single-use bootstrap token"); expect(text).toContain("**Important:** Run `/pair cleanup` after pairing finishes."); diff --git a/extensions/device-pair/index.ts b/extensions/device-pair/index.ts index 87412a695dba..8a5a8d645c60 100644 --- a/extensions/device-pair/index.ts +++ b/extensions/device-pair/index.ts @@ -7,6 +7,7 @@ import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { buildDevicePairPairingQrChannelData } from "./pairing-qr-channel-data.js"; type DevicePairApiModule = typeof import("./api.js"); type NotifyModule = typeof import("./notify.js"); @@ -834,10 +835,9 @@ export default definePluginEntry({ api.logger.info?.(`device-pair: QR fallback channel=${channel} target=${target}`); if (channel === "webchat") { - let qrDataUrl: string; try { const { renderQrPngDataUrl } = await loadDevicePairApiModule(); - qrDataUrl = await renderQrPngDataUrl(setupCode); + await renderQrPngDataUrl(setupCode); } catch (err) { const { revokeDeviceBootstrapToken } = await loadDevicePairApiModule(); api.logger.warn?.( @@ -862,7 +862,10 @@ export default definePluginEntry({ expiresAtMs: payload.expiresAtMs, }), ].join("\n"), - mediaUrl: qrDataUrl, + channelData: buildDevicePairPairingQrChannelData({ + setupCode, + expiresAtMs: payload.expiresAtMs, + }), sensitiveMedia: true, }; } diff --git a/extensions/device-pair/pairing-qr-channel-data.ts b/extensions/device-pair/pairing-qr-channel-data.ts new file mode 100644 index 000000000000..9f639f73d7de --- /dev/null +++ b/extensions/device-pair/pairing-qr-channel-data.ts @@ -0,0 +1,19 @@ +// Private device-pair -> Gateway live-display envelope. +// Keep this local so pairing QR metadata does not become public Plugin SDK API. +export const DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY = "openclawPairingQr"; + +export type DevicePairPairingQrChannelData = { + setupCode: string; + expiresAtMs: number; +}; + +export function buildDevicePairPairingQrChannelData( + params: DevicePairPairingQrChannelData, +): Record { + return { + [DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY]: { + setupCode: params.setupCode, + expiresAtMs: params.expiresAtMs, + }, + }; +} diff --git a/src/auto-reply/reply-payload.test.ts b/src/auto-reply/reply-payload.test.ts new file mode 100644 index 000000000000..eb3edaba1ae8 --- /dev/null +++ b/src/auto-reply/reply-payload.test.ts @@ -0,0 +1,30 @@ +// Reply payload tests cover internal reply metadata contracts. +import { describe, expect, it } from "vitest"; +import { buildPairingQrReplyChannelData, readPairingQrReplyChannelData } from "./reply-payload.js"; + +describe("pairing QR reply channel data", () => { + it("builds and reads the private pairing QR payload metadata", () => { + const channelData = buildPairingQrReplyChannelData({ + setupCode: "setup-code", + expiresAtMs: 1_800_000_000_000, + }); + + expect(readPairingQrReplyChannelData({ channelData })).toEqual({ + setupCode: "setup-code", + expiresAtMs: 1_800_000_000_000, + }); + }); + + it("ignores malformed pairing QR metadata", () => { + expect( + readPairingQrReplyChannelData({ + channelData: { + openclawPairingQr: { + setupCode: "", + expiresAtMs: 0, + }, + }, + }), + ).toBeUndefined(); + }); +}); diff --git a/src/auto-reply/reply-payload.ts b/src/auto-reply/reply-payload.ts index 2c3c7f2da7ae..6d36db62c551 100644 --- a/src/auto-reply/reply-payload.ts +++ b/src/auto-reply/reply-payload.ts @@ -62,6 +62,47 @@ export type ReplyPayload = { channelData?: Record; }; +// Private device-pair -> Gateway live-display envelope key. Do not re-export +// through Plugin SDK; this is not a third-party plugin contract. +export const PAIRING_QR_REPLY_CHANNEL_DATA_KEY = "openclawPairingQr"; + +export type PairingQrReplyChannelData = { + setupCode: string; + expiresAtMs: number; +}; + +function normalizePairingQrSetupCode(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +function normalizePairingQrExpiresAtMs(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; +} + +export function buildPairingQrReplyChannelData( + params: PairingQrReplyChannelData, +): Record { + return { + [PAIRING_QR_REPLY_CHANNEL_DATA_KEY]: { + setupCode: params.setupCode, + expiresAtMs: params.expiresAtMs, + }, + }; +} + +export function readPairingQrReplyChannelData( + payload: Pick, +): PairingQrReplyChannelData | undefined { + const raw = payload.channelData?.[PAIRING_QR_REPLY_CHANNEL_DATA_KEY]; + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + return undefined; + } + const record = raw as Record; + const setupCode = normalizePairingQrSetupCode(record.setupCode); + const expiresAtMs = normalizePairingQrExpiresAtMs(record.expiresAtMs); + return setupCode && expiresAtMs ? { setupCode, expiresAtMs } : undefined; +} + /** Metadata for fast-auto progress notices. */ export const FAST_MODE_AUTO_PROGRESS_KIND = "fast-mode-auto"; diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index ee8b35d00daf..537035631d46 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -14,7 +14,10 @@ import { import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js"; import { CHAT_SEND_SESSION_KEY_MAX_LENGTH } from "../../../packages/gateway-protocol/src/schema.js"; import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js"; -import { setReplyPayloadMetadata } from "../../auto-reply/reply-payload.js"; +import { + buildPairingQrReplyChannelData, + setReplyPayloadMetadata, +} from "../../auto-reply/reply-payload.js"; import type { MsgContext } from "../../auto-reply/templating.js"; import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js"; import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js"; @@ -33,6 +36,7 @@ const mockState = vi.hoisted(() => ({ text?: string; mediaUrl?: string; mediaUrls?: string[]; + channelData?: Record; spokenText?: string; audioAsVoice?: boolean; trustedLocalMedia?: boolean; @@ -48,6 +52,7 @@ const mockState = vi.hoisted(() => ({ text?: string; mediaUrl?: string; mediaUrls?: string[]; + channelData?: Record; spokenText?: string; ttsSupplement?: { spokenText: string }; audioAsVoice?: boolean; @@ -2454,6 +2459,54 @@ describe("chat directive tag stripping for non-streaming final payloads", () => expect(transcriptUpdate).toBeTruthy(); }); + it("broadcasts sensitive pairing QR display without persisting QR content", async () => { + createTranscriptFixture("openclaw-chat-send-command-pair-qr-"); + const setupCode = "openclaw-test-pairing-setup-code"; + mockState.dispatchedReplies = [ + { + kind: "final", + payload: { + text: "Scan this QR code with the OpenClaw iOS app:", + channelData: buildPairingQrReplyChannelData({ + setupCode, + expiresAtMs: Date.now() + 10 * 60_000, + }), + sensitiveMedia: true, + }, + }, + ]; + const respond = vi.fn(); + const context = createChatContext(); + + const payload = await runNonStreamingChatSend({ + context, + respond, + idempotencyKey: "idem-command-pair-qr", + message: "/pair qr", + }); + + const content = getMessageContent(payload); + expect(content[0]).toEqual({ + type: "text", + text: "Scan this QR code with the OpenClaw iOS app:", + }); + expect(content[1]).toEqual( + expect.objectContaining({ + type: "openclaw_pairing_qr", + image_url: expect.stringMatching(/^data:image\/png;base64,/u), + terminalText: expect.stringContaining("█"), + sensitive: true, + }), + ); + const transcriptMessages = await readActiveAssistantTranscriptMessages(); + const serializedTranscript = JSON.stringify(transcriptMessages); + expect(serializedTranscript).toContain("Scan this QR code with the OpenClaw iOS app:"); + expect(serializedTranscript).not.toContain("openclaw_pairing_qr"); + expect(serializedTranscript).not.toContain("data:image/png"); + expect(serializedTranscript).not.toContain("terminalText"); + expect(serializedTranscript).not.toContain(setupCode); + }); + it("keeps visible slash-command finals alongside earlier block text", async () => { createTranscriptFixture("openclaw-chat-send-command-block-text-final-"); mockState.dispatchedReplies = [ diff --git a/src/gateway/server-methods/chat.ts b/src/gateway/server-methods/chat.ts index 62567ec6e6e3..c14e799da7f9 100644 --- a/src/gateway/server-methods/chat.ts +++ b/src/gateway/server-methods/chat.ts @@ -51,6 +51,7 @@ import { dispatchInboundMessage } from "../../auto-reply/dispatch.js"; import { getReplyPayloadMetadata, isReplyPayloadStatusNotice, + readPairingQrReplyChannelData, type ReplyPayload, } from "../../auto-reply/reply-payload.js"; import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js"; @@ -84,6 +85,8 @@ import { } from "../../media/local-roots.js"; import { parseInboundMediaUri } from "../../media/media-reference.js"; import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; +import { renderQrPngDataUrl } from "../../media/qr-image.js"; +import { renderQrTerminal } from "../../media/qr-terminal.js"; import { deleteMediaBuffer, MEDIA_MAX_BYTES, @@ -857,12 +860,35 @@ function buildTranscriptReplyText(payloads: ReplyPayload[]): string { function hasSensitiveMediaPayload(payloads: ReplyPayload[]): boolean { return payloads.some( - (payload) => payload.sensitiveMedia === true && isMediaBearingPayload(payload), + (payload) => + payload.sensitiveMedia === true && + (isMediaBearingPayload(payload) || Boolean(readPairingQrReplyChannelData(payload))), ); } type AssistantDisplayContentBlock = Record; +async function buildPairingQrAssistantContentBlock( + payload: ReplyPayload, +): Promise { + const qr = readPairingQrReplyChannelData(payload); + if (!qr) { + return undefined; + } + const [imageUrl, terminalText] = await Promise.all([ + renderQrPngDataUrl(qr.setupCode), + renderQrTerminal(qr.setupCode, { small: true }), + ]); + return { + type: "openclaw_pairing_qr", + image_url: imageUrl, + terminalText, + alt: "OpenClaw pairing QR code", + expiresAtMs: qr.expiresAtMs, + sensitive: true, + }; +} + function sanitizeAssistantDisplayText(value?: string | null): string | undefined { if (!value) { return undefined; @@ -896,8 +922,10 @@ async function buildAssistantDisplayContentFromReplyPayloads(params: { payloads: ReplyPayload[]; managedImageLocalRoots?: Parameters[0]["localRoots"]; includeSensitiveMedia?: boolean; + includeSensitiveDisplay?: boolean; onLocalAudioAccessDenied?: (message: string) => void; onManagedImagePrepareError?: (message: string) => void; + onSensitiveDisplayPrepareError?: (message: string) => void; }): Promise { const rawTextPayloadCount = params.payloads.filter( (payload) => @@ -919,6 +947,16 @@ async function buildAssistantDisplayContentFromReplyPayloads(params: { } else if (typeof payload.text === "string" && payload.text.trim().length > 0) { strippedTextPayloadCount += 1; } + if (params.includeSensitiveDisplay === true) { + try { + const pairingQrBlock = await buildPairingQrAssistantContentBlock(payload); + if (pairingQrBlock) { + content.push(pairingQrBlock); + } + } catch (err) { + params.onSensitiveDisplayPrepareError?.(formatForLog(err)); + } + } if (params.includeSensitiveMedia === false && payload.sensitiveMedia === true) { continue; } @@ -4642,6 +4680,7 @@ export const chatHandlers: GatewayRequestHandlers = { payloads: finalPayloads, managedImageLocalRoots: mediaLocalRoots, includeSensitiveMedia: false, + includeSensitiveDisplay: true, onLocalAudioAccessDenied: (message) => { context.logGateway.warn( `webchat audio embedding denied local path: ${message}`, @@ -4652,6 +4691,11 @@ export const chatHandlers: GatewayRequestHandlers = { `webchat image embedding skipped attachment: ${message}`, ); }, + onSensitiveDisplayPrepareError: (message) => { + context.logGateway.warn( + `webchat sensitive display skipped attachment: ${message}`, + ); + }, }); const mediaMessage = await buildWebchatAssistantMediaMessage(finalPayloads, { localRoots: mediaLocalRoots, diff --git a/src/tui/tui-formatters.ts b/src/tui/tui-formatters.ts index 17c9dbd0fb22..452ec63f13b3 100644 --- a/src/tui/tui-formatters.ts +++ b/src/tui/tui-formatters.ts @@ -364,12 +364,38 @@ export function extractContentFromMessage(message: unknown): string { function extractAssistantRenderableContent(record: Record): string { const visible = sanitizeRenderableText(extractAssistantVisibleText(record) ?? "").trim(); - if (visible) { - return visible; + const pairingQr = extractPairingQrTerminalText(record); + const content = [visible, pairingQr].filter(Boolean).join("\n\n").trim(); + if (content) { + return content; } return formatAssistantErrorFromRecord(record); } +function extractPairingQrTerminalText(record: Record): string { + const content = record.content; + if (!Array.isArray(content)) { + return ""; + } + const parts: string[] = []; + for (const block of content) { + if (!block || typeof block !== "object") { + continue; + } + const blockRecord = block as Record; + if ( + blockRecord.type === "openclaw_pairing_qr" && + typeof blockRecord.terminalText === "string" + ) { + const text = sanitizeRenderableText(blockRecord.terminalText).trim(); + if (text) { + parts.push(text); + } + } + } + return parts.join("\n\n").trim(); +} + function extractTextBlocks(content: unknown, opts?: { includeThinking?: boolean }): string { if (typeof content === "string") { return sanitizeRenderableText(content).trim(); diff --git a/src/tui/tui-stream-assembler.test.ts b/src/tui/tui-stream-assembler.test.ts index ab23c01f3328..2767016220c8 100644 --- a/src/tui/tui-stream-assembler.test.ts +++ b/src/tui/tui-stream-assembler.test.ts @@ -5,6 +5,8 @@ import { TuiStreamAssembler } from "./tui-stream-assembler.js"; const text = (value: string) => ({ type: "text", text: value }) as const; const thinking = (value: string) => ({ type: "thinking", thinking: value }) as const; const toolUse = () => ({ type: "tool_use", name: "search" }) as const; +const pairingQr = (terminalText: string) => + ({ type: "openclaw_pairing_qr", terminalText }) as const; const messageWithContent = (content: readonly Record[]) => ({ @@ -87,6 +89,22 @@ describe("TuiStreamAssembler", () => { expect(finalText).toBe("Streamed"); }); + it("renders pairing QR terminal text from final assistant content", () => { + const assembler = new TuiStreamAssembler(); + const finalText = assembler.finalize( + "run-pair-qr", + messageWithContent([ + text("Scan this QR code with the OpenClaw iOS app:"), + pairingQr("\u001b[47m\u001b[30m█ ▄\u001b[0m"), + ]), + false, + ); + expect(finalText).toContain("Scan this QR code with the OpenClaw iOS app:"); + expect(finalText).toContain("█ ▄"); + expect(finalText).not.toContain("\u001b[47m"); + expect(finalText).not.toBe("(no output)"); + }); + it("falls back to event error message when final payload has no renderable text", () => { const assembler = new TuiStreamAssembler(); const finalText = assembler.finalize( diff --git a/ui/src/ui/chat/grouped-render.test.ts b/ui/src/ui/chat/grouped-render.test.ts index 5a20c8e705da..dc3efc002663 100644 --- a/ui/src/ui/chat/grouped-render.test.ts +++ b/ui/src/ui/chat/grouped-render.test.ts @@ -1674,6 +1674,27 @@ describe("grouped chat rendering", () => { ?.getAttribute("src"), ).toBe("data:image/png;base64,cG5n"); + const pairingQrContainer = document.createElement("div"); + renderAssistantMessage( + pairingQrContainer, + { + role: "assistant", + content: [ + { + type: "openclaw_pairing_qr", + image_url: "data:image/png;base64,cXJwbmc=", + alt: "OpenClaw pairing QR code", + }, + ], + timestamp: Date.now(), + }, + { showToolCalls: false }, + ); + const pairingQrImage = + pairingQrContainer.querySelector(".chat-message-image"); + expect(pairingQrImage?.getAttribute("src")).toBe("data:image/png;base64,cXJwbmc="); + expect(pairingQrImage?.getAttribute("alt")).toBe("OpenClaw pairing QR code"); + container = renderUserMedia({ id: "user-history-image-blocked", role: "user", diff --git a/ui/src/ui/chat/grouped-render.ts b/ui/src/ui/chat/grouped-render.ts index 5eb0f8a4a2a2..d9569134b030 100644 --- a/ui/src/ui/chat/grouped-render.ts +++ b/ui/src/ui/chat/grouped-render.ts @@ -318,6 +318,14 @@ function extractImages(message: unknown): ImageBlock[] { }), }); } + } else if (b.type === "openclaw_pairing_qr") { + const imageUrl = b.image_url; + if (typeof imageUrl === "string") { + appendImageBlock(images, { + url: imageUrl, + alt: typeof b.alt === "string" ? b.alt : undefined, + }); + } } } }