mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 01:53:54 +00:00
fix: /pair qr fails to show QR code in chat surfaces (#97933)
* fix(pair): render qr per chat surface * test(plugin-sdk): update reply helper surface budget * fix(pair): keep qr metadata internal
This commit is contained in:
@@ -290,7 +290,12 @@ describe("device-pair /pair qr", () => {
|
||||
gatewayClientScopes: INTERNAL_SETUP_SCOPES,
|
||||
}),
|
||||
);
|
||||
const payload = result as { text?: string; mediaUrl?: string; sensitiveMedia?: boolean };
|
||||
const payload = result as {
|
||||
text?: string;
|
||||
mediaUrl?: string;
|
||||
channelData?: Record<string, unknown>;
|
||||
sensitiveMedia?: boolean;
|
||||
};
|
||||
const text = requireText(result);
|
||||
|
||||
expect(pluginApiMocks.renderQrPngDataUrl).toHaveBeenCalledTimes(1);
|
||||
@@ -301,7 +306,11 @@ describe("device-pair /pair qr", () => {
|
||||
},
|
||||
});
|
||||
expect(text).toContain("Scan this QR code with the OpenClaw iOS app:");
|
||||
expect(payload.mediaUrl).toBe("data:image/png;base64,ZmFrZXBuZw==");
|
||||
expect(payload.mediaUrl).toBeUndefined();
|
||||
expect(payload.channelData?.openclawPairingQr).toEqual({
|
||||
setupCode: expect.any(String),
|
||||
expiresAtMs: expect.any(Number),
|
||||
});
|
||||
expect(payload.sensitiveMedia).toBe(true);
|
||||
expect(text).toContain("- Security: single-use bootstrap token");
|
||||
expect(text).toContain("**Important:** Run `/pair cleanup` after pairing finishes.");
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { buildDevicePairPairingQrChannelData } from "./pairing-qr-channel-data.js";
|
||||
|
||||
type DevicePairApiModule = typeof import("./api.js");
|
||||
type NotifyModule = typeof import("./notify.js");
|
||||
@@ -834,10 +835,9 @@ export default definePluginEntry({
|
||||
|
||||
api.logger.info?.(`device-pair: QR fallback channel=${channel} target=${target}`);
|
||||
if (channel === "webchat") {
|
||||
let qrDataUrl: string;
|
||||
try {
|
||||
const { renderQrPngDataUrl } = await loadDevicePairApiModule();
|
||||
qrDataUrl = await renderQrPngDataUrl(setupCode);
|
||||
await renderQrPngDataUrl(setupCode);
|
||||
} catch (err) {
|
||||
const { revokeDeviceBootstrapToken } = await loadDevicePairApiModule();
|
||||
api.logger.warn?.(
|
||||
@@ -862,7 +862,10 @@ export default definePluginEntry({
|
||||
expiresAtMs: payload.expiresAtMs,
|
||||
}),
|
||||
].join("\n"),
|
||||
mediaUrl: qrDataUrl,
|
||||
channelData: buildDevicePairPairingQrChannelData({
|
||||
setupCode,
|
||||
expiresAtMs: payload.expiresAtMs,
|
||||
}),
|
||||
sensitiveMedia: true,
|
||||
};
|
||||
}
|
||||
|
||||
19
extensions/device-pair/pairing-qr-channel-data.ts
Normal file
19
extensions/device-pair/pairing-qr-channel-data.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
// Private device-pair -> Gateway live-display envelope.
|
||||
// Keep this local so pairing QR metadata does not become public Plugin SDK API.
|
||||
export const DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY = "openclawPairingQr";
|
||||
|
||||
export type DevicePairPairingQrChannelData = {
|
||||
setupCode: string;
|
||||
expiresAtMs: number;
|
||||
};
|
||||
|
||||
export function buildDevicePairPairingQrChannelData(
|
||||
params: DevicePairPairingQrChannelData,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
[DEVICE_PAIR_PAIRING_QR_CHANNEL_DATA_KEY]: {
|
||||
setupCode: params.setupCode,
|
||||
expiresAtMs: params.expiresAtMs,
|
||||
},
|
||||
};
|
||||
}
|
||||
30
src/auto-reply/reply-payload.test.ts
Normal file
30
src/auto-reply/reply-payload.test.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
// Reply payload tests cover internal reply metadata contracts.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildPairingQrReplyChannelData, readPairingQrReplyChannelData } from "./reply-payload.js";
|
||||
|
||||
describe("pairing QR reply channel data", () => {
|
||||
it("builds and reads the private pairing QR payload metadata", () => {
|
||||
const channelData = buildPairingQrReplyChannelData({
|
||||
setupCode: "setup-code",
|
||||
expiresAtMs: 1_800_000_000_000,
|
||||
});
|
||||
|
||||
expect(readPairingQrReplyChannelData({ channelData })).toEqual({
|
||||
setupCode: "setup-code",
|
||||
expiresAtMs: 1_800_000_000_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores malformed pairing QR metadata", () => {
|
||||
expect(
|
||||
readPairingQrReplyChannelData({
|
||||
channelData: {
|
||||
openclawPairingQr: {
|
||||
setupCode: "",
|
||||
expiresAtMs: 0,
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -62,6 +62,47 @@ export type ReplyPayload = {
|
||||
channelData?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
// Private device-pair -> Gateway live-display envelope key. Do not re-export
|
||||
// through Plugin SDK; this is not a third-party plugin contract.
|
||||
export const PAIRING_QR_REPLY_CHANNEL_DATA_KEY = "openclawPairingQr";
|
||||
|
||||
export type PairingQrReplyChannelData = {
|
||||
setupCode: string;
|
||||
expiresAtMs: number;
|
||||
};
|
||||
|
||||
function normalizePairingQrSetupCode(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.trim() ? value : undefined;
|
||||
}
|
||||
|
||||
function normalizePairingQrExpiresAtMs(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
export function buildPairingQrReplyChannelData(
|
||||
params: PairingQrReplyChannelData,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
[PAIRING_QR_REPLY_CHANNEL_DATA_KEY]: {
|
||||
setupCode: params.setupCode,
|
||||
expiresAtMs: params.expiresAtMs,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function readPairingQrReplyChannelData(
|
||||
payload: Pick<ReplyPayload, "channelData">,
|
||||
): PairingQrReplyChannelData | undefined {
|
||||
const raw = payload.channelData?.[PAIRING_QR_REPLY_CHANNEL_DATA_KEY];
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = raw as Record<string, unknown>;
|
||||
const setupCode = normalizePairingQrSetupCode(record.setupCode);
|
||||
const expiresAtMs = normalizePairingQrExpiresAtMs(record.expiresAtMs);
|
||||
return setupCode && expiresAtMs ? { setupCode, expiresAtMs } : undefined;
|
||||
}
|
||||
|
||||
/** Metadata for fast-auto progress notices. */
|
||||
export const FAST_MODE_AUTO_PROGRESS_KIND = "fast-mode-auto";
|
||||
|
||||
|
||||
@@ -14,7 +14,10 @@ import {
|
||||
import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import { CHAT_SEND_SESSION_KEY_MAX_LENGTH } from "../../../packages/gateway-protocol/src/schema.js";
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import { setReplyPayloadMetadata } from "../../auto-reply/reply-payload.js";
|
||||
import {
|
||||
buildPairingQrReplyChannelData,
|
||||
setReplyPayloadMetadata,
|
||||
} from "../../auto-reply/reply-payload.js";
|
||||
import type { MsgContext } from "../../auto-reply/templating.js";
|
||||
import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js";
|
||||
import { resolveMirroredTranscriptText } from "../../config/sessions/transcript-mirror.js";
|
||||
@@ -33,6 +36,7 @@ const mockState = vi.hoisted(() => ({
|
||||
text?: string;
|
||||
mediaUrl?: string;
|
||||
mediaUrls?: string[];
|
||||
channelData?: Record<string, unknown>;
|
||||
spokenText?: string;
|
||||
audioAsVoice?: boolean;
|
||||
trustedLocalMedia?: boolean;
|
||||
@@ -48,6 +52,7 @@ const mockState = vi.hoisted(() => ({
|
||||
text?: string;
|
||||
mediaUrl?: string;
|
||||
mediaUrls?: string[];
|
||||
channelData?: Record<string, unknown>;
|
||||
spokenText?: string;
|
||||
ttsSupplement?: { spokenText: string };
|
||||
audioAsVoice?: boolean;
|
||||
@@ -2454,6 +2459,54 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
||||
expect(transcriptUpdate).toBeTruthy();
|
||||
});
|
||||
|
||||
it("broadcasts sensitive pairing QR display without persisting QR content", async () => {
|
||||
createTranscriptFixture("openclaw-chat-send-command-pair-qr-");
|
||||
const setupCode = "openclaw-test-pairing-setup-code";
|
||||
mockState.dispatchedReplies = [
|
||||
{
|
||||
kind: "final",
|
||||
payload: {
|
||||
text: "Scan this QR code with the OpenClaw iOS app:",
|
||||
channelData: buildPairingQrReplyChannelData({
|
||||
setupCode,
|
||||
expiresAtMs: Date.now() + 10 * 60_000,
|
||||
}),
|
||||
sensitiveMedia: true,
|
||||
},
|
||||
},
|
||||
];
|
||||
const respond = vi.fn();
|
||||
const context = createChatContext();
|
||||
|
||||
const payload = await runNonStreamingChatSend({
|
||||
context,
|
||||
respond,
|
||||
idempotencyKey: "idem-command-pair-qr",
|
||||
message: "/pair qr",
|
||||
});
|
||||
|
||||
const content = getMessageContent(payload);
|
||||
expect(content[0]).toEqual({
|
||||
type: "text",
|
||||
text: "Scan this QR code with the OpenClaw iOS app:",
|
||||
});
|
||||
expect(content[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
type: "openclaw_pairing_qr",
|
||||
image_url: expect.stringMatching(/^data:image\/png;base64,/u),
|
||||
terminalText: expect.stringContaining("█"),
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
const transcriptMessages = await readActiveAssistantTranscriptMessages();
|
||||
const serializedTranscript = JSON.stringify(transcriptMessages);
|
||||
expect(serializedTranscript).toContain("Scan this QR code with the OpenClaw iOS app:");
|
||||
expect(serializedTranscript).not.toContain("openclaw_pairing_qr");
|
||||
expect(serializedTranscript).not.toContain("data:image/png");
|
||||
expect(serializedTranscript).not.toContain("terminalText");
|
||||
expect(serializedTranscript).not.toContain(setupCode);
|
||||
});
|
||||
|
||||
it("keeps visible slash-command finals alongside earlier block text", async () => {
|
||||
createTranscriptFixture("openclaw-chat-send-command-block-text-final-");
|
||||
mockState.dispatchedReplies = [
|
||||
|
||||
@@ -51,6 +51,7 @@ import { dispatchInboundMessage } from "../../auto-reply/dispatch.js";
|
||||
import {
|
||||
getReplyPayloadMetadata,
|
||||
isReplyPayloadStatusNotice,
|
||||
readPairingQrReplyChannelData,
|
||||
type ReplyPayload,
|
||||
} from "../../auto-reply/reply-payload.js";
|
||||
import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.js";
|
||||
@@ -84,6 +85,8 @@ import {
|
||||
} from "../../media/local-roots.js";
|
||||
import { parseInboundMediaUri } from "../../media/media-reference.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import { renderQrPngDataUrl } from "../../media/qr-image.js";
|
||||
import { renderQrTerminal } from "../../media/qr-terminal.js";
|
||||
import {
|
||||
deleteMediaBuffer,
|
||||
MEDIA_MAX_BYTES,
|
||||
@@ -857,12 +860,35 @@ function buildTranscriptReplyText(payloads: ReplyPayload[]): string {
|
||||
|
||||
function hasSensitiveMediaPayload(payloads: ReplyPayload[]): boolean {
|
||||
return payloads.some(
|
||||
(payload) => payload.sensitiveMedia === true && isMediaBearingPayload(payload),
|
||||
(payload) =>
|
||||
payload.sensitiveMedia === true &&
|
||||
(isMediaBearingPayload(payload) || Boolean(readPairingQrReplyChannelData(payload))),
|
||||
);
|
||||
}
|
||||
|
||||
type AssistantDisplayContentBlock = Record<string, unknown>;
|
||||
|
||||
async function buildPairingQrAssistantContentBlock(
|
||||
payload: ReplyPayload,
|
||||
): Promise<AssistantDisplayContentBlock | undefined> {
|
||||
const qr = readPairingQrReplyChannelData(payload);
|
||||
if (!qr) {
|
||||
return undefined;
|
||||
}
|
||||
const [imageUrl, terminalText] = await Promise.all([
|
||||
renderQrPngDataUrl(qr.setupCode),
|
||||
renderQrTerminal(qr.setupCode, { small: true }),
|
||||
]);
|
||||
return {
|
||||
type: "openclaw_pairing_qr",
|
||||
image_url: imageUrl,
|
||||
terminalText,
|
||||
alt: "OpenClaw pairing QR code",
|
||||
expiresAtMs: qr.expiresAtMs,
|
||||
sensitive: true,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeAssistantDisplayText(value?: string | null): string | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
@@ -896,8 +922,10 @@ async function buildAssistantDisplayContentFromReplyPayloads(params: {
|
||||
payloads: ReplyPayload[];
|
||||
managedImageLocalRoots?: Parameters<typeof createManagedOutgoingImageBlocks>[0]["localRoots"];
|
||||
includeSensitiveMedia?: boolean;
|
||||
includeSensitiveDisplay?: boolean;
|
||||
onLocalAudioAccessDenied?: (message: string) => void;
|
||||
onManagedImagePrepareError?: (message: string) => void;
|
||||
onSensitiveDisplayPrepareError?: (message: string) => void;
|
||||
}): Promise<AssistantDisplayContentBlock[] | undefined> {
|
||||
const rawTextPayloadCount = params.payloads.filter(
|
||||
(payload) =>
|
||||
@@ -919,6 +947,16 @@ async function buildAssistantDisplayContentFromReplyPayloads(params: {
|
||||
} else if (typeof payload.text === "string" && payload.text.trim().length > 0) {
|
||||
strippedTextPayloadCount += 1;
|
||||
}
|
||||
if (params.includeSensitiveDisplay === true) {
|
||||
try {
|
||||
const pairingQrBlock = await buildPairingQrAssistantContentBlock(payload);
|
||||
if (pairingQrBlock) {
|
||||
content.push(pairingQrBlock);
|
||||
}
|
||||
} catch (err) {
|
||||
params.onSensitiveDisplayPrepareError?.(formatForLog(err));
|
||||
}
|
||||
}
|
||||
if (params.includeSensitiveMedia === false && payload.sensitiveMedia === true) {
|
||||
continue;
|
||||
}
|
||||
@@ -4642,6 +4680,7 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
payloads: finalPayloads,
|
||||
managedImageLocalRoots: mediaLocalRoots,
|
||||
includeSensitiveMedia: false,
|
||||
includeSensitiveDisplay: true,
|
||||
onLocalAudioAccessDenied: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat audio embedding denied local path: ${message}`,
|
||||
@@ -4652,6 +4691,11 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
`webchat image embedding skipped attachment: ${message}`,
|
||||
);
|
||||
},
|
||||
onSensitiveDisplayPrepareError: (message) => {
|
||||
context.logGateway.warn(
|
||||
`webchat sensitive display skipped attachment: ${message}`,
|
||||
);
|
||||
},
|
||||
});
|
||||
const mediaMessage = await buildWebchatAssistantMediaMessage(finalPayloads, {
|
||||
localRoots: mediaLocalRoots,
|
||||
|
||||
@@ -364,12 +364,38 @@ export function extractContentFromMessage(message: unknown): string {
|
||||
|
||||
function extractAssistantRenderableContent(record: Record<string, unknown>): string {
|
||||
const visible = sanitizeRenderableText(extractAssistantVisibleText(record) ?? "").trim();
|
||||
if (visible) {
|
||||
return visible;
|
||||
const pairingQr = extractPairingQrTerminalText(record);
|
||||
const content = [visible, pairingQr].filter(Boolean).join("\n\n").trim();
|
||||
if (content) {
|
||||
return content;
|
||||
}
|
||||
return formatAssistantErrorFromRecord(record);
|
||||
}
|
||||
|
||||
function extractPairingQrTerminalText(record: Record<string, unknown>): string {
|
||||
const content = record.content;
|
||||
if (!Array.isArray(content)) {
|
||||
return "";
|
||||
}
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const blockRecord = block as Record<string, unknown>;
|
||||
if (
|
||||
blockRecord.type === "openclaw_pairing_qr" &&
|
||||
typeof blockRecord.terminalText === "string"
|
||||
) {
|
||||
const text = sanitizeRenderableText(blockRecord.terminalText).trim();
|
||||
if (text) {
|
||||
parts.push(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
return parts.join("\n\n").trim();
|
||||
}
|
||||
|
||||
function extractTextBlocks(content: unknown, opts?: { includeThinking?: boolean }): string {
|
||||
if (typeof content === "string") {
|
||||
return sanitizeRenderableText(content).trim();
|
||||
|
||||
@@ -5,6 +5,8 @@ import { TuiStreamAssembler } from "./tui-stream-assembler.js";
|
||||
const text = (value: string) => ({ type: "text", text: value }) as const;
|
||||
const thinking = (value: string) => ({ type: "thinking", thinking: value }) as const;
|
||||
const toolUse = () => ({ type: "tool_use", name: "search" }) as const;
|
||||
const pairingQr = (terminalText: string) =>
|
||||
({ type: "openclaw_pairing_qr", terminalText }) as const;
|
||||
|
||||
const messageWithContent = (content: readonly Record<string, unknown>[]) =>
|
||||
({
|
||||
@@ -87,6 +89,22 @@ describe("TuiStreamAssembler", () => {
|
||||
expect(finalText).toBe("Streamed");
|
||||
});
|
||||
|
||||
it("renders pairing QR terminal text from final assistant content", () => {
|
||||
const assembler = new TuiStreamAssembler();
|
||||
const finalText = assembler.finalize(
|
||||
"run-pair-qr",
|
||||
messageWithContent([
|
||||
text("Scan this QR code with the OpenClaw iOS app:"),
|
||||
pairingQr("\u001b[47m\u001b[30m█ ▄\u001b[0m"),
|
||||
]),
|
||||
false,
|
||||
);
|
||||
expect(finalText).toContain("Scan this QR code with the OpenClaw iOS app:");
|
||||
expect(finalText).toContain("█ ▄");
|
||||
expect(finalText).not.toContain("\u001b[47m");
|
||||
expect(finalText).not.toBe("(no output)");
|
||||
});
|
||||
|
||||
it("falls back to event error message when final payload has no renderable text", () => {
|
||||
const assembler = new TuiStreamAssembler();
|
||||
const finalText = assembler.finalize(
|
||||
|
||||
@@ -1674,6 +1674,27 @@ describe("grouped chat rendering", () => {
|
||||
?.getAttribute("src"),
|
||||
).toBe("data:image/png;base64,cG5n");
|
||||
|
||||
const pairingQrContainer = document.createElement("div");
|
||||
renderAssistantMessage(
|
||||
pairingQrContainer,
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "openclaw_pairing_qr",
|
||||
image_url: "data:image/png;base64,cXJwbmc=",
|
||||
alt: "OpenClaw pairing QR code",
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{ showToolCalls: false },
|
||||
);
|
||||
const pairingQrImage =
|
||||
pairingQrContainer.querySelector<HTMLImageElement>(".chat-message-image");
|
||||
expect(pairingQrImage?.getAttribute("src")).toBe("data:image/png;base64,cXJwbmc=");
|
||||
expect(pairingQrImage?.getAttribute("alt")).toBe("OpenClaw pairing QR code");
|
||||
|
||||
container = renderUserMedia({
|
||||
id: "user-history-image-blocked",
|
||||
role: "user",
|
||||
|
||||
@@ -318,6 +318,14 @@ function extractImages(message: unknown): ImageBlock[] {
|
||||
}),
|
||||
});
|
||||
}
|
||||
} else if (b.type === "openclaw_pairing_qr") {
|
||||
const imageUrl = b.image_url;
|
||||
if (typeof imageUrl === "string") {
|
||||
appendImageBlock(images, {
|
||||
url: imageUrl,
|
||||
alt: typeof b.alt === "string" ? b.alt : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user