From cf029ea29e9efcb4d28cc96905ddacada2a0e6da Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 00:14:29 -0700 Subject: [PATCH] feat(agents): screen tool drives Control UI layout --- .../openclaw/app/gateway/GatewayProtocol.kt | 2 + .../OpenClawKit/Resources/tool-display.json | 9 + .../OpenClawProtocol/GatewayModels.swift | 195 ++++++++++++++++++ packages/gateway-protocol/src/client-info.ts | 1 + packages/gateway-protocol/src/index.ts | 3 + packages/gateway-protocol/src/schema.ts | 1 + .../src/schema/protocol-schemas.ts | 20 ++ .../gateway-protocol/src/schema/ui-command.ts | 51 +++++ .../protocol-event-coverage.allowlist.json | 2 + src/agents/openclaw-tools.client-caps.test.ts | 9 + src/agents/openclaw-tools.ts | 4 + src/agents/system-prompt.test.ts | 17 ++ src/agents/system-prompt.ts | 5 + src/agents/tool-catalog.test.ts | 1 + src/agents/tool-catalog.ts | 8 + src/agents/tool-display-config.ts | 5 + src/agents/tools/screen-tool.test.ts | 74 +++++++ src/agents/tools/screen-tool.ts | 115 +++++++++++ src/gateway/methods/core-descriptors.ts | 1 + src/gateway/server-broadcast.ts | 1 + src/gateway/server-methods-list.test.ts | 5 +- src/gateway/server-methods-list.ts | 1 + src/gateway/server-methods.ts | 8 + src/gateway/server-methods/ui-command.test.ts | 83 ++++++++ src/gateway/server-methods/ui-command.ts | 47 +++++ src/gateway/server-request-context.ts | 19 +- src/gateway/tool-resolution.exclude.test.ts | 2 + src/security/dangerous-tools.ts | 1 + ui/src/api/gateway.node.test.ts | 1 + ui/src/api/gateway.ts | 1 + ui/src/app/app-host.test.ts | 64 ++++++ ui/src/app/app-host.ts | 49 ++++- .../components/browser/browser-panel.test.ts | 25 ++- ui/src/components/browser/browser-panel.ts | 4 + ui/src/components/panel-toggle-contract.ts | 5 + .../terminal/terminal-panel-readiness.test.ts | 12 ++ ui/src/components/terminal/terminal-panel.ts | 4 + ui/src/pages/chat/chat-page.test.ts | 61 ++++++ ui/src/pages/chat/chat-page.ts | 59 ++++++ ui/src/pages/chat/split-layout.test.ts | 32 +++ ui/src/pages/chat/split-layout.ts | 31 +++ 41 files changed, 1032 insertions(+), 6 deletions(-) create mode 100644 packages/gateway-protocol/src/schema/ui-command.ts create mode 100644 src/agents/tools/screen-tool.test.ts create mode 100644 src/agents/tools/screen-tool.ts create mode 100644 src/gateway/server-methods/ui-command.test.ts create mode 100644 src/gateway/server-methods/ui-command.ts diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt index e69d3fffb5f0..a04508496780 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt @@ -377,6 +377,7 @@ enum class GatewayMethod( ModelsProbe("models.probe"), MigrationsMemoryPlan("migrations.memory.plan"), MigrationsMemoryApply("migrations.memory.apply"), + UiCommand("ui.command"), } enum class GatewayEvent( @@ -385,6 +386,7 @@ enum class GatewayEvent( ConnectChallenge("connect.challenge"), Agent("agent"), Chat("chat"), + UiCommand("ui.command"), SessionApproval("session.approval"), SessionMessage("session.message"), SessionOperation("session.operation"), diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json index e050c6ef924f..94d969fb2ceb 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json @@ -44,6 +44,15 @@ "screenIndex" ] }, + "screen": { + "emoji": "🖥️", + "title": "Screen", + "detailKeys": [ + "action", + "sessionKey", + "dock" + ] + }, "process": { "emoji": "🧰", "title": "Process", diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index dcd542e6d3cb..019fe5b6c833 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -2564,6 +2564,158 @@ public struct PushTestResult: Codable, Sendable { } } +public struct UiSplitCommand: Codable, Sendable { + public let kind: String + public let direction: AnyCodable + public let sessionkey: String + + public init( + kind: String, + direction: AnyCodable, + sessionkey: String) + { + self.kind = kind + self.direction = direction + self.sessionkey = sessionkey + } + + private enum CodingKeys: String, CodingKey { + case kind + case direction + case sessionkey = "sessionKey" + } +} + +public struct UiClosePaneCommand: Codable, Sendable { + public let kind: String + public let sessionkey: String + + public init( + kind: String, + sessionkey: String) + { + self.kind = kind + self.sessionkey = sessionkey + } + + private enum CodingKeys: String, CodingKey { + case kind + case sessionkey = "sessionKey" + } +} + +public struct UiFocusCommand: Codable, Sendable { + public let kind: String + public let sessionkey: String + + public init( + kind: String, + sessionkey: String) + { + self.kind = kind + self.sessionkey = sessionkey + } + + private enum CodingKeys: String, CodingKey { + case kind + case sessionkey = "sessionKey" + } +} + +public struct UiSidebarCommand: Codable, Sendable { + public let kind: String + public let visible: Bool + + public init( + kind: String, + visible: Bool) + { + self.kind = kind + self.visible = visible + } + + private enum CodingKeys: String, CodingKey { + case kind + case visible + } +} + +public struct UiPanelCommand: Codable, Sendable { + public let kind: String + public let panel: AnyCodable + public let _open: Bool + public let dock: AnyCodable? + + public init( + kind: String, + panel: AnyCodable, + _open: Bool, + dock: AnyCodable? = nil) + { + self.kind = kind + self.panel = panel + self._open = _open + self.dock = dock + } + + private enum CodingKeys: String, CodingKey { + case kind + case panel + case _open = "open" + case dock + } +} + +public struct UiNavigateCommand: Codable, Sendable { + public let kind: String + public let sessionkey: String + + public init( + kind: String, + sessionkey: String) + { + self.kind = kind + self.sessionkey = sessionkey + } + + private enum CodingKeys: String, CodingKey { + case kind + case sessionkey = "sessionKey" + } +} + +public struct UiCommandParams: Codable, Sendable { + public let command: UiCommand + public let sessionkey: String? + + public init( + command: UiCommand, + sessionkey: String? = nil) + { + self.command = command + self.sessionkey = sessionkey + } + + private enum CodingKeys: String, CodingKey { + case command + case sessionkey = "sessionKey" + } +} + +public struct UiCommandResult: Codable, Sendable { + public let ok: Bool + + public init( + ok: Bool) + { + self.ok = ok + } + + private enum CodingKeys: String, CodingKey { + case ok + } +} + public struct SecretsReloadParams: Codable, Sendable {} public struct SecretsResolveParams: Codable, Sendable { @@ -13653,6 +13805,49 @@ public enum GatewaySuspendStatusResult: Codable, Sendable { } } +public enum UiCommand: Codable, Sendable { + case split(UiSplitCommand) + case closePane(UiClosePaneCommand) + case focus(UiFocusCommand) + case sidebar(UiSidebarCommand) + case panel(UiPanelCommand) + case navigate(UiNavigateCommand) + + private enum CodingKeys: String, CodingKey { + case discriminator = "kind" + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let discriminator = try container.decode(String.self, forKey: .discriminator) + switch discriminator { + case "split": self = try .split(UiSplitCommand(from: decoder)) + case "close-pane": self = try .closePane(UiClosePaneCommand(from: decoder)) + case "focus": self = try .focus(UiFocusCommand(from: decoder)) + case "sidebar": self = try .sidebar(UiSidebarCommand(from: decoder)) + case "panel": self = try .panel(UiPanelCommand(from: decoder)) + case "navigate": self = try .navigate(UiNavigateCommand(from: decoder)) + default: + throw DecodingError.dataCorruptedError( + forKey: .discriminator, + in: container, + debugDescription: "Unknown UiCommand discriminator value" + ) + } + } + + public func encode(to encoder: Encoder) throws { + switch self { + case .split(let value): try value.encode(to: encoder) + case .closePane(let value): try value.encode(to: encoder) + case .focus(let value): try value.encode(to: encoder) + case .sidebar(let value): try value.encode(to: encoder) + case .panel(let value): try value.encode(to: encoder) + case .navigate(let value): try value.encode(to: encoder) + } + } +} + public enum SessionPlacement: Codable, Sendable { case local(LocalSessionPlacement) case requested(RequestedSessionPlacement) diff --git a/packages/gateway-protocol/src/client-info.ts b/packages/gateway-protocol/src/client-info.ts index d39b6b46c871..29dcf2de7f47 100644 --- a/packages/gateway-protocol/src/client-info.ts +++ b/packages/gateway-protocol/src/client-info.ts @@ -83,6 +83,7 @@ export const GATEWAY_CLIENT_CAPS = { TASK_SUGGESTIONS: "task-suggestions", TERMINAL_OFFSET_SEQ: "terminal-offset-seq", TOOL_EVENTS: "tool-events", + UI_COMMANDS: "ui-commands", } as const; /** Optional capability advertised by clients during gateway handshake. */ diff --git a/packages/gateway-protocol/src/index.ts b/packages/gateway-protocol/src/index.ts index 474dc8b3d75f..1a1a2237cd97 100644 --- a/packages/gateway-protocol/src/index.ts +++ b/packages/gateway-protocol/src/index.ts @@ -8,6 +8,7 @@ import { lazyCompile } from "./protocol-validator.js"; export type { ProtocolValidator } from "./protocol-validator.js"; export * from "./schema/worker-inference.js"; export * from "./schema/skill-history.js"; +export * from "./schema/ui-command.js"; export * from "./migration-api.js"; export type * from "./public-session-catalog.js"; import { @@ -269,6 +270,7 @@ import { TerminalTextResultSchema, TerminalUploadParamsSchema, TerminalUploadResultSchema, + UiCommandParamsSchema, ModelsListParamsSchema, AuthProbeStatusSchema, ModelsProbeParamsSchema, @@ -833,6 +835,7 @@ export const validateChatEvent = lazyCompile(ChatEventSchema); export const validateChatMessageGetResult = lazyCompile(ChatMessageGetResultSchema); export const validateUpdateStatusParams = lazyCompile(UpdateStatusParamsSchema); export const validateUpdateRunParams = lazyCompile(UpdateRunParamsSchema); +export const validateUiCommandParams = lazyCompile(UiCommandParamsSchema); export const validateWebLoginStartParams = lazyCompile(WebLoginStartParamsSchema); export const validateWebLoginWaitParams = lazyCompile(WebLoginWaitParamsSchema); diff --git a/packages/gateway-protocol/src/schema.ts b/packages/gateway-protocol/src/schema.ts index 5d9fccc7bff9..b05c2fcd77cb 100644 --- a/packages/gateway-protocol/src/schema.ts +++ b/packages/gateway-protocol/src/schema.ts @@ -42,6 +42,7 @@ export * from "./schema/system-event.js"; export * from "./schema/task-suggestions.js"; export * from "./schema/tasks.js"; export * from "./schema/terminal.js"; +export * from "./schema/ui-command.js"; export * from "./schema/plugin-approvals.js"; export * from "./schema/plugins.js"; export * from "./schema/wizard.js"; diff --git a/packages/gateway-protocol/src/schema/protocol-schemas.ts b/packages/gateway-protocol/src/schema/protocol-schemas.ts index c2ce245fa3d2..f4bd5113d1fe 100644 --- a/packages/gateway-protocol/src/schema/protocol-schemas.ts +++ b/packages/gateway-protocol/src/schema/protocol-schemas.ts @@ -461,6 +461,17 @@ import { TaskSummarySchema, } from "./tasks.js"; import { TerminalProtocolSchemas } from "./terminal-protocol-schemas.js"; +import { + UiClosePaneCommandSchema, + UiCommandParamsSchema, + UiCommandResultSchema, + UiCommandSchema, + UiFocusCommandSchema, + UiNavigateCommandSchema, + UiPanelCommandSchema, + UiSidebarCommandSchema, + UiSplitCommandSchema, +} from "./ui-command.js"; import { WizardCancelParamsSchema, WizardNextParamsSchema, @@ -582,6 +593,15 @@ export const ProtocolSchemas = { // Push and secret-resolution payloads used by mobile/control integrations. PushTestParams: PushTestParamsSchema, PushTestResult: PushTestResultSchema, + UiSplitCommand: UiSplitCommandSchema, + UiClosePaneCommand: UiClosePaneCommandSchema, + UiFocusCommand: UiFocusCommandSchema, + UiSidebarCommand: UiSidebarCommandSchema, + UiPanelCommand: UiPanelCommandSchema, + UiNavigateCommand: UiNavigateCommandSchema, + UiCommand: UiCommandSchema, + UiCommandParams: UiCommandParamsSchema, + UiCommandResult: UiCommandResultSchema, SecretsReloadParams: SecretsReloadParamsSchema, SecretsResolveParams: SecretsResolveParamsSchema, SecretsResolveAssignment: SecretsResolveAssignmentSchema, diff --git a/packages/gateway-protocol/src/schema/ui-command.ts b/packages/gateway-protocol/src/schema/ui-command.ts new file mode 100644 index 000000000000..343eb24297dc --- /dev/null +++ b/packages/gateway-protocol/src/schema/ui-command.ts @@ -0,0 +1,51 @@ +import type { Static } from "typebox"; +import { Type } from "typebox"; +import { closedObject } from "./closed-object.js"; +import { NonEmptyString } from "./primitives.js"; + +export const UiSplitCommandSchema = closedObject({ + kind: Type.Literal("split"), + direction: Type.Union([Type.Literal("right"), Type.Literal("down")]), + sessionKey: NonEmptyString, +}); +export const UiClosePaneCommandSchema = closedObject({ + kind: Type.Literal("close-pane"), + sessionKey: NonEmptyString, +}); +export const UiFocusCommandSchema = closedObject({ + kind: Type.Literal("focus"), + sessionKey: NonEmptyString, +}); +export const UiSidebarCommandSchema = closedObject({ + kind: Type.Literal("sidebar"), + visible: Type.Boolean(), +}); +export const UiPanelCommandSchema = closedObject({ + kind: Type.Literal("panel"), + panel: Type.Union([Type.Literal("terminal"), Type.Literal("browser")]), + open: Type.Boolean(), + dock: Type.Optional(Type.Union([Type.Literal("bottom"), Type.Literal("right")])), +}); +export const UiNavigateCommandSchema = closedObject({ + kind: Type.Literal("navigate"), + sessionKey: NonEmptyString, +}); + +export const UiCommandSchema = Type.Union([ + UiSplitCommandSchema, + UiClosePaneCommandSchema, + UiFocusCommandSchema, + UiSidebarCommandSchema, + UiPanelCommandSchema, + UiNavigateCommandSchema, +]); +export type UiCommand = Static; + +export const UiCommandParamsSchema = closedObject({ + command: UiCommandSchema, + sessionKey: Type.Optional(NonEmptyString), +}); +export type UiCommandParams = Static; + +export const UiCommandResultSchema = closedObject({ ok: Type.Boolean() }); +export type UiCommandResult = Static; diff --git a/scripts/protocol-event-coverage.allowlist.json b/scripts/protocol-event-coverage.allowlist.json index 6c660a732f4a..65d573b65f1a 100644 --- a/scripts/protocol-event-coverage.allowlist.json +++ b/scripts/protocol-event-coverage.allowlist.json @@ -4,6 +4,7 @@ "session.operation": "Chat UI derives run state from chat/agent events; no session.operation consumer yet.", "session.tool": "Session tool stream is not rendered by the iOS chat surface yet.", "task.suggestion": "Task suggestion cards are a Control UI-only surface; iOS does not render them.", + "ui.command": "Web Control UI-only layout commands; iOS does not consume them.", "presence": "Presence roster is a control-UI (web/desktop) surface; iOS does not render it.", "shutdown": "iOS relies on socket close plus reconnect/backoff instead of the shutdown notice.", "heartbeat": "iOS liveness uses tick and WebSocket-level ping; heartbeat is unused.", @@ -28,6 +29,7 @@ "session.operation": "Chat UI derives run state from chat/agent events; no session.operation consumer yet.", "session.tool": "Session tool stream is not rendered by the Android chat surface yet.", "task.suggestion": "Task suggestion cards are a Control UI-only surface; Android does not render them.", + "ui.command": "Web Control UI-only layout commands; Android does not consume them.", "presence": "Presence roster is a control-UI (web/desktop) surface; Android does not render it.", "talk.mode": "Android toggles talk mode locally; gateway talk.mode sync is not consumed.", "shutdown": "Android relies on socket close plus reconnect/backoff instead of the shutdown notice.", diff --git a/src/agents/openclaw-tools.client-caps.test.ts b/src/agents/openclaw-tools.client-caps.test.ts index 66bb831bf3f1..37d86b7a4b2d 100644 --- a/src/agents/openclaw-tools.client-caps.test.ts +++ b/src/agents/openclaw-tools.client-caps.test.ts @@ -21,6 +21,10 @@ function hasWidget(tools: readonly { name: string }[]): boolean { return tools.some((tool) => tool.name === "show_widget"); } +function hasScreen(tools: readonly { name: string }[]): boolean { + return tools.some((tool) => tool.name === "screen"); +} + describe("gateway client capability tool filtering", () => { it("excludes capability-gated tools when no gateway client caps exist", () => { expect(hasWidget(createOpenClawTools())).toBe(false); @@ -36,6 +40,11 @@ describe("gateway client capability tool filtering", () => { ); }); + it("only exposes screen to UI-command clients", () => { + expect(hasScreen(createOpenClawTools())).toBe(false); + expect(hasScreen(createOpenClawTools({ clientCaps: ["ui-commands"] }))).toBe(true); + }); + it("does not let tools.allow resurrect a gated tool for a channel run", () => { const tools = createOpenClawCodingTools({ messageProvider: "telegram", diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 60f2a1913a5a..b575f7ac03bd 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -61,6 +61,7 @@ import { createMusicGenerateTool } from "./tools/music-generate-tool.js"; import { createNodesTool } from "./tools/nodes-tool.js"; import { createOpenClawDelegateToolsForRun } from "./tools/openclaw-delegate-tool.js"; import { createPdfTool } from "./tools/pdf-tool.js"; +import { createScreenTool } from "./tools/screen-tool.js"; import { createSessionStatusTool } from "./tools/session-status-tool.js"; import { createSessionsHistoryTool } from "./tools/sessions-history-tool.js"; import { createSessionsListTool } from "./tools/sessions-list-tool.js"; @@ -482,6 +483,9 @@ export function createOpenClawTools( sandboxed: options?.sandboxed, config: resolvedConfig, }), + createScreenTool({ + agentSessionKey: options?.runSessionKey ?? options?.agentSessionKey, + }), ]), ...(!embedded && taskSuggestionSessionKey && options?.taskSuggestionDeliveryMode === "gateway" ? createTaskSuggestionTools({ diff --git a/src/agents/system-prompt.test.ts b/src/agents/system-prompt.test.ts index 21154eb3120d..815d012339eb 100644 --- a/src/agents/system-prompt.test.ts +++ b/src/agents/system-prompt.test.ts @@ -384,6 +384,23 @@ describe("buildAgentSystemPrompt", () => { expect(withYield).toContain("wait with `sessions_yield`"); }); + it("limits screen guidance to web/app tool surfaces", () => { + const withoutScreen = buildAgentSystemPrompt({ + workspaceDir: "/tmp/openclaw", + toolNames: ["sessions"], + }); + const withScreen = buildAgentSystemPrompt({ + workspaceDir: "/tmp/openclaw", + toolNames: ["sessions", "screen"], + }); + + expect(withoutScreen).not.toContain("web/app turn may drive UI"); + expect(withScreen).toContain("- screen: Drive operator web UI"); + expect(withScreen).toContain( + "`screen` present: web/app turn may drive UI; messaging turn: don't.", + ); + }); + it("lists available tools when provided", () => { const prompt = buildAgentSystemPrompt({ workspaceDir: "/tmp/openclaw", diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index 9854b3e2d4b3..980d705c6acd 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -780,6 +780,7 @@ export function buildAgentSystemPrompt(params: { web_fetch: "Fetch/extract URL", // Channel docking: add login tools here when a channel needs interactive linking. browser: "Control browser", + screen: "Drive operator web UI", canvas: "Present/eval/snapshot Canvas", nodes: "Paired node status/control/media", cron: "Schedule/wake. Reminder text must read as reminder when fired; mention reminder for delayed gaps; include useful recent context.", @@ -817,6 +818,7 @@ export function buildAgentSystemPrompt(params: { "web_search", "web_fetch", "browser", + "screen", "canvas", "nodes", "cron", @@ -1078,6 +1080,9 @@ export function buildAgentSystemPrompt(params: { "Large work: `sessions_spawn`; completion push-based.", '`sessions_spawn`: omit `context`; transcript needed => `context:"fork"`.', ...(hasSessionsSpawn ? ["`visible:true` only web/app user or asked."] : []), + ...(availableTools.has("screen") + ? ["`screen` present: web/app turn may drive UI; messaging turn: don't."] + : []), ] : []), ...nativeCommandGuidanceLines, diff --git a/src/agents/tool-catalog.test.ts b/src/agents/tool-catalog.test.ts index 3bb6fbc22a5a..86d7cf5a353f 100644 --- a/src/agents/tool-catalog.test.ts +++ b/src/agents/tool-catalog.test.ts @@ -48,6 +48,7 @@ describe("tool-catalog", () => { "session_status", "spawn_task", "dismiss_task", + "screen", "cron", "get_goal", "create_goal", diff --git a/src/agents/tool-catalog.ts b/src/agents/tool-catalog.ts index 340be5d3d17b..8a918e475859 100644 --- a/src/agents/tool-catalog.ts +++ b/src/agents/tool-catalog.ts @@ -247,6 +247,14 @@ const CORE_TOOL_DEFINITIONS: CoreToolDefinition[] = [ profiles: [], includeInOpenClawGroup: true, }, + { + id: "screen", + label: "screen", + description: "Drive operator web UI", + sectionId: "ui", + profiles: ["coding"], + includeInOpenClawGroup: true, + }, { id: "canvas", label: "canvas", diff --git a/src/agents/tool-display-config.ts b/src/agents/tool-display-config.ts index c141a61fc809..87beb80803bb 100644 --- a/src/agents/tool-display-config.ts +++ b/src/agents/tool-display-config.ts @@ -54,6 +54,11 @@ export const TOOL_DISPLAY_CONFIG: ToolDisplayConfig = { title: "Computer", detailKeys: ["action", "coordinate", "text", "node", "nodeId", "screenIndex"], }, + screen: { + emoji: "🖥️", + title: "Screen", + detailKeys: ["action", "sessionKey", "dock"], + }, process: { emoji: "🧰", title: "Process", diff --git a/src/agents/tools/screen-tool.test.ts b/src/agents/tools/screen-tool.test.ts new file mode 100644 index 000000000000..229835bce7b8 --- /dev/null +++ b/src/agents/tools/screen-tool.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js"; +import type { InProcessGatewayCaller } from "./in-process-gateway.js"; +import { createScreenTool } from "./screen-tool.js"; + +function createGatewayRecorder() { + const calls: Array<[string, Record]> = []; + const callGateway: InProcessGatewayCaller = async ( + method: string, + params: Record, + ): Promise => { + calls.push([method, params]); + return { ok: true } as T; + }; + return { callGateway, calls }; +} + +describe("screen tool", () => { + it("uses a flat action enum and requires the UI capability", () => { + const tool = createScreenTool(); + expect(tool.requiredClientCaps).toEqual([GATEWAY_CLIENT_CAPS.UI_COMMANDS]); + expect(tool.parameters).toMatchObject({ + properties: { + action: { + type: "string", + enum: expect.arrayContaining(["split_right", "terminal_show", "navigate"]), + }, + }, + }); + }); + + it.each([ + ["split_right", { kind: "split", direction: "right", sessionKey: "agent:main:main" }], + ["split_down", { kind: "split", direction: "down", sessionKey: "agent:main:main" }], + ["close_pane", { kind: "close-pane", sessionKey: "agent:main:main" }], + ["focus", { kind: "focus", sessionKey: "agent:main:main" }], + ["sidebar_show", { kind: "sidebar", visible: true }], + ["sidebar_hide", { kind: "sidebar", visible: false }], + ["terminal_hide", { kind: "panel", panel: "terminal", open: false }], + ["browser_hide", { kind: "panel", panel: "browser", open: false }], + ["navigate", { kind: "navigate", sessionKey: "agent:main:main" }], + ])("maps %s to a UI command", async (action, command) => { + const { callGateway, calls } = createGatewayRecorder(); + const tool = createScreenTool({ + agentSessionKey: "agent:main:main", + callGateway, + }); + + await tool.execute("call", { action }); + + expect(calls).toEqual([["ui.command", { command, sessionKey: "agent:main:main" }]]); + }); + + it.each(["terminal_show", "browser_show"])("passes dock for %s", async (action) => { + const { callGateway, calls } = createGatewayRecorder(); + const tool = createScreenTool({ callGateway }); + + await tool.execute("call", { action, dock: "right" }); + + expect(calls).toEqual([ + [ + "ui.command", + { + command: { + kind: "panel", + panel: action === "terminal_show" ? "terminal" : "browser", + open: true, + dock: "right", + }, + }, + ], + ]); + }); +}); diff --git a/src/agents/tools/screen-tool.ts b/src/agents/tools/screen-tool.ts new file mode 100644 index 000000000000..a52d8e37bf60 --- /dev/null +++ b/src/agents/tools/screen-tool.ts @@ -0,0 +1,115 @@ +import { Type } from "typebox"; +import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js"; +import type { UiCommand, UiCommandParams } from "../../../packages/gateway-protocol/src/index.js"; +import type { AnyAgentTool } from "./common.js"; +import { jsonResult, readStringParam, ToolInputError } from "./common.js"; +import { callInProcessGatewayTool, type InProcessGatewayCaller } from "./in-process-gateway.js"; + +const ACTIONS = [ + "split_right", + "split_down", + "close_pane", + "focus", + "sidebar_show", + "sidebar_hide", + "terminal_show", + "terminal_hide", + "browser_show", + "browser_hide", + "navigate", +] as const; + +const ScreenToolSchema = Type.Object( + { + action: Type.String({ enum: [...ACTIONS], description: "Action" }), + sessionKey: Type.Optional(Type.String({ description: "Session. Default: current" })), + dock: Type.Optional( + Type.String({ enum: ["bottom", "right"], description: "Panel dock on show" }), + ), + }, + { additionalProperties: false }, +); + +type ScreenToolOptions = { + agentSessionKey?: string; + callGateway?: InProcessGatewayCaller; +}; + +function resolveSessionKey( + params: Record, + agentSessionKey: string | undefined, +): string { + const sessionKey = readStringParam(params, "sessionKey") ?? agentSessionKey?.trim(); + if (!sessionKey) { + throw new ToolInputError("sessionKey required"); + } + return sessionKey; +} + +function readDock(params: Record): "bottom" | "right" | undefined { + const dock = readStringParam(params, "dock"); + if (dock === undefined || dock === "bottom" || dock === "right") { + return dock; + } + throw new ToolInputError("dock must be bottom or right"); +} + +function commandForAction( + action: string, + params: Record, + agentSessionKey: string | undefined, +): UiCommand { + if (action === "split_right" || action === "split_down") { + return { + kind: "split", + direction: action === "split_right" ? "right" : "down", + sessionKey: resolveSessionKey(params, agentSessionKey), + }; + } + if (action === "close_pane" || action === "focus" || action === "navigate") { + return { + kind: action === "close_pane" ? "close-pane" : action, + sessionKey: resolveSessionKey(params, agentSessionKey), + }; + } + if (action === "sidebar_show" || action === "sidebar_hide") { + return { kind: "sidebar", visible: action === "sidebar_show" }; + } + if ( + action === "terminal_show" || + action === "terminal_hide" || + action === "browser_show" || + action === "browser_hide" + ) { + const open = action.endsWith("_show"); + const dock = open ? readDock(params) : undefined; + return { + kind: "panel", + panel: action.startsWith("terminal_") ? "terminal" : "browser", + open, + ...(dock ? { dock } : {}), + }; + } + throw new ToolInputError(`Unknown action: ${action}`); +} + +export function createScreenTool(opts: ScreenToolOptions = {}): AnyAgentTool { + const gatewayCall = opts.callGateway ?? callInProcessGatewayTool; + return { + label: "Screen", + name: "screen", + description: + "Drive operator web UI. Split panes, focus, panels, sidebar, navigate. Needs connected web client.", + parameters: ScreenToolSchema, + requiredClientCaps: [GATEWAY_CLIENT_CAPS.UI_COMMANDS], + execute: async (_toolCallId, rawArgs) => { + const params = rawArgs as Record; + const action = readStringParam(params, "action", { required: true }); + const payload: UiCommandParams = { + command: commandForAction(action, params, opts.agentSessionKey), + ...(opts.agentSessionKey ? { sessionKey: opts.agentSessionKey } : {}), + }; + return jsonResult(await gatewayCall("ui.command", payload)); + }, + }; +} diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index 732cd3f4a5b7..4f2887f919ee 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -365,6 +365,7 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [ // Memory migration reads host assistant state and writes agent workspaces. { name: "migrations.memory.plan", scope: "operator.admin" }, { name: "migrations.memory.apply", scope: "operator.admin", controlPlaneWrite: true }, + { name: "ui.command", scope: "operator.write" }, ] as const; const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap = new Map( diff --git a/src/gateway/server-broadcast.ts b/src/gateway/server-broadcast.ts index b764eefc63b1..12b1888a00d9 100644 --- a/src/gateway/server-broadcast.ts +++ b/src/gateway/server-broadcast.ts @@ -26,6 +26,7 @@ import { logWs, shouldLogWs, summarizeAgentEventForWsLog } from "./ws-log.js"; const EVENT_SCOPE_GUARDS: Record = { agent: [READ_SCOPE], chat: [READ_SCOPE], + "ui.command": [READ_SCOPE], "chat.send_timing": [READ_SCOPE], "chat.side_result": [READ_SCOPE], cron: [READ_SCOPE], diff --git a/src/gateway/server-methods-list.test.ts b/src/gateway/server-methods-list.test.ts index 379d433f121b..53ccb5e14f1e 100644 --- a/src/gateway/server-methods-list.test.ts +++ b/src/gateway/server-methods-list.test.ts @@ -42,10 +42,11 @@ describe("listGatewayMethods", () => { }); it("appends memory migration after model probing without shifting older method indices", () => { - expect(listGatewayMethods().slice(-3)).toEqual([ + expect(listGatewayMethods().slice(-4)).toEqual([ "models.probe", "migrations.memory.plan", "migrations.memory.apply", + "ui.command", ]); }); @@ -95,7 +96,6 @@ describe("listGatewayMethods", () => { ]); expect(methods).toContain("tts.speak"); expect(coreMethods.slice(-10)).toEqual([ - "sessions.catalog.continue", "sessions.catalog.archive", "approval.get", "approval.resolve", @@ -105,6 +105,7 @@ describe("listGatewayMethods", () => { "models.probe", "migrations.memory.plan", "migrations.memory.apply", + "ui.command", ]); expect(methods.indexOf("approval.get")).toBeGreaterThan(methods.indexOf("tts.speak")); expect(methods.indexOf("approval.resolve")).toBe(methods.indexOf("approval.get") + 1); diff --git a/src/gateway/server-methods-list.ts b/src/gateway/server-methods-list.ts index a2757cce458f..161c01d110ad 100644 --- a/src/gateway/server-methods-list.ts +++ b/src/gateway/server-methods-list.ts @@ -40,6 +40,7 @@ export const GATEWAY_EVENTS = [ "connect.challenge", "agent", "chat", + "ui.command", "session.approval", "session.message", "session.operation", diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts index 5cadd21814a1..90de311b142c 100644 --- a/src/gateway/server-methods.ts +++ b/src/gateway/server-methods.ts @@ -132,6 +132,10 @@ const loadTerminalHandlers = lazyHandlerModule( () => import("./server-methods/terminal.js"), (module) => module.terminalHandlers, ); +const loadUiCommandHandlers = lazyHandlerModule( + () => import("./server-methods/ui-command.js"), + (module) => module.uiCommandHandlers, +); const loadModelsAuthStatusHandlers = lazyHandlerModule( () => import("./server-methods/models-auth-status.js"), (module) => module.modelsAuthStatusHandlers, @@ -344,6 +348,10 @@ export const coreGatewayHandlers: GatewayRequestHandlers = { ], loadHandlers: loadTerminalHandlers, }), + ...createLazyCoreHandlers({ + methods: ["ui.command"], + loadHandlers: loadUiCommandHandlers, + }), ...createLazyCoreHandlers({ methods: ["voicewake.get", "voicewake.set"], loadHandlers: loadVoicewakeHandlers, diff --git a/src/gateway/server-methods/ui-command.test.ts b/src/gateway/server-methods/ui-command.test.ts new file mode 100644 index 000000000000..d2e9d10c6b57 --- /dev/null +++ b/src/gateway/server-methods/ui-command.test.ts @@ -0,0 +1,83 @@ +import { expectDefined } from "@openclaw/normalization-core"; +import { describe, expect, it, vi } from "vitest"; +import { + GATEWAY_CLIENT_CAPS, + GATEWAY_CLIENT_IDS, +} from "../../../packages/gateway-protocol/src/client-info.js"; +import type { GatewayClient } from "./types.js"; +import { uiCommandHandlers } from "./ui-command.js"; + +function client(connId: string, id: string, caps: string[] = []): GatewayClient { + return { + connId, + connect: { + client: { id, version: "test", platform: "web", mode: "ui" }, + caps, + }, + } as GatewayClient; +} + +async function call(params: unknown, clients: GatewayClient[]) { + const respond = vi.fn(); + const broadcastToConnIds = vi.fn(); + await expectDefined( + uiCommandHandlers["ui.command"], + "ui.command", + )({ + params, + respond, + context: { + broadcastToConnIds, + getClientConnIds: (filter?: (client: GatewayClient) => boolean) => + new Set( + clients + .filter((entry) => filter?.(entry) !== false) + .flatMap((entry) => (entry.connId ? [entry.connId] : [])), + ), + }, + } as never); + return { respond, broadcastToConnIds }; +} + +describe("ui.command gateway method", () => { + it("rejects invalid params", async () => { + const result = await call({ command: { kind: "sidebar", visible: "yes" } }, []); + + expect(result.respond).toHaveBeenCalledWith(false, undefined, expect.any(Object)); + expect(result.broadcastToConnIds).not.toHaveBeenCalled(); + }); + + it("reports when no capable Control UI client is connected", async () => { + const result = await call({ command: { kind: "sidebar", visible: true } }, [ + client("legacy-ui", GATEWAY_CLIENT_IDS.CONTROL_UI), + client("capable-cli", GATEWAY_CLIENT_IDS.CLI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]), + ]); + + expect(result.respond).toHaveBeenCalledWith( + false, + undefined, + expect.objectContaining({ message: "no ui client" }), + ); + expect(result.broadcastToConnIds).not.toHaveBeenCalled(); + }); + + it("delivers only to capable Control UI connections", async () => { + const params = { + command: { kind: "split", direction: "right", sessionKey: "agent:main:other" }, + sessionKey: "agent:main:main", + }; + const result = await call(params, [ + client("ui-one", GATEWAY_CLIENT_IDS.CONTROL_UI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]), + client("legacy-ui", GATEWAY_CLIENT_IDS.CONTROL_UI), + client("ui-two", GATEWAY_CLIENT_IDS.CONTROL_UI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]), + client("cli", GATEWAY_CLIENT_IDS.CLI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]), + ]); + + expect(result.broadcastToConnIds).toHaveBeenCalledWith( + "ui.command", + params, + new Set(["ui-one", "ui-two"]), + ); + expect(result.respond).toHaveBeenCalledWith(true, { ok: true }); + }); +}); diff --git a/src/gateway/server-methods/ui-command.ts b/src/gateway/server-methods/ui-command.ts new file mode 100644 index 000000000000..3272c86a82cd --- /dev/null +++ b/src/gateway/server-methods/ui-command.ts @@ -0,0 +1,47 @@ +import { + GATEWAY_CLIENT_CAPS, + GATEWAY_CLIENT_IDS, + hasGatewayClientCap, +} from "../../../packages/gateway-protocol/src/client-info.js"; +import { + ErrorCodes, + errorShape, + formatValidationErrors, + type UiCommandParams, + validateUiCommandParams, +} from "../../../packages/gateway-protocol/src/index.js"; +import type { GatewayRequestContextWithClientLookup } from "../server-request-context.js"; +import type { GatewayRequestHandlers } from "./types.js"; + +export const uiCommandHandlers: GatewayRequestHandlers = { + "ui.command": ({ params, respond, context }) => { + if (!validateUiCommandParams(params)) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + `invalid ui.command params: ${formatValidationErrors(validateUiCommandParams.errors)}`, + ), + ); + return; + } + + const commandParams = params as UiCommandParams; + const clientContext = context as GatewayRequestContextWithClientLookup; + // v1 intentionally fans out to every capable Control UI; session-targeted routing is out of scope. + const connIds = + clientContext.getClientConnIds?.( + (client) => + client.connect.client.id === GATEWAY_CLIENT_IDS.CONTROL_UI && + hasGatewayClientCap(client.connect.caps, GATEWAY_CLIENT_CAPS.UI_COMMANDS), + ) ?? new Set(); + if (connIds.size === 0) { + respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "no ui client")); + return; + } + + context.broadcastToConnIds("ui.command", commandParams, connIds); + respond(true, { ok: true }); + }, +}; diff --git a/src/gateway/server-request-context.ts b/src/gateway/server-request-context.ts index e9d12f87c46a..42deca7efc50 100644 --- a/src/gateway/server-request-context.ts +++ b/src/gateway/server-request-context.ts @@ -136,9 +136,13 @@ function canDeliverApprovals( ); } +export type GatewayRequestContextWithClientLookup = GatewayRequestContext & { + getClientConnIds?: (filter?: (client: GatewayClient) => boolean) => ReadonlySet; +}; + export function createGatewayRequestContext( params: GatewayRequestContextParams, -): GatewayRequestContext { +): GatewayRequestContextWithClientLookup { return { deps: params.deps, // Keep cron reads live so config hot reload can swap cron/store state without rebuilding @@ -207,6 +211,19 @@ export function createGatewayRequestContext( } return connIds; }, + getClientConnIds: (filter) => { + const connIds = new Set(); + for (const gatewayClient of params.clients) { + if (!gatewayClient.connId || gatewayClient.invalidated) { + continue; + } + if (filter && !filter(gatewayClient)) { + continue; + } + connIds.add(gatewayClient.connId); + } + return connIds; + }, hasConnectedClientsForDevice: (deviceId: string) => { for (const gatewayClient of params.clients) { if (gatewayClient.connect.device?.id === deviceId && !gatewayClient.invalidated) { diff --git a/src/gateway/tool-resolution.exclude.test.ts b/src/gateway/tool-resolution.exclude.test.ts index ccd55a5ca702..8d73d02bc199 100644 --- a/src/gateway/tool-resolution.exclude.test.ts +++ b/src/gateway/tool-resolution.exclude.test.ts @@ -154,6 +154,7 @@ describe("resolveGatewayScopedTools excludeToolNames", () => { "cron", "gateway", "sessions", + "screen", "nodes", "computer", "openclaw", @@ -162,6 +163,7 @@ describe("resolveGatewayScopedTools excludeToolNames", () => { "cron", "gateway", "sessions", + "screen", "nodes", "computer", "openclaw", diff --git a/src/security/dangerous-tools.ts b/src/security/dangerous-tools.ts index 3365afcf1204..a684d953417e 100644 --- a/src/security/dangerous-tools.ts +++ b/src/security/dangerous-tools.ts @@ -50,6 +50,7 @@ export const GATEWAY_CONTROL_PLANE_TOOLS = ["cron", "gateway"] as const; export const GATEWAY_OWNER_ONLY_CORE_TOOLS = [ ...GATEWAY_CONTROL_PLANE_TOOLS, "sessions", + "screen", "nodes", "computer", "openclaw", diff --git a/ui/src/api/gateway.node.test.ts b/ui/src/api/gateway.node.test.ts index 8ca892ececfa..3798a266b12d 100644 --- a/ui/src/api/gateway.node.test.ts +++ b/ui/src/api/gateway.node.test.ts @@ -428,6 +428,7 @@ describe("GatewayBrowserClient", () => { GATEWAY_CLIENT_CAPS.TERMINAL_OFFSET_SEQ, GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, + GATEWAY_CLIENT_CAPS.UI_COMMANDS, ]); expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]); }); diff --git a/ui/src/api/gateway.ts b/ui/src/api/gateway.ts index ecb246b3b912..beeb9854d0b0 100644 --- a/ui/src/api/gateway.ts +++ b/ui/src/api/gateway.ts @@ -462,6 +462,7 @@ export class GatewayBrowserClient { GATEWAY_CLIENT_CAPS.TERMINAL_OFFSET_SEQ, GATEWAY_CLIENT_CAPS.TOOL_EVENTS, GATEWAY_CLIENT_CAPS.INLINE_WIDGETS, + GATEWAY_CLIENT_CAPS.UI_COMMANDS, ], auth: buildGatewayConnectAuth(selectedAuth), userAgent: navigator.userAgent, diff --git a/ui/src/app/app-host.test.ts b/ui/src/app/app-host.test.ts index 5982497569d4..911cd12d7854 100644 --- a/ui/src/app/app-host.test.ts +++ b/ui/src/app/app-host.test.ts @@ -6,6 +6,7 @@ import type { GatewayBrowserClient } from "../api/gateway.ts"; import { BROWSER_PANEL_TOGGLE_EVENT, TERMINAL_PANEL_TOGGLE_EVENT, + UI_COMMAND_EVENT, } from "../components/panel-toggle-contract.ts"; import "./app-host.ts"; import type { @@ -57,6 +58,10 @@ type ShellLazySurfaceState = ShellKeyboardState & { terminalPanelElement: TestOptionalCustomElement; }; +type ShellUiCommandState = ShellKeyboardState & { + handleGatewayEvent: (event: { event: string; payload: unknown }) => void; +}; + let lazyElementSequence = 0; function createLazyElementSpec(label: string): TestOptionalCustomElement { @@ -404,6 +409,65 @@ describe("OpenClaw shell keyboard shortcuts", () => { }); }); + it("routes UI commands to navigation, panels, and chat fallback", () => { + const update = vi.fn(); + const setSessionKey = vi.fn(); + const navigate = vi.fn(); + const panelEvent = vi.fn(); + const uiCommandEvent = vi.fn(); + window.addEventListener(TERMINAL_PANEL_TOGGLE_EVENT, panelEvent); + window.addEventListener(UI_COMMAND_EVENT, uiCommandEvent); + const shell = document.createElement("openclaw-app-shell") as unknown as ShellUiCommandState; + shell.runtime = { + context: { + navigation: { update }, + gateway: { setSessionKey }, + navigate, + } as unknown as ApplicationContext, + }; + + shell.handleGatewayEvent({ + event: "ui.command", + payload: { command: { kind: "sidebar", visible: false } }, + }); + shell.handleGatewayEvent({ + event: "ui.command", + payload: { + command: { kind: "panel", panel: "terminal", open: false, dock: "right" }, + }, + }); + shell.handleGatewayEvent({ + event: "ui.command", + payload: { + command: { kind: "split", direction: "right", sessionKey: "agent:main:other" }, + }, + }); + shell.handleGatewayEvent({ + event: "ui.command", + payload: { + command: { kind: "focus", sessionKey: "agent:main:other" }, + sessionKey: "agent:main:source", + }, + }); + + expect(update).toHaveBeenCalledWith({ navCollapsed: true }); + expect(panelEvent).toHaveBeenCalledWith( + expect.objectContaining({ detail: { open: false, dock: "right" } }), + ); + expect(setSessionKey).toHaveBeenCalledWith("agent:main:other"); + expect(navigate).toHaveBeenCalledWith("chat", { search: "?session=agent%3Amain%3Aother" }); + expect(uiCommandEvent).toHaveBeenLastCalledWith( + expect.objectContaining({ + detail: { + command: { kind: "focus", sessionKey: "agent:main:other" }, + sessionKey: "agent:main:source", + }, + }), + ); + window.removeEventListener(TERMINAL_PANEL_TOGGLE_EVENT, panelEvent); + window.removeEventListener(UI_COMMAND_EVENT, uiCommandEvent); + }); + it("opens Settings with Shift-Command-Comma", () => { const navigate = vi.fn(); const shell = document.createElement("openclaw-app-shell") as unknown as ShellKeyboardState; diff --git a/ui/src/app/app-host.ts b/ui/src/app/app-host.ts index 3112a9ebb284..01debcc86041 100644 --- a/ui/src/app/app-host.ts +++ b/ui/src/app/app-host.ts @@ -1,8 +1,13 @@ import { ContextProvider } from "@lit/context"; +import type { UiCommandParams } from "@openclaw/gateway-protocol"; import type { RouteLocation, RouterState } from "@openclaw/uirouter"; import { html, nothing } from "lit"; import { property, query, state } from "lit/decorators.js"; -import { hasStoredGatewayAuth, type GatewayBrowserClient } from "../api/gateway.ts"; +import { + type GatewayEventFrame, + hasStoredGatewayAuth, + type GatewayBrowserClient, +} from "../api/gateway.ts"; import "../components/app-sidebar.ts"; import "../components/app-topbar.ts"; import "../components/connection-banner.ts"; @@ -28,6 +33,7 @@ import { BROWSER_PANEL_TOGGLE_EVENT, isTerminalPanelShortcut, TERMINAL_PANEL_TOGGLE_EVENT, + UI_COMMAND_EVENT, type PanelToggleElement, } from "../components/panel-toggle-contract.ts"; import { renderSettingsSidebar } from "../components/settings-sidebar.ts"; @@ -490,6 +496,10 @@ class OpenClawShell extends OpenClawLightDomElement { (gateway, notify) => gateway.subscribe(notify), (gateway) => this.synchronizeGateway(gateway.snapshot), ) + .effect( + () => this.context?.gateway, + (gateway) => gateway.subscribeEvents(this.handleGatewayEvent), + ) .watch( () => this.context?.config, (config, notify) => config.subscribe(notify), @@ -586,6 +596,43 @@ class OpenClawShell extends OpenClawLightDomElement { this.settingsPreloadTimers.clear(); } + private readonly handleGatewayEvent = (event: GatewayEventFrame) => { + if (event.event !== "ui.command" || !event.payload) { + return; + } + const context = this.context; + if (!context) { + return; + } + const commandParams = event.payload as UiCommandParams; + const { command } = commandParams; + if (!command) { + return; + } + if (command.kind === "sidebar") { + context.navigation.update({ navCollapsed: !command.visible }); + return; + } + if (command.kind === "panel") { + window.dispatchEvent( + new CustomEvent( + command.panel === "terminal" ? TERMINAL_PANEL_TOGGLE_EVENT : BROWSER_PANEL_TOGGLE_EVENT, + { detail: { open: command.open, ...(command.dock ? { dock: command.dock } : {}) } }, + ), + ); + return; + } + + const handled = !window.dispatchEvent( + new CustomEvent(UI_COMMAND_EVENT, { detail: commandParams, cancelable: true }), + ); + if (handled || (command.kind !== "navigate" && command.kind !== "split")) { + return; + } + context.gateway.setSessionKey(command.sessionKey); + this.navigate("chat", { search: searchForSession(command.sessionKey) }); + }; + private readonly handleThemeChange = (event: CustomEvent) => { const context = this.context; if (!context) { diff --git a/ui/src/components/browser/browser-panel.test.ts b/ui/src/components/browser/browser-panel.test.ts index 26e9160421cf..515a9694e0ad 100644 --- a/ui/src/components/browser/browser-panel.test.ts +++ b/ui/src/components/browser/browser-panel.test.ts @@ -1,11 +1,16 @@ -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createStorageMock } from "../../test-helpers/storage.ts"; import "./browser-panel.ts"; import { normalizeBrowserUrlDraft } from "./browser-url.ts"; describe("normalizeBrowserUrlDraft", () => { + beforeEach(() => { + vi.stubGlobal("localStorage", createStorageMock()); + }); + afterEach(() => { document.body.replaceChildren(); - localStorage.clear(); + vi.unstubAllGlobals(); }); it("prefixes bare hosts with https", () => { expect(normalizeBrowserUrlDraft("example.com")).toBe("https://example.com/"); @@ -50,4 +55,20 @@ describe("normalizeBrowserUrlDraft", () => { await panel.updateComplete; expect((panel as unknown as { open: boolean }).open).toBe(true); }); + + it("keeps an already closed panel closed for an explicit close request", () => { + const panel = document.createElement("openclaw-browser-panel") as unknown as HTMLElement & { + available: boolean; + open: boolean; + handleToggleRequest: (event: Event) => void; + }; + panel.available = true; + document.body.append(panel); + + panel.handleToggleRequest( + new CustomEvent("openclaw:browser-toggle", { detail: { open: false } }), + ); + + expect(panel.open).toBe(false); + }); }); diff --git a/ui/src/components/browser/browser-panel.ts b/ui/src/components/browser/browser-panel.ts index 1282ebe8a833..1db56a46b10c 100644 --- a/ui/src/components/browser/browser-panel.ts +++ b/ui/src/components/browser/browser-panel.ts @@ -284,6 +284,10 @@ class OpenClawBrowserPanel extends OpenClawLitElement { if (detail?.dock === "right" || detail?.dock === "bottom") { this.dock = detail.dock; } + if (detail?.open === false) { + this.closePanel(); + return; + } const url = typeof detail?.url === "string" ? normalizeBrowserUrlDraft(detail.url) : null; if (url || detail?.open === true) { if (!this.available) { diff --git a/ui/src/components/panel-toggle-contract.ts b/ui/src/components/panel-toggle-contract.ts index 16d2a2fc5496..9ef5bf971675 100644 --- a/ui/src/components/panel-toggle-contract.ts +++ b/ui/src/components/panel-toggle-contract.ts @@ -1,5 +1,10 @@ +import type { UiCommandParams } from "@openclaw/gateway-protocol"; + export const TERMINAL_PANEL_TOGGLE_EVENT = "openclaw:terminal-toggle"; export const BROWSER_PANEL_TOGGLE_EVENT = "openclaw:browser-toggle"; +export const UI_COMMAND_EVENT = "openclaw:ui-command"; + +export type UiCommandDetail = UiCommandParams; export type TerminalPanelToggleDetail = { dock?: "bottom" | "right"; diff --git a/ui/src/components/terminal/terminal-panel-readiness.test.ts b/ui/src/components/terminal/terminal-panel-readiness.test.ts index ba53d43f8d3d..28236960c041 100644 --- a/ui/src/components/terminal/terminal-panel-readiness.test.ts +++ b/ui/src/components/terminal/terminal-panel-readiness.test.ts @@ -70,6 +70,18 @@ describe("terminal panel readiness", () => { await i18n.setLocale("en"); }); + it("keeps an already closed panel closed for an explicit close request", () => { + const panel = document.createElement(TERMINAL_PANEL_ELEMENT_NAME) as OpenClawTerminalPanel; + panel.available = true; + document.body.append(panel); + + panel.handleToggleRequest( + new CustomEvent("openclaw:terminal-toggle", { detail: { open: false } }), + ); + + expect((panel as unknown as { open: boolean }).open).toBe(false); + }); + it("shows a connecting animation while a terminal open is in flight", async () => { const open = deferred<{ sessionId: string; diff --git a/ui/src/components/terminal/terminal-panel.ts b/ui/src/components/terminal/terminal-panel.ts index d783a1e96621..3f678fa22f0e 100644 --- a/ui/src/components/terminal/terminal-panel.ts +++ b/ui/src/components/terminal/terminal-panel.ts @@ -350,6 +350,10 @@ export class OpenClawTerminalPanel extends OpenClawLitElement { if (dock) { this.dock = dock; } + if (detail?.open === false) { + this.closePanel(); + return; + } if (detail?.catalog || detail?.open === true) { if (!this.available) { return; diff --git a/ui/src/pages/chat/chat-page.test.ts b/ui/src/pages/chat/chat-page.test.ts index 655116b6ab4c..038b0e07d583 100644 --- a/ui/src/pages/chat/chat-page.test.ts +++ b/ui/src/pages/chat/chat-page.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("./chat-pane.ts", () => ({})); import { loadSettings } from "../../app/settings.ts"; +import { UI_COMMAND_EVENT } from "../../components/panel-toggle-contract.ts"; import { SESSION_DRAG_MIME } from "../../lib/sessions/drag.ts"; import { searchForSession } from "../../lib/sessions/index.ts"; import { createStorageMock } from "../../test-helpers/storage.ts"; @@ -183,6 +184,66 @@ describe("chat page split layout host", () => { expect(survivingPane.classList.contains("chat-split-view__pane")).toBe(false); }); + it("applies mounted UI split, focus, and close commands", () => { + const page = new ChatPage(); + page.data = { sessionKey: "main" }; + const navigation = setNavigationContext(page); + document.body.append(page); + + const split = new CustomEvent(UI_COMMAND_EVENT, { + detail: { + command: { kind: "split", direction: "right", sessionKey: "agent:main:work" }, + sessionKey: "main", + }, + cancelable: true, + }); + window.dispatchEvent(split); + expect(split.defaultPrevented).toBe(true); + expect(getLayout(page)?.columns.at(1)?.panes.at(0)?.sessionKey).toBe("agent:main:work"); + expect(navigation.replace).toHaveBeenLastCalledWith("chat", { + search: searchForSession("agent:main:work"), + }); + + window.dispatchEvent( + new CustomEvent(UI_COMMAND_EVENT, { + detail: { command: { kind: "focus", sessionKey: "main" }, sessionKey: "main" }, + cancelable: true, + }), + ); + expect(getLayout(page)?.activePaneId).toBe("p1"); + + window.dispatchEvent( + new CustomEvent(UI_COMMAND_EVENT, { + detail: { + command: { kind: "close-pane", sessionKey: "agent:main:work" }, + sessionKey: "main", + }, + cancelable: true, + }), + ); + expect(getLayout(page)).toBeUndefined(); + }); + + it("leaves UI split commands unhandled on narrow viewports", () => { + stubMatchMedia(true); + const page = new ChatPage(); + page.data = { sessionKey: "main" }; + setNavigationContext(page); + document.body.append(page); + + const split = new CustomEvent(UI_COMMAND_EVENT, { + detail: { + command: { kind: "split", direction: "right", sessionKey: "agent:main:work" }, + sessionKey: "main", + }, + cancelable: true, + }); + window.dispatchEvent(split); + // Unhandled so the app host falls back to navigating to the session. + expect(split.defaultPrevented).toBe(false); + expect(getLayout(page)).toBeUndefined(); + }); + it("withholds the split-view opener on narrow single-pane viewports", async () => { stubMatchMedia(true); const page = new ChatPage(); diff --git a/ui/src/pages/chat/chat-page.ts b/ui/src/pages/chat/chat-page.ts index af5b298661be..fadd10fd66b9 100644 --- a/ui/src/pages/chat/chat-page.ts +++ b/ui/src/pages/chat/chat-page.ts @@ -6,6 +6,7 @@ import { repeat } from "lit/directives/repeat.js"; import { applicationContext, type ApplicationContext } from "../../app/context.ts"; import { loadSettings, patchSettings } from "../../app/settings.ts"; import "../../components/resizable-divider.ts"; +import { UI_COMMAND_EVENT, type UiCommandDetail } from "../../components/panel-toggle-contract.ts"; import { t } from "../../i18n/index.ts"; import { resolveSessionDisplayName } from "../../lib/session-display.ts"; import { readSessionDragData, sessionDragActive } from "../../lib/sessions/drag.ts"; @@ -22,6 +23,7 @@ import { type SplitDropZone, } from "./split-drop-zone.ts"; import { + applyUiCommandToSplitLayout, closePane, findPane, insertPane, @@ -88,6 +90,7 @@ export class ChatPage extends OpenClawLightDomElement { this.addEventListener("dragleave", this.handleDragLeave); this.addEventListener("drop", this.handleDrop); window.addEventListener("dragend", this.handleWindowDragEnd); + window.addEventListener(UI_COMMAND_EVENT, this.handleUiCommand); this.syncRouteToActivePane(); } @@ -100,6 +103,7 @@ export class ChatPage extends OpenClawLightDomElement { this.removeEventListener("dragleave", this.handleDragLeave); this.removeEventListener("drop", this.handleDrop); window.removeEventListener("dragend", this.handleWindowDragEnd); + window.removeEventListener(UI_COMMAND_EVENT, this.handleUiCommand); this.clearDropIndicator(); super.disconnectedCallback(); } @@ -133,6 +137,61 @@ export class ChatPage extends OpenClawLightDomElement { } }; + private readonly handleUiCommand = (event: Event) => { + if (!(event instanceof CustomEvent)) { + return; + } + const { command, sessionKey: sourceSessionKey } = event.detail as UiCommandDetail; + if (command.kind === "navigate") { + event.preventDefault(); + this.updateRoute(command.sessionKey); + return; + } + if (command.kind !== "split" && command.kind !== "close-pane" && command.kind !== "focus") { + return; + } + // Narrow viewports render single-pane and disable interactive splitting; + // leave programmatic splits unhandled so the host falls back to navigation. + if (command.kind === "split" && this.narrow) { + return; + } + + const currentSessionKey = this.data?.sessionKey?.trim(); + const layout = + this.layout ?? + (command.kind === "split" && currentSessionKey + ? this.classicLayout(currentSessionKey) + : undefined); + if (!layout) { + return; + } + const targetPane = + command.kind === "split" + ? undefined + : panesOf(layout).find((pane) => pane.sessionKey === command.sessionKey); + const survivingPane = + command.kind === "close-pane" && targetPane + ? panesOf(layout).find((pane) => pane.id !== targetPane.id) + : undefined; + const next = applyUiCommandToSplitLayout(layout, command, sourceSessionKey); + if (next === layout) { + return; + } + event.preventDefault(); + if (!next && survivingPane) { + const survivingLocation = findPane(layout, survivingPane.id); + if (survivingLocation) { + this.classicColumnId = survivingLocation.column.id; + this.classicPaneId = survivingPane.id; + } + } + this.persistLayout(next); + const activePane = next ? findPane(next, next.activePaneId)?.pane : survivingPane; + if (activePane) { + this.updateRoute(activePane.sessionKey, true); + } + }; + private readonly handleDragEnter = (event: DragEvent) => { if (this.narrow || !sessionDragActive(event.dataTransfer)) { return; diff --git a/ui/src/pages/chat/split-layout.test.ts b/ui/src/pages/chat/split-layout.test.ts index 7e61dbfa7f50..2873a1041dbc 100644 --- a/ui/src/pages/chat/split-layout.test.ts +++ b/ui/src/pages/chat/split-layout.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + applyUiCommandToSplitLayout, closePane, findPane, insertPane, @@ -129,6 +130,37 @@ describe("chat split layout", () => { expect(panesOf(layout)).not.toBe(layout.columns.at(0)?.panes); }); + it("maps UI split, focus, and close commands onto pane state", () => { + const initial = setActivePane(setPaneSession(createSplitLayout("main"), "p2", "source"), "p1"); + const split = applyUiCommandToSplitLayout( + initial, + { + kind: "split", + direction: "down", + sessionKey: "agent:main:new", + }, + "source", + ); + expect(split && panesOf(split).map((pane) => pane.sessionKey)).toEqual([ + "main", + "source", + "agent:main:new", + ]); + expect(split?.activePaneId).toBe("p3"); + + const focused = applyUiCommandToSplitLayout(split!, { + kind: "focus", + sessionKey: "main", + }); + expect(focused?.activePaneId).toBe("p1"); + + const closed = applyUiCommandToSplitLayout(focused!, { + kind: "close-pane", + sessionKey: "agent:main:new", + }); + expect(closed && panesOf(closed).map((pane) => pane.sessionKey)).toEqual(["main", "source"]); + }); + it("resizes only a boundary pair and clamps each side to fifteen percent", () => { const layout = insertPane(createSplitLayout("main"), "p1", "third", "right"); const columns = resizeColumns(layout, 0, 0.8); diff --git a/ui/src/pages/chat/split-layout.ts b/ui/src/pages/chat/split-layout.ts index 08beb61406eb..29a876e0e138 100644 --- a/ui/src/pages/chat/split-layout.ts +++ b/ui/src/pages/chat/split-layout.ts @@ -1,3 +1,4 @@ +import type { UiCommand } from "@openclaw/gateway-protocol"; import { expectDefined, isRecord } from "@openclaw/normalization-core"; export type ChatSplitPane = { id: string; sessionKey: string }; @@ -180,6 +181,36 @@ export function setActivePane(layout: ChatSplitLayout, paneId: string): ChatSpli return next; } +type UiSplitLayoutCommand = Extract; + +export function applyUiCommandToSplitLayout( + layout: ChatSplitLayout, + command: UiSplitLayoutCommand, + sourceSessionKey?: string, +): ChatSplitLayout | undefined { + if (command.kind === "split") { + const sourcePane = sourceSessionKey + ? panesOf(layout).find((entry) => entry.sessionKey === sourceSessionKey) + : undefined; + if (sourceSessionKey && !sourcePane) { + return layout; + } + return insertPane( + layout, + sourcePane?.id ?? layout.activePaneId, + command.sessionKey, + command.direction, + ); + } + const pane = panesOf(layout).find((entry) => entry.sessionKey === command.sessionKey); + if (!pane) { + return layout; + } + return command.kind === "close-pane" + ? closePane(layout, pane.id) + : setActivePane(layout, pane.id); +} + function resizePair(weights: number[], boundaryIndex: number, pairRatio: number): number[] { const next = [...weights]; if (boundaryIndex < 0 || boundaryIndex + 1 >= weights.length) {