mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-01 22:21:34 +00:00
feat(agents): screen tool drives Control UI layout
This commit is contained in:
@@ -377,6 +377,7 @@ enum class GatewayMethod(
|
||||
ModelsProbe("models.probe"),
|
||||
MigrationsMemoryPlan("migrations.memory.plan"),
|
||||
MigrationsMemoryApply("migrations.memory.apply"),
|
||||
UiCommand("ui.command"),
|
||||
}
|
||||
|
||||
enum class GatewayEvent(
|
||||
@@ -385,6 +386,7 @@ enum class GatewayEvent(
|
||||
ConnectChallenge("connect.challenge"),
|
||||
Agent("agent"),
|
||||
Chat("chat"),
|
||||
UiCommand("ui.command"),
|
||||
SessionApproval("session.approval"),
|
||||
SessionMessage("session.message"),
|
||||
SessionOperation("session.operation"),
|
||||
|
||||
@@ -44,6 +44,15 @@
|
||||
"screenIndex"
|
||||
]
|
||||
},
|
||||
"screen": {
|
||||
"emoji": "🖥️",
|
||||
"title": "Screen",
|
||||
"detailKeys": [
|
||||
"action",
|
||||
"sessionKey",
|
||||
"dock"
|
||||
]
|
||||
},
|
||||
"process": {
|
||||
"emoji": "🧰",
|
||||
"title": "Process",
|
||||
|
||||
@@ -2564,6 +2564,158 @@ public struct PushTestResult: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiSplitCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let direction: AnyCodable
|
||||
public let sessionkey: String
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
direction: AnyCodable,
|
||||
sessionkey: String)
|
||||
{
|
||||
self.kind = kind
|
||||
self.direction = direction
|
||||
self.sessionkey = sessionkey
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case direction
|
||||
case sessionkey = "sessionKey"
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiClosePaneCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let sessionkey: String
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
sessionkey: String)
|
||||
{
|
||||
self.kind = kind
|
||||
self.sessionkey = sessionkey
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case sessionkey = "sessionKey"
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiFocusCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let sessionkey: String
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
sessionkey: String)
|
||||
{
|
||||
self.kind = kind
|
||||
self.sessionkey = sessionkey
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case sessionkey = "sessionKey"
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiSidebarCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let visible: Bool
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
visible: Bool)
|
||||
{
|
||||
self.kind = kind
|
||||
self.visible = visible
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case visible
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiPanelCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let panel: AnyCodable
|
||||
public let _open: Bool
|
||||
public let dock: AnyCodable?
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
panel: AnyCodable,
|
||||
_open: Bool,
|
||||
dock: AnyCodable? = nil)
|
||||
{
|
||||
self.kind = kind
|
||||
self.panel = panel
|
||||
self._open = _open
|
||||
self.dock = dock
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case panel
|
||||
case _open = "open"
|
||||
case dock
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiNavigateCommand: Codable, Sendable {
|
||||
public let kind: String
|
||||
public let sessionkey: String
|
||||
|
||||
public init(
|
||||
kind: String,
|
||||
sessionkey: String)
|
||||
{
|
||||
self.kind = kind
|
||||
self.sessionkey = sessionkey
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case sessionkey = "sessionKey"
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiCommandParams: Codable, Sendable {
|
||||
public let command: UiCommand
|
||||
public let sessionkey: String?
|
||||
|
||||
public init(
|
||||
command: UiCommand,
|
||||
sessionkey: String? = nil)
|
||||
{
|
||||
self.command = command
|
||||
self.sessionkey = sessionkey
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case command
|
||||
case sessionkey = "sessionKey"
|
||||
}
|
||||
}
|
||||
|
||||
public struct UiCommandResult: Codable, Sendable {
|
||||
public let ok: Bool
|
||||
|
||||
public init(
|
||||
ok: Bool)
|
||||
{
|
||||
self.ok = ok
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case ok
|
||||
}
|
||||
}
|
||||
|
||||
public struct SecretsReloadParams: Codable, Sendable {}
|
||||
|
||||
public struct SecretsResolveParams: Codable, Sendable {
|
||||
@@ -13653,6 +13805,49 @@ public enum GatewaySuspendStatusResult: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public enum UiCommand: Codable, Sendable {
|
||||
case split(UiSplitCommand)
|
||||
case closePane(UiClosePaneCommand)
|
||||
case focus(UiFocusCommand)
|
||||
case sidebar(UiSidebarCommand)
|
||||
case panel(UiPanelCommand)
|
||||
case navigate(UiNavigateCommand)
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case discriminator = "kind"
|
||||
}
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
let discriminator = try container.decode(String.self, forKey: .discriminator)
|
||||
switch discriminator {
|
||||
case "split": self = try .split(UiSplitCommand(from: decoder))
|
||||
case "close-pane": self = try .closePane(UiClosePaneCommand(from: decoder))
|
||||
case "focus": self = try .focus(UiFocusCommand(from: decoder))
|
||||
case "sidebar": self = try .sidebar(UiSidebarCommand(from: decoder))
|
||||
case "panel": self = try .panel(UiPanelCommand(from: decoder))
|
||||
case "navigate": self = try .navigate(UiNavigateCommand(from: decoder))
|
||||
default:
|
||||
throw DecodingError.dataCorruptedError(
|
||||
forKey: .discriminator,
|
||||
in: container,
|
||||
debugDescription: "Unknown UiCommand discriminator value"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
switch self {
|
||||
case .split(let value): try value.encode(to: encoder)
|
||||
case .closePane(let value): try value.encode(to: encoder)
|
||||
case .focus(let value): try value.encode(to: encoder)
|
||||
case .sidebar(let value): try value.encode(to: encoder)
|
||||
case .panel(let value): try value.encode(to: encoder)
|
||||
case .navigate(let value): try value.encode(to: encoder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum SessionPlacement: Codable, Sendable {
|
||||
case local(LocalSessionPlacement)
|
||||
case requested(RequestedSessionPlacement)
|
||||
|
||||
@@ -83,6 +83,7 @@ export const GATEWAY_CLIENT_CAPS = {
|
||||
TASK_SUGGESTIONS: "task-suggestions",
|
||||
TERMINAL_OFFSET_SEQ: "terminal-offset-seq",
|
||||
TOOL_EVENTS: "tool-events",
|
||||
UI_COMMANDS: "ui-commands",
|
||||
} as const;
|
||||
|
||||
/** Optional capability advertised by clients during gateway handshake. */
|
||||
|
||||
@@ -8,6 +8,7 @@ import { lazyCompile } from "./protocol-validator.js";
|
||||
export type { ProtocolValidator } from "./protocol-validator.js";
|
||||
export * from "./schema/worker-inference.js";
|
||||
export * from "./schema/skill-history.js";
|
||||
export * from "./schema/ui-command.js";
|
||||
export * from "./migration-api.js";
|
||||
export type * from "./public-session-catalog.js";
|
||||
import {
|
||||
@@ -269,6 +270,7 @@ import {
|
||||
TerminalTextResultSchema,
|
||||
TerminalUploadParamsSchema,
|
||||
TerminalUploadResultSchema,
|
||||
UiCommandParamsSchema,
|
||||
ModelsListParamsSchema,
|
||||
AuthProbeStatusSchema,
|
||||
ModelsProbeParamsSchema,
|
||||
@@ -833,6 +835,7 @@ export const validateChatEvent = lazyCompile(ChatEventSchema);
|
||||
export const validateChatMessageGetResult = lazyCompile(ChatMessageGetResultSchema);
|
||||
export const validateUpdateStatusParams = lazyCompile(UpdateStatusParamsSchema);
|
||||
export const validateUpdateRunParams = lazyCompile(UpdateRunParamsSchema);
|
||||
export const validateUiCommandParams = lazyCompile(UiCommandParamsSchema);
|
||||
export const validateWebLoginStartParams = lazyCompile(WebLoginStartParamsSchema);
|
||||
export const validateWebLoginWaitParams = lazyCompile(WebLoginWaitParamsSchema);
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ export * from "./schema/system-event.js";
|
||||
export * from "./schema/task-suggestions.js";
|
||||
export * from "./schema/tasks.js";
|
||||
export * from "./schema/terminal.js";
|
||||
export * from "./schema/ui-command.js";
|
||||
export * from "./schema/plugin-approvals.js";
|
||||
export * from "./schema/plugins.js";
|
||||
export * from "./schema/wizard.js";
|
||||
|
||||
@@ -461,6 +461,17 @@ import {
|
||||
TaskSummarySchema,
|
||||
} from "./tasks.js";
|
||||
import { TerminalProtocolSchemas } from "./terminal-protocol-schemas.js";
|
||||
import {
|
||||
UiClosePaneCommandSchema,
|
||||
UiCommandParamsSchema,
|
||||
UiCommandResultSchema,
|
||||
UiCommandSchema,
|
||||
UiFocusCommandSchema,
|
||||
UiNavigateCommandSchema,
|
||||
UiPanelCommandSchema,
|
||||
UiSidebarCommandSchema,
|
||||
UiSplitCommandSchema,
|
||||
} from "./ui-command.js";
|
||||
import {
|
||||
WizardCancelParamsSchema,
|
||||
WizardNextParamsSchema,
|
||||
@@ -582,6 +593,15 @@ export const ProtocolSchemas = {
|
||||
// Push and secret-resolution payloads used by mobile/control integrations.
|
||||
PushTestParams: PushTestParamsSchema,
|
||||
PushTestResult: PushTestResultSchema,
|
||||
UiSplitCommand: UiSplitCommandSchema,
|
||||
UiClosePaneCommand: UiClosePaneCommandSchema,
|
||||
UiFocusCommand: UiFocusCommandSchema,
|
||||
UiSidebarCommand: UiSidebarCommandSchema,
|
||||
UiPanelCommand: UiPanelCommandSchema,
|
||||
UiNavigateCommand: UiNavigateCommandSchema,
|
||||
UiCommand: UiCommandSchema,
|
||||
UiCommandParams: UiCommandParamsSchema,
|
||||
UiCommandResult: UiCommandResultSchema,
|
||||
SecretsReloadParams: SecretsReloadParamsSchema,
|
||||
SecretsResolveParams: SecretsResolveParamsSchema,
|
||||
SecretsResolveAssignment: SecretsResolveAssignmentSchema,
|
||||
|
||||
51
packages/gateway-protocol/src/schema/ui-command.ts
Normal file
51
packages/gateway-protocol/src/schema/ui-command.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import type { Static } from "typebox";
|
||||
import { Type } from "typebox";
|
||||
import { closedObject } from "./closed-object.js";
|
||||
import { NonEmptyString } from "./primitives.js";
|
||||
|
||||
export const UiSplitCommandSchema = closedObject({
|
||||
kind: Type.Literal("split"),
|
||||
direction: Type.Union([Type.Literal("right"), Type.Literal("down")]),
|
||||
sessionKey: NonEmptyString,
|
||||
});
|
||||
export const UiClosePaneCommandSchema = closedObject({
|
||||
kind: Type.Literal("close-pane"),
|
||||
sessionKey: NonEmptyString,
|
||||
});
|
||||
export const UiFocusCommandSchema = closedObject({
|
||||
kind: Type.Literal("focus"),
|
||||
sessionKey: NonEmptyString,
|
||||
});
|
||||
export const UiSidebarCommandSchema = closedObject({
|
||||
kind: Type.Literal("sidebar"),
|
||||
visible: Type.Boolean(),
|
||||
});
|
||||
export const UiPanelCommandSchema = closedObject({
|
||||
kind: Type.Literal("panel"),
|
||||
panel: Type.Union([Type.Literal("terminal"), Type.Literal("browser")]),
|
||||
open: Type.Boolean(),
|
||||
dock: Type.Optional(Type.Union([Type.Literal("bottom"), Type.Literal("right")])),
|
||||
});
|
||||
export const UiNavigateCommandSchema = closedObject({
|
||||
kind: Type.Literal("navigate"),
|
||||
sessionKey: NonEmptyString,
|
||||
});
|
||||
|
||||
export const UiCommandSchema = Type.Union([
|
||||
UiSplitCommandSchema,
|
||||
UiClosePaneCommandSchema,
|
||||
UiFocusCommandSchema,
|
||||
UiSidebarCommandSchema,
|
||||
UiPanelCommandSchema,
|
||||
UiNavigateCommandSchema,
|
||||
]);
|
||||
export type UiCommand = Static<typeof UiCommandSchema>;
|
||||
|
||||
export const UiCommandParamsSchema = closedObject({
|
||||
command: UiCommandSchema,
|
||||
sessionKey: Type.Optional(NonEmptyString),
|
||||
});
|
||||
export type UiCommandParams = Static<typeof UiCommandParamsSchema>;
|
||||
|
||||
export const UiCommandResultSchema = closedObject({ ok: Type.Boolean() });
|
||||
export type UiCommandResult = Static<typeof UiCommandResultSchema>;
|
||||
@@ -4,6 +4,7 @@
|
||||
"session.operation": "Chat UI derives run state from chat/agent events; no session.operation consumer yet.",
|
||||
"session.tool": "Session tool stream is not rendered by the iOS chat surface yet.",
|
||||
"task.suggestion": "Task suggestion cards are a Control UI-only surface; iOS does not render them.",
|
||||
"ui.command": "Web Control UI-only layout commands; iOS does not consume them.",
|
||||
"presence": "Presence roster is a control-UI (web/desktop) surface; iOS does not render it.",
|
||||
"shutdown": "iOS relies on socket close plus reconnect/backoff instead of the shutdown notice.",
|
||||
"heartbeat": "iOS liveness uses tick and WebSocket-level ping; heartbeat is unused.",
|
||||
@@ -28,6 +29,7 @@
|
||||
"session.operation": "Chat UI derives run state from chat/agent events; no session.operation consumer yet.",
|
||||
"session.tool": "Session tool stream is not rendered by the Android chat surface yet.",
|
||||
"task.suggestion": "Task suggestion cards are a Control UI-only surface; Android does not render them.",
|
||||
"ui.command": "Web Control UI-only layout commands; Android does not consume them.",
|
||||
"presence": "Presence roster is a control-UI (web/desktop) surface; Android does not render it.",
|
||||
"talk.mode": "Android toggles talk mode locally; gateway talk.mode sync is not consumed.",
|
||||
"shutdown": "Android relies on socket close plus reconnect/backoff instead of the shutdown notice.",
|
||||
|
||||
@@ -21,6 +21,10 @@ function hasWidget(tools: readonly { name: string }[]): boolean {
|
||||
return tools.some((tool) => tool.name === "show_widget");
|
||||
}
|
||||
|
||||
function hasScreen(tools: readonly { name: string }[]): boolean {
|
||||
return tools.some((tool) => tool.name === "screen");
|
||||
}
|
||||
|
||||
describe("gateway client capability tool filtering", () => {
|
||||
it("excludes capability-gated tools when no gateway client caps exist", () => {
|
||||
expect(hasWidget(createOpenClawTools())).toBe(false);
|
||||
@@ -36,6 +40,11 @@ describe("gateway client capability tool filtering", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("only exposes screen to UI-command clients", () => {
|
||||
expect(hasScreen(createOpenClawTools())).toBe(false);
|
||||
expect(hasScreen(createOpenClawTools({ clientCaps: ["ui-commands"] }))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not let tools.allow resurrect a gated tool for a channel run", () => {
|
||||
const tools = createOpenClawCodingTools({
|
||||
messageProvider: "telegram",
|
||||
|
||||
@@ -61,6 +61,7 @@ import { createMusicGenerateTool } from "./tools/music-generate-tool.js";
|
||||
import { createNodesTool } from "./tools/nodes-tool.js";
|
||||
import { createOpenClawDelegateToolsForRun } from "./tools/openclaw-delegate-tool.js";
|
||||
import { createPdfTool } from "./tools/pdf-tool.js";
|
||||
import { createScreenTool } from "./tools/screen-tool.js";
|
||||
import { createSessionStatusTool } from "./tools/session-status-tool.js";
|
||||
import { createSessionsHistoryTool } from "./tools/sessions-history-tool.js";
|
||||
import { createSessionsListTool } from "./tools/sessions-list-tool.js";
|
||||
@@ -482,6 +483,9 @@ export function createOpenClawTools(
|
||||
sandboxed: options?.sandboxed,
|
||||
config: resolvedConfig,
|
||||
}),
|
||||
createScreenTool({
|
||||
agentSessionKey: options?.runSessionKey ?? options?.agentSessionKey,
|
||||
}),
|
||||
]),
|
||||
...(!embedded && taskSuggestionSessionKey && options?.taskSuggestionDeliveryMode === "gateway"
|
||||
? createTaskSuggestionTools({
|
||||
|
||||
@@ -384,6 +384,23 @@ describe("buildAgentSystemPrompt", () => {
|
||||
expect(withYield).toContain("wait with `sessions_yield`");
|
||||
});
|
||||
|
||||
it("limits screen guidance to web/app tool surfaces", () => {
|
||||
const withoutScreen = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
toolNames: ["sessions"],
|
||||
});
|
||||
const withScreen = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
toolNames: ["sessions", "screen"],
|
||||
});
|
||||
|
||||
expect(withoutScreen).not.toContain("web/app turn may drive UI");
|
||||
expect(withScreen).toContain("- screen: Drive operator web UI");
|
||||
expect(withScreen).toContain(
|
||||
"`screen` present: web/app turn may drive UI; messaging turn: don't.",
|
||||
);
|
||||
});
|
||||
|
||||
it("lists available tools when provided", () => {
|
||||
const prompt = buildAgentSystemPrompt({
|
||||
workspaceDir: "/tmp/openclaw",
|
||||
|
||||
@@ -780,6 +780,7 @@ export function buildAgentSystemPrompt(params: {
|
||||
web_fetch: "Fetch/extract URL",
|
||||
// Channel docking: add login tools here when a channel needs interactive linking.
|
||||
browser: "Control browser",
|
||||
screen: "Drive operator web UI",
|
||||
canvas: "Present/eval/snapshot Canvas",
|
||||
nodes: "Paired node status/control/media",
|
||||
cron: "Schedule/wake. Reminder text must read as reminder when fired; mention reminder for delayed gaps; include useful recent context.",
|
||||
@@ -817,6 +818,7 @@ export function buildAgentSystemPrompt(params: {
|
||||
"web_search",
|
||||
"web_fetch",
|
||||
"browser",
|
||||
"screen",
|
||||
"canvas",
|
||||
"nodes",
|
||||
"cron",
|
||||
@@ -1078,6 +1080,9 @@ export function buildAgentSystemPrompt(params: {
|
||||
"Large work: `sessions_spawn`; completion push-based.",
|
||||
'`sessions_spawn`: omit `context`; transcript needed => `context:"fork"`.',
|
||||
...(hasSessionsSpawn ? ["`visible:true` only web/app user or asked."] : []),
|
||||
...(availableTools.has("screen")
|
||||
? ["`screen` present: web/app turn may drive UI; messaging turn: don't."]
|
||||
: []),
|
||||
]
|
||||
: []),
|
||||
...nativeCommandGuidanceLines,
|
||||
|
||||
@@ -48,6 +48,7 @@ describe("tool-catalog", () => {
|
||||
"session_status",
|
||||
"spawn_task",
|
||||
"dismiss_task",
|
||||
"screen",
|
||||
"cron",
|
||||
"get_goal",
|
||||
"create_goal",
|
||||
|
||||
@@ -247,6 +247,14 @@ const CORE_TOOL_DEFINITIONS: CoreToolDefinition[] = [
|
||||
profiles: [],
|
||||
includeInOpenClawGroup: true,
|
||||
},
|
||||
{
|
||||
id: "screen",
|
||||
label: "screen",
|
||||
description: "Drive operator web UI",
|
||||
sectionId: "ui",
|
||||
profiles: ["coding"],
|
||||
includeInOpenClawGroup: true,
|
||||
},
|
||||
{
|
||||
id: "canvas",
|
||||
label: "canvas",
|
||||
|
||||
@@ -54,6 +54,11 @@ export const TOOL_DISPLAY_CONFIG: ToolDisplayConfig = {
|
||||
title: "Computer",
|
||||
detailKeys: ["action", "coordinate", "text", "node", "nodeId", "screenIndex"],
|
||||
},
|
||||
screen: {
|
||||
emoji: "🖥️",
|
||||
title: "Screen",
|
||||
detailKeys: ["action", "sessionKey", "dock"],
|
||||
},
|
||||
process: {
|
||||
emoji: "🧰",
|
||||
title: "Process",
|
||||
|
||||
74
src/agents/tools/screen-tool.test.ts
Normal file
74
src/agents/tools/screen-tool.test.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import type { InProcessGatewayCaller } from "./in-process-gateway.js";
|
||||
import { createScreenTool } from "./screen-tool.js";
|
||||
|
||||
function createGatewayRecorder() {
|
||||
const calls: Array<[string, Record<string, unknown>]> = [];
|
||||
const callGateway: InProcessGatewayCaller = async <T>(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
): Promise<T> => {
|
||||
calls.push([method, params]);
|
||||
return { ok: true } as T;
|
||||
};
|
||||
return { callGateway, calls };
|
||||
}
|
||||
|
||||
describe("screen tool", () => {
|
||||
it("uses a flat action enum and requires the UI capability", () => {
|
||||
const tool = createScreenTool();
|
||||
expect(tool.requiredClientCaps).toEqual([GATEWAY_CLIENT_CAPS.UI_COMMANDS]);
|
||||
expect(tool.parameters).toMatchObject({
|
||||
properties: {
|
||||
action: {
|
||||
type: "string",
|
||||
enum: expect.arrayContaining(["split_right", "terminal_show", "navigate"]),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["split_right", { kind: "split", direction: "right", sessionKey: "agent:main:main" }],
|
||||
["split_down", { kind: "split", direction: "down", sessionKey: "agent:main:main" }],
|
||||
["close_pane", { kind: "close-pane", sessionKey: "agent:main:main" }],
|
||||
["focus", { kind: "focus", sessionKey: "agent:main:main" }],
|
||||
["sidebar_show", { kind: "sidebar", visible: true }],
|
||||
["sidebar_hide", { kind: "sidebar", visible: false }],
|
||||
["terminal_hide", { kind: "panel", panel: "terminal", open: false }],
|
||||
["browser_hide", { kind: "panel", panel: "browser", open: false }],
|
||||
["navigate", { kind: "navigate", sessionKey: "agent:main:main" }],
|
||||
])("maps %s to a UI command", async (action, command) => {
|
||||
const { callGateway, calls } = createGatewayRecorder();
|
||||
const tool = createScreenTool({
|
||||
agentSessionKey: "agent:main:main",
|
||||
callGateway,
|
||||
});
|
||||
|
||||
await tool.execute("call", { action });
|
||||
|
||||
expect(calls).toEqual([["ui.command", { command, sessionKey: "agent:main:main" }]]);
|
||||
});
|
||||
|
||||
it.each(["terminal_show", "browser_show"])("passes dock for %s", async (action) => {
|
||||
const { callGateway, calls } = createGatewayRecorder();
|
||||
const tool = createScreenTool({ callGateway });
|
||||
|
||||
await tool.execute("call", { action, dock: "right" });
|
||||
|
||||
expect(calls).toEqual([
|
||||
[
|
||||
"ui.command",
|
||||
{
|
||||
command: {
|
||||
kind: "panel",
|
||||
panel: action === "terminal_show" ? "terminal" : "browser",
|
||||
open: true,
|
||||
dock: "right",
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
});
|
||||
});
|
||||
115
src/agents/tools/screen-tool.ts
Normal file
115
src/agents/tools/screen-tool.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { Type } from "typebox";
|
||||
import { GATEWAY_CLIENT_CAPS } from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import type { UiCommand, UiCommandParams } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { AnyAgentTool } from "./common.js";
|
||||
import { jsonResult, readStringParam, ToolInputError } from "./common.js";
|
||||
import { callInProcessGatewayTool, type InProcessGatewayCaller } from "./in-process-gateway.js";
|
||||
|
||||
const ACTIONS = [
|
||||
"split_right",
|
||||
"split_down",
|
||||
"close_pane",
|
||||
"focus",
|
||||
"sidebar_show",
|
||||
"sidebar_hide",
|
||||
"terminal_show",
|
||||
"terminal_hide",
|
||||
"browser_show",
|
||||
"browser_hide",
|
||||
"navigate",
|
||||
] as const;
|
||||
|
||||
const ScreenToolSchema = Type.Object(
|
||||
{
|
||||
action: Type.String({ enum: [...ACTIONS], description: "Action" }),
|
||||
sessionKey: Type.Optional(Type.String({ description: "Session. Default: current" })),
|
||||
dock: Type.Optional(
|
||||
Type.String({ enum: ["bottom", "right"], description: "Panel dock on show" }),
|
||||
),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
type ScreenToolOptions = {
|
||||
agentSessionKey?: string;
|
||||
callGateway?: InProcessGatewayCaller;
|
||||
};
|
||||
|
||||
function resolveSessionKey(
|
||||
params: Record<string, unknown>,
|
||||
agentSessionKey: string | undefined,
|
||||
): string {
|
||||
const sessionKey = readStringParam(params, "sessionKey") ?? agentSessionKey?.trim();
|
||||
if (!sessionKey) {
|
||||
throw new ToolInputError("sessionKey required");
|
||||
}
|
||||
return sessionKey;
|
||||
}
|
||||
|
||||
function readDock(params: Record<string, unknown>): "bottom" | "right" | undefined {
|
||||
const dock = readStringParam(params, "dock");
|
||||
if (dock === undefined || dock === "bottom" || dock === "right") {
|
||||
return dock;
|
||||
}
|
||||
throw new ToolInputError("dock must be bottom or right");
|
||||
}
|
||||
|
||||
function commandForAction(
|
||||
action: string,
|
||||
params: Record<string, unknown>,
|
||||
agentSessionKey: string | undefined,
|
||||
): UiCommand {
|
||||
if (action === "split_right" || action === "split_down") {
|
||||
return {
|
||||
kind: "split",
|
||||
direction: action === "split_right" ? "right" : "down",
|
||||
sessionKey: resolveSessionKey(params, agentSessionKey),
|
||||
};
|
||||
}
|
||||
if (action === "close_pane" || action === "focus" || action === "navigate") {
|
||||
return {
|
||||
kind: action === "close_pane" ? "close-pane" : action,
|
||||
sessionKey: resolveSessionKey(params, agentSessionKey),
|
||||
};
|
||||
}
|
||||
if (action === "sidebar_show" || action === "sidebar_hide") {
|
||||
return { kind: "sidebar", visible: action === "sidebar_show" };
|
||||
}
|
||||
if (
|
||||
action === "terminal_show" ||
|
||||
action === "terminal_hide" ||
|
||||
action === "browser_show" ||
|
||||
action === "browser_hide"
|
||||
) {
|
||||
const open = action.endsWith("_show");
|
||||
const dock = open ? readDock(params) : undefined;
|
||||
return {
|
||||
kind: "panel",
|
||||
panel: action.startsWith("terminal_") ? "terminal" : "browser",
|
||||
open,
|
||||
...(dock ? { dock } : {}),
|
||||
};
|
||||
}
|
||||
throw new ToolInputError(`Unknown action: ${action}`);
|
||||
}
|
||||
|
||||
export function createScreenTool(opts: ScreenToolOptions = {}): AnyAgentTool {
|
||||
const gatewayCall = opts.callGateway ?? callInProcessGatewayTool;
|
||||
return {
|
||||
label: "Screen",
|
||||
name: "screen",
|
||||
description:
|
||||
"Drive operator web UI. Split panes, focus, panels, sidebar, navigate. Needs connected web client.",
|
||||
parameters: ScreenToolSchema,
|
||||
requiredClientCaps: [GATEWAY_CLIENT_CAPS.UI_COMMANDS],
|
||||
execute: async (_toolCallId, rawArgs) => {
|
||||
const params = rawArgs as Record<string, unknown>;
|
||||
const action = readStringParam(params, "action", { required: true });
|
||||
const payload: UiCommandParams = {
|
||||
command: commandForAction(action, params, opts.agentSessionKey),
|
||||
...(opts.agentSessionKey ? { sessionKey: opts.agentSessionKey } : {}),
|
||||
};
|
||||
return jsonResult(await gatewayCall("ui.command", payload));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -365,6 +365,7 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
|
||||
// Memory migration reads host assistant state and writes agent workspaces.
|
||||
{ name: "migrations.memory.plan", scope: "operator.admin" },
|
||||
{ name: "migrations.memory.apply", scope: "operator.admin", controlPlaneWrite: true },
|
||||
{ name: "ui.command", scope: "operator.write" },
|
||||
] as const;
|
||||
|
||||
const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap<string, CoreGatewayMethodSpec> = new Map(
|
||||
|
||||
@@ -26,6 +26,7 @@ import { logWs, shouldLogWs, summarizeAgentEventForWsLog } from "./ws-log.js";
|
||||
const EVENT_SCOPE_GUARDS: Record<string, string[]> = {
|
||||
agent: [READ_SCOPE],
|
||||
chat: [READ_SCOPE],
|
||||
"ui.command": [READ_SCOPE],
|
||||
"chat.send_timing": [READ_SCOPE],
|
||||
"chat.side_result": [READ_SCOPE],
|
||||
cron: [READ_SCOPE],
|
||||
|
||||
@@ -42,10 +42,11 @@ describe("listGatewayMethods", () => {
|
||||
});
|
||||
|
||||
it("appends memory migration after model probing without shifting older method indices", () => {
|
||||
expect(listGatewayMethods().slice(-3)).toEqual([
|
||||
expect(listGatewayMethods().slice(-4)).toEqual([
|
||||
"models.probe",
|
||||
"migrations.memory.plan",
|
||||
"migrations.memory.apply",
|
||||
"ui.command",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -95,7 +96,6 @@ describe("listGatewayMethods", () => {
|
||||
]);
|
||||
expect(methods).toContain("tts.speak");
|
||||
expect(coreMethods.slice(-10)).toEqual([
|
||||
"sessions.catalog.continue",
|
||||
"sessions.catalog.archive",
|
||||
"approval.get",
|
||||
"approval.resolve",
|
||||
@@ -105,6 +105,7 @@ describe("listGatewayMethods", () => {
|
||||
"models.probe",
|
||||
"migrations.memory.plan",
|
||||
"migrations.memory.apply",
|
||||
"ui.command",
|
||||
]);
|
||||
expect(methods.indexOf("approval.get")).toBeGreaterThan(methods.indexOf("tts.speak"));
|
||||
expect(methods.indexOf("approval.resolve")).toBe(methods.indexOf("approval.get") + 1);
|
||||
|
||||
@@ -40,6 +40,7 @@ export const GATEWAY_EVENTS = [
|
||||
"connect.challenge",
|
||||
"agent",
|
||||
"chat",
|
||||
"ui.command",
|
||||
"session.approval",
|
||||
"session.message",
|
||||
"session.operation",
|
||||
|
||||
@@ -132,6 +132,10 @@ const loadTerminalHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/terminal.js"),
|
||||
(module) => module.terminalHandlers,
|
||||
);
|
||||
const loadUiCommandHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/ui-command.js"),
|
||||
(module) => module.uiCommandHandlers,
|
||||
);
|
||||
const loadModelsAuthStatusHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/models-auth-status.js"),
|
||||
(module) => module.modelsAuthStatusHandlers,
|
||||
@@ -344,6 +348,10 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
|
||||
],
|
||||
loadHandlers: loadTerminalHandlers,
|
||||
}),
|
||||
...createLazyCoreHandlers({
|
||||
methods: ["ui.command"],
|
||||
loadHandlers: loadUiCommandHandlers,
|
||||
}),
|
||||
...createLazyCoreHandlers({
|
||||
methods: ["voicewake.get", "voicewake.set"],
|
||||
loadHandlers: loadVoicewakeHandlers,
|
||||
|
||||
83
src/gateway/server-methods/ui-command.test.ts
Normal file
83
src/gateway/server-methods/ui-command.test.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
GATEWAY_CLIENT_CAPS,
|
||||
GATEWAY_CLIENT_IDS,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import type { GatewayClient } from "./types.js";
|
||||
import { uiCommandHandlers } from "./ui-command.js";
|
||||
|
||||
function client(connId: string, id: string, caps: string[] = []): GatewayClient {
|
||||
return {
|
||||
connId,
|
||||
connect: {
|
||||
client: { id, version: "test", platform: "web", mode: "ui" },
|
||||
caps,
|
||||
},
|
||||
} as GatewayClient;
|
||||
}
|
||||
|
||||
async function call(params: unknown, clients: GatewayClient[]) {
|
||||
const respond = vi.fn();
|
||||
const broadcastToConnIds = vi.fn();
|
||||
await expectDefined(
|
||||
uiCommandHandlers["ui.command"],
|
||||
"ui.command",
|
||||
)({
|
||||
params,
|
||||
respond,
|
||||
context: {
|
||||
broadcastToConnIds,
|
||||
getClientConnIds: (filter?: (client: GatewayClient) => boolean) =>
|
||||
new Set(
|
||||
clients
|
||||
.filter((entry) => filter?.(entry) !== false)
|
||||
.flatMap((entry) => (entry.connId ? [entry.connId] : [])),
|
||||
),
|
||||
},
|
||||
} as never);
|
||||
return { respond, broadcastToConnIds };
|
||||
}
|
||||
|
||||
describe("ui.command gateway method", () => {
|
||||
it("rejects invalid params", async () => {
|
||||
const result = await call({ command: { kind: "sidebar", visible: "yes" } }, []);
|
||||
|
||||
expect(result.respond).toHaveBeenCalledWith(false, undefined, expect.any(Object));
|
||||
expect(result.broadcastToConnIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports when no capable Control UI client is connected", async () => {
|
||||
const result = await call({ command: { kind: "sidebar", visible: true } }, [
|
||||
client("legacy-ui", GATEWAY_CLIENT_IDS.CONTROL_UI),
|
||||
client("capable-cli", GATEWAY_CLIENT_IDS.CLI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]),
|
||||
]);
|
||||
|
||||
expect(result.respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ message: "no ui client" }),
|
||||
);
|
||||
expect(result.broadcastToConnIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("delivers only to capable Control UI connections", async () => {
|
||||
const params = {
|
||||
command: { kind: "split", direction: "right", sessionKey: "agent:main:other" },
|
||||
sessionKey: "agent:main:main",
|
||||
};
|
||||
const result = await call(params, [
|
||||
client("ui-one", GATEWAY_CLIENT_IDS.CONTROL_UI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]),
|
||||
client("legacy-ui", GATEWAY_CLIENT_IDS.CONTROL_UI),
|
||||
client("ui-two", GATEWAY_CLIENT_IDS.CONTROL_UI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]),
|
||||
client("cli", GATEWAY_CLIENT_IDS.CLI, [GATEWAY_CLIENT_CAPS.UI_COMMANDS]),
|
||||
]);
|
||||
|
||||
expect(result.broadcastToConnIds).toHaveBeenCalledWith(
|
||||
"ui.command",
|
||||
params,
|
||||
new Set(["ui-one", "ui-two"]),
|
||||
);
|
||||
expect(result.respond).toHaveBeenCalledWith(true, { ok: true });
|
||||
});
|
||||
});
|
||||
47
src/gateway/server-methods/ui-command.ts
Normal file
47
src/gateway/server-methods/ui-command.ts
Normal file
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
GATEWAY_CLIENT_CAPS,
|
||||
GATEWAY_CLIENT_IDS,
|
||||
hasGatewayClientCap,
|
||||
} from "../../../packages/gateway-protocol/src/client-info.js";
|
||||
import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
formatValidationErrors,
|
||||
type UiCommandParams,
|
||||
validateUiCommandParams,
|
||||
} from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { GatewayRequestContextWithClientLookup } from "../server-request-context.js";
|
||||
import type { GatewayRequestHandlers } from "./types.js";
|
||||
|
||||
export const uiCommandHandlers: GatewayRequestHandlers = {
|
||||
"ui.command": ({ params, respond, context }) => {
|
||||
if (!validateUiCommandParams(params)) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`invalid ui.command params: ${formatValidationErrors(validateUiCommandParams.errors)}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const commandParams = params as UiCommandParams;
|
||||
const clientContext = context as GatewayRequestContextWithClientLookup;
|
||||
// v1 intentionally fans out to every capable Control UI; session-targeted routing is out of scope.
|
||||
const connIds =
|
||||
clientContext.getClientConnIds?.(
|
||||
(client) =>
|
||||
client.connect.client.id === GATEWAY_CLIENT_IDS.CONTROL_UI &&
|
||||
hasGatewayClientCap(client.connect.caps, GATEWAY_CLIENT_CAPS.UI_COMMANDS),
|
||||
) ?? new Set<string>();
|
||||
if (connIds.size === 0) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, "no ui client"));
|
||||
return;
|
||||
}
|
||||
|
||||
context.broadcastToConnIds("ui.command", commandParams, connIds);
|
||||
respond(true, { ok: true });
|
||||
},
|
||||
};
|
||||
@@ -136,9 +136,13 @@ function canDeliverApprovals(
|
||||
);
|
||||
}
|
||||
|
||||
export type GatewayRequestContextWithClientLookup = GatewayRequestContext & {
|
||||
getClientConnIds?: (filter?: (client: GatewayClient) => boolean) => ReadonlySet<string>;
|
||||
};
|
||||
|
||||
export function createGatewayRequestContext(
|
||||
params: GatewayRequestContextParams,
|
||||
): GatewayRequestContext {
|
||||
): GatewayRequestContextWithClientLookup {
|
||||
return {
|
||||
deps: params.deps,
|
||||
// Keep cron reads live so config hot reload can swap cron/store state without rebuilding
|
||||
@@ -207,6 +211,19 @@ export function createGatewayRequestContext(
|
||||
}
|
||||
return connIds;
|
||||
},
|
||||
getClientConnIds: (filter) => {
|
||||
const connIds = new Set<string>();
|
||||
for (const gatewayClient of params.clients) {
|
||||
if (!gatewayClient.connId || gatewayClient.invalidated) {
|
||||
continue;
|
||||
}
|
||||
if (filter && !filter(gatewayClient)) {
|
||||
continue;
|
||||
}
|
||||
connIds.add(gatewayClient.connId);
|
||||
}
|
||||
return connIds;
|
||||
},
|
||||
hasConnectedClientsForDevice: (deviceId: string) => {
|
||||
for (const gatewayClient of params.clients) {
|
||||
if (gatewayClient.connect.device?.id === deviceId && !gatewayClient.invalidated) {
|
||||
|
||||
@@ -154,6 +154,7 @@ describe("resolveGatewayScopedTools excludeToolNames", () => {
|
||||
"cron",
|
||||
"gateway",
|
||||
"sessions",
|
||||
"screen",
|
||||
"nodes",
|
||||
"computer",
|
||||
"openclaw",
|
||||
@@ -162,6 +163,7 @@ describe("resolveGatewayScopedTools excludeToolNames", () => {
|
||||
"cron",
|
||||
"gateway",
|
||||
"sessions",
|
||||
"screen",
|
||||
"nodes",
|
||||
"computer",
|
||||
"openclaw",
|
||||
|
||||
@@ -50,6 +50,7 @@ export const GATEWAY_CONTROL_PLANE_TOOLS = ["cron", "gateway"] as const;
|
||||
export const GATEWAY_OWNER_ONLY_CORE_TOOLS = [
|
||||
...GATEWAY_CONTROL_PLANE_TOOLS,
|
||||
"sessions",
|
||||
"screen",
|
||||
"nodes",
|
||||
"computer",
|
||||
"openclaw",
|
||||
|
||||
@@ -428,6 +428,7 @@ describe("GatewayBrowserClient", () => {
|
||||
GATEWAY_CLIENT_CAPS.TERMINAL_OFFSET_SEQ,
|
||||
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
|
||||
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
|
||||
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
|
||||
]);
|
||||
expect(connectFrame.params?.scopes).toEqual([...CONTROL_UI_OPERATOR_SCOPES]);
|
||||
});
|
||||
|
||||
@@ -462,6 +462,7 @@ export class GatewayBrowserClient {
|
||||
GATEWAY_CLIENT_CAPS.TERMINAL_OFFSET_SEQ,
|
||||
GATEWAY_CLIENT_CAPS.TOOL_EVENTS,
|
||||
GATEWAY_CLIENT_CAPS.INLINE_WIDGETS,
|
||||
GATEWAY_CLIENT_CAPS.UI_COMMANDS,
|
||||
],
|
||||
auth: buildGatewayConnectAuth(selectedAuth),
|
||||
userAgent: navigator.userAgent,
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { GatewayBrowserClient } from "../api/gateway.ts";
|
||||
import {
|
||||
BROWSER_PANEL_TOGGLE_EVENT,
|
||||
TERMINAL_PANEL_TOGGLE_EVENT,
|
||||
UI_COMMAND_EVENT,
|
||||
} from "../components/panel-toggle-contract.ts";
|
||||
import "./app-host.ts";
|
||||
import type {
|
||||
@@ -57,6 +58,10 @@ type ShellLazySurfaceState = ShellKeyboardState & {
|
||||
terminalPanelElement: TestOptionalCustomElement;
|
||||
};
|
||||
|
||||
type ShellUiCommandState = ShellKeyboardState & {
|
||||
handleGatewayEvent: (event: { event: string; payload: unknown }) => void;
|
||||
};
|
||||
|
||||
let lazyElementSequence = 0;
|
||||
|
||||
function createLazyElementSpec(label: string): TestOptionalCustomElement {
|
||||
@@ -404,6 +409,65 @@ describe("OpenClaw shell keyboard shortcuts", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes UI commands to navigation, panels, and chat fallback", () => {
|
||||
const update = vi.fn();
|
||||
const setSessionKey = vi.fn();
|
||||
const navigate = vi.fn();
|
||||
const panelEvent = vi.fn();
|
||||
const uiCommandEvent = vi.fn();
|
||||
window.addEventListener(TERMINAL_PANEL_TOGGLE_EVENT, panelEvent);
|
||||
window.addEventListener(UI_COMMAND_EVENT, uiCommandEvent);
|
||||
const shell = document.createElement("openclaw-app-shell") as unknown as ShellUiCommandState;
|
||||
shell.runtime = {
|
||||
context: {
|
||||
navigation: { update },
|
||||
gateway: { setSessionKey },
|
||||
navigate,
|
||||
} as unknown as ApplicationContext,
|
||||
};
|
||||
|
||||
shell.handleGatewayEvent({
|
||||
event: "ui.command",
|
||||
payload: { command: { kind: "sidebar", visible: false } },
|
||||
});
|
||||
shell.handleGatewayEvent({
|
||||
event: "ui.command",
|
||||
payload: {
|
||||
command: { kind: "panel", panel: "terminal", open: false, dock: "right" },
|
||||
},
|
||||
});
|
||||
shell.handleGatewayEvent({
|
||||
event: "ui.command",
|
||||
payload: {
|
||||
command: { kind: "split", direction: "right", sessionKey: "agent:main:other" },
|
||||
},
|
||||
});
|
||||
shell.handleGatewayEvent({
|
||||
event: "ui.command",
|
||||
payload: {
|
||||
command: { kind: "focus", sessionKey: "agent:main:other" },
|
||||
sessionKey: "agent:main:source",
|
||||
},
|
||||
});
|
||||
|
||||
expect(update).toHaveBeenCalledWith({ navCollapsed: true });
|
||||
expect(panelEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ detail: { open: false, dock: "right" } }),
|
||||
);
|
||||
expect(setSessionKey).toHaveBeenCalledWith("agent:main:other");
|
||||
expect(navigate).toHaveBeenCalledWith("chat", { search: "?session=agent%3Amain%3Aother" });
|
||||
expect(uiCommandEvent).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
detail: {
|
||||
command: { kind: "focus", sessionKey: "agent:main:other" },
|
||||
sessionKey: "agent:main:source",
|
||||
},
|
||||
}),
|
||||
);
|
||||
window.removeEventListener(TERMINAL_PANEL_TOGGLE_EVENT, panelEvent);
|
||||
window.removeEventListener(UI_COMMAND_EVENT, uiCommandEvent);
|
||||
});
|
||||
|
||||
it("opens Settings with Shift-Command-Comma", () => {
|
||||
const navigate = vi.fn();
|
||||
const shell = document.createElement("openclaw-app-shell") as unknown as ShellKeyboardState;
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { ContextProvider } from "@lit/context";
|
||||
import type { UiCommandParams } from "@openclaw/gateway-protocol";
|
||||
import type { RouteLocation, RouterState } from "@openclaw/uirouter";
|
||||
import { html, nothing } from "lit";
|
||||
import { property, query, state } from "lit/decorators.js";
|
||||
import { hasStoredGatewayAuth, type GatewayBrowserClient } from "../api/gateway.ts";
|
||||
import {
|
||||
type GatewayEventFrame,
|
||||
hasStoredGatewayAuth,
|
||||
type GatewayBrowserClient,
|
||||
} from "../api/gateway.ts";
|
||||
import "../components/app-sidebar.ts";
|
||||
import "../components/app-topbar.ts";
|
||||
import "../components/connection-banner.ts";
|
||||
@@ -28,6 +33,7 @@ import {
|
||||
BROWSER_PANEL_TOGGLE_EVENT,
|
||||
isTerminalPanelShortcut,
|
||||
TERMINAL_PANEL_TOGGLE_EVENT,
|
||||
UI_COMMAND_EVENT,
|
||||
type PanelToggleElement,
|
||||
} from "../components/panel-toggle-contract.ts";
|
||||
import { renderSettingsSidebar } from "../components/settings-sidebar.ts";
|
||||
@@ -490,6 +496,10 @@ class OpenClawShell extends OpenClawLightDomElement {
|
||||
(gateway, notify) => gateway.subscribe(notify),
|
||||
(gateway) => this.synchronizeGateway(gateway.snapshot),
|
||||
)
|
||||
.effect(
|
||||
() => this.context?.gateway,
|
||||
(gateway) => gateway.subscribeEvents(this.handleGatewayEvent),
|
||||
)
|
||||
.watch(
|
||||
() => this.context?.config,
|
||||
(config, notify) => config.subscribe(notify),
|
||||
@@ -586,6 +596,43 @@ class OpenClawShell extends OpenClawLightDomElement {
|
||||
this.settingsPreloadTimers.clear();
|
||||
}
|
||||
|
||||
private readonly handleGatewayEvent = (event: GatewayEventFrame) => {
|
||||
if (event.event !== "ui.command" || !event.payload) {
|
||||
return;
|
||||
}
|
||||
const context = this.context;
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
const commandParams = event.payload as UiCommandParams;
|
||||
const { command } = commandParams;
|
||||
if (!command) {
|
||||
return;
|
||||
}
|
||||
if (command.kind === "sidebar") {
|
||||
context.navigation.update({ navCollapsed: !command.visible });
|
||||
return;
|
||||
}
|
||||
if (command.kind === "panel") {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent(
|
||||
command.panel === "terminal" ? TERMINAL_PANEL_TOGGLE_EVENT : BROWSER_PANEL_TOGGLE_EVENT,
|
||||
{ detail: { open: command.open, ...(command.dock ? { dock: command.dock } : {}) } },
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const handled = !window.dispatchEvent(
|
||||
new CustomEvent(UI_COMMAND_EVENT, { detail: commandParams, cancelable: true }),
|
||||
);
|
||||
if (handled || (command.kind !== "navigate" && command.kind !== "split")) {
|
||||
return;
|
||||
}
|
||||
context.gateway.setSessionKey(command.sessionKey);
|
||||
this.navigate("chat", { search: searchForSession(command.sessionKey) });
|
||||
};
|
||||
|
||||
private readonly handleThemeChange = (event: CustomEvent<ThemeModeChangeDetail>) => {
|
||||
const context = this.context;
|
||||
if (!context) {
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createStorageMock } from "../../test-helpers/storage.ts";
|
||||
import "./browser-panel.ts";
|
||||
import { normalizeBrowserUrlDraft } from "./browser-url.ts";
|
||||
|
||||
describe("normalizeBrowserUrlDraft", () => {
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("localStorage", createStorageMock());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
document.body.replaceChildren();
|
||||
localStorage.clear();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
it("prefixes bare hosts with https", () => {
|
||||
expect(normalizeBrowserUrlDraft("example.com")).toBe("https://example.com/");
|
||||
@@ -50,4 +55,20 @@ describe("normalizeBrowserUrlDraft", () => {
|
||||
await panel.updateComplete;
|
||||
expect((panel as unknown as { open: boolean }).open).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps an already closed panel closed for an explicit close request", () => {
|
||||
const panel = document.createElement("openclaw-browser-panel") as unknown as HTMLElement & {
|
||||
available: boolean;
|
||||
open: boolean;
|
||||
handleToggleRequest: (event: Event) => void;
|
||||
};
|
||||
panel.available = true;
|
||||
document.body.append(panel);
|
||||
|
||||
panel.handleToggleRequest(
|
||||
new CustomEvent("openclaw:browser-toggle", { detail: { open: false } }),
|
||||
);
|
||||
|
||||
expect(panel.open).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -284,6 +284,10 @@ class OpenClawBrowserPanel extends OpenClawLitElement {
|
||||
if (detail?.dock === "right" || detail?.dock === "bottom") {
|
||||
this.dock = detail.dock;
|
||||
}
|
||||
if (detail?.open === false) {
|
||||
this.closePanel();
|
||||
return;
|
||||
}
|
||||
const url = typeof detail?.url === "string" ? normalizeBrowserUrlDraft(detail.url) : null;
|
||||
if (url || detail?.open === true) {
|
||||
if (!this.available) {
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import type { UiCommandParams } from "@openclaw/gateway-protocol";
|
||||
|
||||
export const TERMINAL_PANEL_TOGGLE_EVENT = "openclaw:terminal-toggle";
|
||||
export const BROWSER_PANEL_TOGGLE_EVENT = "openclaw:browser-toggle";
|
||||
export const UI_COMMAND_EVENT = "openclaw:ui-command";
|
||||
|
||||
export type UiCommandDetail = UiCommandParams;
|
||||
|
||||
export type TerminalPanelToggleDetail = {
|
||||
dock?: "bottom" | "right";
|
||||
|
||||
@@ -70,6 +70,18 @@ describe("terminal panel readiness", () => {
|
||||
await i18n.setLocale("en");
|
||||
});
|
||||
|
||||
it("keeps an already closed panel closed for an explicit close request", () => {
|
||||
const panel = document.createElement(TERMINAL_PANEL_ELEMENT_NAME) as OpenClawTerminalPanel;
|
||||
panel.available = true;
|
||||
document.body.append(panel);
|
||||
|
||||
panel.handleToggleRequest(
|
||||
new CustomEvent("openclaw:terminal-toggle", { detail: { open: false } }),
|
||||
);
|
||||
|
||||
expect((panel as unknown as { open: boolean }).open).toBe(false);
|
||||
});
|
||||
|
||||
it("shows a connecting animation while a terminal open is in flight", async () => {
|
||||
const open = deferred<{
|
||||
sessionId: string;
|
||||
|
||||
@@ -350,6 +350,10 @@ export class OpenClawTerminalPanel extends OpenClawLitElement {
|
||||
if (dock) {
|
||||
this.dock = dock;
|
||||
}
|
||||
if (detail?.open === false) {
|
||||
this.closePanel();
|
||||
return;
|
||||
}
|
||||
if (detail?.catalog || detail?.open === true) {
|
||||
if (!this.available) {
|
||||
return;
|
||||
|
||||
@@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
vi.mock("./chat-pane.ts", () => ({}));
|
||||
|
||||
import { loadSettings } from "../../app/settings.ts";
|
||||
import { UI_COMMAND_EVENT } from "../../components/panel-toggle-contract.ts";
|
||||
import { SESSION_DRAG_MIME } from "../../lib/sessions/drag.ts";
|
||||
import { searchForSession } from "../../lib/sessions/index.ts";
|
||||
import { createStorageMock } from "../../test-helpers/storage.ts";
|
||||
@@ -183,6 +184,66 @@ describe("chat page split layout host", () => {
|
||||
expect(survivingPane.classList.contains("chat-split-view__pane")).toBe(false);
|
||||
});
|
||||
|
||||
it("applies mounted UI split, focus, and close commands", () => {
|
||||
const page = new ChatPage();
|
||||
page.data = { sessionKey: "main" };
|
||||
const navigation = setNavigationContext(page);
|
||||
document.body.append(page);
|
||||
|
||||
const split = new CustomEvent(UI_COMMAND_EVENT, {
|
||||
detail: {
|
||||
command: { kind: "split", direction: "right", sessionKey: "agent:main:work" },
|
||||
sessionKey: "main",
|
||||
},
|
||||
cancelable: true,
|
||||
});
|
||||
window.dispatchEvent(split);
|
||||
expect(split.defaultPrevented).toBe(true);
|
||||
expect(getLayout(page)?.columns.at(1)?.panes.at(0)?.sessionKey).toBe("agent:main:work");
|
||||
expect(navigation.replace).toHaveBeenLastCalledWith("chat", {
|
||||
search: searchForSession("agent:main:work"),
|
||||
});
|
||||
|
||||
window.dispatchEvent(
|
||||
new CustomEvent(UI_COMMAND_EVENT, {
|
||||
detail: { command: { kind: "focus", sessionKey: "main" }, sessionKey: "main" },
|
||||
cancelable: true,
|
||||
}),
|
||||
);
|
||||
expect(getLayout(page)?.activePaneId).toBe("p1");
|
||||
|
||||
window.dispatchEvent(
|
||||
new CustomEvent(UI_COMMAND_EVENT, {
|
||||
detail: {
|
||||
command: { kind: "close-pane", sessionKey: "agent:main:work" },
|
||||
sessionKey: "main",
|
||||
},
|
||||
cancelable: true,
|
||||
}),
|
||||
);
|
||||
expect(getLayout(page)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("leaves UI split commands unhandled on narrow viewports", () => {
|
||||
stubMatchMedia(true);
|
||||
const page = new ChatPage();
|
||||
page.data = { sessionKey: "main" };
|
||||
setNavigationContext(page);
|
||||
document.body.append(page);
|
||||
|
||||
const split = new CustomEvent(UI_COMMAND_EVENT, {
|
||||
detail: {
|
||||
command: { kind: "split", direction: "right", sessionKey: "agent:main:work" },
|
||||
sessionKey: "main",
|
||||
},
|
||||
cancelable: true,
|
||||
});
|
||||
window.dispatchEvent(split);
|
||||
// Unhandled so the app host falls back to navigating to the session.
|
||||
expect(split.defaultPrevented).toBe(false);
|
||||
expect(getLayout(page)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("withholds the split-view opener on narrow single-pane viewports", async () => {
|
||||
stubMatchMedia(true);
|
||||
const page = new ChatPage();
|
||||
|
||||
@@ -6,6 +6,7 @@ import { repeat } from "lit/directives/repeat.js";
|
||||
import { applicationContext, type ApplicationContext } from "../../app/context.ts";
|
||||
import { loadSettings, patchSettings } from "../../app/settings.ts";
|
||||
import "../../components/resizable-divider.ts";
|
||||
import { UI_COMMAND_EVENT, type UiCommandDetail } from "../../components/panel-toggle-contract.ts";
|
||||
import { t } from "../../i18n/index.ts";
|
||||
import { resolveSessionDisplayName } from "../../lib/session-display.ts";
|
||||
import { readSessionDragData, sessionDragActive } from "../../lib/sessions/drag.ts";
|
||||
@@ -22,6 +23,7 @@ import {
|
||||
type SplitDropZone,
|
||||
} from "./split-drop-zone.ts";
|
||||
import {
|
||||
applyUiCommandToSplitLayout,
|
||||
closePane,
|
||||
findPane,
|
||||
insertPane,
|
||||
@@ -88,6 +90,7 @@ export class ChatPage extends OpenClawLightDomElement {
|
||||
this.addEventListener("dragleave", this.handleDragLeave);
|
||||
this.addEventListener("drop", this.handleDrop);
|
||||
window.addEventListener("dragend", this.handleWindowDragEnd);
|
||||
window.addEventListener(UI_COMMAND_EVENT, this.handleUiCommand);
|
||||
this.syncRouteToActivePane();
|
||||
}
|
||||
|
||||
@@ -100,6 +103,7 @@ export class ChatPage extends OpenClawLightDomElement {
|
||||
this.removeEventListener("dragleave", this.handleDragLeave);
|
||||
this.removeEventListener("drop", this.handleDrop);
|
||||
window.removeEventListener("dragend", this.handleWindowDragEnd);
|
||||
window.removeEventListener(UI_COMMAND_EVENT, this.handleUiCommand);
|
||||
this.clearDropIndicator();
|
||||
super.disconnectedCallback();
|
||||
}
|
||||
@@ -133,6 +137,61 @@ export class ChatPage extends OpenClawLightDomElement {
|
||||
}
|
||||
};
|
||||
|
||||
private readonly handleUiCommand = (event: Event) => {
|
||||
if (!(event instanceof CustomEvent)) {
|
||||
return;
|
||||
}
|
||||
const { command, sessionKey: sourceSessionKey } = event.detail as UiCommandDetail;
|
||||
if (command.kind === "navigate") {
|
||||
event.preventDefault();
|
||||
this.updateRoute(command.sessionKey);
|
||||
return;
|
||||
}
|
||||
if (command.kind !== "split" && command.kind !== "close-pane" && command.kind !== "focus") {
|
||||
return;
|
||||
}
|
||||
// Narrow viewports render single-pane and disable interactive splitting;
|
||||
// leave programmatic splits unhandled so the host falls back to navigation.
|
||||
if (command.kind === "split" && this.narrow) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentSessionKey = this.data?.sessionKey?.trim();
|
||||
const layout =
|
||||
this.layout ??
|
||||
(command.kind === "split" && currentSessionKey
|
||||
? this.classicLayout(currentSessionKey)
|
||||
: undefined);
|
||||
if (!layout) {
|
||||
return;
|
||||
}
|
||||
const targetPane =
|
||||
command.kind === "split"
|
||||
? undefined
|
||||
: panesOf(layout).find((pane) => pane.sessionKey === command.sessionKey);
|
||||
const survivingPane =
|
||||
command.kind === "close-pane" && targetPane
|
||||
? panesOf(layout).find((pane) => pane.id !== targetPane.id)
|
||||
: undefined;
|
||||
const next = applyUiCommandToSplitLayout(layout, command, sourceSessionKey);
|
||||
if (next === layout) {
|
||||
return;
|
||||
}
|
||||
event.preventDefault();
|
||||
if (!next && survivingPane) {
|
||||
const survivingLocation = findPane(layout, survivingPane.id);
|
||||
if (survivingLocation) {
|
||||
this.classicColumnId = survivingLocation.column.id;
|
||||
this.classicPaneId = survivingPane.id;
|
||||
}
|
||||
}
|
||||
this.persistLayout(next);
|
||||
const activePane = next ? findPane(next, next.activePaneId)?.pane : survivingPane;
|
||||
if (activePane) {
|
||||
this.updateRoute(activePane.sessionKey, true);
|
||||
}
|
||||
};
|
||||
|
||||
private readonly handleDragEnter = (event: DragEvent) => {
|
||||
if (this.narrow || !sessionDragActive(event.dataTransfer)) {
|
||||
return;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyUiCommandToSplitLayout,
|
||||
closePane,
|
||||
findPane,
|
||||
insertPane,
|
||||
@@ -129,6 +130,37 @@ describe("chat split layout", () => {
|
||||
expect(panesOf(layout)).not.toBe(layout.columns.at(0)?.panes);
|
||||
});
|
||||
|
||||
it("maps UI split, focus, and close commands onto pane state", () => {
|
||||
const initial = setActivePane(setPaneSession(createSplitLayout("main"), "p2", "source"), "p1");
|
||||
const split = applyUiCommandToSplitLayout(
|
||||
initial,
|
||||
{
|
||||
kind: "split",
|
||||
direction: "down",
|
||||
sessionKey: "agent:main:new",
|
||||
},
|
||||
"source",
|
||||
);
|
||||
expect(split && panesOf(split).map((pane) => pane.sessionKey)).toEqual([
|
||||
"main",
|
||||
"source",
|
||||
"agent:main:new",
|
||||
]);
|
||||
expect(split?.activePaneId).toBe("p3");
|
||||
|
||||
const focused = applyUiCommandToSplitLayout(split!, {
|
||||
kind: "focus",
|
||||
sessionKey: "main",
|
||||
});
|
||||
expect(focused?.activePaneId).toBe("p1");
|
||||
|
||||
const closed = applyUiCommandToSplitLayout(focused!, {
|
||||
kind: "close-pane",
|
||||
sessionKey: "agent:main:new",
|
||||
});
|
||||
expect(closed && panesOf(closed).map((pane) => pane.sessionKey)).toEqual(["main", "source"]);
|
||||
});
|
||||
|
||||
it("resizes only a boundary pair and clamps each side to fifteen percent", () => {
|
||||
const layout = insertPane(createSplitLayout("main"), "p1", "third", "right");
|
||||
const columns = resizeColumns(layout, 0, 0.8);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { UiCommand } from "@openclaw/gateway-protocol";
|
||||
import { expectDefined, isRecord } from "@openclaw/normalization-core";
|
||||
|
||||
export type ChatSplitPane = { id: string; sessionKey: string };
|
||||
@@ -180,6 +181,36 @@ export function setActivePane(layout: ChatSplitLayout, paneId: string): ChatSpli
|
||||
return next;
|
||||
}
|
||||
|
||||
type UiSplitLayoutCommand = Extract<UiCommand, { kind: "split" | "close-pane" | "focus" }>;
|
||||
|
||||
export function applyUiCommandToSplitLayout(
|
||||
layout: ChatSplitLayout,
|
||||
command: UiSplitLayoutCommand,
|
||||
sourceSessionKey?: string,
|
||||
): ChatSplitLayout | undefined {
|
||||
if (command.kind === "split") {
|
||||
const sourcePane = sourceSessionKey
|
||||
? panesOf(layout).find((entry) => entry.sessionKey === sourceSessionKey)
|
||||
: undefined;
|
||||
if (sourceSessionKey && !sourcePane) {
|
||||
return layout;
|
||||
}
|
||||
return insertPane(
|
||||
layout,
|
||||
sourcePane?.id ?? layout.activePaneId,
|
||||
command.sessionKey,
|
||||
command.direction,
|
||||
);
|
||||
}
|
||||
const pane = panesOf(layout).find((entry) => entry.sessionKey === command.sessionKey);
|
||||
if (!pane) {
|
||||
return layout;
|
||||
}
|
||||
return command.kind === "close-pane"
|
||||
? closePane(layout, pane.id)
|
||||
: setActivePane(layout, pane.id);
|
||||
}
|
||||
|
||||
function resizePair(weights: number[], boundaryIndex: number, pairRatio: number): number[] {
|
||||
const next = [...weights];
|
||||
if (boundaryIndex < 0 || boundaryIndex + 1 >= weights.length) {
|
||||
|
||||
Reference in New Issue
Block a user