From da44d52ac6c068d7fe79b6cbb7b1bc8a2de3d752 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 22:24:17 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20ask=5Fuser=20=E2=80=94=20structured=20q?= =?UTF-8?q?uestions=20from=20the=20agent=20with=20web=20card,=20channel=20?= =?UTF-8?q?buttons,=20and=20text=20answers=20(#109922)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gateway): add transient question runtime (question.* methods + broadcasts) * feat(agents): add blocking ask_user question tool with chat prompt delivery and text-reply claim * feat(ui): interactive in-thread question cards for ask_user * feat(channels): native tap-to-answer buttons for ask_user on Telegram, Discord, and Slack * feat(ui): unify codex and gateway question cards with interactive gateway answering * refactor(agents): collapse ask_user pending state to one registry; docs for ask_user * fix(agents): include ask_user in normal gateway runs; add question-flow control-ui e2e * test(ui): avoid credential-shaped fixture in question card test * refactor(ui): reorder stream-group context keys * fix(gateway,ui): validate question answers at resolve; reject secret/duplicate-label questions; UI retry and reconnect hardening * fix(gateway,agents): canonicalize accepted option answers; bound ask_user option labels to 64 chars * chore(ci): prune unused question exports, allowlist mobile question events, fix discord lint * chore(ci): regenerate protocol/i18n/docs/tool-display artifacts for question surface * fix(protocol): flatten QuestionRecord for native codegen; drop TS-only alias from schema registry * chore(android): regenerate ask-user localization resources * docs: regenerate docs map after rebase * fix(ci): avoid stale read-only dependency disks * test: remove stale reef lint suppression ratchet * fix(ci): keep source locale drift advisory in release gates * fix(ci): scope locale advisory handling to parity check --- .github/actions/setup-node-env/action.yml | 33 +- .github/workflows/ci.yml | 6 +- .../openclaw/app/gateway/GatewayProtocol.kt | 7 + .../app/i18n/NativeStringResources.kt | 1 + .../app/src/main/res/values-ar/strings.xml | 1 + .../app/src/main/res/values-de/strings.xml | 1 + .../app/src/main/res/values-es/strings.xml | 1 + .../app/src/main/res/values-fa/strings.xml | 1 + .../app/src/main/res/values-fr/strings.xml | 1 + .../app/src/main/res/values-hi/strings.xml | 1 + .../app/src/main/res/values-in/strings.xml | 1 + .../app/src/main/res/values-it/strings.xml | 1 + .../app/src/main/res/values-ja/strings.xml | 1 + .../app/src/main/res/values-ko/strings.xml | 1 + .../app/src/main/res/values-nl/strings.xml | 1 + .../app/src/main/res/values-pl/strings.xml | 1 + .../src/main/res/values-pt-rBR/strings.xml | 1 + .../app/src/main/res/values-ru/strings.xml | 1 + .../app/src/main/res/values-sv/strings.xml | 1 + .../app/src/main/res/values-th/strings.xml | 1 + .../app/src/main/res/values-tr/strings.xml | 1 + .../app/src/main/res/values-uk/strings.xml | 1 + .../app/src/main/res/values-vi/strings.xml | 1 + .../src/main/res/values-zh-rCN/strings.xml | 1 + .../src/main/res/values-zh-rTW/strings.xml | 1 + .../app/src/main/res/values/strings.xml | 1 + .../OpenClawKit/Resources/tool-display.json | 7 + .../OpenClawProtocol/GatewayModels.swift | 271 +++++++ .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/ci.md | 2 +- docs/docs.json | 1 + docs/docs_map.md | 9 + docs/plugins/message-presentation.md | 11 + docs/plugins/sdk-subpaths.md | 1 + docs/tools/ask-user.md | 91 +++ docs/tools/index.md | 1 + .../src/monitor/provider.interactions.ts | 15 + .../discord/src/monitor/questions.test.ts | 94 +++ extensions/discord/src/monitor/questions.ts | 101 +++ .../discord/src/question-custom-id.test.ts | 23 + extensions/discord/src/question-custom-id.ts | 35 + .../discord/src/shared-interactive.test.ts | 31 + extensions/discord/src/shared-interactive.ts | 18 +- extensions/slack/src/blocks-render.ts | 38 +- .../events/interactions.block-actions.ts | 21 + .../src/monitor/events/interactions.test.ts | 57 ++ extensions/slack/src/question-actions.test.ts | 61 ++ extensions/slack/src/question-actions.ts | 67 ++ extensions/slack/src/reply-action-ids.ts | 8 + .../slack/src/shared-interactive.test.ts | 32 + .../src/bot-handlers.authorization.runtime.ts | 12 +- ...andlers.callback-questions.runtime.test.ts | 53 ++ ...bot-handlers.callback-questions.runtime.ts | 38 + .../src/bot-handlers.callback.runtime.ts | 67 +- .../src/bot.create-telegram-bot.test.ts | 52 +- extensions/telegram/src/bot.test.ts | 52 ++ extensions/telegram/src/button-types.test.ts | 51 ++ extensions/telegram/src/button-types.ts | 19 +- .../src/question-callback-data.test.ts | 28 + .../telegram/src/question-callback-data.ts | 42 ++ .../telegram/src/sequential-key.test.ts | 11 + extensions/telegram/src/sequential-key.ts | 7 + package.json | 4 + packages/gateway-protocol/src/index.ts | 66 ++ .../src/questions-validators.test.ts | 80 ++ packages/gateway-protocol/src/schema.ts | 1 + .../src/schema/protocol-schemas.ts | 38 + .../gateway-protocol/src/schema/questions.ts | 143 ++++ scripts/lib/plugin-sdk-entrypoints.json | 1 + scripts/plugin-sdk-surface-report.mjs | 9 +- .../protocol-event-coverage.allowlist.json | 4 + src/agents/core-tool-factory-descriptors.ts | 1 + .../run/attempt-stream-prepare.ts | 7 +- .../run/attempt.queue-message.ts | 39 +- ...ded-agent-subscribe.handlers.tools.test.ts | 356 +++++++++ ...embedded-agent-subscribe.handlers.tools.ts | 159 +++- src/agents/harness/user-input-bridge.ts | 1 + src/agents/openclaw-tools.registration.ts | 18 + src/agents/openclaw-tools.ts | 17 + src/agents/openclaw-tools.update-plan.test.ts | 43 +- src/agents/runtime-plan/types.ts | 6 + src/agents/tool-description-presets.ts | 12 + src/agents/tool-display-config.ts | 5 + .../tools/ask-user-tool.test-support.ts | 19 + src/agents/tools/ask-user-tool.test.ts | 667 +++++++++++++++++ src/agents/tools/ask-user-tool.ts | 694 +++++++++++++++++ src/agents/tools/gateway.test.ts | 1 + src/agents/tools/message-tool.test.ts | 1 + src/agents/tools/message-tool.ts | 4 +- src/auto-reply/reply-payload.ts | 2 + .../reply/agent-runner.media-paths.test.ts | 8 +- src/auto-reply/reply/agent-runner.ts | 1 + src/auto-reply/reply/block-reply-pipeline.ts | 2 + src/auto-reply/reply/commands-steer.test.ts | 6 + src/auto-reply/reply/commands-steer.ts | 1 + .../reply/dispatch-from-config.payloads.ts | 1 + src/auto-reply/reply/dispatch-from-config.ts | 27 +- src/auto-reply/reply/reply-run-registry.ts | 2 + src/gateway/call.test.ts | 2 + src/gateway/gateway-misc.test.ts | 16 + .../http-utils.request-context.test.ts | 1 + src/gateway/method-scopes.test.ts | 19 + src/gateway/method-scopes.ts | 12 +- src/gateway/methods/core-descriptors.ts | 5 + src/gateway/operator-scopes.ts | 3 + src/gateway/question-manager.test.ts | 228 ++++++ src/gateway/question-manager.ts | 329 +++++++++ src/gateway/server-aux-handlers.ts | 15 + src/gateway/server-aux-methods.ts | 5 + src/gateway/server-broadcast.ts | 3 + src/gateway/server-methods-list.test.ts | 14 + src/gateway/server-methods-list.ts | 2 + src/gateway/server-methods/question.test.ts | 187 +++++ src/gateway/server-methods/question.ts | 177 +++++ .../plugin-route-runtime-scopes.test.ts | 2 + src/infra/outbound/deliver.test.ts | 98 +++ src/infra/outbound/deliver.ts | 30 +- src/infra/outbound/payloads.ts | 9 + src/infra/outbound/reply-payload-normalize.ts | 3 + src/infra/question-gateway-resolver.test.ts | 134 ++++ src/infra/question-gateway-resolver.ts | 102 +++ src/interactive/payload.test.ts | 35 + src/interactive/payload.ts | 28 +- src/plugin-sdk/question-gateway-runtime.ts | 6 + src/plugin-sdk/reply-payload.test.ts | 2 + test/scripts/ci-workflow-guards.test.ts | 13 +- ui/src/api/gateway.node.test.ts | 3 + ui/src/api/gateway.ts | 2 + ui/src/app/question-prompt.test.ts | 697 ++++++++++++++++++ ui/src/app/question-prompt.ts | 625 ++++++++++++++++ ui/src/e2e/question-flow.e2e.test.ts | 261 +++++++ ui/src/i18n/locales/en.ts | 11 + ui/src/lib/chat/chat-types.ts | 1 + ui/src/pages/chat/chat-pane.ts | 29 + .../pages/chat/chat-thread.question.test.ts | 79 ++ ui/src/pages/chat/chat-thread.ts | 41 +- ui/src/pages/chat/chat-view.ts | 7 + ui/src/pages/chat/components/chat-composer.ts | 11 +- ui/src/pages/chat/components/chat-message.ts | 53 +- .../components/chat-question-card.test.ts | 453 +++++++----- .../chat/components/chat-question-card.ts | 377 ++++++++-- ui/src/pages/chat/components/chat-thread.ts | 19 + ui/src/styles/chat/question-card.css | 53 ++ 143 files changed, 7814 insertions(+), 334 deletions(-) create mode 100644 docs/tools/ask-user.md create mode 100644 extensions/discord/src/monitor/questions.test.ts create mode 100644 extensions/discord/src/monitor/questions.ts create mode 100644 extensions/discord/src/question-custom-id.test.ts create mode 100644 extensions/discord/src/question-custom-id.ts create mode 100644 extensions/slack/src/question-actions.test.ts create mode 100644 extensions/slack/src/question-actions.ts create mode 100644 extensions/telegram/src/bot-handlers.callback-questions.runtime.test.ts create mode 100644 extensions/telegram/src/bot-handlers.callback-questions.runtime.ts create mode 100644 extensions/telegram/src/question-callback-data.test.ts create mode 100644 extensions/telegram/src/question-callback-data.ts create mode 100644 packages/gateway-protocol/src/questions-validators.test.ts create mode 100644 packages/gateway-protocol/src/schema/questions.ts create mode 100644 src/agents/tools/ask-user-tool.test-support.ts create mode 100644 src/agents/tools/ask-user-tool.test.ts create mode 100644 src/agents/tools/ask-user-tool.ts create mode 100644 src/gateway/question-manager.test.ts create mode 100644 src/gateway/question-manager.ts create mode 100644 src/gateway/server-methods/question.test.ts create mode 100644 src/gateway/server-methods/question.ts create mode 100644 src/infra/question-gateway-resolver.test.ts create mode 100644 src/infra/question-gateway-resolver.ts create mode 100644 src/plugin-sdk/question-gateway-runtime.ts create mode 100644 ui/src/app/question-prompt.test.ts create mode 100644 ui/src/app/question-prompt.ts create mode 100644 ui/src/e2e/question-flow.e2e.test.ts create mode 100644 ui/src/pages/chat/chat-thread.question.test.ts diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml index 9ef74de87abb..2f9791125838 100644 --- a/.github/actions/setup-node-env/action.yml +++ b/.github/actions/setup-node-env/action.yml @@ -345,6 +345,31 @@ runs: ;; esac + sticky_marker="$STICKY_ROOT/.openclaw-deps-fingerprint" + sticky_fingerprint="" + sticky_snapshot_matches="false" + if [ "$STICKY_DISK" = "true" ] && [ -f "$sticky_marker" ]; then + sticky_fingerprint="$(<"$sticky_marker")" + fi + if [ "$STICKY_DISK" = "true" ] && [ -n "$sticky_fingerprint" ] && + [ "$sticky_fingerprint" = "${OPENCLAW_STICKY_DEPS_FINGERPRINT:?}" ]; then + sticky_snapshot_matches="true" + fi + if [ "$STICKY_DISK" = "true" ] && [ "$STICKY_WRITER" != "true" ] && + [ "$sticky_snapshot_matches" != "true" ]; then + # Read-only PR clones cannot refresh a stale snapshot. Installing into + # that clone can saturate its ext4 device until short jobs time out. + # Detach only the workspace bind; the action still discards its clone. + sudo umount "$GITHUB_WORKSPACE/node_modules" + rm -rf "$GITHUB_WORKSPACE/node_modules" + mkdir -p "$GITHUB_WORKSPACE/node_modules" + ephemeral_store="${RUNNER_TEMP:?}/openclaw-pnpm-store" + mkdir -p "$ephemeral_store" + export PNPM_CONFIG_STORE_DIR="$ephemeral_store" + echo "PNPM_CONFIG_STORE_DIR=$ephemeral_store" >> "$GITHUB_ENV" + echo "Sticky dependency snapshot is stale; using runner-local storage for this read-only run" + fi + install_args=( install --prefer-offline @@ -374,13 +399,7 @@ runs: ln -sfn . "$PNPM_CONFIG_MODULES_DIR/node_modules" export NODE_PATH="$PNPM_CONFIG_MODULES_DIR${NODE_PATH:+:$NODE_PATH}" fi - sticky_marker="$STICKY_ROOT/.openclaw-deps-fingerprint" - sticky_fingerprint="" - if [ "$STICKY_DISK" = "true" ] && [ -f "$sticky_marker" ]; then - sticky_fingerprint="$(<"$sticky_marker")" - fi - if [ "$STICKY_DISK" = "true" ] && [ -n "$sticky_fingerprint" ] && - [ "$sticky_fingerprint" = "${OPENCLAW_STICKY_DEPS_FINGERPRINT:?}" ]; then + if [ "$sticky_snapshot_matches" = "true" ]; then bash "$GITHUB_ACTION_PATH/sticky-importers.sh" restore "$STICKY_ROOT" "$GITHUB_WORKSPACE" echo "Sticky dependency snapshot matches the install fingerprint; skipping pnpm install" else diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7124d08563da..106ff17fe22c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1301,9 +1301,6 @@ jobs: name: control-ui-i18n needs: [preflight] if: needs.preflight.outputs.run_control_ui_i18n == 'true' - # Source PR drift stays advisory because the post-merge bot owns repair. - # Generated locale PRs and release CI must pass the strict catalog gate. - continue-on-error: ${{ github.event_name != 'workflow_dispatch' && needs.preflight.outputs.strict_control_ui_i18n != 'true' }} runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04') }} timeout-minutes: 10 steps: @@ -1320,6 +1317,9 @@ jobs: use-actions-cache: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'openclaw/openclaw') && 'false' || 'true' }} - name: Check Control UI locale parity + # Source-only drift stays advisory because the post-merge bot owns + # repair. Generated locale changes and full release CI remain strict. + continue-on-error: ${{ needs.preflight.outputs.strict_control_ui_i18n != 'true' }} run: pnpm ui:i18n:check checks-fast-core: diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt index 690573c2bb90..7c4997f45161 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt @@ -129,6 +129,11 @@ enum class GatewayMethod( ExecApprovalRequest("exec.approval.request"), ExecApprovalWaitDecision("exec.approval.waitDecision"), ExecApprovalResolve("exec.approval.resolve"), + QuestionRequest("question.request"), + QuestionWaitAnswer("question.waitAnswer"), + QuestionResolve("question.resolve"), + QuestionGet("question.get"), + QuestionList("question.list"), PluginApprovalList("plugin.approval.list"), PluginApprovalRequest("plugin.approval.request"), PluginApprovalWaitDecision("plugin.approval.waitDecision"), @@ -423,6 +428,8 @@ enum class GatewayEvent( VoicewakeRoutingChanged("voicewake.routing.changed"), ExecApprovalRequested("exec.approval.requested"), ExecApprovalResolved("exec.approval.resolved"), + QuestionRequested("question.requested"), + QuestionResolved("question.resolved"), PluginApprovalRequested("plugin.approval.requested"), PluginApprovalResolved("plugin.approval.resolved"), OpenclawApprovalRequested("openclaw.approval.requested"), diff --git a/apps/android/app/src/main/java/ai/openclaw/app/i18n/NativeStringResources.kt b/apps/android/app/src/main/java/ai/openclaw/app/i18n/NativeStringResources.kt index a03f9cc84a99..a56e5b6745d1 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/i18n/NativeStringResources.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/i18n/NativeStringResources.kt @@ -177,6 +177,7 @@ internal val nativeStringResourceIds: Map = "Arguments" to R.string.native_7f816072c1c6a23b, "Ask OpenClaw anything" to R.string.native_8998824610d49175, "Ask OpenClaw to use Android capabilities." to R.string.native_e2d1ec2328a8146f, + "Ask User" to R.string.native_9309a242422af6f3, "Assistant speech muted" to R.string.native_a8bbbd7231a6c191, "Assistant working" to R.string.native_e02b4aac2b335f6f, "Attach" to R.string.native_d406ade2958cb5b6, diff --git a/apps/android/app/src/main/res/values-ar/strings.xml b/apps/android/app/src/main/res/values-ar/strings.xml index 44ef6b99ab73..d04cbb39c030 100644 --- a/apps/android/app/src/main/res/values-ar/strings.xml +++ b/apps/android/app/src/main/res/values-ar/strings.xml @@ -853,6 +853,7 @@ "كل %1$s يوم" "مفعّل" "متصل وجاهز" + "Ask User" "خطأ في الدردشة" "%1$s من %2$s" "خطّط للعمل" diff --git a/apps/android/app/src/main/res/values-de/strings.xml b/apps/android/app/src/main/res/values-de/strings.xml index e3857b718edf..36c06ad62ca2 100644 --- a/apps/android/app/src/main/res/values-de/strings.xml +++ b/apps/android/app/src/main/res/values-de/strings.xml @@ -853,6 +853,7 @@ "Alle %1$s Tage" "Aktiviert" "Online und bereit" + "Ask User" "Chatfehler" "%1$s von %2$s" "Arbeit planen" diff --git a/apps/android/app/src/main/res/values-es/strings.xml b/apps/android/app/src/main/res/values-es/strings.xml index 702685ed4472..ab9eb2412e75 100644 --- a/apps/android/app/src/main/res/values-es/strings.xml +++ b/apps/android/app/src/main/res/values-es/strings.xml @@ -853,6 +853,7 @@ "Cada %1$s d" "Activado" "En línea y listo" + "Ask User" "Error de chat" "%1$s de %2$s" "Planifica el trabajo" diff --git a/apps/android/app/src/main/res/values-fa/strings.xml b/apps/android/app/src/main/res/values-fa/strings.xml index 2dd54eeba9b6..dc8902f52917 100644 --- a/apps/android/app/src/main/res/values-fa/strings.xml +++ b/apps/android/app/src/main/res/values-fa/strings.xml @@ -853,6 +853,7 @@ "هر %1$s روز" "فعال" "آنلاین و آماده" + "Ask User" "خطای چت" "%1$s از %2$s" "برنامه‌ریزی کار" diff --git a/apps/android/app/src/main/res/values-fr/strings.xml b/apps/android/app/src/main/res/values-fr/strings.xml index be42b422547b..943d3fe9872d 100644 --- a/apps/android/app/src/main/res/values-fr/strings.xml +++ b/apps/android/app/src/main/res/values-fr/strings.xml @@ -853,6 +853,7 @@ "Tous les %1$s j" "Activé" "En ligne et prêt" + "Ask User" "Erreur de chat" "%1$s sur %2$s" "Planifier le travail" diff --git a/apps/android/app/src/main/res/values-hi/strings.xml b/apps/android/app/src/main/res/values-hi/strings.xml index 8374a9d8cf0b..654343e5c81f 100644 --- a/apps/android/app/src/main/res/values-hi/strings.xml +++ b/apps/android/app/src/main/res/values-hi/strings.xml @@ -853,6 +853,7 @@ "हर %1$s दिन" "सक्षम" "ऑनलाइन और तैयार" + "Ask User" "चैट त्रुटि" "%1$s में से %2$s" "काम की योजना बनाएँ" diff --git a/apps/android/app/src/main/res/values-in/strings.xml b/apps/android/app/src/main/res/values-in/strings.xml index 545a3616af74..736a4f91ae4b 100644 --- a/apps/android/app/src/main/res/values-in/strings.xml +++ b/apps/android/app/src/main/res/values-in/strings.xml @@ -853,6 +853,7 @@ "Setiap %1$s hari" "Diaktifkan" "Online dan siap" + "Ask User" "Kesalahan chat" "%1$s dari %2$s" "Rencanakan pekerjaan" diff --git a/apps/android/app/src/main/res/values-it/strings.xml b/apps/android/app/src/main/res/values-it/strings.xml index 2746736ee193..c3e574d9bb03 100644 --- a/apps/android/app/src/main/res/values-it/strings.xml +++ b/apps/android/app/src/main/res/values-it/strings.xml @@ -853,6 +853,7 @@ "Ogni %1$s g" "Abilitato" "Online e pronto" + "Ask User" "Errore chat" "%1$s di %2$s" "Pianifica il lavoro" diff --git a/apps/android/app/src/main/res/values-ja/strings.xml b/apps/android/app/src/main/res/values-ja/strings.xml index 93f54cc3e947..01772c469d25 100644 --- a/apps/android/app/src/main/res/values-ja/strings.xml +++ b/apps/android/app/src/main/res/values-ja/strings.xml @@ -853,6 +853,7 @@ "%1$s日ごと" "有効" "オンラインで準備完了" + "Ask User" "チャット エラー" "%1$s / %2$s" "作業を計画" diff --git a/apps/android/app/src/main/res/values-ko/strings.xml b/apps/android/app/src/main/res/values-ko/strings.xml index 367c85b9421c..7b7936429d0f 100644 --- a/apps/android/app/src/main/res/values-ko/strings.xml +++ b/apps/android/app/src/main/res/values-ko/strings.xml @@ -853,6 +853,7 @@ "%1$s일마다" "활성화됨" "온라인 및 준비됨" + "Ask User" "채팅 오류" "%1$s / %2$s" "작업 계획 세우기" diff --git a/apps/android/app/src/main/res/values-nl/strings.xml b/apps/android/app/src/main/res/values-nl/strings.xml index d6016d460956..138dee18b1a8 100644 --- a/apps/android/app/src/main/res/values-nl/strings.xml +++ b/apps/android/app/src/main/res/values-nl/strings.xml @@ -853,6 +853,7 @@ "Elke %1$s d" "Ingeschakeld" "Online en gereed" + "Ask User" "Chatfout" "%1$s van %2$s" "Plan het werk" diff --git a/apps/android/app/src/main/res/values-pl/strings.xml b/apps/android/app/src/main/res/values-pl/strings.xml index 9d2990cdb267..95ac22f8349b 100644 --- a/apps/android/app/src/main/res/values-pl/strings.xml +++ b/apps/android/app/src/main/res/values-pl/strings.xml @@ -853,6 +853,7 @@ "Co %1$s d" "Włączone" "Online i gotowy" + "Ask User" "Błąd czatu" "%1$s z %2$s" "Zaplanuj pracę" diff --git a/apps/android/app/src/main/res/values-pt-rBR/strings.xml b/apps/android/app/src/main/res/values-pt-rBR/strings.xml index 179e2cdb3b64..c3f6fd2b386c 100644 --- a/apps/android/app/src/main/res/values-pt-rBR/strings.xml +++ b/apps/android/app/src/main/res/values-pt-rBR/strings.xml @@ -853,6 +853,7 @@ "A cada %1$sd" "Ativado" "Online e pronto" + "Ask User" "Erro no chat" "%1$s de %2$s" "Planeje o trabalho" diff --git a/apps/android/app/src/main/res/values-ru/strings.xml b/apps/android/app/src/main/res/values-ru/strings.xml index 8c6ba751be7c..7506538664e9 100644 --- a/apps/android/app/src/main/res/values-ru/strings.xml +++ b/apps/android/app/src/main/res/values-ru/strings.xml @@ -853,6 +853,7 @@ "Каждые %1$s дн." "Включено" "В сети и готово" + "Ask User" "Ошибка чата" "%1$s из %2$s" "Спланировать работу" diff --git a/apps/android/app/src/main/res/values-sv/strings.xml b/apps/android/app/src/main/res/values-sv/strings.xml index 209b5d00eed9..8e5f846f4a2a 100644 --- a/apps/android/app/src/main/res/values-sv/strings.xml +++ b/apps/android/app/src/main/res/values-sv/strings.xml @@ -853,6 +853,7 @@ "Var %1$s dag" "Aktiverad" "Online och redo" + "Ask User" "Chattfel" "%1$s av %2$s" "Planera arbetet" diff --git a/apps/android/app/src/main/res/values-th/strings.xml b/apps/android/app/src/main/res/values-th/strings.xml index 8eaf4a6c5322..fdc47dd76cbf 100644 --- a/apps/android/app/src/main/res/values-th/strings.xml +++ b/apps/android/app/src/main/res/values-th/strings.xml @@ -853,6 +853,7 @@ "ทุก %1$s วัน" "เปิดใช้งานแล้ว" "ออนไลน์และพร้อมใช้งาน" + "Ask User" "ข้อผิดพลาดของแชท" "%1$s จาก %2$s" "วางแผนงาน" diff --git a/apps/android/app/src/main/res/values-tr/strings.xml b/apps/android/app/src/main/res/values-tr/strings.xml index 5bf010a1cc29..a8a9e6820676 100644 --- a/apps/android/app/src/main/res/values-tr/strings.xml +++ b/apps/android/app/src/main/res/values-tr/strings.xml @@ -853,6 +853,7 @@ "Her %1$s günde bir" "Etkin" "Çevrimiçi ve hazır" + "Ask User" "Sohbet hatası" "%1$s / %2$s" "Çalışmayı planlayın" diff --git a/apps/android/app/src/main/res/values-uk/strings.xml b/apps/android/app/src/main/res/values-uk/strings.xml index 9a0a52bd4229..ca8c61fffb46 100644 --- a/apps/android/app/src/main/res/values-uk/strings.xml +++ b/apps/android/app/src/main/res/values-uk/strings.xml @@ -853,6 +853,7 @@ "Кожні %1$s дн." "Увімкнено" "У мережі й готово" + "Ask User" "Помилка чату" "%1$s з %2$s" "Спланувати роботу" diff --git a/apps/android/app/src/main/res/values-vi/strings.xml b/apps/android/app/src/main/res/values-vi/strings.xml index a2a9ea98eb54..2f84e76994b3 100644 --- a/apps/android/app/src/main/res/values-vi/strings.xml +++ b/apps/android/app/src/main/res/values-vi/strings.xml @@ -853,6 +853,7 @@ "Mỗi %1$s ngày" "Đã bật" "Đang trực tuyến và sẵn sàng" + "Ask User" "Lỗi trò chuyện" "%1$s trên %2$s" "Lập kế hoạch công việc" diff --git a/apps/android/app/src/main/res/values-zh-rCN/strings.xml b/apps/android/app/src/main/res/values-zh-rCN/strings.xml index f161520d7053..11df60072e27 100644 --- a/apps/android/app/src/main/res/values-zh-rCN/strings.xml +++ b/apps/android/app/src/main/res/values-zh-rCN/strings.xml @@ -853,6 +853,7 @@ "每 %1$s 天" "已启用" "在线且已就绪" + "Ask User" "聊天错误" "%1$s / %2$s" "规划工作" diff --git a/apps/android/app/src/main/res/values-zh-rTW/strings.xml b/apps/android/app/src/main/res/values-zh-rTW/strings.xml index 9a0ef9eec629..cb0ff7b6b3ad 100644 --- a/apps/android/app/src/main/res/values-zh-rTW/strings.xml +++ b/apps/android/app/src/main/res/values-zh-rTW/strings.xml @@ -853,6 +853,7 @@ "每 %1$s 天" "已啟用" "已上線並準備就緒" + "Ask User" "聊天錯誤" "%1$s / %2$s" "規劃工作" diff --git a/apps/android/app/src/main/res/values/strings.xml b/apps/android/app/src/main/res/values/strings.xml index e48bdb2cf89a..df5259396d7b 100644 --- a/apps/android/app/src/main/res/values/strings.xml +++ b/apps/android/app/src/main/res/values/strings.xml @@ -853,6 +853,7 @@ "Every %1$sd" "Enabled" "Online and ready" + "Ask User" "Chat error" "%1$s of %2$s" "Plan the work" diff --git a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json index 642a1bb7b49d..36fc1402492e 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json +++ b/apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json @@ -440,6 +440,13 @@ "plan.0.step" ] }, + "ask_user": { + "emoji": "❓", + "title": "Ask User", + "detailKeys": [ + "questions.0.question" + ] + }, "spawn_task": { "emoji": "✨", "title": "Suggest Task", diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 11a9619a9152..fe74309213ac 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -178,6 +178,13 @@ public enum ApprovalTerminalReason: String, Codable, Sendable { case storageCorrupt = "storage-corrupt" } +public enum QuestionStatus: String, Codable, Sendable { + case pending = "pending" + case answered = "answered" + case cancelled = "cancelled" + case expired = "expired" +} + public struct ConnectParams: Codable, Sendable { public let minprotocol: Int public let maxprotocol: Int @@ -12616,6 +12623,270 @@ public struct ExecApprovalResolveParams: Codable, Sendable { } } +public struct QuestionOption: Codable, Sendable { + public let label: String + public let description: String? + + public init( + label: String, + description: String? = nil) + { + self.label = label + self.description = description + } + + private enum CodingKeys: String, CodingKey { + case label + case description + } +} + +public struct Question: Codable, Sendable { + public let id: String + public let header: String + public let question: String + public let options: [QuestionOption] + public let multiselect: Bool? + public let isother: Bool? + public let issecret: Bool? + + public init( + id: String, + header: String, + question: String, + options: [QuestionOption], + multiselect: Bool? = nil, + isother: Bool? = nil, + issecret: Bool? = nil) + { + self.id = id + self.header = header + self.question = question + self.options = options + self.multiselect = multiselect + self.isother = isother + self.issecret = issecret + } + + private enum CodingKeys: String, CodingKey { + case id + case header + case question + case options + case multiselect = "multiSelect" + case isother = "isOther" + case issecret = "isSecret" + } +} + +public struct QuestionRequestQuestion: Codable, Sendable { + public let id: String + public let header: String + public let question: String + public let options: [QuestionOption] + public let multiselect: Bool? + public let isother: Bool? + public let issecret: Bool? + + public init( + id: String, + header: String, + question: String, + options: [QuestionOption], + multiselect: Bool? = nil, + isother: Bool? = nil, + issecret: Bool? = nil) + { + self.id = id + self.header = header + self.question = question + self.options = options + self.multiselect = multiselect + self.isother = isother + self.issecret = issecret + } + + private enum CodingKeys: String, CodingKey { + case id + case header + case question + case options + case multiselect = "multiSelect" + case isother = "isOther" + case issecret = "isSecret" + } +} + +public struct QuestionAnswers: Codable, Sendable { + public let answers: [String: AnyCodable] + + public init( + answers: [String: AnyCodable]) + { + self.answers = answers + } + + private enum CodingKeys: String, CodingKey { + case answers + } +} + +public struct QuestionRecord: Codable, Sendable { + public let id: String + public let questions: [Question] + public let agentid: String? + public let sessionkey: String? + public let createdatms: Int + public let expiresatms: Int + public let status: QuestionStatus + public let answers: QuestionAnswers? + public let resolvedby: String? + + public init( + id: String, + questions: [Question], + agentid: String? = nil, + sessionkey: String? = nil, + createdatms: Int, + expiresatms: Int, + status: QuestionStatus, + answers: QuestionAnswers? = nil, + resolvedby: String? = nil) + { + self.id = id + self.questions = questions + self.agentid = agentid + self.sessionkey = sessionkey + self.createdatms = createdatms + self.expiresatms = expiresatms + self.status = status + self.answers = answers + self.resolvedby = resolvedby + } + + private enum CodingKeys: String, CodingKey { + case id + case questions + case agentid = "agentId" + case sessionkey = "sessionKey" + case createdatms = "createdAtMs" + case expiresatms = "expiresAtMs" + case status + case answers + case resolvedby = "resolvedBy" + } +} + +public struct QuestionRequestParams: Codable, Sendable { + public let id: String? + public let questions: [QuestionRequestQuestion] + public let agentid: String? + public let sessionkey: String? + public let timeoutms: Int? + + public init( + id: String? = nil, + questions: [QuestionRequestQuestion], + agentid: String? = nil, + sessionkey: String? = nil, + timeoutms: Int? = nil) + { + self.id = id + self.questions = questions + self.agentid = agentid + self.sessionkey = sessionkey + self.timeoutms = timeoutms + } + + private enum CodingKeys: String, CodingKey { + case id + case questions + case agentid = "agentId" + case sessionkey = "sessionKey" + case timeoutms = "timeoutMs" + } +} + +public struct QuestionRequestResult: Codable, Sendable { + public let id: String + public let expiresatms: Int + + public init( + id: String, + expiresatms: Int) + { + self.id = id + self.expiresatms = expiresatms + } + + private enum CodingKeys: String, CodingKey { + case id + case expiresatms = "expiresAtMs" + } +} + +public struct QuestionWaitAnswerParams: Codable, Sendable { + public let id: String + public let timeoutms: Int? + + public init( + id: String, + timeoutms: Int? = nil) + { + self.id = id + self.timeoutms = timeoutms + } + + private enum CodingKeys: String, CodingKey { + case id + case timeoutms = "timeoutMs" + } +} + +public struct QuestionGetParams: Codable, Sendable { + public let id: String + + public init( + id: String) + { + self.id = id + } + + private enum CodingKeys: String, CodingKey { + case id + } +} + +public struct QuestionGetResult: Codable, Sendable { + public let question: QuestionRecord + + public init( + question: QuestionRecord) + { + self.question = question + } + + private enum CodingKeys: String, CodingKey { + case question + } +} + +public struct QuestionListParams: Codable, Sendable {} + +public struct QuestionListResult: Codable, Sendable { + public let questions: [QuestionRecord] + + public init( + questions: [QuestionRecord]) + { + self.questions = questions + } + + private enum CodingKeys: String, CodingKey { + case questions + } +} + public struct PluginApprovalRequestParams: Codable, Sendable { public let pluginid: String? public let title: String diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 57c62fed4d4f..a80e4e83f8d2 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -f08ff5ef3bead5b08631b72d347884808c262aab7d33c8ae085407ba55e44402 plugin-sdk-api-baseline.json -d62e193e943766c515978531ef88463eba94094bd77eecef397cef391d841a67 plugin-sdk-api-baseline.jsonl +7e717544b401b5ca3447d5383e241f678b6c343c089b3f430c2fc48244f1b3c8 plugin-sdk-api-baseline.json +0b4bc1cfc71177b5f5e0d146c48d86d48ff0eea1a075358adb7583ee35af54b9 plugin-sdk-api-baseline.jsonl diff --git a/docs/ci.md b/docs/ci.md index 4c9392cdf073..4c22af31be5e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -115,7 +115,7 @@ The slowest Node test families are split or balanced so each job stays small wit - The full Node matrix admits the consistently slow serial tooling, auto-reply command shards, and broad core-fast cache writer first. This keeps the 28-job cap while preventing critical-path work and the next run's transform seed from slipping into a later wave. - Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. - Linux Node shard jobs persist Vitest's experimental filesystem module cache. Trusted Blacksmith jobs clone one protected disk per platform and Node line; pull requests write only to their private clone and discard it, so PR traffic cannot allocate backing disks or publish feature-branch transforms. GitHub-hosted and fork jobs use an `actions/cache` fallback with coarse PR-scoped restore prefixes. The planner marks the broad `core-unit-fast` graph as the single writer without coupling cache ownership to matrix order. Concurrent Vitest workers retain separate live directories. A transform-input fingerprint clears incompatible lockfile, package, tsconfig, and Vitest-config generations inside the stable disk. Only a protected writer scans and prunes the cache to 75% after it exceeds 2 GiB. A non-cancelling daily or default-branch repository-dispatch warmer refreshes the protected seed; GitHub's normal cache eviction expires fallback PR archives. -- Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. After that exact restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. The non-cancelled cache warmer is the only writer: successful `main` CI completion coalesces a dependency refresh, while the daily run remains a deadline fallback. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. +- Trusted Linux Node jobs also bind the pnpm store and `node_modules` from one protected dependency disk per supported Node line. Package manifests, install settings, runner platform, and the exact Node patch stay out of the disk key; an exact runtime and install-input fingerprint decides whether a job reuses the tree or reinstalls and refreshes the same disk. A pull request whose read-only snapshot has a different fingerprint detaches the workspace bind and installs into runner-local storage, avoiding slow writes to a clone it cannot publish. After an exact restore or frozen-lockfile install, setup disables pnpm's redundant pre-run dependency check: the repository intentionally prunes plugin-local `node_modules`, which pnpm otherwise treats as stale and repairs through unsafe concurrent implicit installs during shard fanout. The non-cancelled cache warmer is the only writer: successful `main` CI completion coalesces a dependency refresh, while the daily run remains a deadline fallback. Required CI jobs and pull requests get disposable clones, so dependency changes do not create new disks, competing snapshots, or a cache lock that can cancel builds. - Node shard and build-artifact jobs also restore Node's portable on-disk compile cache. Independent `test` and `build` namespaces prevent their writers from replacing each other's snapshots: the scheduled test warmer owns the protected test seed, while `build-artifacts` publishes the protected build seed only from trusted `main` pushes. PR jobs read protected snapshots without publishing feature-branch bytecode; fallback archives remain PR-scoped. This reuses V8 bytecode for Node-loaded orchestration, build tooling, and external dependencies across different checkout paths, including when only part of the source graph changes. Vitest child processes disable an inherited compile cache because coverage can be enabled inside dynamic configs and V8 coverage can lose source-position precision when scripts are deserialized from bytecode. - The build-artifact job also persists content-fingerprinted `build-all` step outputs. CI's self-built plugin SDK declarations hash the complete repository-owned TypeScript/JSON source graph, exclude installed and generated directories, and restore both flat declarations and package bridges after `tsdown` clears `dist`. Documentation, workflow, plugin, and other changes outside that graph can reuse the declaration snapshot; source changes rebuild it before the export gate runs. - Full declaration builds split `tsdown` into AI, workspace-package, and unified groups. Each group caches declarations only, then still rebuilds runtime JavaScript before restoring those declarations. Core or plugin changes therefore invalidate only the large unified graph, while workspace-package changes conservatively invalidate every dependent declaration group. Public full builds generally use an immutable Actions cache; coarse restore keys seed partial changes, per-group content fingerprints reject stale data, and GitHub's cache quota evicts old generations. The weekly Node 22 lane instead publishes a 14-day artifact after successful `main` runs and restores only artifacts whose immutable producer identity resolves to that workflow on `main`, avoiding quota churn without allowing PR code to write a shared cache. Private-QA declarations are never persisted in Actions caches because cache namespaces are not confidentiality boundaries. diff --git a/docs/docs.json b/docs/docs.json index 79a8fbbe2945..bc5c55b0098f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1358,6 +1358,7 @@ "group": "Tools", "pages": [ "tools/apply-patch", + "tools/ask-user", "tools/btw", "tools/code-execution", "tools/code-mode", diff --git a/docs/docs_map.md b/docs/docs_map.md index bd5f80a38070..be66456626cf 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -9493,6 +9493,15 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Example - H2: Related +## tools/ask-user.md + +- Route: /tools/ask-user +- Headings: + - H2: Answer a question + - H2: Timeout and no answer + - H2: Tool schema + - H2: Model guidance + ## tools/brave-search.md - Route: /tools/brave-search diff --git a/docs/plugins/message-presentation.md b/docs/plugins/message-presentation.md index 45ed28417f43..5d3ab9a739d0 100644 --- a/docs/plugins/message-presentation.md +++ b/docs/plugins/message-presentation.md @@ -77,6 +77,11 @@ type MessagePresentationAction = approvalKind: "exec" | "plugin"; decision: "allow-once" | "allow-always" | "deny"; } + | { + type: "question"; + questionId: string; + optionValue: string; + } | { type: "url"; url: string } | { type: "web-app"; @@ -136,6 +141,12 @@ Button semantics: encode that action into a transport-private callback and resolve it through the approval service; they must not parse `/approve` command text or infer kind from the ID. +- `action.type: "question"` identifies one choice for a live, runtime-authored + `ask_user` question. Like `approval`, this is an OpenClaw runtime action; + agents and plugins must not synthesize question IDs. Telegram, Discord, and + Slack map it to transport-private native callbacks and resolve the choice + through the Gateway. Other channels degrade the controls to label text, and + the user can answer with a plain-text reply. - `action.type: "url"` opens a normal link. - `action.type: "web-app"` launches a channel-native web app. Set `url` for a URL-backed app or `widgetId` for an OpenClaw-hosted widget whose launch diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index abc3af2a68f1..6629dd8d294d 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -136,6 +136,7 @@ deprecated for new code; see the per-row notes below. | `plugin-sdk/telegram-account` | Deprecated Telegram account-resolution compatibility facade for tracked owner compatibility; new plugins should use injected runtime helpers or generic channel SDK subpaths | | `plugin-sdk/zalouser` | Deprecated Zalo Personal compatibility facade for published Lark/Zalo packages that still import sender command authorization; new plugins should use generic channel SDK subpaths | | `plugin-sdk/interactive-runtime` | Semantic message presentation, delivery, and legacy interactive reply helpers. See [Message Presentation](/plugins/message-presentation) | + | `plugin-sdk/question-gateway-runtime` | Resolve runtime-authored `ask_user` choices through the Gateway from channel interaction handlers | | `plugin-sdk/channel-inbound` | Shared inbound helpers for event classification, context building, formatting, roots, debounce, mention matching, mention-policy, and inbound logging | | `plugin-sdk/channel-inbound-debounce` | Narrow inbound debounce helpers | | `plugin-sdk/channel-mention-gating` | Narrow mention-policy, mention marker, and mention text helpers without the broader inbound runtime surface | diff --git a/docs/tools/ask-user.md b/docs/tools/ask-user.md new file mode 100644 index 000000000000..3a649af9a16f --- /dev/null +++ b/docs/tools/ask-user.md @@ -0,0 +1,91 @@ +--- +summary: "How ask_user pauses an agent turn for a structured human decision" +read_when: + - You want an agent to ask the user a structured question + - You are answering or debugging an ask_user prompt + - You need the ask_user schema, timeout, or channel behavior +title: "Ask user" +--- + +`ask_user` lets the agent ask the human one to three structured questions and +wait for the answers. It is for decisions that genuinely belong to the user, +not routine confirmation or information the agent can derive from the request, +code, or a sensible default. + +The tool is available only in the main session. Subagents and other non-primary +runs do not receive it. + +## Answer a question + +You can answer from any supported conversation surface: + +- The web Control UI shows one unified question card. +- Telegram, Discord, and Slack render native buttons for a single-choice, + single-question prompt. +- A plain-text reply works on any channel. Reply with a number, an option label, + or your own answer. + +OpenClaw always enables a free-text **Other** answer. The agent must not add an +`Other` option to the authored option list. + +Prompts that cannot use native buttons, including multi-question and +multi-select prompts, degrade to readable text. The Control UI keeps the full +structured card. + +## Timeout and no answer + +The default timeout is 900 seconds. `timeoutSeconds` is clamped to the range +30 through 3600 seconds. + +If the question expires or is cancelled before an answer arrives, the tool +returns `status: "no_answer"`. The agent then continues with its best judgment. +An aborted agent run cancels its pending Gateway question. + +## Tool schema + +```ts +{ + questions: Array<{ + id: string; // unique snake_case answer key + header: string; // short label; truncated to 12 characters + question: string; // one sentence + options: Array<{ + label: string; + description?: string; + }>; // 2-4 options + multiSelect?: boolean; + }>; // 1-3 questions + timeoutSeconds?: number; // integer; default 900, clamped to 30-3600 +} +``` + +With `multiSelect: true`, the user can choose more than one option. Answer +values are returned as an array for every question. + +Example answered result: + +```json +{ + "status": "answered", + "answers": { + "answers": { + "deploy_target": { + "answers": ["Staging (Recommended)"] + } + } + } +} +``` + +## Model guidance + +The model-facing contract tells the agent to: + +- ask only when blocked on a genuinely user-owned decision; +- prefer one question and use no more than three; +- put the recommended option first and suffix its label with `(Recommended)`; +- omit an authored `Other` option because free text is added automatically; +- continue with best judgment after `no_answer`. + +The agent should not use `ask_user` to ask whether it may proceed or to confirm +its own plan. diff --git a/docs/tools/index.md b/docs/tools/index.md index 7722f48f06b2..8d6757cc38fc 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -85,6 +85,7 @@ semantics, use [Tools and custom providers](/gateway/config-tools). | ----------------------- | --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | | Runtime | Run commands, manage processes, or use provider-backed Python analysis | `exec`, `process`, `terminal`, `code_execution` | [Exec](/tools/exec), [Control UI terminal](/web/control-ui#operator-terminal), [Code execution](/tools/code-execution) | | Files | Read and change workspace files | `read`, `write`, `edit`, `apply_patch` | [Apply patch](/tools/apply-patch) | +| Human input | Pause for a structured decision owned by the user | `ask_user` | [Ask user](/tools/ask-user) | | Web | Search the web, search X posts, or fetch readable page content | `web_search`, `x_search`, `web_fetch` | [Web tools](/tools/web), [Web fetch](/tools/web-fetch) | | Browser | Operate a browser session | `browser` | [Browser](/tools/browser) | | Messaging and channels | Send replies or channel actions | `message` | [Agent send](/tools/agent-send) | diff --git a/extensions/discord/src/monitor/provider.interactions.ts b/extensions/discord/src/monitor/provider.interactions.ts index 2333700f2a8b..211e6a4767a8 100644 --- a/extensions/discord/src/monitor/provider.interactions.ts +++ b/extensions/discord/src/monitor/provider.interactions.ts @@ -27,6 +27,7 @@ import { createDiscordModelPickerFallbackSelect, createDiscordNativeCommand, } from "./native-command.js"; +import { createDiscordQuestionButton } from "./questions.js"; import type { ThreadBindingManager } from "./thread-bindings.types.js"; type DiscordVoiceManager = import("../voice/manager.js").DiscordVoiceManager; @@ -111,6 +112,20 @@ export function createDiscordProviderInteractionSurface(params: { } const components: BaseMessageInteractiveComponent[] = [ + createDiscordQuestionButton({ + cfg: params.cfg, + accountId: params.accountId, + authContext: { + cfg: params.cfg, + accountId: params.accountId, + discordConfig: params.discordConfig, + runtime: params.runtime, + token: params.token, + guildEntries: params.guildEntries, + allowFrom: params.allowFrom, + dmPolicy: params.dmPolicy, + }, + }), createDiscordCommandArgFallbackButton({ cfg: params.cfg, discordConfig: params.discordConfig, diff --git a/extensions/discord/src/monitor/questions.test.ts b/extensions/discord/src/monitor/questions.test.ts new file mode 100644 index 000000000000..c77e22ae0a52 --- /dev/null +++ b/extensions/discord/src/monitor/questions.test.ts @@ -0,0 +1,94 @@ +// Discord question component feedback tests. +import { describe, expect, it, vi } from "vitest"; +import type { ButtonInteraction } from "../internal/discord.js"; +import { createDiscordQuestionButton } from "./questions.js"; + +type InteractionHarness = { + interaction: ButtonInteraction; + acknowledge: ReturnType; + followUp: ReturnType; +}; + +function createInteraction(): InteractionHarness { + const acknowledge = vi.fn(); + const followUp = vi.fn(); + const interaction = { + userId: "user-1", + reply: vi.fn(), + acknowledge, + followUp, + } as unknown as ButtonInteraction; + return { interaction, acknowledge, followUp }; +} + +describe("Discord question button", () => { + it.each([ + [{ status: "answered", questionId: "target", optionValue: "Production" }, "Answer submitted."], + [ + { status: "already-terminal", reason: "already-terminal" }, + "This question was already answered.", + ], + ] as const)("shows ephemeral outcome feedback", async (result, expectedText) => { + const { interaction, acknowledge, followUp } = createInteraction(); + const button = createDiscordQuestionButton({ + cfg: {} as never, + accountId: "default", + authorizeQuestion: vi.fn(async () => true), + resolveQuestion: vi.fn(async () => result), + }); + + await button.run(interaction, { + id: "ask_0123456789abcdef0123456789abcdef", + i: "1", + }); + + expect(acknowledge).toHaveBeenCalledOnce(); + expect(followUp).toHaveBeenCalledWith({ content: expectedText, ephemeral: true }); + }); + + it("does not resolve unauthorized clicks", async () => { + const { interaction, acknowledge } = createInteraction(); + const resolveQuestion = vi.fn(); + const button = createDiscordQuestionButton({ + cfg: {} as never, + accountId: "default", + authorizeQuestion: vi.fn(async () => false), + resolveQuestion, + }); + + await button.run(interaction, { + id: "ask_0123456789abcdef0123456789abcdef", + i: "1", + }); + + expect(resolveQuestion).not.toHaveBeenCalled(); + expect(acknowledge).not.toHaveBeenCalled(); + }); + + it("does not turn a committed answer into an error when feedback fails", async () => { + const { interaction, followUp } = createInteraction(); + followUp.mockRejectedValue(new Error("receipt failed")); + const button = createDiscordQuestionButton({ + cfg: {} as never, + accountId: "default", + authorizeQuestion: vi.fn(async () => true), + resolveQuestion: vi.fn(async () => ({ + status: "answered" as const, + questionId: "target", + optionValue: "Production", + })), + }); + + await expect( + button.run(interaction, { + id: "ask_0123456789abcdef0123456789abcdef", + i: "1", + }), + ).resolves.toBeUndefined(); + expect(followUp).toHaveBeenCalledOnce(); + expect(followUp).toHaveBeenCalledWith({ + content: "Answer submitted.", + ephemeral: true, + }); + }); +}); diff --git a/extensions/discord/src/monitor/questions.ts b/extensions/discord/src/monitor/questions.ts new file mode 100644 index 000000000000..66666eab611a --- /dev/null +++ b/extensions/discord/src/monitor/questions.ts @@ -0,0 +1,101 @@ +// Discord ask_user component dispatch and ephemeral feedback. +import { ButtonStyle } from "discord-api-types/v10"; +import { + resolveQuestionOverGateway, + type ResolveQuestionOverGatewayParams, +} from "openclaw/plugin-sdk/question-gateway-runtime"; +import { Button, type ButtonInteraction, type ComponentData } from "../internal/discord.js"; +import { parseDiscordQuestionData } from "../question-custom-id.js"; +import { + type AgentComponentContext, + resolveAuthorizedComponentInteraction, +} from "./agent-components-helpers.js"; + +type QuestionResolver = ( + params: ResolveQuestionOverGatewayParams, +) => ReturnType; + +class QuestionButton extends Button { + override label = "question"; + customId = "ocq:id=seed;i=0"; + override style = ButtonStyle.Primary; + + constructor( + private readonly ctx: { + cfg: ResolveQuestionOverGatewayParams["cfg"]; + accountId: string; + resolveQuestion: QuestionResolver; + authorizeQuestion: (interaction: ButtonInteraction) => Promise; + }, + ) { + super(); + } + + override async run(interaction: ButtonInteraction, data: ComponentData): Promise { + const callback = parseDiscordQuestionData(data); + if (!callback) { + await interaction.reply({ content: "This question is no longer valid.", ephemeral: true }); + return; + } + if (!(await this.ctx.authorizeQuestion(interaction))) { + return; + } + try { + await interaction.acknowledge(); + } catch {} + let result: Awaited>; + try { + result = await this.ctx.resolveQuestion({ + cfg: this.ctx.cfg, + questionId: callback.questionId, + optionIndex: callback.optionIndex, + senderId: interaction.userId, + clientDisplayName: `Discord question (${this.ctx.accountId})`, + }); + } catch { + try { + await interaction.followUp({ content: "Could not submit this answer.", ephemeral: true }); + } catch {} + return; + } + try { + await interaction.followUp({ + content: + result.status === "answered" + ? "Answer submitted." + : "This question was already answered.", + ephemeral: true, + }); + } catch { + // Gateway state already committed; receipt delivery is best-effort. + } + } +} + +export function createDiscordQuestionButton(params: { + cfg: ResolveQuestionOverGatewayParams["cfg"]; + accountId: string; + authContext?: AgentComponentContext; + authorizeQuestion?: (interaction: ButtonInteraction) => Promise; + resolveQuestion?: QuestionResolver; +}): Button { + const authContext = params.authContext ?? { cfg: params.cfg, accountId: params.accountId }; + return new QuestionButton({ + cfg: params.cfg, + accountId: params.accountId, + resolveQuestion: params.resolveQuestion ?? resolveQuestionOverGateway, + authorizeQuestion: + params.authorizeQuestion ?? + (async (interaction) => + Boolean( + await resolveAuthorizedComponentInteraction({ + ctx: authContext, + interaction, + label: "discord question", + componentLabel: "button", + unauthorizedReply: "You are not authorized to answer this question.", + defer: false, + }), + )), + }); +} diff --git a/extensions/discord/src/question-custom-id.test.ts b/extensions/discord/src/question-custom-id.test.ts new file mode 100644 index 000000000000..026835db6c1a --- /dev/null +++ b/extensions/discord/src/question-custom-id.test.ts @@ -0,0 +1,23 @@ +// Discord question custom-id envelope tests. +import { describe, expect, it } from "vitest"; +import { parseCustomId } from "./internal/discord.js"; +import { buildDiscordQuestionCustomId, parseDiscordQuestionData } from "./question-custom-id.js"; + +describe("question custom id", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + + it("round-trips a compact option index within Discord's character limit", () => { + const customId = buildDiscordQuestionCustomId({ questionId, optionIndex: 3 }); + + expect(customId).toBe(`ocq:id=${questionId};i=3`); + expect(customId).toHaveLength(47); + expect(customId?.length).toBeLessThanOrEqual(100); + const parsed = parseCustomId(customId ?? ""); + expect(parsed.key).toBe("ocq"); + expect(parseDiscordQuestionData(parsed.data)).toEqual({ questionId, optionIndex: 3 }); + }); + + it("rejects malformed indices", () => { + expect(parseDiscordQuestionData(parseCustomId(`ocq:id=${questionId};i=4`).data)).toBeNull(); + }); +}); diff --git a/extensions/discord/src/question-custom-id.ts b/extensions/discord/src/question-custom-id.ts new file mode 100644 index 000000000000..ffa1d7c7c921 --- /dev/null +++ b/extensions/discord/src/question-custom-id.ts @@ -0,0 +1,35 @@ +// Discord-private ask_user component envelope. +import type { ComponentData } from "./internal/discord.js"; + +const DISCORD_QUESTION_CUSTOM_ID_MAX_CHARS = 100; +const QUESTION_RECORD_ID_PATTERN = /^ask_[a-f0-9]{32}$/u; + +type DiscordQuestionCallback = { + questionId: string; + optionIndex: number; +}; + +export function buildDiscordQuestionCustomId( + callback: DiscordQuestionCallback, +): string | undefined { + if ( + !QUESTION_RECORD_ID_PATTERN.test(callback.questionId) || + !Number.isInteger(callback.optionIndex) || + callback.optionIndex < 0 || + callback.optionIndex > 3 + ) { + return undefined; + } + const customId = `ocq:id=${callback.questionId};i=${callback.optionIndex}`; + return customId.length <= DISCORD_QUESTION_CUSTOM_ID_MAX_CHARS ? customId : undefined; +} + +export function parseDiscordQuestionData(data: ComponentData): DiscordQuestionCallback | null { + const questionId = typeof data.id === "string" ? data.id : ""; + const rawIndex = + typeof data.i === "string" ? data.i : typeof data.i === "number" ? String(data.i) : ""; + if (!QUESTION_RECORD_ID_PATTERN.test(questionId) || !/^[0-3]$/u.test(rawIndex)) { + return null; + } + return { questionId, optionIndex: Number(rawIndex) }; +} diff --git a/extensions/discord/src/shared-interactive.test.ts b/extensions/discord/src/shared-interactive.test.ts index 9b4b0ed005eb..5c9eb1f42353 100644 --- a/extensions/discord/src/shared-interactive.test.ts +++ b/extensions/discord/src/shared-interactive.test.ts @@ -271,6 +271,37 @@ describe("buildDiscordInteractiveComponents", () => { }); }); + it("renders question choices with compact option indices", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + expect( + buildDiscordPresentationComponents({ + blocks: [ + { + type: "buttons", + buttons: ["Staging", "Production"].map((label) => ({ + label, + action: { type: "question" as const, questionId, optionValue: label }, + })), + }, + ], + }), + ).toEqual({ + blocks: [ + { + type: "actions", + buttons: [ + { label: "Staging", style: "secondary", internalCustomId: `ocq:id=${questionId};i=0` }, + { + label: "Production", + style: "secondary", + internalCustomId: `ocq:id=${questionId};i=1`, + }, + ], + }, + ], + }); + }); + it("rejects malformed approval custom ids and compacts overlong canonical ids", () => { expect( parseExecApprovalData(parseCustomId("execapproval:kind=exec;id=%zz;action=allow-once").data), diff --git a/extensions/discord/src/shared-interactive.ts b/extensions/discord/src/shared-interactive.ts index 0b28cf273133..3c430f0a38b0 100644 --- a/extensions/discord/src/shared-interactive.ts +++ b/extensions/discord/src/shared-interactive.ts @@ -21,6 +21,7 @@ import type { DiscordComponentButtonStyle, DiscordComponentMessageSpec, } from "./components.types.js"; +import { buildDiscordQuestionCustomId } from "./question-custom-id.js"; function resolveDiscordInteractiveButtonStyle( style?: InteractiveButtonStyle, @@ -65,6 +66,7 @@ const DISCORD_INTERACTIVE_BUTTON_ROW_SIZE = 5; function buildDiscordButtonComponent( button: MessagePresentationButton, + optionIndex: number, ): DiscordComponentButtonSpec | undefined { const action = resolveMessagePresentationButtonAction(button); if (!action) { @@ -82,6 +84,20 @@ function buildDiscordButtonComponent( ...(button.disabled === true ? { disabled: true } : {}), }; } + if (action.type === "question") { + const internalCustomId = buildDiscordQuestionCustomId({ + questionId: action.questionId, + optionIndex, + }); + return internalCustomId + ? { + label: button.label, + style: resolveDiscordInteractiveButtonStyle(button.style), + internalCustomId, + ...(button.disabled === true ? { disabled: true } : {}), + } + : undefined; + } if ( action.type === "web-app" && action.widgetId && @@ -127,7 +143,7 @@ function appendDiscordButtonBlocks( buttons: readonly MessagePresentationButton[], ): void { const components = buttons - .map((button) => buildDiscordButtonComponent(button)) + .map((button, optionIndex) => buildDiscordButtonComponent(button, optionIndex)) .filter((button): button is DiscordComponentButtonSpec => Boolean(button)); for (let index = 0; index < components.length; index += DISCORD_INTERACTIVE_BUTTON_ROW_SIZE) { blocks.push({ diff --git a/extensions/slack/src/blocks-render.ts b/extensions/slack/src/blocks-render.ts index 81bf59aefbc1..7236b285553c 100644 --- a/extensions/slack/src/blocks-render.ts +++ b/extensions/slack/src/blocks-render.ts @@ -39,6 +39,7 @@ import { SLACK_SECTION_TEXT_MAX, SLACK_STATIC_SELECT_OPTIONS_MAX, } from "./presentation.js"; +import { encodeSlackQuestionAction } from "./question-actions.js"; import { SLACK_APPROVAL_BUTTON_ACTION_ID, SLACK_APPROVAL_SELECT_ACTION_ID, @@ -47,6 +48,7 @@ import { SLACK_REPLY_BUTTON_ACTION_ID, SLACK_REPLY_LINK_ACTION_ID, SLACK_REPLY_SELECT_ACTION_ID, + SLACK_QUESTION_BUTTON_ACTION_ID, } from "./reply-action-ids.js"; import { truncateSlackText } from "./truncate.js"; @@ -89,6 +91,10 @@ function buildSlackCallbackSelectActionId(selectIndex: number): string { return `${SLACK_CALLBACK_SELECT_ACTION_ID}:${String(selectIndex)}`; } +function buildSlackQuestionButtonActionId(buttonIndex: number, choiceIndex: number): string { + return `${SLACK_QUESTION_BUTTON_ACTION_ID}:${String(buttonIndex)}:${String(choiceIndex + 1)}`; +} + function resolveSlackButtonStyle( style: "primary" | "secondary" | "success" | "danger" | undefined, ) { @@ -105,10 +111,12 @@ type SlackActionTarget = | { kind: "approval"; value: string } | { kind: "callback"; value: string } | { kind: "link"; url: string } + | { kind: "question"; value: string } | { kind: "reply"; value: string }; function resolveSlackActionTarget( action: MessagePresentationAction | undefined, + optionIndex?: number, ): SlackActionTarget | undefined { if (!action) { return undefined; @@ -116,6 +124,13 @@ function resolveSlackActionTarget( if (action.type === "approval") { return { kind: "approval", value: encodeSlackApprovalAction(action) }; } + if (action.type === "question") { + const value = + optionIndex === undefined + ? undefined + : encodeSlackQuestionAction({ questionId: action.questionId, optionIndex }); + return value ? { kind: "question", value } : undefined; + } if (action.type === "url" || action.type === "web-app") { const url = normalizeOptionalString(action.url); return url ? { kind: "link", url } : undefined; @@ -134,10 +149,11 @@ function resolveSlackActionTarget( function resolveSlackButtonTarget( button: MessagePresentationButtonsBlock["buttons"][number], + optionIndex?: number, ): SlackActionTarget | undefined { if (button.action !== undefined) { const action = resolveMessagePresentationButtonAction(button); - return action ? resolveSlackActionTarget(action) : undefined; + return action ? resolveSlackActionTarget(action, optionIndex) : undefined; } // Legacy buttons could carry both a URL and callback fallback. Preserve the @@ -157,11 +173,11 @@ function resolveSlackButtonTarget( function resolveSlackOptionTarget( option: MessagePresentationSelectBlock["options"][number], -): Exclude | undefined { +): Exclude | undefined { if (option.action !== undefined) { const action = resolveMessagePresentationOptionAction(option); const target = action ? resolveSlackActionTarget(action) : undefined; - return target?.kind === "link" ? undefined : target; + return target?.kind === "link" || target?.kind === "question" ? undefined : target; } const value = normalizeOptionalString(option.value); return value ? { kind: "reply", value } : undefined; @@ -254,7 +270,7 @@ export function buildSlackInteractiveBlocks( if (block.type === "buttons") { const elements = block.buttons .flatMap((button, choiceIndex) => { - const target = resolveSlackButtonTarget(button); + const target = resolveSlackButtonTarget(button, choiceIndex); if ( !target || (target.kind === "link" @@ -275,7 +291,9 @@ export function buildSlackInteractiveBlocks( ? buildSlackApprovalButtonActionId(state.buttonIndex + 1, choiceIndex) : target.kind === "callback" ? buildSlackCallbackButtonActionId(state.buttonIndex + 1, choiceIndex) - : buildSlackReplyButtonActionId(state.buttonIndex + 1, choiceIndex), + : target.kind === "question" + ? buildSlackQuestionButtonActionId(state.buttonIndex + 1, choiceIndex) + : buildSlackReplyButtonActionId(state.buttonIndex + 1, choiceIndex), text: { type: "plain_text" as const, text: truncateSlackText(button.label, SLACK_ACTION_LABEL_MAX), @@ -461,7 +479,7 @@ function buildSlackPresentationButtonBlock( ): SlackBlock | undefined { const elements = block.buttons .flatMap((button, choiceIndex) => { - const target = resolveSlackButtonTarget(button); + const target = resolveSlackButtonTarget(button, choiceIndex); if ( !target || (target.kind === "link" @@ -482,7 +500,9 @@ function buildSlackPresentationButtonBlock( ? buildSlackApprovalButtonActionId(buttonIndex, choiceIndex) : target.kind === "callback" ? buildSlackCallbackButtonActionId(buttonIndex, choiceIndex) - : buildSlackReplyButtonActionId(buttonIndex, choiceIndex), + : target.kind === "question" + ? buildSlackQuestionButtonActionId(buttonIndex, choiceIndex) + : buildSlackReplyButtonActionId(buttonIndex, choiceIndex), text: { type: "plain_text" as const, text: truncateSlackText(button.label, SLACK_ACTION_LABEL_MAX), @@ -546,11 +566,11 @@ export function canRenderSlackPresentation( if (block.type === "buttons") { const allButtonsRenderable = block.buttons.length <= SLACK_ACTION_BLOCK_ELEMENTS_MAX && - block.buttons.every((button) => { + block.buttons.every((button, choiceIndex) => { if (!isWithinSlackLimit(button.label, SLACK_ACTION_LABEL_MAX)) { return false; } - const target = resolveSlackButtonTarget(button); + const target = resolveSlackButtonTarget(button, choiceIndex); return target ? target.kind === "link" ? isWithinSlackLimit(target.url, SLACK_BUTTON_URL_MAX) diff --git a/extensions/slack/src/monitor/events/interactions.block-actions.ts b/extensions/slack/src/monitor/events/interactions.block-actions.ts index d453726e08f3..0a9843e14b27 100644 --- a/extensions/slack/src/monitor/events/interactions.block-actions.ts +++ b/extensions/slack/src/monitor/events/interactions.block-actions.ts @@ -19,9 +19,11 @@ import { decodeSlackApprovalAction, type SlackApprovalAction } from "../../appro import { isSlackApprovalAuthorizedSender } from "../../approval-auth.js"; import { isSlackExecApprovalAuthorizedSender } from "../../exec-approvals.js"; import { dispatchSlackPluginInteractiveHandler } from "../../interactive-dispatch.js"; +import { decodeSlackQuestionAction, resolveSlackQuestionAction } from "../../question-actions.js"; import { isSlackApprovalActionId, isSlackCallbackActionId, + isSlackQuestionActionId, SLACK_REPLY_BUTTON_ACTION_ID, SLACK_REPLY_LINK_ACTION_ID, SLACK_REPLY_SELECT_ACTION_ID, @@ -1071,6 +1073,25 @@ async function handleSlackBlockAction(params: { }); return; } + if (isSlackQuestionActionId(parsed.actionId)) { + const question = decodeSlackQuestionAction(parsed.actionSummary.value); + if (!question) { + await respondEphemeral(respond, "This question action is invalid or expired."); + return; + } + const auth = await authorizeSlackBlockAction({ ctx: params.ctx, parsed, respond }); + if (!auth.allowed) { + return; + } + await resolveSlackQuestionAction({ + action: question, + cfg: params.ctx.cfg, + accountId: params.ctx.accountId, + userId: parsed.userId, + respond: async (text) => await respondEphemeral(respond, text), + }); + return; + } const pluginInteractionData = buildSlackPluginInteractionData({ actionId: parsed.actionId, summary: parsed.actionSummary, diff --git a/extensions/slack/src/monitor/events/interactions.test.ts b/extensions/slack/src/monitor/events/interactions.test.ts index 9dff30b22684..9a7a17927607 100644 --- a/extensions/slack/src/monitor/events/interactions.test.ts +++ b/extensions/slack/src/monitor/events/interactions.test.ts @@ -32,6 +32,13 @@ const resolveApprovalOverGatewayMock = vi.hoisted(() => approval: { status: "allowed", decision: "allow-once" }, })), ); +const resolveQuestionOverGatewayMock = vi.hoisted(() => + vi.fn(async (_arg: unknown) => ({ + status: "answered" as const, + questionId: "target", + optionValue: "Production", + })), +); let registerSlackInteractionEvents: typeof import("./interactions.js").registerSlackInteractionEvents; @@ -55,6 +62,10 @@ vi.mock("openclaw/plugin-sdk/approval-gateway-runtime", () => ({ resolveApprovalOverGateway: (arg: unknown) => resolveApprovalOverGatewayMock(arg), })); +vi.mock("openclaw/plugin-sdk/question-gateway-runtime", () => ({ + resolveQuestionOverGateway: (arg: unknown) => resolveQuestionOverGatewayMock(arg), +})); + vi.mock("../../interactive-dispatch.js", () => ({ dispatchSlackPluginInteractiveHandler: (params: { data: string; @@ -425,6 +436,12 @@ describe("registerSlackInteractionEvents", () => { applied: true, approval: { status: "allowed", decision: "allow-once" }, }); + resolveQuestionOverGatewayMock.mockClear(); + resolveQuestionOverGatewayMock.mockResolvedValue({ + status: "answered", + questionId: "target", + optionValue: "Production", + }); dispatchPluginInteractiveHandlerMock.mockResolvedValue({ matched: false, handled: false, @@ -1285,6 +1302,46 @@ describe("registerSlackInteractionEvents", () => { expect(respond).not.toHaveBeenCalled(); }); + it("resolves typed question buttons without enqueueing an agent interaction", async () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + const { ctx, getHandler } = createContext(); + registerSlackInteractionEvents({ ctx: ctx as never }); + + const ack = vi.fn().mockResolvedValue(undefined); + const respond = vi.fn().mockResolvedValue(undefined); + await getHandler()({ + ack, + respond, + body: { + user: { id: "U123" }, + channel: { id: "C1" }, + container: { channel_id: "C1", message_ts: "100.200" }, + message: { ts: "100.200", text: "Question", blocks: [] }, + }, + action: { + type: "button", + action_id: "openclaw:question_button:1:2", + block_id: "openclaw_reply_buttons_1", + value: `slq1:${questionId}:1`, + text: { type: "plain_text", text: "Production" }, + }, + }); + + expect(ack).toHaveBeenCalledOnce(); + expect(resolveQuestionOverGatewayMock).toHaveBeenCalledWith({ + cfg: ctx.cfg, + questionId, + optionIndex: 1, + senderId: "U123", + clientDisplayName: "Slack question (default)", + }); + expect(respond).toHaveBeenCalledWith({ + text: "Answer submitted.", + response_type: "ephemeral", + }); + expect(enqueueSystemEventMock).not.toHaveBeenCalled(); + }); + it("cleans stale typed buttons and shows the canonical first-answer winner", async () => { resolveApprovalOverGatewayMock.mockResolvedValueOnce({ applied: false, diff --git a/extensions/slack/src/question-actions.test.ts b/extensions/slack/src/question-actions.test.ts new file mode 100644 index 000000000000..06853056a814 --- /dev/null +++ b/extensions/slack/src/question-actions.test.ts @@ -0,0 +1,61 @@ +// Slack question envelope and feedback tests. +import { describe, expect, it, vi } from "vitest"; +import { + decodeSlackQuestionAction, + encodeSlackQuestionAction, + resolveSlackQuestionAction, +} from "./question-actions.js"; + +describe("Slack question actions", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + + it("round-trips a compact option index within Slack's value limit", () => { + const value = encodeSlackQuestionAction({ questionId, optionIndex: 3 }); + + expect(value).toBe(`slq1:${questionId}:3`); + expect(value).toHaveLength(43); + expect(value?.length).toBeLessThanOrEqual(2000); + expect(decodeSlackQuestionAction(value)).toEqual({ questionId, optionIndex: 3 }); + }); + + it.each([ + [{ status: "answered", questionId: "target", optionValue: "Production" }, "Answer submitted."], + [{ status: "already-terminal", reason: "not-found" }, "This question was already answered."], + ] as const)("shows ephemeral-ready outcome feedback", async (result, expectedText) => { + const respond = vi.fn(async () => undefined); + + await resolveSlackQuestionAction({ + action: { questionId, optionIndex: 1 }, + cfg: {} as never, + accountId: "default", + userId: "U123", + respond, + resolveQuestion: vi.fn(async () => result), + }); + + expect(respond).toHaveBeenCalledWith(expectedText); + }); + + it("does not turn a committed answer into an error when feedback fails", async () => { + const respond = vi.fn(async () => { + throw new Error("receipt failed"); + }); + + await expect( + resolveSlackQuestionAction({ + action: { questionId, optionIndex: 1 }, + cfg: {} as never, + accountId: "default", + userId: "U123", + respond, + resolveQuestion: vi.fn(async () => ({ + status: "answered" as const, + questionId: "target", + optionValue: "Production", + })), + }), + ).resolves.toBeUndefined(); + expect(respond).toHaveBeenCalledOnce(); + expect(respond).toHaveBeenCalledWith("Answer submitted."); + }); +}); diff --git a/extensions/slack/src/question-actions.ts b/extensions/slack/src/question-actions.ts new file mode 100644 index 000000000000..e726ea037135 --- /dev/null +++ b/extensions/slack/src/question-actions.ts @@ -0,0 +1,67 @@ +// Slack-private ask_user button envelope and resolution feedback. +import { + resolveQuestionOverGateway, + type ResolveQuestionOverGatewayParams, +} from "openclaw/plugin-sdk/question-gateway-runtime"; +import { SLACK_BUTTON_VALUE_MAX } from "./presentation.js"; + +const SLACK_QUESTION_VALUE_PREFIX = "slq1:"; +const QUESTION_RECORD_ID_PATTERN = /^ask_[a-f0-9]{32}$/u; + +type SlackQuestionAction = { + questionId: string; + optionIndex: number; +}; + +export function encodeSlackQuestionAction(action: SlackQuestionAction): string | undefined { + if ( + !QUESTION_RECORD_ID_PATTERN.test(action.questionId) || + !Number.isInteger(action.optionIndex) || + action.optionIndex < 0 || + action.optionIndex > 3 + ) { + return undefined; + } + const value = `${SLACK_QUESTION_VALUE_PREFIX}${action.questionId}:${action.optionIndex}`; + return value.length <= SLACK_BUTTON_VALUE_MAX ? value : undefined; +} + +export function decodeSlackQuestionAction(value: unknown): SlackQuestionAction | null { + if (typeof value !== "string" || value.length > SLACK_BUTTON_VALUE_MAX) { + return null; + } + const match = /^slq1:(ask_[a-f0-9]{32}):([0-3])$/u.exec(value); + return match?.[1] && match[2] ? { questionId: match[1], optionIndex: Number(match[2]) } : null; +} + +type QuestionResolver = ( + params: ResolveQuestionOverGatewayParams, +) => ReturnType; + +export async function resolveSlackQuestionAction(params: { + action: SlackQuestionAction; + cfg: ResolveQuestionOverGatewayParams["cfg"]; + accountId: string; + userId: string; + respond: (text: string) => Promise; + resolveQuestion?: QuestionResolver; +}): Promise { + let result: Awaited>; + try { + result = await (params.resolveQuestion ?? resolveQuestionOverGateway)({ + cfg: params.cfg, + questionId: params.action.questionId, + optionIndex: params.action.optionIndex, + senderId: params.userId, + clientDisplayName: `Slack question (${params.accountId})`, + }); + } catch { + await params.respond("Could not submit this answer.").catch(() => {}); + return; + } + await params + .respond( + result.status === "answered" ? "Answer submitted." : "This question was already answered.", + ) + .catch(() => {}); +} diff --git a/extensions/slack/src/reply-action-ids.ts b/extensions/slack/src/reply-action-ids.ts index 0fb957580863..65a79c8003cf 100644 --- a/extensions/slack/src/reply-action-ids.ts +++ b/extensions/slack/src/reply-action-ids.ts @@ -6,6 +6,14 @@ export const SLACK_CALLBACK_BUTTON_ACTION_ID = "openclaw:callback_button"; export const SLACK_CALLBACK_SELECT_ACTION_ID = "openclaw:callback_select"; export const SLACK_APPROVAL_BUTTON_ACTION_ID = "openclaw:approval_button"; export const SLACK_APPROVAL_SELECT_ACTION_ID = "openclaw:approval_select"; +export const SLACK_QUESTION_BUTTON_ACTION_ID = "openclaw:question_button"; + +export function isSlackQuestionActionId(actionId: string): boolean { + return ( + actionId === SLACK_QUESTION_BUTTON_ACTION_ID || + actionId.startsWith(`${SLACK_QUESTION_BUTTON_ACTION_ID}:`) + ); +} export function isSlackApprovalActionId(actionId: string): boolean { return ( diff --git a/extensions/slack/src/shared-interactive.test.ts b/extensions/slack/src/shared-interactive.test.ts index ab3c4b2c9b34..192d466fb1f3 100644 --- a/extensions/slack/src/shared-interactive.test.ts +++ b/extensions/slack/src/shared-interactive.test.ts @@ -342,6 +342,38 @@ describe("buildSlackInteractiveBlocks", () => { }); describe("buildSlackPresentationBlocks", () => { + it("renders question choices with compact private indices", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + expect( + buildSlackPresentationBlocks({ + blocks: [ + { + type: "buttons", + buttons: ["Staging", "Production"].map((label) => ({ + label, + action: { type: "question" as const, questionId, optionValue: label }, + })), + }, + ], + }), + ).toEqual([ + { + type: "actions", + block_id: "openclaw_reply_buttons_1", + elements: [ + expect.objectContaining({ + action_id: "openclaw:question_button:1:1", + value: `slq1:${questionId}:0`, + }), + expect.objectContaining({ + action_id: "openclaw:question_button:1:2", + value: `slq1:${questionId}:1`, + }), + ], + }, + ]); + }); + it("renders presentation blocks in authored order", () => { const blocks = buildSlackPresentationBlocks({ blocks: [ diff --git a/extensions/telegram/src/bot-handlers.authorization.runtime.ts b/extensions/telegram/src/bot-handlers.authorization.runtime.ts index 933537dfd620..fbc17cf569e0 100644 --- a/extensions/telegram/src/bot-handlers.authorization.runtime.ts +++ b/extensions/telegram/src/bot-handlers.authorization.runtime.ts @@ -27,7 +27,11 @@ import { resolveTelegramEventIngressAuthorization, } from "./ingress.js"; -export type TelegramEventAuthorizationMode = "reaction" | "callback-scope" | "callback-allowlist"; +export type TelegramEventAuthorizationMode = + | "reaction" + | "callback-scope" + | "callback-allowlist" + | "callback-runtime-allowlist"; export function createTelegramHandlerAuthorizationRuntime({ accountId, @@ -77,6 +81,12 @@ export function createTelegramHandlerAuthorizationRuntime({ deniedDmReason: "callback unauthorized by inlineButtonsScope allowlist", deniedGroupReason: "callback unauthorized by inlineButtonsScope allowlist", }, + "callback-runtime-allowlist": { + enforceDirectAuthorization: true, + enforceGroupAllowlistAuthorization: true, + deniedDmReason: "runtime callback unauthorized by allowlist", + deniedGroupReason: "runtime callback unauthorized by group allowlist", + }, }; // Authorization owns one ingress snapshot. The agent turn intentionally diff --git a/extensions/telegram/src/bot-handlers.callback-questions.runtime.test.ts b/extensions/telegram/src/bot-handlers.callback-questions.runtime.test.ts new file mode 100644 index 000000000000..d79278428034 --- /dev/null +++ b/extensions/telegram/src/bot-handlers.callback-questions.runtime.test.ts @@ -0,0 +1,53 @@ +// Telegram question callback feedback tests. +import { describe, expect, it, vi } from "vitest"; +import { handleTelegramQuestionCallback } from "./bot-handlers.callback-questions.runtime.js"; + +const callback = { + questionId: "ask_0123456789abcdef0123456789abcdef", + optionIndex: 1, +}; + +describe("handleTelegramQuestionCallback", () => { + it.each([ + [{ status: "answered", questionId: "target", optionValue: "Production" }, "Answer submitted."], + [ + { status: "already-terminal", reason: "already-terminal" }, + "This question was already answered.", + ], + ] as const)("shows outcome feedback", async (result, expectedText) => { + const feedback = vi.fn(async () => undefined); + const resolveQuestion = vi.fn(async () => result); + + await handleTelegramQuestionCallback({ + callback, + cfg: {} as never, + senderId: "42", + feedback, + resolveQuestion, + }); + + expect(feedback).toHaveBeenCalledWith(expectedText, true); + }); + + it("does not turn a committed answer into an error when feedback fails", async () => { + const feedback = vi.fn(async () => { + throw new Error("receipt failed"); + }); + + await expect( + handleTelegramQuestionCallback({ + callback, + cfg: {} as never, + senderId: "42", + feedback, + resolveQuestion: vi.fn(async () => ({ + status: "answered" as const, + questionId: "target", + optionValue: "Production", + })), + }), + ).resolves.toBeUndefined(); + expect(feedback).toHaveBeenCalledOnce(); + expect(feedback).toHaveBeenCalledWith("Answer submitted.", true); + }); +}); diff --git a/extensions/telegram/src/bot-handlers.callback-questions.runtime.ts b/extensions/telegram/src/bot-handlers.callback-questions.runtime.ts new file mode 100644 index 000000000000..10e066d930a7 --- /dev/null +++ b/extensions/telegram/src/bot-handlers.callback-questions.runtime.ts @@ -0,0 +1,38 @@ +// Telegram ask_user callback resolution and toast feedback. +import { + resolveQuestionOverGateway, + type ResolveQuestionOverGatewayParams, +} from "openclaw/plugin-sdk/question-gateway-runtime"; +import type { TelegramQuestionCallback } from "./question-callback-data.js"; + +type QuestionResolver = ( + params: ResolveQuestionOverGatewayParams, +) => ReturnType; + +export async function handleTelegramQuestionCallback(params: { + callback: TelegramQuestionCallback; + cfg: ResolveQuestionOverGatewayParams["cfg"]; + senderId: string; + feedback: (text: string, terminal: boolean) => Promise; + resolveQuestion?: QuestionResolver; +}): Promise { + let result: Awaited>; + try { + result = await (params.resolveQuestion ?? resolveQuestionOverGateway)({ + cfg: params.cfg, + questionId: params.callback.questionId, + optionIndex: params.callback.optionIndex, + senderId: params.senderId, + clientDisplayName: "Telegram question", + }); + } catch (error) { + await params.feedback("Could not submit this answer.", false).catch(() => {}); + throw error; + } + await params + .feedback( + result.status === "answered" ? "Answer submitted." : "This question was already answered.", + true, + ) + .catch(() => {}); +} diff --git a/extensions/telegram/src/bot-handlers.callback.runtime.ts b/extensions/telegram/src/bot-handlers.callback.runtime.ts index f571cecaa141..625cf4ecd425 100644 --- a/extensions/telegram/src/bot-handlers.callback.runtime.ts +++ b/extensions/telegram/src/bot-handlers.callback.runtime.ts @@ -18,6 +18,7 @@ import { } from "./bot-handlers.callback-errors.runtime.js"; import { handleTelegramInteractiveCallback } from "./bot-handlers.callback-interactions.runtime.js"; import { handleTelegramModelCallback } from "./bot-handlers.callback-model.runtime.js"; +import { handleTelegramQuestionCallback } from "./bot-handlers.callback-questions.runtime.js"; import type { TelegramHandlerMessageRuntime } from "./bot-handlers.message.runtime.js"; import { parseTelegramNativeCommandCallbackData } from "./bot-native-commands.js"; import type { RegisterTelegramHandlerParams } from "./bot-native-commands.js"; @@ -30,6 +31,10 @@ import type { TelegramGetChat } from "./bot/types.js"; import { getTelegramCallbackQueryAnswerPromise } from "./callback-query-answer-state.js"; import { resolveTelegramInlineButtonsScope } from "./inline-buttons.js"; import { parseTelegramOpaqueCallbackData } from "./native-command-callback-data.js"; +import { + hasTelegramQuestionCallbackPrefix, + parseTelegramQuestionCallbackData, +} from "./question-callback-data.js"; export function registerTelegramCallbackQueryHandler( { accountId, bot, runtime, telegramDeps, shouldSkipUpdate }: RegisterTelegramHandlerParams, @@ -47,26 +52,46 @@ export function registerTelegramCallbackQueryHandler( bot.on("callback_query", async (ctx) => { const callback = ctx.callbackQuery; - if (!callback || shouldSkipUpdate(ctx)) { + if (!callback) { return; } - const answerCallbackQuery = async () => { + let callbackAnswered = false; + const answerCallbackQuery = async (text?: string) => { // Callback answers prevent Telegram retries while the routed action runs. await withTelegramApiErrorLogging({ operation: "answerCallbackQuery", runtime, - fn: () => bot.api.answerCallbackQuery(callback.id), + fn: () => + text + ? bot.api.answerCallbackQuery(callback.id, { text }) + : bot.api.answerCallbackQuery(callback.id), }).catch(() => {}); + callbackAnswered = true; }; + if (shouldSkipUpdate(ctx)) { + const earlyAnswerPromise = getTelegramCallbackQueryAnswerPromise(ctx); + if (earlyAnswerPromise) { + await earlyAnswerPromise.catch(async () => await answerCallbackQuery()); + } else { + await answerCallbackQuery(); + } + return; + } + const data = (callback.data ?? "").trim(); + const typedQuestionCallback = parseTelegramQuestionCallbackData(data); const earlyAnswerPromise = getTelegramCallbackQueryAnswerPromise(ctx); if (earlyAnswerPromise) { - await earlyAnswerPromise.catch(answerCallbackQuery); + try { + await earlyAnswerPromise; + callbackAnswered = true; + } catch { + await answerCallbackQuery(); + } } else { await answerCallbackQuery(); } try { - const data = (callback.data ?? "").trim(); const callbackMessage = callback.message; if (!data || !callbackMessage) { return; @@ -80,18 +105,20 @@ export function registerTelegramCallbackQueryHandler( const callbackCommandText = nativeCallbackCommand ?? (opaqueCallbackData ? "" : genericCallbackText); const hasReservedApprovalPrefix = hasTelegramApprovalCallbackPrefix(data); + const hasReservedQuestionPrefix = hasTelegramQuestionCallbackPrefix(data); const typedApprovalCallback = parseTelegramApprovalCallbackData(data); const legacyApprovalCallback = parseExecApprovalCommandText( nativeCallbackCommand ?? (opaqueCallbackData ? "" : data), ); const isApprovalCallback = hasReservedApprovalPrefix || legacyApprovalCallback !== null; + const isRuntimeControlCallback = isApprovalCallback || hasReservedQuestionPrefix; const authorizationCfg = telegramDeps.getRuntimeConfig(); const inlineButtonsScope = resolveTelegramInlineButtonsScope({ cfg: authorizationCfg, accountId, }); - // Approval controls retain their kind-specific authorization after capability changes. - if (!isApprovalCallback) { + // Runtime controls retain their authorization after inline-button capability changes. + if (!isRuntimeControlCallback) { if ( inlineButtonsScope === "off" || (inlineButtonsScope === "dm" && isGroup) || @@ -128,8 +155,9 @@ export function registerTelegramCallbackQueryHandler( ); return; } - const authorizationMode: TelegramEventAuthorizationMode = - !isGroup || (!isApprovalCallback && inlineButtonsScope === "allowlist") + const authorizationMode: TelegramEventAuthorizationMode = hasReservedQuestionPrefix + ? "callback-runtime-allowlist" + : !isGroup || (!isRuntimeControlCallback && inlineButtonsScope === "allowlist") ? "callback-allowlist" : "callback-scope"; const senderAuthorization = await authorizeTelegramEventSender({ @@ -175,6 +203,23 @@ export function registerTelegramCallbackQueryHandler( await approvalRuntime.handleCanonical(typedApprovalCallback); return; } + if (typedQuestionCallback) { + await handleTelegramQuestionCallback({ + callback: typedQuestionCallback, + cfg: runtimeCfg, + senderId, + feedback: async (text, terminal) => { + if (terminal) { + await actions.clearCallbackButtons().catch(() => {}); + } + await actions.replyToCallbackChat(text); + }, + }); + return; + } + if (hasReservedQuestionPrefix) { + return; + } if (hasReservedApprovalPrefix) { await approvalRuntime.handleMalformedReserved(); return; @@ -258,6 +303,10 @@ export function registerTelegramCallbackQueryHandler( if (isTelegramSpooledReplayUpdate(ctx.update)) { recordTelegramMessageProcessingResult({ kind: "failed-retryable", error: err }); } + } finally { + if (typedQuestionCallback && !callbackAnswered) { + await answerCallbackQuery(); + } } }); } diff --git a/extensions/telegram/src/bot.create-telegram-bot.test.ts b/extensions/telegram/src/bot.create-telegram-bot.test.ts index f4d132a3e9cb..b27ea136caa0 100644 --- a/extensions/telegram/src/bot.create-telegram-bot.test.ts +++ b/extensions/telegram/src/bot.create-telegram-bot.test.ts @@ -491,6 +491,53 @@ describe("createTelegramBot", () => { expect(answerCallbackQuerySpy).toHaveBeenCalledTimes(1); }); + it("acknowledges question callbacks before their handler completes", async () => { + installPerKeySequentializer(); + loadConfig.mockReturnValue({ channels: { telegram: { dmPolicy: "disabled" } } }); + createTelegramBot({ token: "tok" }); + const callbackHandler = requireValue( + getOnHandler("callback_query") as + | ((ctx: Record) => Promise) + | undefined, + "callback_query handler", + ); + answerCallbackQuerySpy.mockClear(); + let releaseHandler: (() => void) | undefined; + const handlerGate = new Promise((resolve) => { + releaseHandler = resolve; + }); + const callbackQuery = { + id: "cbq-question-early-ack", + data: "tgq1:ask_0123456789abcdef0123456789abcdef:1", + from: { id: 9, first_name: "Ada", username: "ada_bot" }, + message: { + chat: { id: 1234, type: "private" }, + date: 1736380800, + message_id: 42, + }, + }; + const pending = runTelegramMiddlewareChain({ + ctx: { + update: { update_id: 403, callback_query: callbackQuery }, + callbackQuery, + me: { username: "openclaw_bot" }, + }, + finalHandler: async (ctx) => { + await callbackHandler(ctx); + await handlerGate; + }, + }); + await flushTelegramTestMicrotasks(); + + expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-question-early-ack"); + if (!releaseHandler) { + throw new Error("Expected Telegram question callback release callback to be initialized"); + } + releaseHandler(); + await pending; + expect(answerCallbackQuerySpy).toHaveBeenCalledTimes(1); + }); + it("lets /status bypass a busy Telegram topic lane", async () => { installPerKeySequentializer(); loadConfig.mockReturnValue({ @@ -2884,8 +2931,8 @@ describe("createTelegramBot", () => { await callbackHandler({ update: { update_id: 222 }, callbackQuery: { - id: "cb-1", - data: "ping", + id: "cb-question-duplicate", + data: "tgq1:ask_0123456789abcdef0123456789abcdef:1", from: { id: 789, username: "testuser" }, message: { chat: { id: 123, type: "private" }, @@ -2897,6 +2944,7 @@ describe("createTelegramBot", () => { getFile: async () => ({}), }); expect(replySpy).toHaveBeenCalledTimes(1); + expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cb-question-duplicate"); replySpy.mockClear(); diff --git a/extensions/telegram/src/bot.test.ts b/extensions/telegram/src/bot.test.ts index 11ef7bcf9557..93163acb29d9 100644 --- a/extensions/telegram/src/bot.test.ts +++ b/extensions/telegram/src/bot.test.ts @@ -33,6 +33,19 @@ import { buildTelegramOpaqueCallbackData } from "./native-command-callback-data. import { setTelegramRuntime } from "./runtime.js"; import { clearTelegramRuntimeForTest as clearTelegramRuntime } from "./runtime.test-support.js"; import type { TelegramRuntime } from "./runtime.types.js"; + +const questionGatewayHoisted = vi.hoisted(() => ({ + resolveQuestionOverGatewaySpy: vi.fn(async () => ({ + status: "answered" as const, + questionId: "target", + optionValue: "Production", + })), +})); + +vi.mock("openclaw/plugin-sdk/question-gateway-runtime", () => ({ + resolveQuestionOverGateway: questionGatewayHoisted.resolveQuestionOverGatewaySpy, +})); + const { answerCallbackQuerySpy, commandSpy, @@ -380,6 +393,7 @@ describe("createTelegramBot", () => { beforeEach(() => { setMyCommandsSpy.mockClear(); + questionGatewayHoisted.resolveQuestionOverGatewaySpy.mockClear(); clearPluginInteractiveHandlers(); loadConfig.mockReturnValue({ agents: { @@ -1013,6 +1027,44 @@ describe("createTelegramBot", () => { expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-group-1"); }); + it("keeps group question callbacks on the configured callback allowlist", async () => { + onSpy.mockClear(); + answerCallbackQuerySpy.mockClear(); + + const config = { + channels: { + telegram: { + dmPolicy: "open", + allowFrom: ["9"], + capabilities: { inlineButtons: "all" }, + groupPolicy: "open", + groups: { "*": { requireMention: false, allowFrom: ["9"] } }, + }, + }, + } satisfies NonNullable[0]["config"]>; + loadConfig.mockReturnValue(config); + createTelegramBot({ token: "tok", config }); + const callbackHandler = getTelegramCallbackHandlerForTests(); + + await callbackHandler({ + callbackQuery: { + id: "cbq-question-blocked", + data: "tgq1:ask_0123456789abcdef0123456789abcdef:1", + from: { id: 999, first_name: "Mallory", username: "mallory" }, + message: { + chat: { id: -100999, type: "supergroup", title: "Test Group" }, + date: 1736380800, + message_id: 21, + }, + }, + me: { username: "openclaw_bot" }, + getFile: async () => ({ download: async () => new Uint8Array() }), + }); + + expect(questionGatewayHoisted.resolveQuestionOverGatewaySpy).not.toHaveBeenCalled(); + expect(answerCallbackQuerySpy).toHaveBeenCalledWith("cbq-question-blocked"); + }); + it("replaces legacy approval controls with a visible terminal receipt", async () => { onSpy.mockClear(); editMessageReplyMarkupSpy.mockClear(); diff --git a/extensions/telegram/src/button-types.test.ts b/extensions/telegram/src/button-types.test.ts index 5179afaf8e93..ab1f411eee0b 100644 --- a/extensions/telegram/src/button-types.test.ts +++ b/extensions/telegram/src/button-types.test.ts @@ -54,6 +54,28 @@ describe("buildTelegramPresentationButtons", () => { ]); }); + it("encodes question buttons by record id and option index", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + expect( + buildTelegramPresentationButtons({ + blocks: [ + { + type: "buttons", + buttons: ["Staging", "Production"].map((label) => ({ + label, + action: { type: "question" as const, questionId, optionValue: label }, + })), + }, + ], + }), + ).toEqual([ + [ + { text: "Staging", callback_data: `tgq1:${questionId}:0`, style: undefined }, + { text: "Production", callback_data: `tgq1:${questionId}:1`, style: undefined }, + ], + ]); + }); + it("drops presentation buttons whose callback payload exceeds Telegram limits", () => { expect( buildTelegramPresentationButtons({ @@ -150,6 +172,35 @@ describe("buildTelegramPresentationButtons", () => { expect(parseTelegramOpaqueCallbackData(callbackData)).toBe(value); }); + it("keeps transport-private question callback prefixes opaque for legacy values", () => { + const value = "tgq1:ask_0123456789abcdef0123456789abcdef:0"; + const callbackData = buildTelegramOpaqueCallbackData(value); + + expect( + buildTelegramPresentationButtons({ + blocks: [ + { + type: "buttons", + buttons: [{ label: "Plugin", value }], + }, + ], + }), + ).toEqual([[{ text: "Plugin", callback_data: callbackData, style: undefined }]]); + expect(parseTelegramOpaqueCallbackData(callbackData)).toBe(value); + }); + + it("keeps trimmed transport-private question prefixes opaque", () => { + const value = " tgq1:ask_0123456789abcdef0123456789abcdef:0 "; + const callbackData = buildTelegramOpaqueCallbackData(value); + + expect( + buildTelegramPresentationButtons({ + blocks: [{ type: "buttons", buttons: [{ label: "Plugin", value }] }], + }), + ).toEqual([[{ text: "Plugin", callback_data: callbackData, style: undefined }]]); + expect(parseTelegramOpaqueCallbackData(callbackData)).toBe(value); + }); + it("keeps shortened plugin approval callbacks on the approval bypass path", () => { const approvalId = `plugin:${"a".repeat(36)}`; expect( diff --git a/extensions/telegram/src/button-types.ts b/extensions/telegram/src/button-types.ts index debe3350510b..252205346b97 100644 --- a/extensions/telegram/src/button-types.ts +++ b/extensions/telegram/src/button-types.ts @@ -20,6 +20,10 @@ import { buildTelegramNativeCommandCallbackData, buildTelegramOpaqueCallbackData, } from "./native-command-callback-data.js"; +import { + buildTelegramQuestionCallbackData, + hasTelegramQuestionCallbackPrefix, +} from "./question-callback-data.js"; export type TelegramButtonStyle = "danger" | "success" | "primary"; @@ -43,6 +47,7 @@ function toTelegramButtonStyle( function toTelegramInlineButton( button: MessagePresentationButton, + optionIndex: number, ): TelegramInlineButton | undefined { const style = toTelegramButtonStyle(button.style); const action = resolveMessagePresentationButtonAction(button); @@ -59,6 +64,13 @@ function toTelegramInlineButton( const callbackData = buildTelegramApprovalCallbackData(action); return callbackData ? { text: button.label, callback_data: callbackData, style } : undefined; } + if (action.type === "question") { + const callbackData = buildTelegramQuestionCallbackData({ + questionId: action.questionId, + optionIndex, + }); + return callbackData ? { text: button.label, callback_data: callbackData, style } : undefined; + } if (action.type === "command") { const command = rewriteTelegramApprovalDecisionAlias(action.command.trim()); const nativeCallbackData = command @@ -73,8 +85,11 @@ function toTelegramInlineButton( } // Reserve the full approval prefix, including malformed values, so legacy // plugin callbacks cannot be consumed by the approval handler. + const normalizedCallbackValue = action.value.trim(); const needsOpaqueEnvelope = - Boolean(button.action) || hasTelegramApprovalCallbackPrefix(action.value); + Boolean(button.action) || + hasTelegramApprovalCallbackPrefix(normalizedCallbackValue) || + hasTelegramQuestionCallbackPrefix(normalizedCallbackValue); const callbackData = sanitizeTelegramCallbackData( needsOpaqueEnvelope ? buildTelegramOpaqueCallbackData(action.value) : action.value, ); @@ -88,7 +103,7 @@ function chunkInteractiveButtons( for (let i = 0; i < buttons.length; i += TELEGRAM_INTERACTIVE_ROW_SIZE) { const row = buttons .slice(i, i + TELEGRAM_INTERACTIVE_ROW_SIZE) - .map(toTelegramInlineButton) + .map((button, offset) => toTelegramInlineButton(button, i + offset)) .filter((button): button is TelegramInlineButton => Boolean(button)); if (row.length > 0) { rows.push(row); diff --git a/extensions/telegram/src/question-callback-data.test.ts b/extensions/telegram/src/question-callback-data.test.ts new file mode 100644 index 000000000000..85ab36eacfd1 --- /dev/null +++ b/extensions/telegram/src/question-callback-data.test.ts @@ -0,0 +1,28 @@ +// Telegram question callback envelope tests. +import { describe, expect, it } from "vitest"; +import { + buildTelegramQuestionCallbackData, + parseTelegramQuestionCallbackData, +} from "./question-callback-data.js"; + +describe("question callback data", () => { + const questionId = "ask_0123456789abcdef0123456789abcdef"; + + it("round-trips a compact option index within Telegram's byte limit", () => { + const data = buildTelegramQuestionCallbackData({ questionId, optionIndex: 3 }); + + expect(data).toBe(`tgq1:${questionId}:3`); + expect(Buffer.byteLength(data ?? "", "utf8")).toBe(43); + expect(Buffer.byteLength(data ?? "", "utf8")).toBeLessThanOrEqual(64); + expect(parseTelegramQuestionCallbackData(data)).toEqual({ questionId, optionIndex: 3 }); + }); + + it.each([ + `tgq1:${questionId}:4`, + `tgq2:${questionId}:0`, + "tgq1:ask_short:0", + `tgq1:${questionId}:0:extra`, + ])("rejects malformed data: %s", (data) => { + expect(parseTelegramQuestionCallbackData(data)).toBeNull(); + }); +}); diff --git a/extensions/telegram/src/question-callback-data.ts b/extensions/telegram/src/question-callback-data.ts new file mode 100644 index 000000000000..37a2e629a5f1 --- /dev/null +++ b/extensions/telegram/src/question-callback-data.ts @@ -0,0 +1,42 @@ +// Telegram-private ask_user callback envelope. +const TELEGRAM_QUESTION_CALLBACK_PREFIX = "tgq1:"; +const TELEGRAM_CALLBACK_DATA_MAX_BYTES = 64; +const QUESTION_RECORD_ID_PATTERN = /^ask_[a-f0-9]{32}$/u; + +export type TelegramQuestionCallback = { + questionId: string; + optionIndex: number; +}; + +export function hasTelegramQuestionCallbackPrefix(data?: string | null): boolean { + return data?.startsWith(TELEGRAM_QUESTION_CALLBACK_PREFIX) === true; +} + +export function buildTelegramQuestionCallbackData( + callback: TelegramQuestionCallback, +): string | undefined { + if ( + !QUESTION_RECORD_ID_PATTERN.test(callback.questionId) || + !Number.isInteger(callback.optionIndex) || + callback.optionIndex < 0 || + callback.optionIndex > 3 + ) { + return undefined; + } + const data = `${TELEGRAM_QUESTION_CALLBACK_PREFIX}${callback.questionId}:${callback.optionIndex}`; + return Buffer.byteLength(data, "utf8") <= TELEGRAM_CALLBACK_DATA_MAX_BYTES ? data : undefined; +} + +export function parseTelegramQuestionCallbackData( + data?: string | null, +): TelegramQuestionCallback | null { + if ( + !hasTelegramQuestionCallbackPrefix(data) || + !data || + Buffer.byteLength(data, "utf8") > TELEGRAM_CALLBACK_DATA_MAX_BYTES + ) { + return null; + } + const match = /^tgq1:(ask_[a-f0-9]{32}):([0-3])$/u.exec(data); + return match?.[1] && match[2] ? { questionId: match[1], optionIndex: Number(match[2]) } : null; +} diff --git a/extensions/telegram/src/sequential-key.test.ts b/extensions/telegram/src/sequential-key.test.ts index 49d694735f57..68b36126a62f 100644 --- a/extensions/telegram/src/sequential-key.test.ts +++ b/extensions/telegram/src/sequential-key.test.ts @@ -282,6 +282,17 @@ describe("getTelegramSequentialKey", () => { }, "telegram:789:approval", ], + [ + { + update: { + callback_query: { + message: mockMessage({ chat: mockChat({ id: 321 }) }), + data: "tgq1:ask_0123456789abcdef0123456789abcdef:2", + }, + }, + }, + "telegram:321:question", + ], [ { update: { diff --git a/extensions/telegram/src/sequential-key.ts b/extensions/telegram/src/sequential-key.ts index 88bfd56be623..a54f611da968 100644 --- a/extensions/telegram/src/sequential-key.ts +++ b/extensions/telegram/src/sequential-key.ts @@ -14,6 +14,7 @@ import { resolveTelegramForumThreadId, resolveTelegramMessageForumFlagHint, } from "./bot/helpers.js"; +import { parseTelegramQuestionCallbackData } from "./question-callback-data.js"; const TELEGRAM_READ_ONLY_STATUS_COMMAND_KEYS = new Set([ "commands", @@ -179,6 +180,12 @@ export function getTelegramSequentialKey(ctx: TelegramSequentialKeyContext): str return "telegram:btw"; } const callbackData = ctx.update?.callback_query?.data; + if (parseTelegramQuestionCallbackData(callbackData)) { + if (typeof chatId === "number") { + return `telegram:${chatId}:question`; + } + return "telegram:question"; + } if (callbackData && parseExecApprovalCommandText(callbackData) !== null) { if (typeof chatId === "number") { return `telegram:${chatId}:approval`; diff --git a/package.json b/package.json index 0dc1afb2372f..b1cf83db227e 100644 --- a/package.json +++ b/package.json @@ -330,6 +330,10 @@ "types": "./dist/plugin-sdk/approval-runtime.d.ts", "default": "./dist/plugin-sdk/approval-runtime.js" }, + "./plugin-sdk/question-gateway-runtime": { + "types": "./dist/plugin-sdk/question-gateway-runtime.d.ts", + "default": "./dist/plugin-sdk/question-gateway-runtime.js" + }, "./plugin-sdk/config-runtime": { "types": "./dist/plugin-sdk/config-runtime.d.ts", "default": "./dist/plugin-sdk/config-runtime.js" diff --git a/packages/gateway-protocol/src/index.ts b/packages/gateway-protocol/src/index.ts index 537737646550..1328cedd0565 100644 --- a/packages/gateway-protocol/src/index.ts +++ b/packages/gateway-protocol/src/index.ts @@ -187,6 +187,24 @@ import { ExecApprovalGetParamsSchema, ExecApprovalRequestParamsSchema, ExecApprovalResolveParamsSchema, + QuestionAnswersSchema, + QuestionGetParamsSchema, + QuestionGetResultSchema, + QuestionListParamsSchema, + QuestionListResultSchema, + QuestionOptionSchema, + QuestionRecordSchema, + QuestionRequestedEventSchema, + QuestionRequestParamsSchema, + QuestionRequestQuestionSchema, + QuestionRequestResultSchema, + QuestionResolvedEventSchema, + QuestionResolveParamsSchema, + QuestionResolveResultSchema, + QuestionSchema, + QuestionStatusSchema, + QuestionWaitAnswerParamsSchema, + QuestionWaitAnswerResultSchema, PluginApprovalRequestParamsSchema, PluginApprovalResolveParamsSchema, PluginCatalogEntrySchema, @@ -832,6 +850,18 @@ export const validateExecApprovalsSetParams = lazyCompile(ExecApprovalsSetParams export const validateExecApprovalGetParams = lazyCompile(ExecApprovalGetParamsSchema); export const validateExecApprovalRequestParams = lazyCompile(ExecApprovalRequestParamsSchema); export const validateExecApprovalResolveParams = lazyCompile(ExecApprovalResolveParamsSchema); +export const validateQuestionRequestParams = lazyCompile(QuestionRequestParamsSchema); +export const validateQuestionRequestResult = lazyCompile(QuestionRequestResultSchema); +export const validateQuestionWaitAnswerParams = lazyCompile(QuestionWaitAnswerParamsSchema); +export const validateQuestionWaitAnswerResult = lazyCompile(QuestionWaitAnswerResultSchema); +export const validateQuestionResolveParams = lazyCompile(QuestionResolveParamsSchema); +export const validateQuestionResolveResult = lazyCompile(QuestionResolveResultSchema); +export const validateQuestionGetParams = lazyCompile(QuestionGetParamsSchema); +export const validateQuestionGetResult = lazyCompile(QuestionGetResultSchema); +export const validateQuestionListParams = lazyCompile(QuestionListParamsSchema); +export const validateQuestionListResult = lazyCompile(QuestionListResultSchema); +export const validateQuestionRequestedEvent = lazyCompile(QuestionRequestedEventSchema); +export const validateQuestionResolvedEvent = lazyCompile(QuestionResolvedEventSchema); export const validatePluginApprovalRequestParams = lazyCompile(PluginApprovalRequestParamsSchema); export const validatePluginApprovalResolveParams = lazyCompile(PluginApprovalResolveParamsSchema); export const validatePluginsListParams = lazyCompile(PluginsListParamsSchema); @@ -1297,6 +1327,24 @@ export { ExecApprovalGetParamsSchema, ExecApprovalRequestParamsSchema, ExecApprovalResolveParamsSchema, + QuestionAnswersSchema, + QuestionGetParamsSchema, + QuestionGetResultSchema, + QuestionListParamsSchema, + QuestionListResultSchema, + QuestionOptionSchema, + QuestionRecordSchema, + QuestionRequestedEventSchema, + QuestionRequestParamsSchema, + QuestionRequestQuestionSchema, + QuestionRequestResultSchema, + QuestionResolvedEventSchema, + QuestionResolveParamsSchema, + QuestionResolveResultSchema, + QuestionSchema, + QuestionStatusSchema, + QuestionWaitAnswerParamsSchema, + QuestionWaitAnswerResultSchema, ChatHistoryParamsSchema, ChatMetadataParamsSchema, ChatSendParamsSchema, @@ -1697,6 +1745,24 @@ export type { ExecApprovalGetParams, ExecApprovalRequestParams, ExecApprovalResolveParams, + Question, + QuestionAnswers, + QuestionGetParams, + QuestionGetResult, + QuestionListParams, + QuestionListResult, + QuestionOption, + QuestionRecord, + QuestionRequestedEvent, + QuestionRequestParams, + QuestionRequestQuestion, + QuestionRequestResult, + QuestionResolvedEvent, + QuestionResolveParams, + QuestionResolveResult, + QuestionStatus, + QuestionWaitAnswerParams, + QuestionWaitAnswerResult, LogsTailParams, LogsTailResult, TerminalOpenParams, diff --git a/packages/gateway-protocol/src/questions-validators.test.ts b/packages/gateway-protocol/src/questions-validators.test.ts new file mode 100644 index 000000000000..573ffddc18d9 --- /dev/null +++ b/packages/gateway-protocol/src/questions-validators.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { + validateQuestionGetResult, + validateQuestionListResult, + validateQuestionRequestedEvent, + validateQuestionRequestParams, + validateQuestionRequestResult, + validateQuestionResolvedEvent, + validateQuestionResolveParams, + validateQuestionResolveResult, + validateQuestionWaitAnswerParams, + validateQuestionWaitAnswerResult, +} from "./index.js"; + +const question = { + id: "choice", + header: "Choice", + question: "Which option?", + options: [{ label: "One", description: "First" }, { label: "Two" }], + multiSelect: false, + isOther: true, + isSecret: false, +}; +const answers = { answers: { choice: { answers: ["Two"] } } }; +const pendingRecord = { + id: "question-uuid", + questions: [question], + agentId: "main", + sessionKey: "agent:main:main", + createdAtMs: 1, + expiresAtMs: 2, + status: "pending", +}; + +describe("question protocol validators", () => { + it("round-trips method params and results", () => { + expect( + validateQuestionRequestParams({ + id: "client-question-id", + questions: [question], + timeoutMs: 100, + }), + ).toBe(true); + expect(validateQuestionRequestResult({ id: "question-uuid", expiresAtMs: 2 })).toBe(true); + expect(validateQuestionWaitAnswerParams({ id: "question-uuid", timeoutMs: 50 })).toBe(true); + expect(validateQuestionWaitAnswerResult({ status: "pending" })).toBe(true); + expect(validateQuestionWaitAnswerResult({ status: "answered", answers })).toBe(true); + expect(validateQuestionResolveParams({ id: "question-uuid", answers })).toBe(true); + expect(validateQuestionResolveParams({ id: "question-uuid", cancel: true })).toBe(true); + expect(validateQuestionResolveResult({ status: "cancelled" })).toBe(true); + expect(validateQuestionGetResult({ question: pendingRecord })).toBe(true); + expect(validateQuestionListResult({ questions: [pendingRecord] })).toBe(true); + }); + + it("round-trips requested and resolved events", () => { + expect(validateQuestionRequestedEvent(pendingRecord)).toBe(true); + expect( + validateQuestionResolvedEvent({ id: "question-uuid", status: "answered", answers }), + ).toBe(true); + expect(validateQuestionResolvedEvent({ id: "question-uuid", status: "expired" })).toBe(true); + }); + + it("keeps records normalized while allowing request-boundary header truncation", () => { + expect( + validateQuestionRequestParams({ + questions: [{ ...question, header: "longer than twelve" }], + }), + ).toBe(true); + expect( + validateQuestionRequestedEvent({ + ...pendingRecord, + questions: [{ ...question, header: "longer than twelve" }], + }), + ).toBe(false); + expect(validateQuestionRequestParams({ questions: [] })).toBe(false); + expect( + validateQuestionRequestParams({ questions: [question, question, question, question] }), + ).toBe(false); + }); +}); diff --git a/packages/gateway-protocol/src/schema.ts b/packages/gateway-protocol/src/schema.ts index b05c2fcd77cb..e80d82e0c40f 100644 --- a/packages/gateway-protocol/src/schema.ts +++ b/packages/gateway-protocol/src/schema.ts @@ -31,6 +31,7 @@ export * from "./schema/migrations.js"; export * from "./schema/nodes.js"; export * from "./schema/protocol-schemas.js"; export * from "./schema/push.js"; +export * from "./schema/questions.js"; export * from "./schema/secrets.js"; export * from "./schema/session-placement.js"; export * from "./schema/sessions.js"; diff --git a/packages/gateway-protocol/src/schema/protocol-schemas.ts b/packages/gateway-protocol/src/schema/protocol-schemas.ts index 817dd20c7ba8..1c1f4e9cbdf4 100644 --- a/packages/gateway-protocol/src/schema/protocol-schemas.ts +++ b/packages/gateway-protocol/src/schema/protocol-schemas.ts @@ -369,6 +369,25 @@ import { import { NodeInvokeProtocolSchemas } from "./protocol-schemas-node-invoke.js"; import { NodePresenceProtocolSchemas } from "./protocol-schemas-node-presence.js"; import { PushTestParamsSchema, PushTestResultSchema } from "./push.js"; +import { + QuestionAnswersSchema, + QuestionGetParamsSchema, + QuestionGetResultSchema, + QuestionListParamsSchema, + QuestionListResultSchema, + QuestionOptionSchema, + QuestionRecordSchema, + QuestionRequestParamsSchema, + QuestionRequestQuestionSchema, + QuestionRequestResultSchema, + QuestionResolvedEventSchema, + QuestionResolveParamsSchema, + QuestionResolveResultSchema, + QuestionSchema, + QuestionStatusSchema, + QuestionWaitAnswerParamsSchema, + QuestionWaitAnswerResultSchema, +} from "./questions.js"; import { SecretsReloadParamsSchema, SecretsResolveAssignmentSchema, @@ -938,6 +957,25 @@ export const ProtocolSchemas = { ExecApprovalGetParams: ExecApprovalGetParamsSchema, ExecApprovalRequestParams: ExecApprovalRequestParamsSchema, ExecApprovalResolveParams: ExecApprovalResolveParamsSchema, + QuestionOption: QuestionOptionSchema, + Question: QuestionSchema, + QuestionRequestQuestion: QuestionRequestQuestionSchema, + QuestionAnswers: QuestionAnswersSchema, + QuestionStatus: QuestionStatusSchema, + QuestionRecord: QuestionRecordSchema, + QuestionRequestParams: QuestionRequestParamsSchema, + QuestionRequestResult: QuestionRequestResultSchema, + QuestionWaitAnswerParams: QuestionWaitAnswerParamsSchema, + QuestionWaitAnswerResult: QuestionWaitAnswerResultSchema, + QuestionResolveParams: QuestionResolveParamsSchema, + QuestionResolveResult: QuestionResolveResultSchema, + QuestionGetParams: QuestionGetParamsSchema, + QuestionGetResult: QuestionGetResultSchema, + QuestionListParams: QuestionListParamsSchema, + QuestionListResult: QuestionListResultSchema, + // QuestionRequestedEvent is a TS-only alias of QuestionRecord; registering both + // names makes native codegen reference a type it never emits. + QuestionResolvedEvent: QuestionResolvedEventSchema, PluginApprovalRequestParams: PluginApprovalRequestParamsSchema, PluginApprovalResolveParams: PluginApprovalResolveParamsSchema, PluginCatalogClawHubInstall: PluginCatalogClawHubInstallSchema, diff --git a/packages/gateway-protocol/src/schema/questions.ts b/packages/gateway-protocol/src/schema/questions.ts new file mode 100644 index 000000000000..07108103130b --- /dev/null +++ b/packages/gateway-protocol/src/schema/questions.ts @@ -0,0 +1,143 @@ +// Gateway Protocol schema module defines transient operator questions. +import type { Static } from "typebox"; +import { Type } from "typebox"; +import { closedObject } from "./closed-object.js"; +import { NonEmptyString } from "./primitives.js"; + +const QuestionIdSchema = Type.String({ pattern: "^[a-z][a-z0-9_]*$" }); +const QuestionHeaderSchema = Type.String({ maxLength: 12 }); + +export const QuestionOptionSchema = closedObject({ + label: NonEmptyString, + description: Type.Optional(Type.String()), +}); + +const QuestionInputFields = { + id: QuestionIdSchema, + header: Type.String(), + question: NonEmptyString, + options: Type.Array(QuestionOptionSchema, { maxItems: 4 }), + multiSelect: Type.Optional(Type.Boolean()), + isOther: Type.Optional(Type.Boolean()), + isSecret: Type.Optional(Type.Boolean()), +}; + +/** Unnormalized question accepted by question.request. */ +export const QuestionRequestQuestionSchema = closedObject(QuestionInputFields); + +const QuestionFields = { + ...QuestionInputFields, + header: QuestionHeaderSchema, +}; + +/** Canonical normalized question shown to an operator. */ +export const QuestionSchema = closedObject(QuestionFields); + +export const QuestionAnswersSchema = closedObject({ + answers: Type.Record(QuestionIdSchema, closedObject({ answers: Type.Array(Type.String()) })), +}); + +export const QuestionStatusSchema = Type.Union([ + Type.Literal("pending"), + Type.Literal("answered"), + Type.Literal("cancelled"), + Type.Literal("expired"), +]); + +/** + * One pending or recently resolved transient question request. Flat object with + * optional terminal fields (exec-approval record precedent): native protocol + * codegen cannot emit per-status object unions, and the manager owns the + * status/answers invariant (answers present only when status is "answered"). + */ +export const QuestionRecordSchema = closedObject({ + id: NonEmptyString, + questions: Type.Array(QuestionSchema, { minItems: 1, maxItems: 3 }), + agentId: Type.Optional(NonEmptyString), + sessionKey: Type.Optional(NonEmptyString), + createdAtMs: Type.Integer({ minimum: 0 }), + expiresAtMs: Type.Integer({ minimum: 0 }), + status: QuestionStatusSchema, + answers: Type.Optional(QuestionAnswersSchema), + resolvedBy: Type.Optional(NonEmptyString), +}); + +export const QuestionRequestParamsSchema = closedObject({ + id: Type.Optional(NonEmptyString), + questions: Type.Array(QuestionRequestQuestionSchema, { minItems: 1, maxItems: 3 }), + agentId: Type.Optional(NonEmptyString), + sessionKey: Type.Optional(NonEmptyString), + timeoutMs: Type.Optional(Type.Integer({ minimum: 1 })), +}); + +export const QuestionRequestResultSchema = closedObject({ + id: NonEmptyString, + expiresAtMs: Type.Integer({ minimum: 0 }), +}); + +export const QuestionWaitAnswerParamsSchema = closedObject({ + id: NonEmptyString, + timeoutMs: Type.Optional(Type.Integer({ minimum: 1 })), +}); + +export const QuestionWaitAnswerResultSchema = Type.Union([ + closedObject({ status: Type.Literal("pending") }), + closedObject({ status: Type.Literal("answered"), answers: QuestionAnswersSchema }), + closedObject({ status: Type.Literal("cancelled") }), + closedObject({ status: Type.Literal("expired") }), +]); + +export const QuestionResolveParamsSchema = Type.Union([ + closedObject({ + id: NonEmptyString, + answers: QuestionAnswersSchema, + resolvedBy: Type.Optional(NonEmptyString), + }), + closedObject({ + id: NonEmptyString, + cancel: Type.Literal(true), + resolvedBy: Type.Optional(NonEmptyString), + }), +]); + +export const QuestionResolveResultSchema = Type.Union([ + closedObject({ status: Type.Literal("answered"), answers: QuestionAnswersSchema }), + closedObject({ status: Type.Literal("cancelled") }), +]); + +export const QuestionGetParamsSchema = closedObject({ id: NonEmptyString }); +export const QuestionGetResultSchema = closedObject({ question: QuestionRecordSchema }); +export const QuestionListParamsSchema = closedObject({}); +export const QuestionListResultSchema = closedObject({ + questions: Type.Array(QuestionRecordSchema), +}); + +export const QuestionRequestedEventSchema = QuestionRecordSchema; +export const QuestionResolvedEventSchema = Type.Union([ + closedObject({ + id: NonEmptyString, + status: Type.Literal("answered"), + answers: QuestionAnswersSchema, + }), + closedObject({ id: NonEmptyString, status: Type.Literal("cancelled") }), + closedObject({ id: NonEmptyString, status: Type.Literal("expired") }), +]); + +export type QuestionOption = Static; +export type Question = Static; +export type QuestionRequestQuestion = Static; +export type QuestionAnswers = Static; +export type QuestionStatus = Static; +export type QuestionRecord = Static; +export type QuestionRequestParams = Static; +export type QuestionRequestResult = Static; +export type QuestionWaitAnswerParams = Static; +export type QuestionWaitAnswerResult = Static; +export type QuestionResolveParams = Static; +export type QuestionResolveResult = Static; +export type QuestionGetParams = Static; +export type QuestionGetResult = Static; +export type QuestionListParams = Static; +export type QuestionListResult = Static; +export type QuestionRequestedEvent = Static; +export type QuestionResolvedEvent = Static; diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index bade30f11c58..cd865b8ad259 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -32,6 +32,7 @@ "approval-reaction-runtime", "approval-reply-runtime", "approval-runtime", + "question-gateway-runtime", "config-runtime", "config-contracts", "config-types", diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index efb8fef8af04..e1e26c6ee369 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -214,7 +214,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // Registry sweep: 77 packages, zero fetch failures; retired dead channel-ingress facade. // +1: speech-settings keeps agent prompt imports off the synthesis/runtime graph. // +1: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. - 330, + // +1: question-gateway-runtime resolves ask_user choices for channel plugins. + 331, env, ), // ScopeTree adds six channel-policy exports, mirrored by compat, including three functions. @@ -262,7 +263,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // Harvest: retired dual-field plan payload builder -1. // +23: core channel, envelope, direct-DM, feedback, legacy-payload, and memory contracts. // +81: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. - 8149, + // +3: question-gateway-runtime resolver plus request/result types. + 8152, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -296,7 +298,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // Harvest: retired dual-field plan payload builder -1. // +13: core channel, envelope, direct-DM, feedback, legacy-payload, and memory operations. // +32: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. - 4533, + // +1: question-gateway-runtime resolver. + 4534, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/scripts/protocol-event-coverage.allowlist.json b/scripts/protocol-event-coverage.allowlist.json index 65d573b65f1a..eab88cd2bf61 100644 --- a/scripts/protocol-event-coverage.allowlist.json +++ b/scripts/protocol-event-coverage.allowlist.json @@ -18,6 +18,8 @@ "voicewake.routing.changed": "iOS only consumes voicewake.changed trigger updates; routing changes are not surfaced.", "plugin.approval.requested": "Plugin approval prompts are not implemented on iOS.", "plugin.approval.resolved": "Plugin approval prompts are not implemented on iOS.", + "question.requested": "Interactive question UI ships on web/channels first; mobile apps ignore operator question events until their card UI lands.", + "question.resolved": "Interactive question UI ships on web/channels first; mobile apps ignore operator question events until their card UI lands.", "openclaw.approval.requested": "OpenClaw system-agent config approvals are a web/desktop operator surface; iOS has no operator-approval prompt.", "openclaw.approval.resolved": "OpenClaw system-agent config approvals are a web/desktop operator surface; iOS has no operator-approval prompt.", "terminal.data": "Embedded terminal is a web/desktop surface; iOS has no terminal client.", @@ -44,6 +46,8 @@ "voicewake.routing.changed": "Android reads voicewake state on demand via voicewake.get; no push consumer yet.", "plugin.approval.requested": "Plugin approval prompts are not implemented on Android.", "plugin.approval.resolved": "Plugin approval prompts are not implemented on Android.", + "question.requested": "Interactive question UI ships on web/channels first; mobile apps ignore operator question events until their card UI lands.", + "question.resolved": "Interactive question UI ships on web/channels first; mobile apps ignore operator question events until their card UI lands.", "openclaw.approval.requested": "OpenClaw system-agent config approvals are a web/desktop operator surface; Android has no operator-approval prompt.", "openclaw.approval.resolved": "OpenClaw system-agent config approvals are a web/desktop operator surface; Android has no operator-approval prompt.", "terminal.data": "Embedded terminal is a web/desktop surface; Android has no terminal client.", diff --git a/src/agents/core-tool-factory-descriptors.ts b/src/agents/core-tool-factory-descriptors.ts index 32710b823480..133b5c4d13d0 100644 --- a/src/agents/core-tool-factory-descriptors.ts +++ b/src/agents/core-tool-factory-descriptors.ts @@ -17,6 +17,7 @@ const CORE_TOOL_FACTORY_DESCRIPTORS = [ { name: "exec", family: "shell" }, { name: "process", family: "shell" }, { name: "agents_list", family: "openclaw" }, + { name: "ask_user", family: "openclaw" }, { name: "openclaw", family: "openclaw" }, { name: "computer", family: "openclaw" }, { name: "conversations_list", family: "openclaw" }, diff --git a/src/agents/embedded-agent-runner/run/attempt-stream-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-stream-prepare.ts index 0561e2cc481b..9c11cc855f80 100644 --- a/src/agents/embedded-agent-runner/run/attempt-stream-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-stream-prepare.ts @@ -353,7 +353,12 @@ export function prepareEmbeddedAttemptStream(input: { if (options?.steeringMode) { input.activeSession.agent.steeringMode = options.steeringMode; } - await steerActiveSessionWithOptionalDeliveryWait(input.activeSession, text, options); + await steerActiveSessionWithOptionalDeliveryWait( + input.activeSession, + text, + options, + attempt.sessionKey, + ); }, isStreaming: () => input.activeSession.isStreaming, isStopped: () => diff --git a/src/agents/embedded-agent-runner/run/attempt.queue-message.ts b/src/agents/embedded-agent-runner/run/attempt.queue-message.ts index 1db825f66be1..3d2a60870633 100644 --- a/src/agents/embedded-agent-runner/run/attempt.queue-message.ts +++ b/src/agents/embedded-agent-runner/run/attempt.queue-message.ts @@ -4,7 +4,12 @@ import { toErrorObject } from "../../../infra/errors.js"; import type { ImageContent } from "../../../llm/types.js"; import type { UserTurnTranscriptRecorder } from "../../../sessions/user-turn-transcript.types.js"; +import { + cancelPendingAskUserForSession, + claimPendingAskUserAnswer, +} from "../../tools/ask-user-tool.js"; import { log } from "../logger.js"; +import type { EmbeddedAgentQueueMessageOptions } from "../run-state.js"; /** * Minimal active-session surface needed to steer a running attempt and observe @@ -230,15 +235,33 @@ async function steerAndWaitForTranscriptCommit( export async function steerActiveSessionWithOptionalDeliveryWait( activeSession: EmbeddedAgentActiveSessionSteerTarget, text: string, - options: - | { - deliveryTimeoutMs?: number; - images?: ImageContent[]; - waitForTranscriptCommit?: boolean; - userTurnTranscriptRecorder?: UserTurnTranscriptRecorder; - } - | undefined, + options: EmbeddedAgentQueueMessageOptions | undefined, + sessionKey?: string, ): Promise { + const isInboundUserMessage = options?.isInboundUserMessage === true; + const isPlainTextAnswer = !options?.images?.length; + if (isInboundUserMessage && !isPlainTextAnswer) { + try { + await cancelPendingAskUserForSession({ sessionKey, resolvedBy: "image-reply" }); + } catch (error) { + log.warn(`failed to cancel ask_user before image steering: ${String(error)}`); + } + } + if ( + isInboundUserMessage && + isPlainTextAnswer && + (await claimPendingAskUserAnswer({ + sessionKey, + text, + persist: options.userTurnTranscriptRecorder + ? async () => { + await options.userTurnTranscriptRecorder?.persistApproved(); + } + : undefined, + })) + ) { + return; + } if (options?.waitForTranscriptCommit !== true) { if (options?.userTurnTranscriptRecorder) { await activeSession.steer(text, options.images, options.userTurnTranscriptRecorder); diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts index facf04b50181..a69c7e9364b8 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts @@ -29,11 +29,62 @@ import type { ToolCallSummary, ToolHandlerContext, } from "./embedded-agent-subscribe.handlers.types.js"; +import { + createAskUserTool, + normalizeAskUserParams, + reserveAskUserPromptDelivery, +} from "./tools/ask-user-tool.js"; +import { resetPendingAskUserQuestionsForTest } from "./tools/ask-user-tool.test-support.js"; type ToolExecutionStartEvent = Extract; type ToolExecutionEndEvent = Extract; type PayloadToolMetas = Parameters[0]["toolMetas"]; +const pendingAskUserFinishes = new Set<() => Promise>(); + +async function activateAskUserPrompt(toolCallId: string, args: unknown) { + let questionId: string | undefined; + let resolveAnswer: ((value: { status: "cancelled" }) => void) | undefined; + const tool = createAskUserTool({ + sessionKey: "agent:unit-session", + gatewayCall: async (method, _opts, params) => { + if (method === "question.request") { + if (!params || typeof params !== "object" || !("id" in params)) { + throw new Error("question.request params missing id"); + } + questionId = String(params.id); + return { id: questionId }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + resolveAnswer = resolve; + }); + } + throw new Error(`unexpected method ${method}`); + }, + }); + const pending = tool.execute(toolCallId, args); + let finished = false; + const finish = async () => { + if (finished) { + return; + } + finished = true; + await vi.waitFor(() => expect(resolveAnswer).toBeTypeOf("function")); + resolveAnswer?.({ status: "cancelled" }); + await pending; + pendingAskUserFinishes.delete(finish); + }; + pendingAskUserFinishes.add(finish); + await vi.waitFor(() => expect(questionId).toBeTypeOf("string")); + return { questionId: questionId!, finish }; +} + +afterEach(async () => { + await Promise.all([...pendingAskUserFinishes].map((finish) => finish())); + resetPendingAskUserQuestionsForTest(); +}); + const beforeToolCallTesting = { adjustedParamsByToolCallId, buildAdjustedParamsKey }; function createTestContext(): { @@ -247,6 +298,311 @@ function requireSingleMessagingTarget(ctx: ToolHandlerContext) { } describe("handleToolExecutionStart read path checks", () => { + it("delivers a numbered ask_user prompt with question id association", async () => { + const { ctx } = createTestContext(); + const onToolResult = vi.fn(); + ctx.params.onToolResult = onToolResult; + const args = { + questions: [ + { + id: "deploy_target", + header: "Target", + question: "Where should this deploy?", + options: [ + { label: "Staging (Recommended)", description: "Safer default" }, + { label: "Production" }, + ], + }, + ], + }; + + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-call-1", + args, + }); + const activation = await activateAskUserPrompt("ask-call-1", args); + await vi.waitFor(() => expect(onToolResult).toHaveBeenCalledOnce()); + const { questionId } = activation; + + expect(onToolResult).toHaveBeenCalledWith({ + text: [ + "Question for you:", + "", + "Target", + "Where should this deploy?", + "1. Staging (Recommended) - Safer default", + "2. Production", + "Other: reply with your own answer.", + "", + "Reply with the number, the option text, or your own answer.", + ].join("\n"), + channelData: { + askUser: { + questionId, + }, + }, + presentationTextMode: "fallback", + presentation: { + blocks: [ + { + type: "text", + text: [ + "Where should this deploy?", + "", + "- Staging (Recommended): Safer default", + "- Production", + "", + "Tap an option, or reply with the option text or your own answer.", + ].join("\n"), + }, + { + type: "buttons", + buttons: [ + { + label: "Staging (Recommended)", + action: { + type: "question", + questionId, + optionValue: "Staging (Recommended)", + }, + }, + { + label: "Production", + action: { + type: "question", + questionId, + optionValue: "Production", + }, + }, + ], + }, + ], + }, + }); + await activation.finish(); + }); + + it.each([ + { + name: "multi-question", + questions: [ + { + id: "target", + header: "Target", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + { + id: "region", + header: "Region", + question: "Which region?", + options: [{ label: "EU" }, { label: "US" }], + }, + ], + }, + { + name: "multi-select", + questions: [ + { + id: "targets", + header: "Targets", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + multiSelect: true, + }, + ], + }, + ])("keeps $name ask_user prompts text-only", async ({ questions }) => { + const { ctx } = createTestContext(); + const onToolResult = vi.fn(); + ctx.params.onToolResult = onToolResult; + const toolCallId = `ask-${questions[0]?.id ?? "unknown"}`; + + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId, + args: { questions }, + }); + const activation = await activateAskUserPrompt(toolCallId, { questions }); + await vi.waitFor(() => expect(onToolResult).toHaveBeenCalledOnce()); + + const payload = onToolResult.mock.calls[0]?.[0]; + expect(payload?.text).toContain("Reply with the number, the option text, or your own answer."); + expect(payload).not.toHaveProperty("presentation"); + expect(payload).not.toHaveProperty("presentationTextMode"); + await activation.finish(); + }); + + it("reserves ask_user before awaiting block-reply flush", async () => { + const { ctx, onBlockReplyFlush } = createTestContext(); + const onToolResult = vi.fn(); + ctx.params.onToolResult = onToolResult; + let releaseFlush: (() => void) | undefined; + onBlockReplyFlush.mockImplementation( + () => + new Promise((resolve) => { + releaseFlush = resolve; + }), + ); + const args = { + questions: [ + { + id: "target", + header: "Target", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + ], + }; + + const pending = handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-flush", + args, + }); + const activation = await activateAskUserPrompt("ask-flush", args); + await Promise.resolve(); + expect(onToolResult).not.toHaveBeenCalled(); + + releaseFlush?.(); + await pending; + await vi.waitFor(() => expect(onToolResult).toHaveBeenCalledOnce()); + await activation.finish(); + }); + + it.each(["buffer", "callback"] as const)( + "releases ask_user reservation when the %s flush throws synchronously", + (flushKind) => { + const { ctx, onBlockReplyFlush } = createTestContext(); + ctx.params.onToolResult = vi.fn(); + const failure = new Error("flush failed"); + if (flushKind === "buffer") { + vi.mocked(ctx.flushBlockReplyBuffer).mockImplementation(() => { + throw failure; + }); + } else { + onBlockReplyFlush.mockImplementation(() => { + throw failure; + }); + } + const args = { + questions: [ + { + id: "target", + header: "Target", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + ], + }; + + expect(() => + handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: `ask-${flushKind}-failure`, + args, + }), + ).toThrow(failure); + expect( + reserveAskUserPromptDelivery({ + toolCallId: `ask-${flushKind}-retry`, + sessionKey: "agent:unit-session", + questions: normalizeAskUserParams(args).questions, + }), + ).toBeDefined(); + }, + ); + + it("delivers only the ask_user prompt that reserved the session slot", async () => { + const { ctx } = createTestContext(); + const onToolResult = vi.fn(); + ctx.params.onToolResult = onToolResult; + const args = { + questions: [ + { + id: "target", + header: "Target", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + ], + }; + + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-first", + args, + }); + const activation = await activateAskUserPrompt("ask-first", args); + await vi.waitFor(() => expect(onToolResult).toHaveBeenCalledOnce()); + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-second", + args, + }); + + expect(onToolResult).toHaveBeenCalledTimes(1); + expect(onToolResult).toHaveBeenCalledWith( + expect.objectContaining({ + channelData: { + askUser: { + questionId: activation.questionId, + }, + }, + }), + ); + await activation.finish(); + }); + + it("releases an undelivered ask_user reservation when execution is rejected", async () => { + const { ctx } = createTestContext(); + const onToolResult = vi.fn(); + ctx.params.onToolResult = onToolResult; + const args = { + questions: [ + { + id: "target", + header: "Target", + question: "Where next?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + ], + }; + + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-denied", + args, + }); + await handleToolExecutionEnd(ctx, { + type: "tool_execution_end", + toolName: "ask_user", + toolCallId: "ask-denied", + isError: true, + result: { content: [{ type: "text", text: "denied" }] }, + }); + await Promise.resolve(); + expect(onToolResult).not.toHaveBeenCalled(); + + await handleToolExecutionStart(ctx, { + type: "tool_execution_start", + toolName: "ask_user", + toolCallId: "ask-after-denial", + args, + }); + const activation = await activateAskUserPrompt("ask-after-denial", args); + await vi.waitFor(() => expect(onToolResult).toHaveBeenCalledOnce()); + await activation.finish(); + }); + it("emits trace-only tool start diagnostics when trace logging is enabled", async () => { const { ctx, trace, isEnabled, warn } = createTestContext(); isEnabled.mockImplementation((level: string) => level === "trace"); diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.ts b/src/agents/embedded-agent-subscribe.handlers.tools.ts index e20199f3f1d6..55424688b539 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.ts @@ -12,6 +12,7 @@ import { readStringValue, } from "@openclaw/normalization-core/string-coerce"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import type { QuestionRequestQuestion } from "../../packages/gateway-protocol/src/schema/questions.js"; import { HEARTBEAT_RESPONSE_TOOL_NAME, normalizeHeartbeatToolResponse, @@ -37,7 +38,7 @@ import { parseInteractiveParam, parseJsonMessageParam, } from "../infra/outbound/message-action-params.js"; -import { hasReplyPayloadContent } from "../interactive/payload.js"; +import { hasReplyPayloadContent, type MessagePresentation } from "../interactive/payload.js"; import type { PluginHookAfterToolCallEvent } from "../plugins/types.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; import { hasTopLevelShellControlOperator, splitShellArgs } from "../utils/shell-argv.js"; @@ -89,6 +90,7 @@ import { } from "./embedded-agent-subscribe.tools.js"; import { inferToolMetaFromArgs } from "./embedded-agent-utils.js"; import { parseExecApprovalResultText } from "./exec-approval-result.js"; +import { formatAgentHarnessUserInputPrompt } from "./harness/user-input-bridge.js"; import type { AgentEvent } from "./runtime/index.js"; import { createToolValidationErrorSummary, @@ -98,6 +100,13 @@ import { buildToolMutationState } from "./tool-mutation.js"; import { normalizeToolName } from "./tool-policy.js"; import { readToolResultDetails } from "./tool-result-error.js"; import { createToolTerminalObserver } from "./tool-terminal-outcome.js"; +import { + cancelAskUserPromptDelivery, + normalizeAskUserParams, + reserveAskUserPromptDelivery, + settleAskUserPromptDelivery, + waitForAskUserPromptReady, +} from "./tools/ask-user-tool.js"; type ExecApprovalReplyModule = typeof import("../infra/exec-approval-reply.js"); type HookRunnerGlobalModule = typeof import("../plugins/hook-runner-global.js"); @@ -111,6 +120,45 @@ const execApprovalReplyModuleLoader = createLazyImportLoader( () => import("../plugins/hook-runner-global.js"), ); + +function buildAskUserQuestionPresentation(params: { + questionId: string; + questions: QuestionRequestQuestion[]; +}): MessagePresentation | undefined { + // Button taps resolve atomically, so v1 keeps multi-question records text-only. + if (params.questions.length !== 1) { + return undefined; + } + const [question] = params.questions; + if (!question || question.multiSelect || question.isSecret || question.options.length === 0) { + return undefined; + } + const presentationText = [ + question.question, + "", + ...question.options.map( + (option) => `- ${option.label}${option.description ? `: ${option.description}` : ""}`, + ), + "", + "Tap an option, or reply with the option text or your own answer.", + ].join("\n"); + return { + blocks: [ + { type: "text", text: presentationText }, + { + type: "buttons", + buttons: question.options.map((option) => ({ + label: option.label, + action: { + type: "question", + questionId: params.questionId, + optionValue: option.label, + }, + })), + }, + ], + }; +} const fallbackToolTerminalObservers = new WeakMap< ToolHandlerContext["state"], ReturnType @@ -144,6 +192,24 @@ function readUpdatePlanResult( return { ...(explanation ? { explanation } : {}), steps }; } +function buildAskUserPromptPayload( + toolCallId: string, + sessionKey: string | undefined, + args: unknown, +) { + try { + const { questions } = normalizeAskUserParams(args); + const reservation = reserveAskUserPromptDelivery({ toolCallId, sessionKey, questions }); + if (!reservation) { + return undefined; + } + return reservation; + } catch { + // Argument validation owns malformed calls; do not deliver an unusable prompt first. + return undefined; + } +} + function isMiddlewareToolResultError(result: unknown): boolean { if (!result || typeof result !== "object") { return false; @@ -943,21 +1009,40 @@ export function handleToolExecutionStart( hideFromChannelProgress?: boolean; }, ): void | Promise { - const continueAfterBlockReplyFlush = (): void | Promise => { - const onBlockReplyFlushResult = ctx.params.onBlockReplyFlush?.({ - reason: "tool_start", - assistantMessageIndex: ctx.state.assistantMessageIndex, - }); - if (isPromiseLike(onBlockReplyFlushResult)) { - return onBlockReplyFlushResult.then(() => { - continueToolExecutionStart(); - }); + const startToolName = normalizeToolName(evt.toolName); + const askUserPromptReservation = + startToolName === "ask_user" && ctx.params.onToolResult + ? buildAskUserPromptPayload(evt.toolCallId, ctx.params.sessionKey, evt.args) + : undefined; + const cancelAskUserPromptReservation = () => { + if (askUserPromptReservation) { + cancelAskUserPromptDelivery(evt.toolCallId, ctx.params.sessionKey); } - continueToolExecutionStart(); - return undefined; + }; + const continueAfterBlockReplyFlush = (): void | Promise => { + let onBlockReplyFlushResult: void | Promise; + try { + onBlockReplyFlushResult = ctx.params.onBlockReplyFlush?.({ + reason: "tool_start", + assistantMessageIndex: ctx.state.assistantMessageIndex, + }); + } catch (error) { + cancelAskUserPromptReservation(); + throw error; + } + if (isPromiseLike(onBlockReplyFlushResult)) { + return onBlockReplyFlushResult.then( + () => continueToolExecutionStart(), + (error: unknown) => { + cancelAskUserPromptReservation(); + throw error; + }, + ); + } + return continueToolExecutionStart(); }; - const continueToolExecutionStart = () => { + const continueToolExecutionStart = (): void | Promise => { const rawToolName = evt.toolName; const toolName = normalizeToolName(rawToolName); const hideFromChannelProgress = evt.hideFromChannelProgress === true; @@ -1165,12 +1250,53 @@ export function handleToolExecutionStart( } } } + + if (toolName === "ask_user" && ctx.params.onToolResult) { + const payload = askUserPromptReservation; + if (payload) { + const questionId = payload.questionId; + void waitForAskUserPromptReady(questionId) + .then((questions) => { + if (!questions) { + return; + } + const prompt = formatAgentHarnessUserInputPrompt(questions, { + intro: "Question for you:", + }); + const presentation = buildAskUserQuestionPresentation({ questionId, questions }); + return ctx.params.onToolResult?.({ + text: `${prompt}\n\nReply with the number, the option text, or your own answer.`, + ...(presentation ? { presentation, presentationTextMode: "fallback" as const } : {}), + channelData: { askUser: { questionId } }, + }); + }) + .then( + () => settleAskUserPromptDelivery(questionId), + (error: unknown) => { + settleAskUserPromptDelivery(questionId, error); + ctx.log.warn(`failed to deliver ask_user prompt: ${String(error)}`); + }, + ); + } + } }; // Flush pending block replies to preserve message boundaries before tool execution. - const flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer(); + let flushBlockReplyBufferResult: void | Promise; + try { + flushBlockReplyBufferResult = ctx.flushBlockReplyBuffer(); + } catch (error) { + cancelAskUserPromptReservation(); + throw error; + } if (isPromiseLike(flushBlockReplyBufferResult)) { - return flushBlockReplyBufferResult.then(() => continueAfterBlockReplyFlush()); + return flushBlockReplyBufferResult.then( + () => continueAfterBlockReplyFlush(), + (error: unknown) => { + cancelAskUserPromptReservation(); + throw error; + }, + ); } return continueAfterBlockReplyFlush(); } @@ -1280,6 +1406,9 @@ export async function handleToolExecutionEnd( const toolName = normalizeToolName(rawToolName); const hideFromChannelProgress = evt.hideFromChannelProgress === true; const toolCallId = evt.toolCallId; + if (toolName === "ask_user") { + cancelAskUserPromptDelivery(toolCallId, ctx.params.sessionKey); + } const runId = ctx.params.runId; const isError = evt.isError; const result = evt.result; diff --git a/src/agents/harness/user-input-bridge.ts b/src/agents/harness/user-input-bridge.ts index 0cff1af9b994..78974a3a4856 100644 --- a/src/agents/harness/user-input-bridge.ts +++ b/src/agents/harness/user-input-bridge.ts @@ -10,6 +10,7 @@ export type AgentHarnessUserInputQuestion = { id: string; header: string; question: string; + multiSelect?: boolean; isOther?: boolean; isSecret?: boolean; options?: readonly AgentHarnessUserInputOption[] | null; diff --git a/src/agents/openclaw-tools.registration.ts b/src/agents/openclaw-tools.registration.ts index 45f4fb68aaa3..e4b9753c5e51 100644 --- a/src/agents/openclaw-tools.registration.ts +++ b/src/agents/openclaw-tools.registration.ts @@ -5,6 +5,7 @@ */ import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isPrimaryBootstrapRun } from "./bootstrap-routing.js"; import { isToolAllowedByPolicyName } from "./tool-policy-match.js"; import type { AnyAgentTool } from "./tools/common.js"; @@ -51,3 +52,20 @@ export function shouldIncludeUpdatePlanToolForOpenClawTools(params: { isToolAllowedByPolicyName("update_plan", { deny }) ); } + +/** Includes ask_user only on a primary session and when normal deny policy permits it. */ +export function shouldIncludeAskUserToolForOpenClawTools(params: { + config?: OpenClawConfig; + agentSessionKey?: string; + pluginToolDenylist?: string[]; +}): boolean { + const sessionKey = params.agentSessionKey?.trim(); + if (!sessionKey) { + return false; + } + const deny = uniqueStrings([ + ...(params.config?.tools?.deny ?? []), + ...(params.pluginToolDenylist ?? []), + ]); + return isPrimaryBootstrapRun(sessionKey) && isToolAllowedByPolicyName("ask_user", { deny }); +} diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 38a9a92380cc..12d138018922 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -35,6 +35,7 @@ import { import { applyNodesToolWorkspaceGuard } from "./openclaw-tools.nodes-workspace-guard.js"; import { collectPresentOpenClawTools, + shouldIncludeAskUserToolForOpenClawTools, shouldIncludeUpdatePlanToolForOpenClawTools, } from "./openclaw-tools.registration.js"; import type { SandboxFsBridge } from "./sandbox/fs-bridge.js"; @@ -42,6 +43,7 @@ import type { SpawnedToolContext } from "./spawned-context.js"; import type { ToolFsPolicy } from "./tool-fs-policy.js"; import { resolveToolLoopDetectionConfig } from "./tool-loop-detection-config.js"; import { createAgentsListTool } from "./tools/agents-list-tool.js"; +import { createAskUserTool } from "./tools/ask-user-tool.js"; import type { AnyAgentTool } from "./tools/common.js"; import { createComputerTool } from "./tools/computer-tool.js"; import { @@ -453,6 +455,13 @@ export function createOpenClawTools( pluginToolAllowlist: options?.pluginToolAllowlist, pluginToolDenylist: options?.pluginToolDenylist, }); + // isEmbeddedMode() marks the TUI-embedded host, not the embedded agent runner; + // gating on it would hide ask_user from every normal gateway run. + const includeAskUserTool = shouldIncludeAskUserToolForOpenClawTools({ + config: resolvedConfig, + agentSessionKey: options?.runSessionKey ?? options?.agentSessionKey, + pluginToolDenylist: options?.pluginToolDenylist, + }); const includeTranscriptsTool = resolveTranscriptsConfig(resolvedConfig?.transcripts).enabled; const tools: AnyAgentTool[] = [ ...(embedded @@ -560,6 +569,14 @@ export function createOpenClawTools( }), ]), ...(includeUpdatePlanTool ? [createUpdatePlanTool()] : []), + ...(includeAskUserTool + ? [ + createAskUserTool({ + agentId: sessionAgentId, + sessionKey: options?.runSessionKey ?? options?.agentSessionKey, + }), + ] + : []), createSessionsListTool({ agentSessionKey: options?.agentSessionKey, sandboxed: options?.sandboxed, diff --git a/src/agents/openclaw-tools.update-plan.test.ts b/src/agents/openclaw-tools.update-plan.test.ts index f9c8680b7786..894710978664 100644 --- a/src/agents/openclaw-tools.update-plan.test.ts +++ b/src/agents/openclaw-tools.update-plan.test.ts @@ -5,7 +5,10 @@ import { setEmbeddedMode } from "../infra/embedded-mode.js"; import { isToolWrappedWithBeforeToolCallHook } from "./agent-tools.before-tool-call.js"; import { resolveCoreToolFactoryFamily } from "./core-tool-factory-descriptors.js"; import { createOpenClawTools } from "./openclaw-tools.js"; -import { shouldIncludeUpdatePlanToolForOpenClawTools } from "./openclaw-tools.registration.js"; +import { + shouldIncludeAskUserToolForOpenClawTools, + shouldIncludeUpdatePlanToolForOpenClawTools, +} from "./openclaw-tools.registration.js"; import { createUpdatePlanTool } from "./tools/update-plan-tool.js"; type UpdatePlanGatingParams = Parameters[0]; @@ -82,9 +85,47 @@ describe("openclaw-tools update_plan gating", () => { }; expect(defaultTools).toContain("update_plan"); + expect(defaultTools).not.toContain("ask_user"); expect(shouldIncludeUpdatePlanToolForOpenClawTools(emptyAllowlistParams)).toBe(true); }); + it("keeps ask_user on primary sessions and excludes spawned worker sessions", () => { + expect(shouldIncludeAskUserToolForOpenClawTools({})).toBe(false); + expect(shouldIncludeAskUserToolForOpenClawTools({ agentSessionKey: "agent:main:main" })).toBe( + true, + ); + expect( + shouldIncludeAskUserToolForOpenClawTools({ + agentSessionKey: "agent:main:subagent:worker", + }), + ).toBe(false); + expect( + shouldIncludeAskUserToolForOpenClawTools({ agentSessionKey: "agent:main:acp:worker" }), + ).toBe(false); + // ask_user must not depend on the TUI embedded-host flag; normal gateway + // runs are the primary consumer. + expect( + createFastToolNames({ + config: {} as OpenClawConfig, + runSessionKey: "agent:main:non-embedded", + }), + ).toContain("ask_user"); + setEmbeddedMode(true); + + expect( + createFastToolNames({ + config: {} as OpenClawConfig, + agentSessionKey: "agent:main:subagent:worker", + }), + ).not.toContain("ask_user"); + expect( + createFastToolNames({ + config: {} as OpenClawConfig, + runSessionKey: "agent:main:run", + }), + ).toContain("ask_user"); + }); + it("wraps constructed tools with before-tool-call hooks by default", () => { const tools = createOpenClawTools({ config: {} as OpenClawConfig, diff --git a/src/agents/runtime-plan/types.ts b/src/agents/runtime-plan/types.ts index a27cea120f4d..6fb18aca30d6 100644 --- a/src/agents/runtime-plan/types.ts +++ b/src/agents/runtime-plan/types.ts @@ -112,6 +112,11 @@ type AgentRuntimeMessagePresentationAction = approvalKind: "exec" | "plugin"; decision: "allow-once" | "allow-always" | "deny"; } + | { + type: "question"; + questionId: string; + optionValue: string; + } | { type: "url"; url: string; @@ -264,6 +269,7 @@ type AgentRuntimeReplyPayload = { trustedLocalMedia?: boolean; sensitiveMedia?: boolean; presentation?: AgentRuntimeMessagePresentation; + presentationTextMode?: "fallback"; delivery?: AgentRuntimeReplyPayloadDelivery; /** * @deprecated Use presentation. diff --git a/src/agents/tool-description-presets.ts b/src/agents/tool-description-presets.ts index e703885ab1d4..f7e7c3211841 100644 --- a/src/agents/tool-description-presets.ts +++ b/src/agents/tool-description-presets.ts @@ -11,6 +11,7 @@ export const SESSIONS_SPAWN_TOOL_DISPLAY_SUMMARY = "Spawn subagent or ACP sessio export const SESSIONS_SPAWN_SUBAGENT_TOOL_DISPLAY_SUMMARY = "Spawn subagent session."; export const SESSION_STATUS_TOOL_DISPLAY_SUMMARY = "Show session status/model/usage."; export const UPDATE_PLAN_TOOL_DISPLAY_SUMMARY = "Track short work plan."; +export const ASK_USER_TOOL_DISPLAY_SUMMARY = "Ask the user and wait for an answer."; export const SPAWN_TASK_TOOL_DISPLAY_SUMMARY = "Suggest follow-up work for operator approval."; export const DISMISS_TASK_TOOL_DISPLAY_SUMMARY = "Withdraw a pending task suggestion."; @@ -98,3 +99,14 @@ export function describeSessionStatusTool(): string { export function describeUpdatePlanTool(): string { return "Use for multi-step work. Send the full list each call; keep statuses current and exactly one `in_progress` until done."; } + +/** Describes the ask_user tool and its decision-only use policy. */ +export function describeAskUserTool(): string { + return [ + "Ask the human user 1-3 structured questions and wait for their answer.", + "Use only when blocked on a decision genuinely theirs that cannot be resolved from the request, code, or sensible defaults; never ask whether to proceed or confirm a plan.", + "Prefer one question. Put the recommended option first and suffix its label with ` (Recommended)`.", + "Do not include an Other option; free text is added automatically.", + "If the result is no_answer, continue with best judgment.", + ].join(" "); +} diff --git a/src/agents/tool-display-config.ts b/src/agents/tool-display-config.ts index 73b1b8781fe5..a126009bec7e 100644 --- a/src/agents/tool-display-config.ts +++ b/src/agents/tool-display-config.ts @@ -300,6 +300,11 @@ export const TOOL_DISPLAY_CONFIG: ToolDisplayConfig = { title: "Update Plan", detailKeys: ["explanation", "plan.0.step"], }, + ask_user: { + emoji: "❓", + title: "Ask User", + detailKeys: ["questions.0.question"], + }, spawn_task: { emoji: "✨", title: "Suggest Task", diff --git a/src/agents/tools/ask-user-tool.test-support.ts b/src/agents/tools/ask-user-tool.test-support.ts new file mode 100644 index 000000000000..c152de6d1587 --- /dev/null +++ b/src/agents/tools/ask-user-tool.test-support.ts @@ -0,0 +1,19 @@ +import "./ask-user-tool.js"; + +type AskUserToolTestApi = { + resetPendingAskUserQuestionsForTest(): void; +}; + +function getTestApi(): AskUserToolTestApi { + const api = (globalThis as Record)[ + Symbol.for("openclaw.askUserToolTestApi") + ]; + if (!api) { + throw new Error("ask_user tool test API is unavailable"); + } + return api as AskUserToolTestApi; +} + +export function resetPendingAskUserQuestionsForTest(): void { + getTestApi().resetPendingAskUserQuestionsForTest(); +} diff --git a/src/agents/tools/ask-user-tool.test.ts b/src/agents/tools/ask-user-tool.test.ts new file mode 100644 index 000000000000..7eee82f1967e --- /dev/null +++ b/src/agents/tools/ask-user-tool.test.ts @@ -0,0 +1,667 @@ +import { Value } from "typebox/value"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.types.js"; +import { steerActiveSessionWithOptionalDeliveryWait } from "../embedded-agent-runner/run/attempt.queue-message.js"; +import { + createAskUserTool, + isAskUserPromptActive, + normalizeAskUserParams, + reserveAskUserPromptDelivery, + settleAskUserPromptDelivery, +} from "./ask-user-tool.js"; +import { resetPendingAskUserQuestionsForTest } from "./ask-user-tool.test-support.js"; + +type GatewayCall = NonNullable[0]["gatewayCall"]>; + +const validArgs = { + questions: [ + { + id: "deploy_target", + header: "Deployment target", + question: "Where should this deploy?", + options: [ + { label: "Staging (Recommended)", description: "Safer default" }, + { label: "Production" }, + ], + }, + ], +}; + +function gatewayStub( + implementation: ( + method: string, + opts: Record, + params: Record, + extra?: { signal?: AbortSignal }, + ) => Promise, +) { + const mock = vi.fn(implementation); + return { mock, call: mock as unknown as GatewayCall }; +} + +function requestedQuestionId(mock: ReturnType["mock"]): string { + const requestCall = mock.mock.calls.find(([method]) => method === "question.request"); + const questionId = requestCall?.[2].id; + if (typeof questionId !== "string") { + throw new Error("question.request did not include an id"); + } + return questionId; +} + +afterEach(() => { + resetPendingAskUserQuestionsForTest(); +}); + +describe("ask_user normalization", () => { + it("normalizes headers, forces free text, and clamps timeout", () => { + const normalized = normalizeAskUserParams({ ...validArgs, timeoutSeconds: 5 }); + + expect(normalized.timeoutSeconds).toBe(30); + expect(normalized.questions[0]).toMatchObject({ + id: "deploy_target", + header: "Deployment t", + isOther: true, + }); + expect(normalizeAskUserParams({ ...validArgs, timeoutSeconds: 9_999 }).timeoutSeconds).toBe( + 3_600, + ); + expect(Value.Check(createAskUserTool({}).parameters, validArgs)).toBe(true); + expect( + Value.Check(createAskUserTool({}).parameters, { + questions: [{ ...validArgs.questions[0], isSecret: true }], + }), + ).toBe(false); + expect(normalized.questions[0]).not.toHaveProperty("isSecret"); + }); + + it.each([ + ["empty questions", { questions: [] }, "1 to 3 questions"], + [ + "too many questions", + { questions: Array.from({ length: 4 }, () => validArgs.questions[0]) }, + "1 to 3 questions", + ], + [ + "too few options", + { questions: [{ ...validArgs.questions[0], options: [{ label: "Only" }] }] }, + "2 to 4 options", + ], + [ + "duplicate ids", + { questions: [validArgs.questions[0], validArgs.questions[0]] }, + "duplicate question id 'deploy_target'", + ], + [ + "invalid id", + { questions: [{ ...validArgs.questions[0], id: "Deploy Target" }] }, + "must be snake_case", + ], + ])("rejects %s", (_name, args, error) => { + expect(() => normalizeAskUserParams(args)).toThrow(error); + }); +}); + +describe("ask_user execution", () => { + it("returns answered details plus readable answer lines", async () => { + const answers = { answers: { deploy_target: { answers: ["Staging (Recommended)"] } } }; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id, expiresAtMs: Date.now() + 30_000 }; + } + if (method === "question.waitAnswer") { + return { status: "answered", answers }; + } + throw new Error(`unexpected method ${method}`); + }); + const tool = createAskUserTool({ + agentId: "main", + sessionKey: "agent:main:main", + gatewayCall: gateway.call, + }); + + const result = await tool.execute("call-answered", validArgs); + const questionId = requestedQuestionId(gateway.mock); + + expect(result.details).toEqual({ status: "answered", answers }); + expect(result.content).toEqual([ + expect.objectContaining({ + type: "text", + text: expect.stringContaining("Deployment t: Staging (Recommended)"), + }), + ]); + expect(gateway.mock).toHaveBeenNthCalledWith( + 1, + "question.request", + {}, + expect.objectContaining({ + id: questionId, + agentId: "main", + sessionKey: "agent:main:main", + timeoutMs: 900_000, + }), + undefined, + ); + expect(gateway.mock).toHaveBeenNthCalledWith( + 2, + "question.waitAnswer", + { timeoutMs: 910_000 }, + { id: questionId, timeoutMs: 900_000 }, + undefined, + ); + }); + + it.each([ + ["expired", "No answer arrived"], + ["pending", "No answer arrived"], + ["cancelled", "question was cancelled"], + ] as const)("maps %s to no_answer", async (status, text) => { + const gateway = gatewayStub(async (method, _opts, params) => + method === "question.request" ? { id: params.id } : { status }, + ); + const result = await createAskUserTool({ + sessionKey: `agent:main:${status}`, + gatewayCall: gateway.call, + }).execute(`call-${status}`, validArgs); + const questionId = requestedQuestionId(gateway.mock); + + expect(result.details).toEqual({ status: "no_answer" }); + expect(result.content[0]).toMatchObject({ text: expect.stringContaining(text) }); + if (status === "pending") { + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: questionId, cancel: true, resolvedBy: "wait-timeout" }, + ); + } + }); + + it("rejects a second pending question in the same session", async () => { + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + finishWait?.({ status: "cancelled" }); + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + const tool = createAskUserTool({ + sessionKey: "agent:main:serialized", + gatewayCall: gateway.call, + }); + const first = tool.execute("call-first", validArgs); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + + await expect(tool.execute("call-second", validArgs)).rejects.toThrow( + "already has a pending question", + ); + finishWait?.({ status: "cancelled" }); + await expect(first).resolves.toMatchObject({ details: { status: "no_answer" } }); + }); + + it("cancels the gateway question when the run aborts", async () => { + const controller = new AbortController(); + const gateway = gatewayStub(async (method, _opts, params, extra) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.resolve") { + return { status: "cancelled" }; + } + return await new Promise((_resolve, reject) => { + extra?.signal?.addEventListener("abort", () => reject(new Error("aborted")), { + once: true, + }); + }); + }); + const pending = createAskUserTool({ + sessionKey: "agent:main:abort", + gatewayCall: gateway.call, + }).execute("call-abort", validArgs, controller.signal); + await vi.waitFor(() => + expect(gateway.mock.mock.calls.some((call) => call[0] === "question.waitAnswer")).toBe(true), + ); + const questionId = requestedQuestionId(gateway.mock); + + controller.abort(new Error("stop")); + + await expect(pending).rejects.toThrow("aborted"); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: questionId, cancel: true, resolvedBy: "run-abort" }, + ); + }); + + it("aborts registration and still attempts gateway cancellation", async () => { + const controller = new AbortController(); + const gateway = gatewayStub(async (method, _opts, _params, extra) => { + if (method === "question.resolve") { + return { status: "cancelled" }; + } + return await new Promise((_resolve, reject) => { + extra?.signal?.addEventListener("abort", () => reject(new Error("registration aborted")), { + once: true, + }); + }); + }); + const pending = createAskUserTool({ + sessionKey: "agent:main:register-abort", + gatewayCall: gateway.call, + }).execute("call-register-abort", validArgs, controller.signal); + await vi.waitFor(() => + expect(gateway.mock.mock.calls.some((call) => call[0] === "question.request")).toBe(true), + ); + const questionId = requestedQuestionId(gateway.mock); + + controller.abort(new Error("stop")); + + await expect(pending).rejects.toThrow("registration aborted"); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: questionId, cancel: true, resolvedBy: "run-abort" }, + ); + }); + + it("does not activate prompt delivery when registration ignores an earlier abort", async () => { + const sessionKey = "agent:main:late-registration-abort"; + const reservation = reserveAskUserPromptDelivery({ + toolCallId: "call-late-registration-abort", + sessionKey, + questions: normalizeAskUserParams(validArgs).questions, + }); + if (!reservation) { + throw new Error("expected prompt reservation"); + } + let finishRegistration: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method) => { + if (method === "question.request") { + return await new Promise((resolve) => { + finishRegistration = resolve; + }); + } + if (method === "question.resolve") { + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + const controller = new AbortController(); + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-late-registration-abort", + validArgs, + controller.signal, + ); + await vi.waitFor(() => expect(finishRegistration).toBeTypeOf("function")); + + controller.abort(new Error("stop before registration completed")); + finishRegistration?.({ id: reservation.questionId }); + + await expect(pending).rejects.toThrow("stop before registration completed"); + expect(isAskUserPromptActive(reservation.questionId)).toBe(false); + }); + + it("best-effort cancels a deterministic id after an ambiguous registration failure", async () => { + const sessionKey = "agent:main:registration-loss"; + const gateway = gatewayStub(async (method) => { + if (method === "question.request") { + throw new Error("connection lost after send"); + } + if (method === "question.resolve") { + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + + await expect( + createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-registration-loss", + validArgs, + ), + ).rejects.toThrow("connection lost after send"); + const questionId = requestedQuestionId(gateway.mock); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: questionId, cancel: true, resolvedBy: "registration-failed" }, + ); + }); + + it("cancels instead of waiting when originating prompt delivery fails", async () => { + const sessionKey = "agent:main:delivery-failure"; + const reservation = reserveAskUserPromptDelivery({ + toolCallId: "call-delivery-failure", + sessionKey, + questions: normalizeAskUserParams(validArgs).questions, + }); + if (!reservation) { + throw new Error("expected prompt reservation"); + } + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + finishWait?.({ status: "cancelled" }); + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-delivery-failure", + validArgs, + ); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + + settleAskUserPromptDelivery(reservation.questionId, new Error("channel unavailable")); + + await expect(pending).rejects.toThrow("ask_user prompt delivery failed"); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: reservation.questionId, cancel: true, resolvedBy: "prompt-delivery-failed" }, + ); + expect(gateway.mock.mock.calls.some((call) => call[0] === "question.waitAnswer")).toBe(true); + }); + + it("preserves an answer that wins the prompt-failure cancellation race", async () => { + const sessionKey = "agent:main:delivery-answer-race"; + const reservation = reserveAskUserPromptDelivery({ + toolCallId: "call-delivery-answer-race", + sessionKey, + questions: normalizeAskUserParams(validArgs).questions, + }); + if (!reservation) { + throw new Error("expected prompt reservation"); + } + const answers = { answers: { deploy_target: { answers: ["Production"] } } }; + let waitCalls = 0; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + waitCalls += 1; + if (waitCalls === 1) { + return await new Promise(() => {}); + } + return { status: "answered", answers }; + } + if (method === "question.resolve") { + throw Object.assign(new Error("already answered"), { + name: "GatewayClientRequestError", + details: { reason: "QUESTION_ALREADY_TERMINAL" }, + }); + } + throw new Error(`unexpected method ${method}`); + }); + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-delivery-answer-race", + validArgs, + ); + await vi.waitFor(() => expect(waitCalls).toBe(1)); + + settleAskUserPromptDelivery(reservation.questionId, new Error("channel unavailable")); + + await expect(pending).resolves.toMatchObject({ details: { status: "answered", answers } }); + expect(waitCalls).toBe(2); + }); + + it("aborts while prompt delivery is still pending", async () => { + const sessionKey = "agent:main:delivery-abort"; + const reservation = reserveAskUserPromptDelivery({ + toolCallId: "call-delivery-abort", + sessionKey, + questions: normalizeAskUserParams(validArgs).questions, + }); + if (!reservation) { + throw new Error("expected prompt reservation"); + } + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params, extra) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve, reject) => { + finishWait = resolve; + extra?.signal?.addEventListener("abort", () => reject(new Error("wait aborted")), { + once: true, + }); + }); + } + if (method === "question.resolve") { + finishWait?.({ status: "cancelled" }); + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + const controller = new AbortController(); + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-delivery-abort", + validArgs, + controller.signal, + ); + await vi.waitFor(() => + expect(gateway.mock.mock.calls.some((call) => call[0] === "question.request")).toBe(true), + ); + + controller.abort(new Error("stop during delivery")); + + expect(isAskUserPromptActive(reservation.questionId)).toBe(false); + await expect(pending).rejects.toThrow("stop during delivery"); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { id: reservation.questionId, cancel: true, resolvedBy: "run-abort" }, + ); + }); + + it("claims unmatched plain text as free text without steering it into the run", async () => { + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + const answers = params.answers; + finishWait?.({ status: "answered", answers }); + return { status: "answered", answers }; + } + throw new Error(`unexpected method ${method}`); + }); + const pending = createAskUserTool({ + sessionKey: "agent:main:claim", + gatewayCall: gateway.call, + }).execute("call-claim", validArgs); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + const questionId = requestedQuestionId(gateway.mock); + const steer = vi.fn(async () => undefined); + const activeSession = { steer, subscribe: vi.fn(() => () => undefined) }; + const persistApproved = vi.fn(async () => undefined); + const recorder = { persistApproved } as unknown as UserTurnTranscriptRecorder; + + await steerActiveSessionWithOptionalDeliveryWait( + activeSession, + "A custom destination", + { + isInboundUserMessage: true, + waitForTranscriptCommit: true, + userTurnTranscriptRecorder: recorder, + }, + "agent:main:claim", + ); + + expect(steer).not.toHaveBeenCalled(); + expect(persistApproved).toHaveBeenCalledOnce(); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + {}, + { + id: questionId, + answers: { answers: { deploy_target: { answers: ["A custom destination"] } } }, + resolvedBy: "plain-text", + }, + ); + await expect(pending).resolves.toMatchObject({ details: { status: "answered" } }); + }); + + it.each([ + ["cancellation succeeds", false], + ["cancellation fails", true], + ])("keeps image replies on normal steering when %s", async (_name, cancelFails) => { + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + if (cancelFails) { + throw new Error("gateway unavailable"); + } + finishWait?.({ status: "cancelled" }); + return { status: "cancelled" }; + } + throw new Error(`unexpected method ${method}`); + }); + const suffix = cancelFails ? "image-cancel-failure" : "image-reply"; + const sessionKey = `agent:main:${suffix}`; + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + `call-${suffix}`, + validArgs, + ); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + const questionId = requestedQuestionId(gateway.mock); + const steer = vi.fn(async () => undefined); + const images = [{ type: "image" as const, data: "pixels", mimeType: "image/png" }]; + + await steerActiveSessionWithOptionalDeliveryWait( + { steer, subscribe: vi.fn(() => () => undefined) }, + "Use this image", + { isInboundUserMessage: true, images }, + sessionKey, + ); + + expect(steer).toHaveBeenCalledWith("Use this image", images); + expect(gateway.mock).toHaveBeenCalledWith( + "question.resolve", + { timeoutMs: 10_000 }, + { + id: questionId, + cancel: true, + resolvedBy: "image-reply", + }, + ); + if (cancelFails) { + finishWait?.({ status: "cancelled" }); + } + await pending; + }); + + it("confirms a committed plain-text answer after its resolve response is lost", async () => { + let finishWait: ((value: unknown) => void) | undefined; + let committedAnswers: unknown; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + if (committedAnswers) { + return { status: "answered", answers: committedAnswers }; + } + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + committedAnswers = params.answers; + finishWait?.({ status: "answered", answers: committedAnswers }); + throw new Error("response lost after commit"); + } + throw new Error(`unexpected method ${method}`); + }); + const sessionKey = "agent:main:resolve-loss"; + const pending = createAskUserTool({ sessionKey, gatewayCall: gateway.call }).execute( + "call-resolve-loss", + validArgs, + ); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + const steer = vi.fn(async () => undefined); + const persistApproved = vi.fn(async () => undefined); + + await steerActiveSessionWithOptionalDeliveryWait( + { steer, subscribe: vi.fn(() => () => undefined) }, + "1", + { + isInboundUserMessage: true, + userTurnTranscriptRecorder: { persistApproved } as unknown as UserTurnTranscriptRecorder, + }, + sessionKey, + ); + + expect(steer).not.toHaveBeenCalled(); + expect(persistApproved).toHaveBeenCalledOnce(); + await expect(pending).resolves.toMatchObject({ details: { status: "answered" } }); + }); + + it("falls back to normal steering when the gateway question is already terminal", async () => { + let finishWait: ((value: unknown) => void) | undefined; + const gateway = gatewayStub(async (method, _opts, params) => { + if (method === "question.request") { + return { id: params.id }; + } + if (method === "question.waitAnswer") { + return await new Promise((resolve) => { + finishWait = resolve; + }); + } + if (method === "question.resolve") { + throw Object.assign(new Error("already answered"), { + name: "GatewayClientRequestError", + details: { reason: "QUESTION_ALREADY_TERMINAL" }, + }); + } + throw new Error(`unexpected method ${method}`); + }); + const pending = createAskUserTool({ + sessionKey: "agent:main:terminal-race", + gatewayCall: gateway.call, + }).execute("call-terminal-race", validArgs); + await vi.waitFor(() => expect(finishWait).toBeTypeOf("function")); + const steer = vi.fn(async () => undefined); + + await steerActiveSessionWithOptionalDeliveryWait( + { steer, subscribe: vi.fn(() => () => undefined) }, + "Follow-up message", + { isInboundUserMessage: true }, + "agent:main:terminal-race", + ); + + expect(steer).toHaveBeenCalledWith("Follow-up message", undefined); + finishWait?.({ status: "cancelled" }); + await pending; + }); +}); diff --git a/src/agents/tools/ask-user-tool.ts b/src/agents/tools/ask-user-tool.ts new file mode 100644 index 000000000000..4a0f5ebb7386 --- /dev/null +++ b/src/agents/tools/ask-user-tool.ts @@ -0,0 +1,694 @@ +/** Built-in blocking user-question tool and its active-session answer bridge. */ +import { createHash } from "node:crypto"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { Type } from "typebox"; +import type { + QuestionAnswers, + QuestionRequestQuestion, + QuestionWaitAnswerResult, +} from "../../../packages/gateway-protocol/src/index.js"; +import { + buildAgentHarnessUserInputAnswers, + type AgentHarnessUserInputQuestion, +} from "../harness/user-input-bridge.js"; +import { ASK_USER_TOOL_DISPLAY_SUMMARY, describeAskUserTool } from "../tool-description-presets.js"; +import { type AnyAgentTool, ToolInputError, textResult } from "./common.js"; +import { callGatewayTool, type GatewayCallOptions } from "./gateway.js"; + +const DEFAULT_ASK_USER_TIMEOUT_SECONDS = 900; +const MIN_ASK_USER_TIMEOUT_SECONDS = 30; +const MAX_ASK_USER_TIMEOUT_SECONDS = 3600; +const ASK_USER_RPC_GRACE_MS = 10_000; +const QUESTION_ID_PATTERN = /^[a-z][a-z0-9_]*$/; +const TERMINAL_QUESTION_ERROR_REASONS = new Set([ + "QUESTION_ALREADY_TERMINAL", + "QUESTION_NOT_FOUND", +]); + +const AskUserToolSchema = Type.Object( + { + questions: Type.Array( + Type.Object( + { + id: Type.String({ + minLength: 1, + pattern: "^[a-z][a-z0-9_]*$", + description: "Unique snake_case answer key.", + }), + header: Type.String({ + minLength: 1, + description: "Short chip label; longer input is truncated to 12 characters.", + }), + question: Type.String({ + minLength: 1, + description: "Single-sentence question for the user.", + }), + options: Type.Array( + Type.Object( + { + label: Type.String({ minLength: 1 }), + description: Type.Optional(Type.String()), + }, + { additionalProperties: false }, + ), + { minItems: 2, maxItems: 4 }, + ), + multiSelect: Type.Optional(Type.Boolean()), + }, + { additionalProperties: false }, + ), + { minItems: 1, maxItems: 3 }, + ), + timeoutSeconds: Type.Optional(Type.Integer()), + }, + { additionalProperties: false }, +); + +type AskUserGatewayCall = ( + method: string, + opts: GatewayCallOptions, + params?: unknown, + extra?: { signal?: AbortSignal }, +) => Promise; + +type AskUserQuestionPhase = + | { kind: "reserved" } + | { kind: "registering" } + | { kind: "prompting" } + | { kind: "answerable" } + | { kind: "resolving" } + | { kind: "prompt-failed"; error: unknown }; + +type AskUserQuestionState = { + questionId: string; + sessionKey: string; + questions: QuestionRequestQuestion[]; + phase: AskUserQuestionPhase; + gatewayCall?: AskUserGatewayCall; + answer?: Promise; + waiters: Set<() => void>; +}; + +const askUserQuestions = new Map(); + +type NormalizedAskUserParams = { + questions: QuestionRequestQuestion[]; + timeoutSeconds: number; +}; + +function readRequiredString(value: unknown, label: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new ToolInputError(`${label} must be a non-empty string`); + } + return value.trim(); +} + +function normalizeOption(value: unknown, questionIndex: number, optionIndex: number) { + const labelPrefix = `questions[${questionIndex}].options[${optionIndex}]`; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new ToolInputError(`${labelPrefix} must be an object`); + } + const record = value as Record; + const label = readRequiredString(record.label, `${labelPrefix}.label`); + // Telegram button text caps at 64 chars — the tightest native transport. + // Bounding here keeps schema-valid prompts deliverable on every channel. + if (label.length > 64) { + throw new ToolInputError(`${labelPrefix}.label must be at most 64 characters (use 1-5 words)`); + } + if (record.description !== undefined && typeof record.description !== "string") { + throw new ToolInputError(`${labelPrefix}.description must be a string`); + } + const description = + typeof record.description === "string" ? record.description.trim() : undefined; + return { label, ...(description ? { description } : {}) }; +} + +/** Validates and canonicalizes model-authored ask_user arguments. */ +export function normalizeAskUserParams(value: unknown): NormalizedAskUserParams { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new ToolInputError("ask_user arguments must be an object"); + } + const params = value as Record; + if ( + !Array.isArray(params.questions) || + params.questions.length < 1 || + params.questions.length > 3 + ) { + throw new ToolInputError("questions must contain 1 to 3 questions"); + } + const ids = new Set(); + const questions = params.questions.map( + (questionValue, questionIndex): QuestionRequestQuestion => { + const prefix = `questions[${questionIndex}]`; + if (!questionValue || typeof questionValue !== "object" || Array.isArray(questionValue)) { + throw new ToolInputError(`${prefix} must be an object`); + } + const question = questionValue as Record; + const id = readRequiredString(question.id, `${prefix}.id`); + if (!QUESTION_ID_PATTERN.test(id)) { + throw new ToolInputError(`${prefix}.id must be snake_case (for example, deploy_target)`); + } + if (ids.has(id)) { + throw new ToolInputError(`duplicate question id '${id}'`); + } + ids.add(id); + const header = truncateUtf16Safe(readRequiredString(question.header, `${prefix}.header`), 12); + const questionText = readRequiredString(question.question, `${prefix}.question`); + if ( + !Array.isArray(question.options) || + question.options.length < 2 || + question.options.length > 4 + ) { + throw new ToolInputError(`${prefix}.options must contain 2 to 4 options`); + } + if (question.multiSelect !== undefined && typeof question.multiSelect !== "boolean") { + throw new ToolInputError(`${prefix}.multiSelect must be a boolean`); + } + return { + id, + header, + question: questionText, + options: question.options.map((option, optionIndex) => + normalizeOption(option, questionIndex, optionIndex), + ), + ...(question.multiSelect === true ? { multiSelect: true } : {}), + isOther: true, + }; + }, + ); + + const rawTimeoutSeconds = params.timeoutSeconds; + if ( + rawTimeoutSeconds !== undefined && + (typeof rawTimeoutSeconds !== "number" || + !Number.isFinite(rawTimeoutSeconds) || + !Number.isInteger(rawTimeoutSeconds)) + ) { + throw new ToolInputError("timeoutSeconds must be an integer"); + } + const timeoutSeconds = Math.min( + MAX_ASK_USER_TIMEOUT_SECONDS, + Math.max(MIN_ASK_USER_TIMEOUT_SECONDS, rawTimeoutSeconds ?? DEFAULT_ASK_USER_TIMEOUT_SECONDS), + ); + return { questions, timeoutSeconds }; +} + +/** Stable client-generated gateway question id shared with tool-start delivery. */ +function buildAskUserQuestionId(toolCallId: string, sessionKey?: string): string { + const identity = `${sessionKey?.trim() ?? ""}\0${toolCallId}`; + return `ask_${createHash("sha256").update(identity).digest("hex").slice(0, 32)}`; +} + +function askUserSessionKey(sessionKey: string | undefined, agentId?: string): string { + return sessionKey?.trim() || (agentId?.trim() ? `agent:${agentId.trim()}` : "session:unknown"); +} + +function findAskUserQuestionForSession(sessionKey: string): AskUserQuestionState | undefined { + for (const question of askUserQuestions.values()) { + if (question.sessionKey === sessionKey) { + return question; + } + } + return undefined; +} + +function transitionAskUserQuestion(state: AskUserQuestionState, phase: AskUserQuestionPhase): void { + state.phase = phase; + for (const wake of state.waiters) { + wake(); + } + state.waiters.clear(); +} + +function releaseAskUserQuestion(questionId: string): void { + const state = askUserQuestions.get(questionId); + if (!state) { + return; + } + askUserQuestions.delete(questionId); + for (const wake of state.waiters) { + wake(); + } + state.waiters.clear(); +} + +async function waitForQuestionChange( + state: AskUserQuestionState, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + await new Promise((resolve, reject) => { + const wake = () => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }; + const onAbort = () => { + state.waiters.delete(wake); + reject(signal?.reason instanceof Error ? signal.reason : new Error("ask_user aborted")); + }; + state.waiters.add(wake); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** Reserves one visible ask_user prompt slot before subscriber delivery. */ +export function reserveAskUserPromptDelivery(params: { + toolCallId: string; + sessionKey?: string; + questions: QuestionRequestQuestion[]; +}): { questionId: string } | undefined { + const sessionKey = askUserSessionKey(params.sessionKey); + if (findAskUserQuestionForSession(sessionKey)) { + return undefined; + } + const questionId = buildAskUserQuestionId(params.toolCallId, params.sessionKey); + if (askUserQuestions.has(questionId)) { + return undefined; + } + askUserQuestions.set(questionId, { + questionId, + sessionKey, + questions: params.questions, + phase: { kind: "reserved" }, + waiters: new Set(), + }); + return { questionId }; +} + +/** Waits until policy-accepted tool execution has registered the gateway question. */ +export async function waitForAskUserPromptReady( + questionId: string, +): Promise { + const state = askUserQuestions.get(questionId); + if (!state) { + return undefined; + } + while (askUserQuestions.get(questionId) === state) { + if ( + state.phase.kind === "prompting" || + state.phase.kind === "answerable" || + state.phase.kind === "resolving" || + state.phase.kind === "prompt-failed" + ) { + return state.questions; + } + await waitForQuestionChange(state); + } + return undefined; +} + +/** Opens prompt delivery after question.request succeeds. */ +function markAskUserPromptReady(questionId: string, questions: QuestionRequestQuestion[]): void { + const state = askUserQuestions.get(questionId); + if (!state || (state.phase.kind !== "reserved" && state.phase.kind !== "registering")) { + return; + } + state.questions = questions; + transitionAskUserQuestion(state, { kind: "prompting" }); +} + +/** Records whether the originating-conversation prompt reached its delivery callback. */ +export function settleAskUserPromptDelivery(questionId: string, error?: unknown): void { + const state = askUserQuestions.get(questionId); + if (!state || state.phase.kind !== "prompting") { + return; + } + transitionAskUserQuestion( + state, + error === undefined ? { kind: "answerable" } : { kind: "prompt-failed", error }, + ); +} + +/** Returns whether a question-associated prompt still belongs to a blocking ask_user call. */ +export function isAskUserPromptActive(questionId: string): boolean { + return askUserQuestions.has(questionId); +} + +/** Releases a tool-start reservation when policy rejects execution. */ +export function cancelAskUserPromptDelivery(toolCallId: string, sessionKey?: string): void { + releaseAskUserQuestion(buildAskUserQuestionId(toolCallId, sessionKey)); +} + +function answeredResult(questions: readonly QuestionRequestQuestion[], answers: QuestionAnswers) { + const payload = { status: "answered" as const, answers }; + const lines = questions.map((question) => { + const values = answers.answers[question.id]?.answers ?? []; + return `${question.header}: ${values.length > 0 ? values.join(", ") : "(no answer)"}`; + }); + return textResult(`${lines.join("\n")}\n\n${JSON.stringify(payload, null, 2)}`, payload); +} + +function noAnswerResult(status: Exclude) { + const payload = { status: "no_answer" as const }; + const note = + status === "cancelled" + ? "The question was cancelled; proceed with best judgment." + : "No answer arrived; proceed with best judgment."; + return textResult(`${note}\n\n${JSON.stringify(payload, null, 2)}`, payload); +} + +async function waitForPromptDelivery( + state: AskUserQuestionState, + signal?: AbortSignal, +): Promise<{ error?: unknown }> { + while (askUserQuestions.get(state.questionId) === state) { + if (state.phase.kind === "answerable" || state.phase.kind === "resolving") { + return {}; + } + if (state.phase.kind === "prompt-failed") { + return { error: state.phase.error }; + } + await waitForQuestionChange(state, signal); + } + return { error: new Error("ask_user prompt is no longer active") }; +} + +function readQuestionErrorReason(error: unknown): string | undefined { + if (!error || typeof error !== "object") { + return undefined; + } + const requestError = error as { details?: unknown; name?: unknown }; + if (requestError.name !== "GatewayClientRequestError") { + return undefined; + } + const details = requestError.details; + if (!details || typeof details !== "object" || Array.isArray(details)) { + return undefined; + } + const reason = (details as { reason?: unknown }).reason; + return typeof reason === "string" ? reason : undefined; +} + +function isTerminalQuestionResolveError(error: unknown): boolean { + const reason = readQuestionErrorReason(error); + return reason !== undefined && TERMINAL_QUESTION_ERROR_REASONS.has(reason); +} + +async function observeCommittedAnswer( + answer: Promise | undefined, +): Promise { + if (!answer) { + return false; + } + let timer: ReturnType | undefined; + try { + const result = await Promise.race([ + answer, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), 1_000); + timer.unref?.(); + }), + ]); + return result?.status === "answered"; + } catch { + return false; + } finally { + if (timer) { + clearTimeout(timer); + } + } +} + +/** Claims the next queued plain-text message for this session's active question. */ +export async function claimPendingAskUserAnswer(params: { + sessionKey?: string; + text: string; + persist?: () => Promise; +}): Promise { + const sessionKey = params.sessionKey?.trim(); + if (!sessionKey) { + return false; + } + const state = findAskUserQuestionForSession(sessionKey); + if (!state || state.phase.kind !== "answerable" || !state.gatewayCall) { + return false; + } + transitionAskUserQuestion(state, { kind: "resolving" }); + try { + await params.persist?.(); + } catch (error) { + transitionAskUserQuestion(state, { kind: "answerable" }); + throw error; + } + const answers = buildAgentHarnessUserInputAnswers( + state.questions as AgentHarnessUserInputQuestion[], + params.text, + ); + try { + await state.gatewayCall( + "question.resolve", + {}, + { id: state.questionId, answers, resolvedBy: "plain-text" }, + ); + return true; + } catch (error) { + if (isTerminalQuestionResolveError(error)) { + return false; + } + // The long-lived wait observes a resolve that committed even when its response was lost. + // Reusing it avoids a second gateway read and answer-shape comparison. + if (await observeCommittedAnswer(state.answer)) { + return true; + } + transitionAskUserQuestion(state, { kind: "answerable" }); + throw error; + } +} + +/** Cancels the blocking question before an inbound turn takes another route. */ +export async function cancelPendingAskUserForSession(params: { + sessionKey?: string; + resolvedBy: string; +}): Promise { + const sessionKey = params.sessionKey?.trim(); + if (!sessionKey) { + return false; + } + const state = findAskUserQuestionForSession(sessionKey); + if (!state || state.phase.kind === "reserved" || !state.gatewayCall) { + return false; + } + while (state.phase.kind === "registering") { + await waitForQuestionChange(state); + if (askUserQuestions.get(state.questionId) !== state) { + return false; + } + } + if (state.phase.kind === "resolving" || state.phase.kind === "prompt-failed") { + return false; + } + const previousPhase = state.phase; + transitionAskUserQuestion(state, { kind: "resolving" }); + try { + await state.gatewayCall( + "question.resolve", + { timeoutMs: ASK_USER_RPC_GRACE_MS }, + { id: state.questionId, cancel: true, resolvedBy: params.resolvedBy }, + ); + return true; + } catch (error) { + if (isTerminalQuestionResolveError(error)) { + return true; + } + transitionAskUserQuestion(state, previousPhase); + throw error; + } +} + +function resetPendingAskUserQuestionsForTest(): void { + for (const questionId of askUserQuestions.keys()) { + releaseAskUserQuestion(questionId); + } +} + +if (process.env.VITEST || process.env.NODE_ENV === "test") { + (globalThis as Record)[Symbol.for("openclaw.askUserToolTestApi")] = { + resetPendingAskUserQuestionsForTest, + }; +} + +/** Creates the main-session-only blocking ask_user tool. */ +export function createAskUserTool(params: { + agentId?: string; + sessionKey?: string; + gatewayCall?: AskUserGatewayCall; +}): AnyAgentTool { + const gatewayCall: AskUserGatewayCall = params.gatewayCall ?? callGatewayTool; + return { + label: "Ask User", + name: "ask_user", + displaySummary: ASK_USER_TOOL_DISPLAY_SUMMARY, + description: describeAskUserTool(), + parameters: AskUserToolSchema, + execute: async (toolCallId, args, signal) => { + const questionId = buildAskUserQuestionId(toolCallId, params.sessionKey); + let normalized: NormalizedAskUserParams; + try { + signal?.throwIfAborted(); + normalized = normalizeAskUserParams(args); + } catch (error) { + releaseAskUserQuestion(questionId); + throw error; + } + const sessionKey = askUserSessionKey(params.sessionKey, params.agentId); + const reserved = askUserQuestions.get(questionId); + const existing = findAskUserQuestionForSession(sessionKey); + if ((reserved && reserved.phase.kind !== "reserved") || (existing && existing !== reserved)) { + throw new ToolInputError( + "ask_user already has a pending question for this session; wait for it to resolve before asking another", + ); + } + + const timeoutMs = normalized.timeoutSeconds * 1_000; + const deliverPrompt = reserved?.phase.kind === "reserved"; + const state: AskUserQuestionState = + reserved ?? + ({ + questionId, + sessionKey, + questions: normalized.questions, + phase: { kind: "registering" }, + gatewayCall, + waiters: new Set(), + } satisfies AskUserQuestionState); + state.sessionKey = sessionKey; + state.questions = normalized.questions; + state.gatewayCall = gatewayCall; + transitionAskUserQuestion(state, { kind: "registering" }); + askUserQuestions.set(questionId, state); + + let cancellation: + | Promise | undefined> + | undefined; + let registered = false; + const cancelPendingQuestion = (resolvedBy: string) => { + cancellation ??= (async () => { + try { + await gatewayCall( + "question.resolve", + { timeoutMs: ASK_USER_RPC_GRACE_MS }, + { id: questionId, cancel: true, resolvedBy }, + ); + return undefined; + } catch (error) { + if (!isTerminalQuestionResolveError(error)) { + return undefined; + } + try { + const result = (await gatewayCall( + "question.waitAnswer", + { timeoutMs: ASK_USER_RPC_GRACE_MS }, + { id: questionId, timeoutMs: 1_000 }, + )) as QuestionWaitAnswerResult; + return result.status === "answered" ? result : undefined; + } catch { + return undefined; + } + } + })(); + return cancellation; + }; + const cancelOnAbort = () => { + if (askUserQuestions.get(questionId) === state) { + releaseAskUserQuestion(questionId); + } + void cancelPendingQuestion("run-abort"); + }; + const finishWait = async (result: QuestionWaitAnswerResult) => { + if (result.status === "pending") { + const answered = await cancelPendingQuestion("wait-timeout"); + if (answered) { + return answeredResult(normalized.questions, answered.answers); + } + } + if (result.status === "answered") { + return answeredResult(normalized.questions, result.answers); + } + if ( + result.status === "pending" || + result.status === "expired" || + result.status === "cancelled" + ) { + return noAnswerResult(result.status); + } + throw new Error("question.waitAnswer returned an invalid status"); + }; + + try { + const requestResult = (await gatewayCall( + "question.request", + {}, + { + id: questionId, + questions: normalized.questions, + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + timeoutMs, + }, + signal ? { signal } : undefined, + )) as { id?: unknown }; + registered = true; + if (requestResult.id !== questionId) { + throw new Error("question.request returned an unexpected question id"); + } + signal?.addEventListener("abort", cancelOnAbort, { once: true }); + if (signal?.aborted) { + cancelOnAbort(); + signal.throwIfAborted(); + } + const answerPromise = gatewayCall( + "question.waitAnswer", + { timeoutMs: timeoutMs + ASK_USER_RPC_GRACE_MS }, + { id: questionId, timeoutMs }, + signal ? { signal } : undefined, + ) as Promise; + state.answer = answerPromise; + if (deliverPrompt) { + // Tool-start reserves the prompt, but only a committed Gateway record opens delivery. + // This prevents channels from exposing a question ID that cannot accept an answer. + markAskUserPromptReady(questionId, normalized.questions); + const promptDeliveryPromise = waitForPromptDelivery(state, signal); + const first = await Promise.race([ + promptDeliveryPromise.then((result) => ({ + kind: "delivery" as const, + result, + })), + answerPromise.then((result) => ({ kind: "answer" as const, result })), + ]); + signal?.throwIfAborted(); + if (first.kind === "answer") { + return await finishWait(first.result); + } + const deliveryResult = first.result; + if (deliveryResult.error !== undefined) { + const answered = await cancelPendingQuestion("prompt-delivery-failed"); + if (answered) { + return answeredResult(normalized.questions, answered.answers); + } + throw new Error("ask_user prompt delivery failed", { cause: deliveryResult.error }); + } + } else { + transitionAskUserQuestion(state, { kind: "answerable" }); + } + const result = await state.answer; + signal?.throwIfAborted(); + return await finishWait(result); + } catch (error) { + if (registered || readQuestionErrorReason(error) !== "QUESTION_ID_IN_USE") { + const answered = await cancelPendingQuestion( + signal?.aborted ? "run-abort" : registered ? "tool-error" : "registration-failed", + ); + if (!signal?.aborted && answered) { + return answeredResult(normalized.questions, answered.answers); + } + } + throw error; + } finally { + signal?.removeEventListener("abort", cancelOnAbort); + if (askUserQuestions.get(questionId) === state) { + releaseAskUserQuestion(questionId); + } + } + }, + }; +} diff --git a/src/agents/tools/gateway.test.ts b/src/agents/tools/gateway.test.ts index e17bb2d90917..acc87b5ecb78 100644 --- a/src/agents/tools/gateway.test.ts +++ b/src/agents/tools/gateway.test.ts @@ -299,6 +299,7 @@ describe("gateway tool defaults", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); diff --git a/src/agents/tools/message-tool.test.ts b/src/agents/tools/message-tool.test.ts index c00b088e56ea..ad9554fe9db3 100644 --- a/src/agents/tools/message-tool.test.ts +++ b/src/agents/tools/message-tool.test.ts @@ -2172,6 +2172,7 @@ describe("message tool schema scoping", () => { ]), ); expect(presentationSchemaJson).not.toContain('"const":"approval"'); + expect(presentationSchemaJson).not.toContain('"const":"question"'); expect(presentationSchemaJson).toContain('"chartType"'); expect(presentationSchemaJson).toContain('"pie"'); expect(presentationSchemaJson).toContain('"table"'); diff --git a/src/agents/tools/message-tool.ts b/src/agents/tools/message-tool.ts index d83f521adb91..8abdc4307193 100644 --- a/src/agents/tools/message-tool.ts +++ b/src/agents/tools/message-tool.ts @@ -574,8 +574,8 @@ const presentationCommandOrCallbackActionSchema = Type.Union([ presentationCallbackActionSchema, ]); -// Approval actions carry server-issued IDs and are runtime-authored only. The -// message tool exposes the remaining button actions that models may safely author. +// Approval and question actions carry server-issued IDs and are runtime-authored +// only. The message tool exposes the remaining actions models may safely author. const presentationButtonActionSchema = Type.Union([ presentationCommandActionSchema, presentationCallbackActionSchema, diff --git a/src/auto-reply/reply-payload.ts b/src/auto-reply/reply-payload.ts index 982e1ef7a50b..2ec5e5fa880a 100644 --- a/src/auto-reply/reply-payload.ts +++ b/src/auto-reply/reply-payload.ts @@ -18,6 +18,8 @@ export type ReplyPayload = { sensitiveMedia?: boolean; /** Channel-agnostic rich presentation. Core degrades or asks the channel renderer to map it. */ presentation?: MessagePresentation; + /** Runtime-authored text is the exact fallback, not additional native presentation content. */ + presentationTextMode?: "fallback"; /** Channel-agnostic delivery preferences, e.g. pin the sent message when supported. */ delivery?: ReplyPayloadDelivery; /** diff --git a/src/auto-reply/reply/agent-runner.media-paths.test.ts b/src/auto-reply/reply/agent-runner.media-paths.test.ts index 781cabdc7a6e..d7b3f902c365 100644 --- a/src/auto-reply/reply/agent-runner.media-paths.test.ts +++ b/src/auto-reply/reply/agent-runner.media-paths.test.ts @@ -443,6 +443,7 @@ describe("runReplyAgent media path normalization", () => { "generate chart", { steeringMode: "all", + isInboundUserMessage: true, taskSuggestionDeliveryMode: "gateway", }, ); @@ -478,6 +479,7 @@ describe("runReplyAgent media path normalization", () => { "compare these", { steeringMode: "all", + isInboundUserMessage: true, images, taskSuggestionDeliveryMode: undefined, }, @@ -544,7 +546,11 @@ describe("runReplyAgent media path normalization", () => { expect(queueEmbeddedAgentMessageWithOutcomeAsyncMock).toHaveBeenLastCalledWith( "session", "summarize the audio", - { steeringMode: "all", taskSuggestionDeliveryMode: undefined }, + { + steeringMode: "all", + isInboundUserMessage: true, + taskSuggestionDeliveryMode: undefined, + }, ); expect(enqueueFollowupRunMock).not.toHaveBeenCalled(); }); diff --git a/src/auto-reply/reply/agent-runner.ts b/src/auto-reply/reply/agent-runner.ts index 6b6899b3f070..6d8998c356aa 100644 --- a/src/auto-reply/reply/agent-runner.ts +++ b/src/auto-reply/reply/agent-runner.ts @@ -1412,6 +1412,7 @@ export async function runReplyAgent(params: { followupRun.prompt, { steeringMode: "all", + isInboundUserMessage: true, ...(followupRun.images?.length ? { images: followupRun.images } : {}), ...(turnAdoptionLifecycle ? { waitForTranscriptCommit: true } : {}), ...(resolvedQueue.debounceMs !== undefined ? { debounceMs: resolvedQueue.debounceMs } : {}), diff --git a/src/auto-reply/reply/block-reply-pipeline.ts b/src/auto-reply/reply/block-reply-pipeline.ts index f53c110aa491..f069127a26f2 100644 --- a/src/auto-reply/reply/block-reply-pipeline.ts +++ b/src/auto-reply/reply/block-reply-pipeline.ts @@ -56,6 +56,7 @@ function createBlockReplyPayloadKey(payload: ReplyPayload): string { text: reply.trimmedText, mediaList: reply.mediaUrls, presentation: payload.presentation ?? null, + presentationTextMode: payload.presentationTextMode ?? null, interactive: payload.interactive ?? null, channelData: payload.channelData ?? null, replyToId: payload.replyToId ?? null, @@ -72,6 +73,7 @@ export function createBlockReplyContentKey(payload: ReplyPayload): string { text: reply.trimmedText, mediaList: reply.mediaUrls, presentation: payload.presentation ?? null, + presentationTextMode: payload.presentationTextMode ?? null, interactive: payload.interactive ?? null, channelData: payload.channelData ?? null, }); diff --git a/src/auto-reply/reply/commands-steer.test.ts b/src/auto-reply/reply/commands-steer.test.ts index bca578841155..2b5a324fc1d3 100644 --- a/src/auto-reply/reply/commands-steer.test.ts +++ b/src/auto-reply/reply/commands-steer.test.ts @@ -61,6 +61,7 @@ describe("handleSteerCommand", () => { "keep going", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: undefined, }, @@ -79,6 +80,7 @@ describe("handleSteerCommand", () => { "keep going", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: "gateway", }, @@ -103,6 +105,7 @@ describe("handleSteerCommand", () => { "check the target", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: undefined, }, @@ -126,6 +129,7 @@ describe("handleSteerCommand", () => { "continue from state", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: undefined, }, @@ -165,6 +169,7 @@ describe("handleSteerCommand", () => { "check the active file", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: undefined, }, @@ -194,6 +199,7 @@ describe("handleSteerCommand", () => { "use the active direct lane", { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, taskSuggestionDeliveryMode: undefined, }, diff --git a/src/auto-reply/reply/commands-steer.ts b/src/auto-reply/reply/commands-steer.ts index 9728819d17da..568e1101f3ac 100644 --- a/src/auto-reply/reply/commands-steer.ts +++ b/src/auto-reply/reply/commands-steer.ts @@ -172,6 +172,7 @@ export const handleSteerCommand: CommandHandler = async (params, allowTextComman const queueOutcome = await queueEmbeddedAgentMessageWithOutcomeAsync(sessionId, message, { steeringMode: "all", + isInboundUserMessage: true, debounceMs: 0, ...(params.opts?.sourceReplyDeliveryMode ? { sourceReplyDeliveryMode: params.opts.sourceReplyDeliveryMode } diff --git a/src/auto-reply/reply/dispatch-from-config.payloads.ts b/src/auto-reply/reply/dispatch-from-config.payloads.ts index a4fc5a23ca62..2986d3a1562c 100644 --- a/src/auto-reply/reply/dispatch-from-config.payloads.ts +++ b/src/auto-reply/reply/dispatch-from-config.payloads.ts @@ -25,6 +25,7 @@ export function createFinalDispatchPayloadDedupeKey(payload: ReplyPayload): stri trustedLocalMedia: payload.trustedLocalMedia, sensitiveMedia: payload.sensitiveMedia, presentation: payload.presentation, + presentationTextMode: payload.presentationTextMode, delivery: payload.delivery, interactive: payload.interactive, btw: payload.btw, diff --git a/src/auto-reply/reply/dispatch-from-config.ts b/src/auto-reply/reply/dispatch-from-config.ts index 889590a2f8cb..77b691d84b39 100644 --- a/src/auto-reply/reply/dispatch-from-config.ts +++ b/src/auto-reply/reply/dispatch-from-config.ts @@ -30,6 +30,7 @@ import { } from "../../agents/subagent-capabilities.js"; import { isToolAllowedByPolicies } from "../../agents/tool-policy-match.js"; import { mergeAlsoAllowPolicy, resolveToolProfilePolicy } from "../../agents/tool-policy.js"; +import { isAskUserPromptActive } from "../../agents/tools/ask-user-tool.js"; import { resolveConversationBindingRecord, touchConversationBindingRecord, @@ -1395,12 +1396,24 @@ async function dispatchReplyFromConfigInner( : undefined; return execApproval && typeof execApproval === "object" && !Array.isArray(execApproval); }; + const hasAskUserPayload = (payload: ReplyPayload) => { + const askUser = payload.channelData?.askUser; + return askUser && typeof askUser === "object" && !Array.isArray(askUser); + }; + const isInactiveAskUserPayload = (payload: ReplyPayload) => { + const askUser = payload.channelData?.askUser; + if (!askUser || typeof askUser !== "object" || Array.isArray(askUser)) { + return false; + } + const questionId = (askUser as { questionId?: unknown }).questionId; + return typeof questionId === "string" && !isAskUserPromptActive(questionId); + }; const shouldSuppressLateTextOnlyToolProgress = (payload: ReplyPayload) => { if (!finalReplyDeliveryStarted) { return false; } const reply = resolveSendableOutboundReplyParts(payload); - return !reply.hasMedia && !hasExecApprovalPayload(payload); + return !reply.hasMedia && !hasExecApprovalPayload(payload) && !hasAskUserPayload(payload); }; // Durable inter-tool commentary lane: with verbose progress on, preamble // items become standalone progress messages like tool summaries. The latest @@ -1892,6 +1905,9 @@ async function dispatchReplyFromConfigInner( if (execApproval && typeof execApproval === "object" && !Array.isArray(execApproval)) { return payload; } + if (hasAskUserPayload(payload)) { + return payload; + } if (isFastModeAutoProgressPayload(payload)) { return payload; } @@ -2235,12 +2251,18 @@ async function dispatchReplyFromConfigInner( if (isDispatchOperationAborted()) { return; } + if (isInactiveAskUserPayload(payload)) { + return; + } await waitForPendingDirectBlockReplyDelivery( getDispatchAbortOperation()?.abortSignal, ); if (isDispatchOperationAborted()) { return; } + if (isInactiveAskUserPayload(payload)) { + return; + } markInboundDedupeReplayUnsafe(); // Buffered commentary preceded this tool; land it before the summary. await flushPendingCommentaryProgress(); @@ -2312,6 +2334,9 @@ async function dispatchReplyFromConfigInner( if (isDispatchOperationAborted()) { return; } + if (isInactiveAskUserPayload(deliveryPayload)) { + return; + } if ( shouldSuppressLateTextOnlyToolProgress(deliveryPayload) && !isFastModeAutoProgressPayload(deliveryPayload) && diff --git a/src/auto-reply/reply/reply-run-registry.ts b/src/auto-reply/reply/reply-run-registry.ts index 38313b2e0202..87d5ef39bcd7 100644 --- a/src/auto-reply/reply/reply-run-registry.ts +++ b/src/auto-reply/reply/reply-run-registry.ts @@ -30,6 +30,8 @@ type ReplyBackendCancelReason = "user_abort" | "restart" | "superseded"; export type ReplyBackendQueueMessageOptions = { steeringMode?: "all"; + /** True when this queue item came from the channel's current user turn. */ + isInboundUserMessage?: boolean; debounceMs?: number; /** Ordered current-turn images to inject with the steering text. */ images?: ImageContent[]; diff --git a/src/gateway/call.test.ts b/src/gateway/call.test.ts index 996574631ba6..580a5e5ecf4d 100644 --- a/src/gateway/call.test.ts +++ b/src/gateway/call.test.ts @@ -757,6 +757,7 @@ describe("callGateway url resolution", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); @@ -779,6 +780,7 @@ describe("callGateway url resolution", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); diff --git a/src/gateway/gateway-misc.test.ts b/src/gateway/gateway-misc.test.ts index 764f9f76997b..812b63cf305a 100644 --- a/src/gateway/gateway-misc.test.ts +++ b/src/gateway/gateway-misc.test.ts @@ -468,6 +468,22 @@ describe("gateway broadcaster", () => { expectSentEvents(adminSocket, expectedEvents); }); + it("requires operator.questions for question broadcasts", () => { + const questionSocket: TestSocket = { bufferedAmount: 0, send: vi.fn(), close: vi.fn() }; + const readSocket: TestSocket = { bufferedAmount: 0, send: vi.fn(), close: vi.fn() }; + const clients = new Set([ + makeOperatorWsClient("c-questions", questionSocket, ["operator.questions"]), + makeOperatorWsClient("c-read", readSocket, ["operator.read"]), + ]); + const { broadcast } = createGatewayBroadcaster({ clients }); + + broadcast("question.requested", { id: "question-1" }); + broadcast("question.resolved", { id: "question-1", status: "expired" }); + + expect(questionSocket.send).toHaveBeenCalledTimes(2); + expect(readSocket.send).not.toHaveBeenCalled(); + }); + it("requires operator.read for task ledger broadcast events", () => { const { pairingSocket, nodeSocket, readSocket, writeSocket, adminSocket, broadcast } = makeScopedBroadcastContext(); diff --git a/src/gateway/http-utils.request-context.test.ts b/src/gateway/http-utils.request-context.test.ts index 92328b9a5ed8..31ba0088d053 100644 --- a/src/gateway/http-utils.request-context.test.ts +++ b/src/gateway/http-utils.request-context.test.ts @@ -250,6 +250,7 @@ describe("resolveOpenAiCompatibleHttpOperatorScopes", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); diff --git a/src/gateway/method-scopes.test.ts b/src/gateway/method-scopes.test.ts index b9eb5678955b..e489bcce975c 100644 --- a/src/gateway/method-scopes.test.ts +++ b/src/gateway/method-scopes.test.ts @@ -191,6 +191,7 @@ describe("method scope resolution", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); @@ -389,6 +390,7 @@ describe("method scope resolution", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); @@ -511,6 +513,22 @@ describe("operator scope authorization", () => { }); }); + it.each([ + "question.request", + "question.waitAnswer", + "question.resolve", + "question.get", + "question.list", + ])("requires questions scope for %s", (method) => { + expect(authorizeOperatorScopesForMethod(method, ["operator.write"])).toEqual({ + allowed: false, + missingScope: "operator.questions", + }); + expect(authorizeOperatorScopesForMethod(method, ["operator.questions"])).toEqual({ + allowed: true, + }); + }); + it.each([ "exec.approvals.get", "exec.approvals.set", @@ -641,5 +659,6 @@ describe("CLI default operator scopes", () => { it("includes operator.talk.secrets for node-role device pairing approvals", async () => { const { CLI_DEFAULT_OPERATOR_SCOPES } = await import("./method-scopes.js"); expect(CLI_DEFAULT_OPERATOR_SCOPES).toContain("operator.talk.secrets"); + expect(CLI_DEFAULT_OPERATOR_SCOPES).toContain("operator.questions"); }); }); diff --git a/src/gateway/method-scopes.ts b/src/gateway/method-scopes.ts index 1707acf931e4..3db3143bd658 100644 --- a/src/gateway/method-scopes.ts +++ b/src/gateway/method-scopes.ts @@ -13,6 +13,7 @@ import { ADMIN_SCOPE, APPROVALS_SCOPE, PAIRING_SCOPE, + QUESTIONS_SCOPE, READ_SCOPE, TALK_SECRETS_SCOPE, WRITE_SCOPE, @@ -20,7 +21,15 @@ import { type OperatorScope, } from "./operator-scopes.js"; -export { ADMIN_SCOPE, APPROVALS_SCOPE, PAIRING_SCOPE, READ_SCOPE, WRITE_SCOPE, type OperatorScope }; +export { + ADMIN_SCOPE, + APPROVALS_SCOPE, + PAIRING_SCOPE, + QUESTIONS_SCOPE, + READ_SCOPE, + WRITE_SCOPE, + type OperatorScope, +}; /** Default scopes granted to CLI/operator clients when no narrower local policy is known. */ export const CLI_DEFAULT_OPERATOR_SCOPES: OperatorScope[] = [ @@ -28,6 +37,7 @@ export const CLI_DEFAULT_OPERATOR_SCOPES: OperatorScope[] = [ READ_SCOPE, WRITE_SCOPE, APPROVALS_SCOPE, + QUESTIONS_SCOPE, PAIRING_SCOPE, TALK_SECRETS_SCOPE, ]; diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index e7c9b128d3b7..7f6aa8e85fdf 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -60,6 +60,11 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [ { name: "exec.approval.request", scope: "operator.approvals" }, { name: "exec.approval.waitDecision", scope: "operator.approvals" }, { name: "exec.approval.resolve", scope: "operator.approvals" }, + { name: "question.request", scope: "operator.questions" }, + { name: "question.waitAnswer", scope: "operator.questions" }, + { name: "question.resolve", scope: "operator.questions" }, + { name: "question.get", scope: "operator.questions" }, + { name: "question.list", scope: "operator.questions" }, { name: "plugin.approval.list", scope: "operator.approvals" }, { name: "plugin.approval.request", scope: "operator.approvals" }, { name: "plugin.approval.waitDecision", scope: "operator.approvals" }, diff --git a/src/gateway/operator-scopes.ts b/src/gateway/operator-scopes.ts index 1e6c423fe9f7..4df73a3232d2 100644 --- a/src/gateway/operator-scopes.ts +++ b/src/gateway/operator-scopes.ts @@ -4,6 +4,7 @@ export const ADMIN_SCOPE = "operator.admin" as const; export const READ_SCOPE = "operator.read" as const; export const WRITE_SCOPE = "operator.write" as const; export const APPROVALS_SCOPE = "operator.approvals" as const; +export const QUESTIONS_SCOPE = "operator.questions" as const; export const PAIRING_SCOPE = "operator.pairing" as const; export const TALK_SECRETS_SCOPE = "operator.talk.secrets" as const; @@ -13,6 +14,7 @@ export type OperatorScope = | typeof READ_SCOPE | typeof WRITE_SCOPE | typeof APPROVALS_SCOPE + | typeof QUESTIONS_SCOPE | typeof PAIRING_SCOPE | typeof TALK_SECRETS_SCOPE; @@ -21,6 +23,7 @@ const KNOWN_OPERATOR_SCOPE_VALUES: readonly OperatorScope[] = [ READ_SCOPE, WRITE_SCOPE, APPROVALS_SCOPE, + QUESTIONS_SCOPE, PAIRING_SCOPE, TALK_SECRETS_SCOPE, ]; diff --git a/src/gateway/question-manager.test.ts b/src/gateway/question-manager.test.ts new file mode 100644 index 000000000000..55057082605b --- /dev/null +++ b/src/gateway/question-manager.test.ts @@ -0,0 +1,228 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Question, QuestionAnswers } from "../../packages/gateway-protocol/src/index.js"; +import { + QuestionManager, + QuestionManagerError, + QuestionManagerErrorCodes, +} from "./question-manager.js"; + +const QUESTION_RESOLVED_ENTRY_GRACE_MS = 15_000; + +const questions: Question[] = [ + { + id: "choice", + header: "Choice", + question: "Which option?", + options: [ + { label: "One", description: "First" }, + { label: "Two", description: "Second" }, + ], + isOther: true, + }, +]; +const answers = { answers: { choice: { answers: ["Two"] } } }; + +const invalidAnswerCases: Array<[string, Question[], QuestionAnswers, string]> = [ + ["an empty answer map", questions, { answers: {} }, "choice"], + [ + "an unknown question id", + questions, + { answers: { choice: { answers: ["Two"] }, unknown: { answers: ["value"] } } }, + "unknown", + ], + [ + "a missing question answer", + [...questions, { ...questions[0]!, id: "second" }], + answers, + "second", + ], + ["an empty string", questions, { answers: { choice: { answers: [" "] } } }, "choice"], + [ + "multiple values for a single-select question", + questions, + { answers: { choice: { answers: ["One", "Two"] } } }, + "choice", + ], + [ + "a value outside the declared options", + [{ ...questions[0]!, isOther: false }], + { answers: { choice: { answers: ["Three"] } } }, + "choice", + ], +]; + +let manager: QuestionManager; + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + manager = new QuestionManager(); +}); + +afterEach(() => { + manager.reset(); + vi.useRealTimers(); +}); + +describe("QuestionManager", () => { + it("requests, gets, and deterministically lists pending questions", () => { + const first = manager.request({ questions, timeoutMs: 10_000, agentId: "main" }); + vi.setSystemTime(1_001); + const second = manager.request({ + questions: [{ ...questions[0]!, id: "other" }], + timeoutMs: 10_000, + sessionKey: "agent:main:main", + }); + + expect(manager.get(first.id)).toEqual(first); + expect(manager.list().map((record) => record.id)).toEqual([first.id, second.id]); + }); + + it("accepts a unique client id and rejects reuse during the grace window", () => { + const first = manager.request({ id: "ask_client_id", questions, timeoutMs: 10_000 }); + + expect(first.id).toBe("ask_client_id"); + expect(() => + manager.request({ id: "ask_client_id", questions, timeoutMs: 10_000 }), + ).toThrowError(QuestionManagerError); + try { + manager.request({ id: "ask_client_id", questions, timeoutMs: 10_000 }); + } catch (error) { + expect(error).toMatchObject({ code: QuestionManagerErrorCodes.ID_IN_USE }); + } + }); + + it("releases waitAnswer with the submitted answer", async () => { + const record = manager.request({ questions, timeoutMs: 10_000 }); + const waiting = manager.waitAnswer(record.id); + + expect(manager.resolve(record.id, answers, "control-ui")).toEqual({ + status: "answered", + answers, + }); + await expect(waiting).resolves.toEqual({ status: "answered", answers }); + expect(manager.get(record.id)).toMatchObject({ status: "answered", resolvedBy: "control-ui" }); + }); + + it.each(invalidAnswerCases)( + "rejects %s without terminalizing", + (_name, requestQuestions, invalid, questionId) => { + const record = manager.request({ questions: [...requestQuestions], timeoutMs: 10_000 }); + + expect(() => manager.resolve(record.id, invalid)).toThrow(`question '${questionId}'`); + expect(manager.get(record.id)?.status).toBe("pending"); + }, + ); + + it("accepts trimmed option labels and free text when allowed", () => { + const strict = manager.request({ + questions: [{ ...questions[0]!, isOther: false }], + timeoutMs: 10_000, + }); + expect( + manager.resolve(strict.id, { answers: { choice: { answers: [" Two "] } } }), + ).toMatchObject({ status: "answered" }); + + const open = manager.request({ questions, timeoutMs: 10_000 }); + expect( + manager.resolve(open.id, { answers: { choice: { answers: ["custom"] } } }), + ).toMatchObject({ status: "answered" }); + + const freeText = manager.request({ + questions: [{ ...questions[0]!, options: [], isOther: false }], + timeoutMs: 10_000, + }); + expect( + manager.resolve(freeText.id, { answers: { choice: { answers: ["custom"] } } }), + ).toMatchObject({ status: "answered" }); + }); + + it("times out one waiter without resolving the question", async () => { + const record = manager.request({ questions, timeoutMs: 10_000 }); + const waiting = manager.waitAnswer(record.id, 50); + + await vi.advanceTimersByTimeAsync(50); + + await expect(waiting).resolves.toEqual({ status: "pending" }); + expect(manager.get(record.id)?.status).toBe("pending"); + }); + + it("expires pending questions and emits the terminal event", async () => { + const onResolved = vi.fn(); + const record = manager.request({ questions, timeoutMs: 50, onResolved }); + const waiting = manager.waitAnswer(record.id); + + await vi.advanceTimersByTimeAsync(50); + + await expect(waiting).resolves.toEqual({ status: "expired" }); + expect(manager.get(record.id)?.status).toBe("expired"); + expect(onResolved).toHaveBeenCalledWith({ id: record.id, status: "expired" }); + }); + + it("cancels pending questions", async () => { + const record = manager.request({ questions, timeoutMs: 10_000 }); + const waiting = manager.waitAnswer(record.id); + + expect(manager.cancel(record.id, "agent")).toEqual({ status: "cancelled" }); + await expect(waiting).resolves.toEqual({ status: "cancelled" }); + expect(manager.get(record.id)).toMatchObject({ status: "cancelled", resolvedBy: "agent" }); + }); + + it("rejects double resolve and resolve after expiry with typed errors", async () => { + const answered = manager.request({ questions, timeoutMs: 10_000 }); + manager.resolve(answered.id, answers); + + expect(() => manager.resolve(answered.id, answers)).toThrowError(QuestionManagerError); + try { + manager.resolve(answered.id, answers); + } catch (error) { + expect(error).toMatchObject({ code: QuestionManagerErrorCodes.ALREADY_TERMINAL }); + } + + const expired = manager.request({ questions, timeoutMs: 10 }); + await vi.advanceTimersByTimeAsync(10); + try { + manager.resolve(expired.id, answers); + throw new Error("expected resolve to fail"); + } catch (error) { + expect(error).toMatchObject({ code: QuestionManagerErrorCodes.ALREADY_TERMINAL }); + } + }); + + it("keeps terminal records through the grace window", async () => { + const record = manager.request({ questions, timeoutMs: 10_000 }); + manager.resolve(record.id, answers); + + await vi.advanceTimersByTimeAsync(QUESTION_RESOLVED_ENTRY_GRACE_MS - 1); + expect(manager.get(record.id)?.status).toBe("answered"); + + await vi.advanceTimersByTimeAsync(1); + expect(manager.get(record.id)).toBeNull(); + }); +}); + +describe("answer canonicalization", () => { + it("stores declared option labels for trim-variant submissions", () => { + const localManager = new QuestionManager(); + const record = localManager.request({ + questions: [ + { + id: "pick", + header: "Pick", + question: "Pick one", + options: [{ label: "Two" }, { label: "Three" }], + isOther: false, + }, + ], + timeoutMs: 60_000, + }); + const result = localManager.resolve(record.id, { + answers: { pick: { answers: [" Two "] } }, + }); + expect(result).toEqual({ + status: "answered", + answers: { answers: { pick: { answers: ["Two"] } } }, + }); + localManager.reset(); + }); +}); diff --git a/src/gateway/question-manager.ts b/src/gateway/question-manager.ts new file mode 100644 index 000000000000..5cf872c78680 --- /dev/null +++ b/src/gateway/question-manager.ts @@ -0,0 +1,329 @@ +// Gateway question manager. +// Tracks transient operator questions and short-lived terminal records in memory. +import { randomUUID } from "node:crypto"; +import { resolveExpiresAtMsFromDurationMs } from "@openclaw/normalization-core/number-coercion"; +import type { + Question, + QuestionAnswers, + QuestionRecord, + QuestionResolvedEvent, + QuestionResolveResult, + QuestionWaitAnswerResult, +} from "../../packages/gateway-protocol/src/index.js"; +import { resolveTimerTimeoutMs } from "../shared/number-coercion.js"; + +/** Grace period for late question.waitAnswer and question.get calls. */ +const QUESTION_RESOLVED_ENTRY_GRACE_MS = 15_000; + +export const QuestionManagerErrorCodes = { + NOT_FOUND: "QUESTION_NOT_FOUND", + ALREADY_TERMINAL: "QUESTION_ALREADY_TERMINAL", + ID_IN_USE: "QUESTION_ID_IN_USE", + INVALID_ANSWER: "QUESTION_INVALID_ANSWER", +} as const; + +type QuestionManagerErrorCode = + (typeof QuestionManagerErrorCodes)[keyof typeof QuestionManagerErrorCodes]; + +export class QuestionManagerError extends Error { + constructor( + readonly code: QuestionManagerErrorCode, + message: string, + ) { + super(message); + this.name = "QuestionManagerError"; + } +} + +type QuestionManagerRequest = { + id?: string; + questions: Question[]; + agentId?: string; + sessionKey?: string; + timeoutMs: number; + onResolved?: (event: QuestionResolvedEvent) => void; +}; + +type Waiter = { + resolve: (result: QuestionWaitAnswerResult) => void; + timer: ReturnType | null; +}; + +type QuestionEntry = { + record: QuestionRecord; + expiryTimer: ReturnType; + cleanupTimer: ReturnType | null; + waiters: Set; + onResolved?: (event: QuestionResolvedEvent) => void; +}; + +function unrefTimer(timer: ReturnType): void { + timer.unref?.(); +} + +function waitResult(record: QuestionRecord): QuestionWaitAnswerResult { + switch (record.status) { + case "pending": + return { status: "pending" }; + case "answered": + // The manager only sets status "answered" together with validated answers. + return { status: "answered", answers: record.answers ?? { answers: {} } }; + case "cancelled": + return { status: "cancelled" }; + case "expired": + return { status: "expired" }; + } + return record.status satisfies never; +} + +function resolvedEvent(record: QuestionRecord): QuestionResolvedEvent | null { + if (record.status === "pending") { + return null; + } + return record.status === "answered" + ? { id: record.id, status: record.status, answers: record.answers ?? { answers: {} } } + : { id: record.id, status: record.status }; +} + +/** Process-local lifecycle owner for pending questions. */ +export class QuestionManager { + private readonly entries = new Map(); + + request(params: QuestionManagerRequest): QuestionRecord { + const createdAtMs = Date.now(); + const timeoutMs = resolveTimerTimeoutMs(params.timeoutMs, 1); + const expiresAtMs = resolveExpiresAtMsFromDurationMs(timeoutMs, { nowMs: createdAtMs }); + if (expiresAtMs === undefined) { + throw new Error("question expiry is unavailable"); + } + const id = params.id ?? randomUUID(); + if (this.entries.has(id)) { + throw new QuestionManagerError( + QuestionManagerErrorCodes.ID_IN_USE, + `question '${id}' already exists`, + ); + } + const record: QuestionRecord = { + id, + questions: params.questions, + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + createdAtMs, + expiresAtMs, + status: "pending", + }; + const entry: QuestionEntry = { + record, + expiryTimer: null as unknown as ReturnType, + cleanupTimer: null, + waiters: new Set(), + onResolved: params.onResolved, + }; + this.entries.set(record.id, entry); + entry.expiryTimer = setTimeout(() => this.expire(record.id), timeoutMs); + unrefTimer(entry.expiryTimer); + return record; + } + + get(id: string): QuestionRecord | null { + const entry = this.entries.get(id); + if (!entry) { + return null; + } + if (entry.record.status === "pending" && entry.record.expiresAtMs <= Date.now()) { + this.expire(id); + } + return this.entries.get(id)?.record ?? null; + } + + list(): QuestionRecord[] { + const records: QuestionRecord[] = []; + for (const id of this.entries.keys()) { + const record = this.get(id); + if (record?.status === "pending") { + records.push(record); + } + } + return records.toSorted( + (left, right) => left.createdAtMs - right.createdAtMs || left.id.localeCompare(right.id), + ); + } + + waitAnswer(id: string, timeoutMs?: number): Promise { + const record = this.requireRecord(id); + if (record.status !== "pending") { + return Promise.resolve(waitResult(record)); + } + const entry = this.entries.get(id); + if (!entry) { + throw this.notFound(id); + } + return new Promise((resolve) => { + const waiter: Waiter = { resolve, timer: null }; + entry.waiters.add(waiter); + if (timeoutMs !== undefined) { + waiter.timer = setTimeout( + () => { + entry.waiters.delete(waiter); + resolve({ status: "pending" }); + }, + resolveTimerTimeoutMs(timeoutMs, 1), + ); + unrefTimer(waiter.timer); + } + }); + } + + resolve(id: string, answers: QuestionAnswers, resolvedBy?: string): QuestionResolveResult { + const entry = this.requirePendingEntry(id); + const canonical = this.validateAnswers(entry.record.questions, answers); + entry.record = { + ...entry.record, + status: "answered", + answers: canonical, + ...(resolvedBy ? { resolvedBy } : {}), + }; + this.finish(entry); + return { status: "answered", answers: canonical }; + } + + cancel(id: string, resolvedBy?: string): QuestionResolveResult { + const entry = this.requirePendingEntry(id); + entry.record = { + ...entry.record, + status: "cancelled", + ...(resolvedBy ? { resolvedBy } : {}), + }; + this.finish(entry); + return { status: "cancelled" }; + } + + /** Clears all manager-owned timers and releases waiters. */ + reset(): void { + for (const entry of this.entries.values()) { + clearTimeout(entry.expiryTimer); + if (entry.cleanupTimer) { + clearTimeout(entry.cleanupTimer); + } + for (const waiter of entry.waiters) { + if (waiter.timer) { + clearTimeout(waiter.timer); + } + waiter.resolve(waitResult(entry.record)); + } + entry.waiters.clear(); + } + this.entries.clear(); + } + + private requireRecord(id: string): QuestionRecord { + const record = this.get(id); + if (!record) { + throw this.notFound(id); + } + return record; + } + + private requirePendingEntry(id: string): QuestionEntry { + const record = this.requireRecord(id); + if (record.status !== "pending") { + throw new QuestionManagerError( + QuestionManagerErrorCodes.ALREADY_TERMINAL, + `question '${id}' is already ${record.status}`, + ); + } + const entry = this.entries.get(id); + if (!entry) { + throw this.notFound(id); + } + return entry; + } + + /** Validates answers against stored questions and returns them in canonical form. */ + private validateAnswers(questions: Question[], answers: QuestionAnswers): QuestionAnswers { + const submittedIds = Object.keys(answers.answers); + const questionsById = new Map(questions.map((question) => [question.id, question])); + const unknownId = submittedIds.find((id) => !questionsById.has(id)); + if (unknownId) { + throw this.invalidAnswer(unknownId, "is not part of this request"); + } + const canonical: QuestionAnswers = { answers: {} }; + for (const question of questions) { + const values = answers.answers[question.id]?.answers; + if (!values || values.length === 0) { + throw this.invalidAnswer(question.id, "requires an answer"); + } + if (values.some((value) => !value.trim())) { + throw this.invalidAnswer(question.id, "contains an empty answer"); + } + if (!question.multiSelect && values.length > 1) { + throw this.invalidAnswer(question.id, "does not allow multiple answers"); + } + // Store the declared option label when a value matches trim-insensitively; + // downstream renderers compare answers to option labels exactly. + const canonicalValues = values.map((value) => { + const matched = question.options.find((option) => option.label.trim() === value.trim()); + return matched ? matched.label : value.trim(); + }); + if ( + question.options.length > 0 && + !question.isOther && + canonicalValues.some((value) => !question.options.some((option) => option.label === value)) + ) { + throw this.invalidAnswer(question.id, "contains an unknown option"); + } + canonical.answers[question.id] = { answers: canonicalValues }; + } + return canonical; + } + + private invalidAnswer(id: string, reason: string): QuestionManagerError { + return new QuestionManagerError( + QuestionManagerErrorCodes.INVALID_ANSWER, + `question '${id}' ${reason}`, + ); + } + + private notFound(id: string): QuestionManagerError { + return new QuestionManagerError( + QuestionManagerErrorCodes.NOT_FOUND, + `question '${id}' was not found`, + ); + } + + private expire(id: string): void { + const entry = this.entries.get(id); + if (!entry || entry.record.status !== "pending") { + return; + } + entry.record = { ...entry.record, status: "expired" }; + this.finish(entry); + } + + private finish(entry: QuestionEntry): void { + clearTimeout(entry.expiryTimer); + const result = waitResult(entry.record); + for (const waiter of entry.waiters) { + if (waiter.timer) { + clearTimeout(waiter.timer); + } + waiter.resolve(result); + } + entry.waiters.clear(); + const event = resolvedEvent(entry.record); + if (event) { + try { + entry.onResolved?.(event); + } catch { + // Broadcast fanout is observational and must not change question truth. + } + } + const cleanupTimer = setTimeout(() => { + if (entry.cleanupTimer === cleanupTimer && this.entries.get(entry.record.id) === entry) { + this.entries.delete(entry.record.id); + } + }, QUESTION_RESOLVED_ENTRY_GRACE_MS); + entry.cleanupTimer = cleanupTimer; + unrefTimer(cleanupTimer); + } +} diff --git a/src/gateway/server-aux-handlers.ts b/src/gateway/server-aux-handlers.ts index 23b3a070e467..fae206dcfae9 100644 --- a/src/gateway/server-aux-handlers.ts +++ b/src/gateway/server-aux-handlers.ts @@ -45,6 +45,7 @@ import { closeOrphanedOperatorApprovals, pruneTerminalOperatorApprovals, } from "./operator-approval-store.js"; +import { QuestionManager } from "./question-manager.js"; import type { ChannelAutostartSuppression } from "./server-channels.js"; import { captureSharedGatewaySessionGenerationOwnership, @@ -157,6 +158,14 @@ export function createGatewayAuxHandlers(params: { ), { cacheRejections: true }, ); + const questionManager = new QuestionManager(); + const loadQuestionHandlers = createLazyPromise( + () => + import("./server-methods/question.js").then(({ createQuestionHandlers }) => + createQuestionHandlers(questionManager), + ), + { cacheRejections: true }, + ); const buildReloadPlan = params.buildReloadPlan ?? buildGatewayReloadPlan; const pluginApprovalManager = createApprovalManager( "plugin", @@ -514,6 +523,7 @@ export function createGatewayAuxHandlers(params: { pluginApprovalIosPushDelivery, pluginApprovalManager, systemAgentApprovalManager, + questionManager, extraHandlers: { "exec.approval.get": createLazyHandler("exec.approval.get", loadExecApprovalHandlers), "exec.approval.list": createLazyHandler("exec.approval.list", loadExecApprovalHandlers), @@ -539,6 +549,11 @@ export function createGatewayAuxHandlers(params: { "approval.get": createLazyHandler("approval.get", loadApprovalHandlers), "approval.history": createLazyHandler("approval.history", loadApprovalHandlers), "approval.resolve": createLazyHandler("approval.resolve", loadApprovalHandlers), + "question.request": createLazyHandler("question.request", loadQuestionHandlers), + "question.waitAnswer": createLazyHandler("question.waitAnswer", loadQuestionHandlers), + "question.resolve": createLazyHandler("question.resolve", loadQuestionHandlers), + "question.get": createLazyHandler("question.get", loadQuestionHandlers), + "question.list": createLazyHandler("question.list", loadQuestionHandlers), "secrets.reload": createLazyHandler("secrets.reload", loadSecretsHandlers), "secrets.resolve": createLazyHandler("secrets.resolve", loadSecretsHandlers), }, diff --git a/src/gateway/server-aux-methods.ts b/src/gateway/server-aux-methods.ts index 49fdc9c46f16..7cc724fd4467 100644 --- a/src/gateway/server-aux-methods.ts +++ b/src/gateway/server-aux-methods.ts @@ -14,6 +14,11 @@ export const GATEWAY_AUX_METHODS = [ "approval.get", "approval.history", "approval.resolve", + "question.request", + "question.waitAnswer", + "question.resolve", + "question.get", + "question.list", "secrets.reload", "secrets.resolve", ] as const; diff --git a/src/gateway/server-broadcast.ts b/src/gateway/server-broadcast.ts index 12b1888a00d9..6c916f1f3279 100644 --- a/src/gateway/server-broadcast.ts +++ b/src/gateway/server-broadcast.ts @@ -5,6 +5,7 @@ import { ADMIN_SCOPE, APPROVALS_SCOPE, PAIRING_SCOPE, + QUESTIONS_SCOPE, READ_SCOPE, WRITE_SCOPE, } from "./method-scopes.js"; @@ -33,6 +34,8 @@ const EVENT_SCOPE_GUARDS: Record = { health: [], "exec.approval.requested": [APPROVALS_SCOPE], "exec.approval.resolved": [APPROVALS_SCOPE], + "question.requested": [QUESTIONS_SCOPE], + "question.resolved": [QUESTIONS_SCOPE], heartbeat: [], "plugin.approval.requested": [APPROVALS_SCOPE], "plugin.approval.resolved": [APPROVALS_SCOPE], diff --git a/src/gateway/server-methods-list.test.ts b/src/gateway/server-methods-list.test.ts index 38febd95e323..150c5b923408 100644 --- a/src/gateway/server-methods-list.test.ts +++ b/src/gateway/server-methods-list.test.ts @@ -21,6 +21,20 @@ describe("GATEWAY_EVENTS", () => { it("advertises node presence activity updates", () => { expect(GATEWAY_EVENTS).toContain("node.presence"); }); + + it("advertises question methods and events", () => { + expect(GATEWAY_EVENTS).toContain("question.requested"); + expect(GATEWAY_EVENTS).toContain("question.resolved"); + expect(listGatewayMethods()).toEqual( + expect.arrayContaining([ + "question.request", + "question.waitAnswer", + "question.resolve", + "question.get", + "question.list", + ]), + ); + }); }); describe("listGatewayMethods", () => { diff --git a/src/gateway/server-methods-list.ts b/src/gateway/server-methods-list.ts index 161c01d110ad..b2e69c5fff0f 100644 --- a/src/gateway/server-methods-list.ts +++ b/src/gateway/server-methods-list.ts @@ -68,6 +68,8 @@ export const GATEWAY_EVENTS = [ "voicewake.routing.changed", "exec.approval.requested", "exec.approval.resolved", + "question.requested", + "question.resolved", "plugin.approval.requested", "plugin.approval.resolved", "openclaw.approval.requested", diff --git a/src/gateway/server-methods/question.test.ts b/src/gateway/server-methods/question.test.ts new file mode 100644 index 000000000000..deb5c1855023 --- /dev/null +++ b/src/gateway/server-methods/question.test.ts @@ -0,0 +1,187 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { QuestionManager } from "../question-manager.js"; +import { createQuestionHandlers } from "./question.js"; +import type { GatewayRequestHandlerOptions, RespondFn } from "./types.js"; + +let manager: QuestionManager; +let broadcast: ReturnType; +let handlers: ReturnType; + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(1_000); + manager = new QuestionManager(); + broadcast = vi.fn(); + handlers = createQuestionHandlers(manager); +}); + +afterEach(() => { + manager.reset(); + vi.useRealTimers(); +}); + +async function call(method: string, params: Record) { + const calls: Parameters[] = []; + const respond: RespondFn = (...args) => calls.push(args); + await handlers[method]?.({ + req: { type: "req", id: "request-1", method, params }, + params, + respond, + client: null, + isWebchatConnect: () => false, + context: { broadcast } as unknown as GatewayRequestHandlerOptions["context"], + }); + const response = calls[0]; + if (!response) { + throw new Error(`expected ${method} response`); + } + return response; +} + +const requestParams = { + questions: [ + { + id: "destination", + header: "Longer than twelve", + question: "Where next?", + options: [], + multiSelect: false, + isOther: true, + isSecret: false, + }, + ], + agentId: "main", + sessionKey: "agent:main:main", + timeoutMs: 100, +}; + +describe("question gateway methods", () => { + it("requests normalized questions, then gets and lists them", async () => { + const requested = await call("question.request", { + ...requestParams, + id: "client-question-id", + }); + expect(requested[0]).toBe(true); + const id = (requested[1] as { id: string }).id; + expect(id).toBe("client-question-id"); + expect(broadcast).toHaveBeenCalledWith( + "question.requested", + expect.objectContaining({ + id, + questions: [expect.objectContaining({ header: "Longer than " })], + status: "pending", + }), + ); + + expect(await call("question.get", { id })).toEqual([ + true, + { question: expect.objectContaining({ id, status: "pending" }) }, + undefined, + ]); + expect(await call("question.list", {})).toEqual([ + true, + { questions: [expect.objectContaining({ id })] }, + undefined, + ]); + }); + + it("broadcasts answered and expired terminal states", async () => { + const requested = await call("question.request", requestParams); + const id = (requested[1] as { id: string }).id; + const answers = { answers: { destination: { answers: ["Home"] } } }; + + expect(await call("question.resolve", { id, answers, resolvedBy: "control-ui" })).toEqual([ + true, + { status: "answered", answers }, + undefined, + ]); + expect(broadcast).toHaveBeenCalledWith("question.resolved", { + id, + status: "answered", + answers, + }); + + const expiring = await call("question.request", { ...requestParams, timeoutMs: 10 }); + const expiringId = (expiring[1] as { id: string }).id; + await vi.advanceTimersByTimeAsync(10); + expect(broadcast).toHaveBeenCalledWith("question.resolved", { + id: expiringId, + status: "expired", + }); + }); + + it("rejects duplicate ids and one-option questions at the request boundary", async () => { + const duplicate = await call("question.request", { + questions: [requestParams.questions[0], requestParams.questions[0]], + }); + expect(duplicate[0]).toBe(false); + expect((duplicate[2] as { message: string }).message).toContain("duplicate question id"); + + const oneOption = await call("question.request", { + questions: [{ ...requestParams.questions[0], options: [{ label: "Only" }] }], + }); + expect(oneOption[0]).toBe(false); + expect((oneOption[2] as { message: string }).message).toContain("2 to 4 options"); + + const clientId = "duplicate-client-id"; + expect((await call("question.request", { ...requestParams, id: clientId }))[0]).toBe(true); + const reusedId = await call("question.request", { ...requestParams, id: clientId }); + expect(reusedId[0]).toBe(false); + expect(reusedId[2]).toMatchObject({ + code: "INVALID_REQUEST", + details: { reason: "QUESTION_ID_IN_USE" }, + }); + }); + + it("rejects secret questions and duplicate normalized option labels", async () => { + const secret = await call("question.request", { + ...requestParams, + questions: [{ ...requestParams.questions[0], isSecret: true }], + }); + expect(secret[0]).toBe(false); + expect((secret[2] as { message: string }).message).toContain( + "question 'destination': secret questions are not supported yet", + ); + + const duplicateLabels = await call("question.request", { + ...requestParams, + questions: [ + { + ...requestParams.questions[0], + options: [{ label: " Deploy " }, { label: "deploy" }], + }, + ], + }); + expect(duplicateLabels[0]).toBe(false); + expect((duplicateLabels[2] as { message: string }).message).toContain( + "question 'destination' has duplicate option label", + ); + }); + + it("returns INVALID_REQUEST for answers that violate the stored question", async () => { + const requested = await call("question.request", { + ...requestParams, + questions: [ + { + ...requestParams.questions[0], + options: [{ label: "Home" }, { label: "Office" }], + isOther: false, + }, + ], + }); + const id = (requested[1] as { id: string }).id; + + const resolved = await call("question.resolve", { + id, + answers: { answers: { destination: { answers: ["Somewhere else"] } } }, + }); + + expect(resolved[0]).toBe(false); + expect(resolved[2]).toMatchObject({ + code: "INVALID_REQUEST", + message: expect.stringContaining("question 'destination'"), + details: { reason: "QUESTION_INVALID_ANSWER" }, + }); + expect(manager.get(id)?.status).toBe("pending"); + }); +}); diff --git a/src/gateway/server-methods/question.ts b/src/gateway/server-methods/question.ts new file mode 100644 index 000000000000..28ec0eac4f60 --- /dev/null +++ b/src/gateway/server-methods/question.ts @@ -0,0 +1,177 @@ +// Question gateway methods create, inspect, wait for, and resolve transient prompts. +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { + ErrorCodes, + errorShape, + formatValidationErrors, + type Question, + type QuestionRequestParams, + type QuestionResolveParams, + validateQuestionGetParams, + validateQuestionListParams, + validateQuestionRequestParams, + validateQuestionResolveParams, + validateQuestionWaitAnswerParams, +} from "../../../packages/gateway-protocol/src/index.js"; +import { + QuestionManager, + QuestionManagerError, + QuestionManagerErrorCodes, +} from "../question-manager.js"; +import type { GatewayRequestHandlers, RespondFn } from "./types.js"; + +const DEFAULT_QUESTION_TIMEOUT_MS = 15 * 60 * 1_000; + +class QuestionRequestValidationError extends Error {} + +function validationError( + method: string, + errors: Parameters[0], + respond: RespondFn, +): void { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + `invalid ${method} params: ${formatValidationErrors(errors)}`, + ), + ); +} + +function managerError(error: unknown, respond: RespondFn): boolean { + if (!(error instanceof QuestionManagerError)) { + return false; + } + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, error.message, { details: { reason: error.code } }), + ); + return true; +} + +function normalizeQuestions(params: QuestionRequestParams): Question[] { + const ids = new Set(); + return params.questions.map((question) => { + if (ids.has(question.id)) { + throw new QuestionRequestValidationError(`duplicate question id '${question.id}'`); + } + ids.add(question.id); + if (question.options.length === 1) { + throw new QuestionRequestValidationError( + `question '${question.id}' must have either no options or 2 to 4 options`, + ); + } + if (question.isSecret) { + throw new QuestionRequestValidationError( + `question '${question.id}': secret questions are not supported yet`, + ); + } + const optionLabels = new Set(); + for (const option of question.options) { + const normalizedLabel = option.label.trim().toLowerCase(); + if (optionLabels.has(normalizedLabel)) { + throw new QuestionRequestValidationError( + `question '${question.id}' has duplicate option label '${option.label}'`, + ); + } + optionLabels.add(normalizedLabel); + } + return { + ...question, + header: truncateUtf16Safe(question.header, 12), + }; + }); +} + +/** Creates the lazily loaded question RPC surface for one Gateway lifetime. */ +export function createQuestionHandlers(manager: QuestionManager): GatewayRequestHandlers { + return { + "question.request": ({ params, respond, context }) => { + if (!validateQuestionRequestParams(params)) { + validationError("question.request", validateQuestionRequestParams.errors, respond); + return; + } + const request = params as QuestionRequestParams; + try { + const record = manager.request({ + ...(request.id ? { id: request.id } : {}), + questions: normalizeQuestions(request), + ...(request.agentId ? { agentId: request.agentId } : {}), + ...(request.sessionKey ? { sessionKey: request.sessionKey } : {}), + timeoutMs: request.timeoutMs ?? DEFAULT_QUESTION_TIMEOUT_MS, + onResolved: (event) => context.broadcast("question.resolved", event), + }); + context.broadcast("question.requested", record); + respond(true, { id: record.id, expiresAtMs: record.expiresAtMs }, undefined); + } catch (error) { + if (error instanceof QuestionRequestValidationError) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, error.message)); + return; + } + if (!managerError(error, respond)) { + throw error; + } + } + }, + "question.waitAnswer": async ({ params, respond }) => { + if (!validateQuestionWaitAnswerParams(params)) { + validationError("question.waitAnswer", validateQuestionWaitAnswerParams.errors, respond); + return; + } + const request = params as { id: string; timeoutMs?: number }; + try { + respond(true, await manager.waitAnswer(request.id, request.timeoutMs), undefined); + } catch (error) { + if (!managerError(error, respond)) { + throw error; + } + } + }, + "question.resolve": ({ params, respond }) => { + if (!validateQuestionResolveParams(params)) { + validationError("question.resolve", validateQuestionResolveParams.errors, respond); + return; + } + const request = params as QuestionResolveParams; + try { + const result = + "cancel" in request + ? manager.cancel(request.id, request.resolvedBy) + : manager.resolve(request.id, request.answers, request.resolvedBy); + respond(true, result, undefined); + } catch (error) { + if (!managerError(error, respond)) { + throw error; + } + } + }, + "question.get": ({ params, respond }) => { + if (!validateQuestionGetParams(params)) { + validationError("question.get", validateQuestionGetParams.errors, respond); + return; + } + const id = (params as { id: string }).id; + const question = manager.get(id); + if (!question) { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, `question '${id}' was not found`, { + details: { reason: QuestionManagerErrorCodes.NOT_FOUND }, + }), + ); + return; + } + respond(true, { question }, undefined); + }, + "question.list": ({ params, respond }) => { + if (!validateQuestionListParams(params)) { + validationError("question.list", validateQuestionListParams.errors, respond); + return; + } + respond(true, { questions: manager.list() }, undefined); + }, + }; +} diff --git a/src/gateway/server/plugin-route-runtime-scopes.test.ts b/src/gateway/server/plugin-route-runtime-scopes.test.ts index 2e6369d4c4e1..d332bfbd3168 100644 --- a/src/gateway/server/plugin-route-runtime-scopes.test.ts +++ b/src/gateway/server/plugin-route-runtime-scopes.test.ts @@ -63,6 +63,7 @@ describe("resolvePluginRouteRuntimeOperatorScopes", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); @@ -80,6 +81,7 @@ describe("resolvePluginRouteRuntimeOperatorScopes", () => { "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", "operator.talk.secrets", ]); diff --git a/src/infra/outbound/deliver.test.ts b/src/infra/outbound/deliver.test.ts index a04fc6694a99..66b94b1097d3 100644 --- a/src/infra/outbound/deliver.test.ts +++ b/src/infra/outbound/deliver.test.ts @@ -3436,6 +3436,104 @@ describe("deliverOutboundPayloads", () => { }); }); + it("uses runtime fallback text only when native presentation rendering is unavailable", async () => { + const nativeRender = vi.fn(({ payload }) => ({ + ...payload, + channelData: { native: true }, + })); + const sendPayload = vi.fn().mockResolvedValue({ + channel: "matrix" as const, + messageId: "question-card", + roomId: "!room", + }); + setActivePluginRegistry( + createTestRegistry([ + { + pluginId: "matrix", + source: "test", + plugin: createOutboundTestPlugin({ + id: "matrix", + outbound: { + deliveryMode: "direct", + presentationCapabilities: { supported: true, buttons: true }, + renderPresentation: nativeRender, + sendText: vi.fn(), + sendMedia: vi.fn(), + sendPayload, + }, + }), + }, + ]), + ); + + await deliverOutboundPayloads({ + cfg: {}, + channel: "matrix", + to: "!room", + payloads: [ + { + text: "Question with numbered fallback", + presentationTextMode: "fallback", + presentation: { + blocks: [ + { type: "text", text: "Question" }, + { type: "buttons", buttons: [{ label: "Yes", value: "yes" }] }, + ], + }, + }, + ], + }); + + expect(requireMockCallArg(nativeRender, "native renderer").payload).toMatchObject({ + text: undefined, + presentationTextMode: "fallback", + }); + expect(requireMockCallArg(sendPayload, "send payload").payload).toMatchObject({ + channelData: { native: true }, + text: undefined, + }); + }); + + it("keeps runtime presentation fallback text exact on button-less channels", async () => { + const sendText = vi.fn().mockResolvedValue({ + channel: "matrix" as const, + messageId: "question-text", + roomId: "!room", + }); + setActivePluginRegistry( + createTestRegistry([ + { + pluginId: "matrix", + source: "test", + plugin: createOutboundTestPlugin({ + id: "matrix", + outbound: { deliveryMode: "direct", sendText }, + }), + }, + ]), + ); + + await deliverOutboundPayloads({ + cfg: {}, + channel: "matrix", + to: "!room", + payloads: [ + { + text: "Question\n1. Yes\n2. No", + presentationTextMode: "fallback", + presentation: { + blocks: [ + { type: "text", text: "Question" }, + { type: "buttons", buttons: [{ label: "Yes", value: "yes" }] }, + ], + }, + }, + ], + }); + + expect(requireMockCallArg(sendText, "send text").text).toBe("Question\n1. Yes\n2. No"); + }); + it("runs adapter after-delivery hooks with the payload delivery results", async () => { const afterDeliverPayload = vi.fn(); setActivePluginRegistry( diff --git a/src/infra/outbound/deliver.ts b/src/infra/outbound/deliver.ts index ef9a0b26e45c..d400e7273c80 100644 --- a/src/infra/outbound/deliver.ts +++ b/src/infra/outbound/deliver.ts @@ -1148,21 +1148,37 @@ async function renderPresentationForDelivery( presentation, capabilities: handler.presentationCapabilities, }); - const adaptedPayload = { ...payload, presentation: adaptedPresentation }; + const textIsFallback = payload.presentationTextMode === "fallback"; + const adaptedPayload = { + ...payload, + ...(textIsFallback ? { text: undefined } : {}), + presentation: adaptedPresentation, + }; const rendered = handler.renderPresentation ? await handler.renderPresentation(adaptedPayload) : null; if (rendered) { - const { presentation: _presentation, ...withoutPresentation } = rendered; + const { + presentation: _presentation, + presentationTextMode: _presentationTextMode, + ...withoutPresentation + } = rendered; return withoutPresentation; } - const { presentation: _presentation, ...withoutPresentation } = payload; + const { + presentation: _presentation, + presentationTextMode: _presentationTextMode, + ...withoutPresentation + } = payload; return { ...withoutPresentation, - text: renderMessagePresentationFallbackText({ - text: payload.text, - presentation: adaptedPresentation, - }), + text: textIsFallback + ? (payload.text ?? + renderMessagePresentationFallbackText({ presentation: adaptedPresentation })) + : renderMessagePresentationFallbackText({ + text: payload.text, + presentation: adaptedPresentation, + }), }; } diff --git a/src/infra/outbound/payloads.ts b/src/infra/outbound/payloads.ts index 09885f34e027..04a21b6d1379 100644 --- a/src/infra/outbound/payloads.ts +++ b/src/infra/outbound/payloads.ts @@ -34,6 +34,7 @@ export type NormalizedOutboundPayload = { mediaUrls: string[]; audioAsVoice?: boolean; presentation?: MessagePresentation; + presentationTextMode?: ReplyPayload["presentationTextMode"]; delivery?: ReplyPayloadDelivery; interactive?: LegacyInteractiveReply; channelData?: Record; @@ -49,6 +50,7 @@ export type OutboundPayloadJson = { mediaUrls?: string[]; audioAsVoice?: boolean; presentation?: MessagePresentation; + presentationTextMode?: ReplyPayload["presentationTextMode"]; delivery?: ReplyPayloadDelivery; interactive?: LegacyInteractiveReply; channelData?: Record; @@ -343,6 +345,9 @@ export function projectOutboundPayloadPlanForOutbound( mediaUrls: entry.parts.mediaUrls, audioAsVoice: payload.audioAsVoice === true ? true : undefined, ...(entry.hasPresentation ? { presentation: payload.presentation } : {}), + ...(entry.hasPresentation && payload.presentationTextMode + ? { presentationTextMode: payload.presentationTextMode } + : {}), ...(payload.delivery ? { delivery: payload.delivery } : {}), ...(entry.hasInteractive ? { interactive: payload.interactive } : {}), ...(entry.hasChannelData ? { channelData: payload.channelData } : {}), @@ -365,6 +370,9 @@ export function projectOutboundPayloadPlanForJson( mediaUrls: entry.parts.mediaUrls.length ? entry.parts.mediaUrls : undefined, audioAsVoice: payload.audioAsVoice === true ? true : undefined, presentation: payload.presentation, + ...(payload.presentationTextMode + ? { presentationTextMode: payload.presentationTextMode } + : {}), delivery: payload.delivery, interactive: payload.interactive, channelData: payload.channelData, @@ -403,6 +411,7 @@ export function summarizeOutboundPayloadForTransport( mediaUrls: parts.mediaUrls, audioAsVoice: payload.audioAsVoice === true ? true : undefined, presentation: payload.presentation, + ...(payload.presentationTextMode ? { presentationTextMode: payload.presentationTextMode } : {}), delivery: payload.delivery, interactive: payload.interactive, channelData: payload.channelData, diff --git a/src/infra/outbound/reply-payload-normalize.ts b/src/infra/outbound/reply-payload-normalize.ts index bf917a4da555..72c45844311b 100644 --- a/src/infra/outbound/reply-payload-normalize.ts +++ b/src/infra/outbound/reply-payload-normalize.ts @@ -12,6 +12,7 @@ export type OutboundReplyPayload = { mediaUrls?: string[]; mediaUrl?: string; presentation?: InternalReplyPayload["presentation"]; + presentationTextMode?: InternalReplyPayload["presentationTextMode"]; /** * @deprecated Use presentation. Runtime support remains for legacy producers. */ @@ -41,6 +42,7 @@ export function normalizeOutboundReplyPayload( const presentation = readObjectValue( payload.presentation, ) as OutboundReplyPayload["presentation"]; + const presentationTextMode = payload.presentationTextMode === "fallback" ? "fallback" : undefined; const interactive = readObjectValue(payload.interactive) as OutboundReplyPayload["interactive"]; const channelData = readObjectValue(payload.channelData) as OutboundReplyPayload["channelData"]; const sensitiveMedia = payload.sensitiveMedia === true ? true : undefined; @@ -52,6 +54,7 @@ export function normalizeOutboundReplyPayload( mediaUrls, mediaUrl, presentation, + ...(presentationTextMode ? { presentationTextMode } : {}), interactive, channelData, sensitiveMedia, diff --git a/src/infra/question-gateway-resolver.test.ts b/src/infra/question-gateway-resolver.test.ts new file mode 100644 index 000000000000..a32223923a16 --- /dev/null +++ b/src/infra/question-gateway-resolver.test.ts @@ -0,0 +1,134 @@ +// Covers compact question-button resolution through a stubbed Gateway. +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveQuestionOverGateway } from "./question-gateway-resolver.js"; + +const hoisted = vi.hoisted(() => ({ callGateway: vi.fn() })); + +vi.mock("../gateway/call.js", () => ({ callGateway: hoisted.callGateway })); + +const recordId = "ask_0123456789abcdef0123456789abcdef"; +const pendingRecord = { + id: recordId, + status: "pending", + questions: [ + { + id: "deploy_target", + header: "Target", + question: "Where should this deploy?", + options: [{ label: "Staging" }, { label: "Production" }], + }, + ], + createdAtMs: 1, + expiresAtMs: 2, +} as const; + +function terminalError(reason: "QUESTION_ALREADY_TERMINAL" | "QUESTION_NOT_FOUND") { + return Object.assign(new Error(reason), { + name: "GatewayClientRequestError", + details: { reason }, + }); +} + +describe("resolveQuestionOverGateway", () => { + beforeEach(() => { + hoisted.callGateway.mockReset(); + }); + + it("maps the compact option index to the canonical question id and label", async () => { + hoisted.callGateway.mockResolvedValueOnce({ question: pendingRecord }).mockResolvedValueOnce({ + status: "answered", + answers: { answers: { deploy_target: { answers: ["Production"] } } }, + }); + + await expect( + resolveQuestionOverGateway({ + cfg: {} as never, + questionId: recordId, + optionIndex: 1, + senderId: "telegram:42", + }), + ).resolves.toEqual({ + status: "answered", + questionId: "deploy_target", + optionValue: "Production", + }); + expect(hoisted.callGateway.mock.calls).toEqual([ + [ + expect.objectContaining({ + method: "question.get", + params: { id: recordId }, + scopes: ["operator.questions"], + }), + ], + [ + expect.objectContaining({ + method: "question.resolve", + params: { + id: recordId, + answers: { answers: { deploy_target: { answers: ["Production"] } } }, + resolvedBy: "telegram:42", + }, + }), + ], + ]); + }); + + it.each([ + ["question.get", "QUESTION_NOT_FOUND", "not-found"], + ["question.resolve", "QUESTION_ALREADY_TERMINAL", "already-terminal"], + ] as const)( + "returns a terminal outcome when %s races", + async (method, reason, expectedReason) => { + if (method === "question.resolve") { + hoisted.callGateway.mockResolvedValueOnce({ question: pendingRecord }); + } + hoisted.callGateway.mockRejectedValueOnce(terminalError(reason)); + + await expect( + resolveQuestionOverGateway({ cfg: {} as never, questionId: recordId, optionIndex: 0 }), + ).resolves.toEqual({ status: "already-terminal", reason: expectedReason }); + }, + ); + + it("does not resolve an already-terminal record", async () => { + hoisted.callGateway.mockResolvedValueOnce({ + question: { ...pendingRecord, status: "expired" }, + }); + + await expect( + resolveQuestionOverGateway({ cfg: {} as never, questionId: recordId, optionIndex: 0 }), + ).resolves.toEqual({ status: "already-terminal", reason: "already-terminal" }); + expect(hoisted.callGateway).toHaveBeenCalledOnce(); + }); + + it("rejects invalid indices before resolving", async () => { + hoisted.callGateway.mockResolvedValueOnce({ question: pendingRecord }); + + await expect( + resolveQuestionOverGateway({ cfg: {} as never, questionId: recordId, optionIndex: 2 }), + ).rejects.toThrow("out of range"); + expect(hoisted.callGateway).toHaveBeenCalledOnce(); + }); + + it("never partially resolves a multi-question record", async () => { + hoisted.callGateway.mockResolvedValueOnce({ + question: { + ...pendingRecord, + questions: [ + ...pendingRecord.questions, + { + id: "region", + header: "Region", + question: "Which region?", + options: [{ label: "EU" }, { label: "US" }], + }, + ], + }, + }); + + await expect( + resolveQuestionOverGateway({ cfg: {} as never, questionId: recordId, optionIndex: 0 }), + ).rejects.toThrow("one tappable question"); + expect(hoisted.callGateway).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/infra/question-gateway-resolver.ts b/src/infra/question-gateway-resolver.ts new file mode 100644 index 000000000000..4e08e753791b --- /dev/null +++ b/src/infra/question-gateway-resolver.ts @@ -0,0 +1,102 @@ +// Resolves runtime-authored question choices through the Gateway. +import type { + QuestionGetResult, + QuestionResolveResult, +} from "../../packages/gateway-protocol/src/schema/questions.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { callGateway } from "../gateway/call.js"; + +const QUESTION_RECORD_ID_PATTERN = /^ask_[a-f0-9]{32}$/u; + +export type ResolveQuestionOverGatewayResult = + | { status: "answered"; questionId: string; optionValue: string } + | { status: "already-terminal"; reason: "already-terminal" | "not-found" }; + +export type ResolveQuestionOverGatewayParams = { + cfg: OpenClawConfig; + questionId: string; + optionIndex: number; + senderId?: string | null; + gatewayUrl?: string; + clientDisplayName?: string; +}; + +function readTerminalReason(error: unknown): "already-terminal" | "not-found" | undefined { + if (!(error instanceof Error) || error.name !== "GatewayClientRequestError") { + return undefined; + } + const details = (error as Error & { details?: unknown }).details; + if (!details || typeof details !== "object" || Array.isArray(details)) { + return undefined; + } + const reason = (details as { reason?: unknown }).reason; + if (reason === "QUESTION_ALREADY_TERMINAL") { + return "already-terminal"; + } + return reason === "QUESTION_NOT_FOUND" ? "not-found" : undefined; +} + +/** Maps a compact option index to the canonical label, then resolves the question atomically. */ +export async function resolveQuestionOverGateway( + params: ResolveQuestionOverGatewayParams, +): Promise { + if (!QUESTION_RECORD_ID_PATTERN.test(params.questionId)) { + throw new Error("question resolution requires a valid question record id"); + } + if (!Number.isInteger(params.optionIndex) || params.optionIndex < 0) { + throw new Error("question resolution requires a valid option index"); + } + const gatewayOptions = { + config: params.cfg, + url: params.gatewayUrl, + scopes: ["operator.questions" as const], + clientDisplayName: + params.clientDisplayName ?? `Question (${params.senderId?.trim() || "unknown"})`, + }; + let getResult: QuestionGetResult; + try { + getResult = await callGateway({ + ...gatewayOptions, + method: "question.get", + params: { id: params.questionId }, + }); + } catch (error) { + const reason = readTerminalReason(error); + if (reason) { + return { status: "already-terminal", reason }; + } + throw error; + } + + const record = getResult.question; + if (record.status !== "pending") { + return { status: "already-terminal", reason: "already-terminal" }; + } + const question = record.questions.length === 1 ? record.questions[0] : undefined; + if (!question || question.multiSelect || question.isSecret) { + throw new Error("question button resolution requires one tappable question"); + } + const optionValue = question.options[params.optionIndex]?.label; + if (!optionValue) { + throw new Error("question resolution option index is out of range"); + } + + try { + await callGateway({ + ...gatewayOptions, + method: "question.resolve", + params: { + id: params.questionId, + answers: { answers: { [question.id]: { answers: [optionValue] } } }, + resolvedBy: params.senderId?.trim() || undefined, + }, + }); + } catch (error) { + const reason = readTerminalReason(error); + if (reason) { + return { status: "already-terminal", reason }; + } + throw error; + } + return { status: "answered", questionId: question.id, optionValue }; +} diff --git a/src/interactive/payload.test.ts b/src/interactive/payload.test.ts index 1c9370ffd5b1..e2452f7ab998 100644 --- a/src/interactive/payload.test.ts +++ b/src/interactive/payload.test.ts @@ -498,6 +498,41 @@ describe("interactive payload helpers", () => { }); }); + it("normalizes question actions without exposing their transport data", () => { + const action = { + type: "question" as const, + questionId: "ask_0123456789abcdef0123456789abcdef", + optionValue: "Production", + }; + const presentation = normalizeMessagePresentation({ + blocks: [{ type: "buttons", buttons: [{ label: "Production", action }] }], + }); + + expect(presentation).toEqual({ + blocks: [{ type: "buttons", buttons: [{ label: "Production", action }] }], + }); + expect(resolveMessagePresentationControlValue({ action })).toBeUndefined(); + expect(renderMessagePresentationFallbackText({ presentation: presentation ?? undefined })).toBe( + "- Production", + ); + expect(presentationToInteractiveReply(presentation ?? { blocks: [] })).toEqual({ + blocks: [{ type: "buttons", buttons: [{ label: "Production", action }] }], + }); + }); + + it.each([ + { type: "Question", questionId: "ask_1", optionValue: "Yes" }, + { type: "question", questionId: "", optionValue: "Yes" }, + { type: "question", questionId: "ask_\ud800", optionValue: "Yes" }, + { type: "question", questionId: "ask_1", optionValue: " " }, + ])("rejects malformed question action %#", (action) => { + expect( + normalizeMessagePresentation({ + blocks: [{ type: "buttons", buttons: [{ label: "Yes", action }] }], + }), + ).toBeUndefined(); + }); + it("rejects malformed canonical actions instead of falling back to legacy fields", () => { expect( normalizeMessagePresentation({ diff --git a/src/interactive/payload.ts b/src/interactive/payload.ts index 50bd007e0214..b6c614804107 100644 --- a/src/interactive/payload.ts +++ b/src/interactive/payload.ts @@ -33,6 +33,12 @@ export type MessagePresentationAction = approvalKind: "exec" | "plugin"; decision: "allow-once" | "allow-always" | "deny"; } + | { + /** Resolve one runtime-authored operator question choice. */ + type: "question"; + questionId: string; + optionValue: string; + } | { /** Open a normal external link. */ type: "url"; @@ -365,6 +371,22 @@ function normalizePresentationAction(raw: unknown): MessagePresentationAction | } return { type: "approval", approvalId, approvalKind, decision }; } + if (type === "question") { + if (record.type !== "question") { + return undefined; + } + const questionId = record.questionId; + const optionValue = record.optionValue; + if ( + typeof questionId !== "string" || + !isWellFormedApprovalId(questionId) || + typeof optionValue !== "string" || + !optionValue.trim() + ) { + return undefined; + } + return { type: "question", questionId, optionValue }; + } if (type === "url") { const url = normalizeOptionalString(record.url); return url ? { type: "url", url } : undefined; @@ -842,9 +864,9 @@ export const interactiveReplyToPresentation = legacyInteractiveReplyToPresentati * * Text and context blocks are rendered as-is. Buttons with a `command`-typed * action render as `label: \`command\`` so the value is copyable. URL and web - * app actions include their user-facing URL. Approval, callback, legacy value, - * and select actions render label-only to keep transport data private. Disabled - * buttons render label-only regardless of action type. + * app actions include their user-facing URL. Approval, question, callback, + * legacy value, and select actions render label-only to keep transport data + * private. Disabled buttons render label-only regardless of action type. * * Downstream consumers should not claim a manual command is available unless * they verify one was actually rendered. diff --git a/src/plugin-sdk/question-gateway-runtime.ts b/src/plugin-sdk/question-gateway-runtime.ts new file mode 100644 index 000000000000..6ba9ccf8ca37 --- /dev/null +++ b/src/plugin-sdk/question-gateway-runtime.ts @@ -0,0 +1,6 @@ +/** Runtime SDK subpath for resolving ask_user question choices over the Gateway. */ +export { + resolveQuestionOverGateway, + type ResolveQuestionOverGatewayParams, + type ResolveQuestionOverGatewayResult, +} from "../infra/question-gateway-resolver.js"; diff --git a/src/plugin-sdk/reply-payload.test.ts b/src/plugin-sdk/reply-payload.test.ts index 20d749684212..6cb07104b01a 100644 --- a/src/plugin-sdk/reply-payload.test.ts +++ b/src/plugin-sdk/reply-payload.test.ts @@ -469,6 +469,7 @@ describe("normalizeOutboundReplyPayload", () => { expect( normalizeOutboundReplyPayload({ presentation, + presentationTextMode: "fallback", interactive, channelData, trustedLocalMedia: true, @@ -478,6 +479,7 @@ describe("normalizeOutboundReplyPayload", () => { mediaUrls: undefined, mediaUrl: undefined, presentation, + presentationTextMode: "fallback", interactive, channelData, sensitiveMedia: undefined, diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 6a10af4719de..555c748a3517 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -2066,6 +2066,12 @@ describe("ci workflow guards", () => { expect(installStep.run).toContain( '[ "$sticky_fingerprint" = "${OPENCLAW_STICKY_DEPS_FINGERPRINT:?}" ]', ); + expect(installStep.run).toContain('[ "$STICKY_WRITER" != "true" ]'); + expect(installStep.run).toContain('sudo umount "$GITHUB_WORKSPACE/node_modules"'); + expect(installStep.run).toContain('ephemeral_store="${RUNNER_TEMP:?}/openclaw-pnpm-store"'); + expect(installStep.run).toContain( + "Sticky dependency snapshot is stale; using runner-local storage for this read-only run", + ); expect(installStep.run).toContain( 'bash "$GITHUB_ACTION_PATH/sticky-importers.sh" restore "$STICKY_ROOT" "$GITHUB_WORKSPACE"', ); @@ -3628,7 +3634,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ); }); - it("keeps Control UI locale parity advisory until release CI", () => { + it("keeps source-only Control UI locale drift advisory", () => { const workflow = readCiWorkflow(); const workflowSource = readFileSync(".github/workflows/ci.yml", "utf8"); const buildArtifactSteps = workflow.jobs["build-artifacts"].steps; @@ -3649,8 +3655,9 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ); expect(localeJob.needs).toEqual(["preflight"]); expect(localeJob.if).toBe("needs.preflight.outputs.run_control_ui_i18n == 'true'"); - expect(localeJob["continue-on-error"]).toBe( - "${{ github.event_name != 'workflow_dispatch' && needs.preflight.outputs.strict_control_ui_i18n != 'true' }}", + expect(localeJob["continue-on-error"]).toBeUndefined(); + expect(localeStep["continue-on-error"]).toBe( + "${{ needs.preflight.outputs.strict_control_ui_i18n != 'true' }}", ); expect(localeStep.run).toBe("pnpm ui:i18n:check"); expect(readFileSync(".github/workflows/full-release-validation.yml", "utf8")).toContain( diff --git a/ui/src/api/gateway.node.test.ts b/ui/src/api/gateway.node.test.ts index 3798a266b12d..10f36bd6f3eb 100644 --- a/ui/src/api/gateway.node.test.ts +++ b/ui/src/api/gateway.node.test.ts @@ -28,10 +28,12 @@ const CONTROL_UI_OPERATOR_SCOPES = [ "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", ] as const; const CONTROL_UI_BOOTSTRAP_OPERATOR_SCOPES = [ "operator.approvals", + "operator.questions", "operator.read", "operator.talk.secrets", "operator.write", @@ -1051,6 +1053,7 @@ describe("GatewayBrowserClient", () => { "operator.admin", "operator.approvals", "operator.pairing", + "operator.questions", "operator.read", "operator.write", ], diff --git a/ui/src/api/gateway.ts b/ui/src/api/gateway.ts index beeb9854d0b0..1ef29ccb54d1 100644 --- a/ui/src/api/gateway.ts +++ b/ui/src/api/gateway.ts @@ -138,11 +138,13 @@ const CONTROL_UI_OPERATOR_SCOPES = [ "operator.read", "operator.write", "operator.approvals", + "operator.questions", "operator.pairing", ] as const; const CONTROL_UI_BOOTSTRAP_OPERATOR_SCOPES = [ "operator.approvals", + "operator.questions", "operator.read", "operator.talk.secrets", "operator.write", diff --git a/ui/src/app/question-prompt.test.ts b/ui/src/app/question-prompt.test.ts new file mode 100644 index 000000000000..6b7386807e36 --- /dev/null +++ b/ui/src/app/question-prompt.test.ts @@ -0,0 +1,697 @@ +// Control UI tests cover operator question parsing and lifecycle state. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createQuestionPromptState, + disposeQuestionPromptState, + handleQuestionPromptEvent, + refreshPendingQuestionsWithRetry, + setQuestionPromptClient, + submitQuestionPrompt, +} from "./question-prompt.ts"; + +type RequestFn = (method: string, params?: unknown) => Promise; +type QuestionPromptState = ReturnType; + +const states: QuestionPromptState[] = []; + +function createState(onChange = vi.fn()) { + const state = createQuestionPromptState(onChange); + states.push(state); + return state; +} + +function requestedPayload(overrides: Record = {}) { + return { + id: "question-1", + questions: [ + { + id: "format", + header: "Format", + question: "Which format should I use?", + options: [{ label: "Compact", description: "Keep it brief" }, { label: "Detailed" }], + isOther: true, + }, + ], + agentId: "main", + sessionKey: "agent:main:main", + createdAtMs: 1_000, + expiresAtMs: Date.now() + 60_000, + status: "pending", + ...overrides, + }; +} + +function questionNotFoundError() { + return Object.assign(new Error("question was not found"), { + name: "GatewayClientRequestError", + details: { reason: "QUESTION_NOT_FOUND" }, + }); +} + +afterEach(() => { + for (const state of states.splice(0)) { + disposeQuestionPromptState(state); + } + vi.useRealTimers(); +}); + +describe("question event parsing", () => { + it("round-trips requested and resolved event payloads", () => { + const state = createState(); + expect( + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }), + ).toBe(true); + expect(state.prompts.get("question-1")).toMatchObject({ + id: "question-1", + sessionKey: "agent:main:main", + status: "pending", + questions: [{ id: "format", options: [{ label: "Compact" }, { label: "Detailed" }] }], + }); + expect( + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Compact"] } } }, + }, + }), + ).toBe(true); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answers: { answers: { format: { answers: ["Compact"] } } }, + }); + }); + + it("rejects malformed records and answer maps", () => { + const state = createState(); + expect( + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ id: "" }), + }), + ).toBe(false); + expect( + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ + questions: [{ id: "Bad ID", header: "Bad", question: "Bad?", options: [] }], + }), + }), + ).toBe(false); + expect( + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: "Compact" } } }, + }, + }), + ).toBe(false); + }); +}); + +describe("question prompt state", () => { + it.each(["cancelled", "expired"] as const)("transitions requested to %s", (status) => { + const state = createState(); + expect( + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }), + ).toBe(true); + + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { id: "question-1", status }, + }); + + expect(state.prompts.get("question-1")?.status).toBe(status); + }); + + it("marks answers from another surface", () => { + const state = createState(); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }, + }); + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + }); + + it("marks a locally submitted answer as local when its broadcast arrives", async () => { + let releaseRequest: () => void = () => {}; + const request = vi.fn( + () => + new Promise((resolve) => { + releaseRequest = () => resolve({ status: "answered" }); + }), + ); + const state = createState(); + setQuestionPromptClient(state, { request }); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + const submitting = submitQuestionPrompt(state, "question-1", { format: ["Compact"] }); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Compact"] } } }, + }, + }); + releaseRequest(); + await submitting; + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: false, + localResolutionConfirmed: true, + submitting: false, + }); + }); + + it("marks a concurrent winning answer from another surface", async () => { + let rejectRequest: (error: Error) => void = () => {}; + const request = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectRequest = reject; + }), + ); + const state = createState(); + setQuestionPromptClient(state, { request }); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + const submitting = submitQuestionPrompt(state, "question-1", { format: ["Compact"] }); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }, + }); + rejectRequest(new Error("question already resolved")); + await submitting; + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + localResolutionConfirmed: false, + submitting: false, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + }); + + it("keeps local provenance when the accepted resolve response is lost", async () => { + let rejectRequest: (error: Error) => void = () => {}; + const request = vi.fn( + () => + new Promise((_resolve, reject) => { + rejectRequest = reject; + }), + ); + const state = createState(); + setQuestionPromptClient(state, { request }); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + const submitting = submitQuestionPrompt(state, "question-1", { format: ["Compact"] }); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Compact"] } } }, + }, + }); + rejectRequest(new Error("connection closed")); + await submitting; + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: false, + submitting: false, + answers: { answers: { format: { answers: ["Compact"] } } }, + }); + }); + + it("expires pending cards locally when their countdown ends", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-17T00:00:00.000Z")); + const state = createState(); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ expiresAtMs: Date.now() + 1_000 }), + }); + + vi.advanceTimersByTime(1_000); + + expect(state.prompts.get("question-1")?.status).toBe("expired"); + }); + + it("re-enables a prompt with a non-destructive resolve error", async () => { + const request = vi.fn(async () => { + throw new Error("gateway unavailable"); + }); + const state = createState(); + setQuestionPromptClient(state, { request }); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + await submitQuestionPrompt(state, "question-1", { format: ["Compact"] }); + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "pending", + submitting: false, + error: "gateway unavailable", + }); + }); + + it("surfaces a retryable error when submission happens while disconnected", async () => { + const state = createState(); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + await submitQuestionPrompt(state, "question-1", { format: ["Compact"] }); + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "pending", + submitting: false, + error: "Not connected. Try again after reconnecting.", + }); + }); +}); + +describe("question RPC helpers", () => { + it("sends option labels, free text, and multi-select arrays in the frozen answer shape", async () => { + const request = vi.fn(async () => ({})); + const state = createState(); + setQuestionPromptClient(state, { request }); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + await submitQuestionPrompt(state, "question-1", { + format: ["Compact"], + destination: ["My own target"], + extras: ["Tests", "Docs"], + }); + + expect(request).toHaveBeenCalledWith("question.resolve", { + id: "question-1", + answers: { + answers: { + format: { answers: ["Compact"] }, + destination: { answers: ["My own target"] }, + extras: { answers: ["Tests", "Docs"] }, + }, + }, + }); + }); +}); + +describe("refreshPendingQuestions", () => { + it("hydrates pending questions after connect", async () => { + const request = vi.fn(async () => ({ questions: [requestedPayload()] })); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("pending")); + + expect(request).toHaveBeenCalledWith("question.list", {}); + expect(state.prompts.get("question-1")?.status).toBe("pending"); + }); + + it("retries transient hydration failures while the client remains current", async () => { + vi.useFakeTimers(); + let attempts = 0; + const request = vi.fn(async () => { + attempts += 1; + if (attempts < 5) { + throw new Error("gateway unavailable"); + } + return { questions: [requestedPayload()] }; + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + + refreshPendingQuestionsWithRetry(state, client); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(2_000); + await vi.advanceTimersByTimeAsync(4_000); + await vi.advanceTimersByTimeAsync(4_000); + + expect(request).toHaveBeenCalledTimes(5); + expect(state.prompts.get("question-1")?.status).toBe("pending"); + }); + + it("preserves a resolution received while reconnect refresh is in flight", async () => { + let finishList: (value: unknown) => void = () => {}; + const request = vi.fn( + () => + new Promise((resolve) => { + finishList = resolve; + }), + ); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + refreshPendingQuestionsWithRetry(state, client); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }, + }); + finishList({ questions: [requestedPayload()] }); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + + expect(state.prompts.get("question-1")?.status).toBe("answered"); + }); + + it("retains a resolution that arrives before reconnect hydration", async () => { + let finishList: (value: unknown) => void = () => {}; + const request = vi.fn((method) => + method === "question.list" + ? new Promise((resolve) => { + finishList = resolve; + }) + : Promise.resolve({ + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }), + ); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + + refreshPendingQuestionsWithRetry(state, client); + handleQuestionPromptEvent(state, { + event: "question.resolved", + payload: { + id: "question-1", + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }, + }); + finishList({ questions: [] }); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + + expect(request).toHaveBeenCalledWith("question.get", { id: "question-1" }); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + expect(state.unmatchedResolutions.size).toBe(0); + }); + + it("recovers a terminal answer missed during disconnect", async () => { + const request = vi.fn(async (method) => { + if (method === "question.list") { + return { questions: [] }; + } + return { + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }; + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + + expect(request).toHaveBeenCalledWith("question.get", { id: "question-1" }); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + }); + + it("keeps a missing pending question recoverable after question.get fails", async () => { + let getAttempts = 0; + const request = vi.fn(async (method) => { + if (method === "question.list") { + return { questions: [] }; + } + getAttempts += 1; + if (getAttempts === 1) { + throw new Error("gateway unavailable"); + } + return { + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }; + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(request).toHaveBeenCalledTimes(2)); + expect(state.prompts.get("question-1")?.status).toBe("pending"); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + }); + + it("terminalizes a missing pending question after authoritative not-found", async () => { + const request = vi.fn(async (method) => { + if (method === "question.list") { + return { questions: [] }; + } + throw questionNotFoundError(); + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + answeredElsewhere: true, + locallyExpired: false, + submitting: false, + error: null, + }); + }); + + it("reconciles a locally expired prompt with the authoritative record", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-17T00:00:00.000Z")); + const request = vi.fn(async (method) => + method === "question.list" + ? { questions: [] } + : { + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }, + ); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ expiresAtMs: Date.now() + 1_000 }), + }); + vi.advanceTimersByTime(1_000); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "expired", + locallyExpired: true, + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + locallyExpired: false, + answeredElsewhere: true, + answers: { answers: { format: { answers: ["Detailed"] } } }, + }); + }); + + it("applies a question.get answer when the local timer expires in flight", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-17T00:00:00.000Z")); + let finishGet: (value: unknown) => void = () => {}; + const request = vi.fn((method) => { + if (method === "question.list") { + return Promise.resolve({ questions: [] }); + } + return new Promise((resolve) => { + finishGet = resolve; + }); + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ expiresAtMs: Date.now() + 1_000 }), + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => + expect(request).toHaveBeenCalledWith("question.get", { id: "question-1" }), + ); + await vi.advanceTimersByTimeAsync(1_000); + expect(state.prompts.get("question-1")?.locallyExpired).toBe(true); + finishGet({ + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }); + + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + locallyExpired: false, + answeredElsewhere: true, + }); + }); + + it("reconciles local expiry that occurs while question.list is in flight", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-17T00:00:00.000Z")); + let finishList: (value: unknown) => void = () => {}; + const request = vi.fn((method) => { + if (method === "question.list") { + return new Promise((resolve) => { + finishList = resolve; + }); + } + return Promise.resolve({ + question: requestedPayload({ + status: "answered", + answers: { answers: { format: { answers: ["Detailed"] } } }, + }), + }); + }); + const state = createState(); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload({ expiresAtMs: Date.now() + 1_000 }), + }); + + refreshPendingQuestionsWithRetry(state, client); + await vi.advanceTimersByTimeAsync(1_000); + finishList({ questions: [] }); + + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("answered")); + expect(request).toHaveBeenCalledWith("question.get", { id: "question-1" }); + expect(state.prompts.get("question-1")).toMatchObject({ + status: "answered", + locallyExpired: false, + answeredElsewhere: true, + }); + }); + + it("publishes listed questions before older reconciliation finishes", async () => { + let finishGet: (value: unknown) => void = () => {}; + const request = vi.fn((method) => { + if (method === "question.list") { + return Promise.resolve({ + questions: [requestedPayload({ id: "question-2", createdAtMs: 2_000 })], + }); + } + return new Promise((resolve) => { + finishGet = resolve; + }); + }); + const onChange = vi.fn(); + const state = createState(onChange); + const client = { request }; + setQuestionPromptClient(state, client); + handleQuestionPromptEvent(state, { + event: "question.requested", + payload: requestedPayload(), + }); + onChange.mockClear(); + + refreshPendingQuestionsWithRetry(state, client); + await vi.waitFor(() => + expect(request).toHaveBeenCalledWith("question.get", { id: "question-1" }), + ); + + expect(state.prompts.get("question-2")?.status).toBe("pending"); + expect(onChange).toHaveBeenCalled(); + + finishGet({ question: requestedPayload({ status: "cancelled" }) }); + await vi.waitFor(() => expect(state.prompts.get("question-1")?.status).toBe("cancelled")); + }); +}); diff --git a/ui/src/app/question-prompt.ts b/ui/src/app/question-prompt.ts new file mode 100644 index 000000000000..41c785b2b039 --- /dev/null +++ b/ui/src/app/question-prompt.ts @@ -0,0 +1,625 @@ +// Control UI module owns transient in-thread operator question state. +import type { + Question, + QuestionAnswers, + QuestionRecord, + QuestionResolvedEvent, +} from "../../../packages/gateway-protocol/src/index.js"; +import type { GatewayEventFrame } from "../api/gateway.ts"; + +type QuestionClient = { + request: (method: string, params?: unknown) => Promise; +}; + +type QuestionDraft = { + selected: Set; + freeText: string; +}; + +export type QuestionPrompt = { + id: string; + questions: Question[]; + agentId?: string; + sessionKey?: string; + createdAtMs: number; + expiresAtMs: number; + status: QuestionRecord["status"]; + answers?: QuestionAnswers; + submittedAnswers?: QuestionAnswers; + answeredElsewhere: boolean; + localResolutionConfirmed: boolean; + locallyExpired: boolean; + submitting: boolean; + error: string | null; + drafts: Map; + revision: number; +}; + +type QuestionPromptState = { + client: QuestionClient | null; + prompts: Map; + unmatchedResolutions: Map; + revision: number; + tickTimer: ReturnType | null; + refreshRetryTimer: ReturnType | null; + onChange: () => void; +}; + +type QuestionAnswerValues = Record; + +const REFRESH_RETRY_DELAYS_MS = [1_000, 2_000, 4_000] as const; + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function readNonEmptyString(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +function readTimestamp(value: unknown): number | null { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; +} + +function parseQuestion(value: unknown): Question | null { + if (!isRecord(value)) { + return null; + } + const id = readNonEmptyString(value.id); + const header = typeof value.header === "string" ? value.header : null; + const question = readNonEmptyString(value.question); + if (!id || !/^[a-z][a-z0-9_]*$/.test(id) || header === null || header.length > 12 || !question) { + return null; + } + if (!Array.isArray(value.options) || value.options.length > 4) { + return null; + } + const options = value.options.flatMap((option) => { + if (!isRecord(option)) { + return []; + } + const label = readNonEmptyString(option.label); + if (!label || (option.description !== undefined && typeof option.description !== "string")) { + return []; + } + return [ + { + label, + ...(typeof option.description === "string" ? { description: option.description } : {}), + }, + ]; + }); + if (options.length !== value.options.length) { + return null; + } + for (const field of ["multiSelect", "isOther"] as const) { + if (value[field] !== undefined && typeof value[field] !== "boolean") { + return null; + } + } + return { + id, + header, + question, + options, + ...(value.multiSelect === true ? { multiSelect: true } : {}), + ...(typeof value.isOther === "boolean" ? { isOther: value.isOther } : {}), + }; +} + +function parseQuestionAnswers(value: unknown): QuestionAnswers | null { + if (!isRecord(value) || !isRecord(value.answers)) { + return null; + } + const answers: QuestionAnswers["answers"] = {}; + for (const [id, answerValue] of Object.entries(value.answers)) { + if ( + !/^[a-z][a-z0-9_]*$/.test(id) || + !isRecord(answerValue) || + !Array.isArray(answerValue.answers) + ) { + return null; + } + if (!answerValue.answers.every((answer) => typeof answer === "string")) { + return null; + } + answers[id] = { answers: [...answerValue.answers] }; + } + return { answers }; +} + +function questionAnswersEqual( + left: QuestionAnswers | undefined, + right: QuestionAnswers | undefined, +): boolean { + if (!left || !right) { + return false; + } + const leftIds = Object.keys(left.answers).toSorted(); + const rightIds = Object.keys(right.answers).toSorted(); + return ( + leftIds.length === rightIds.length && + leftIds.every( + (id, index) => + id === rightIds[index] && + left.answers[id]?.answers.length === right.answers[id]?.answers.length && + left.answers[id]?.answers.every((answer, answerIndex) => + Object.is(answer, right.answers[id]?.answers[answerIndex]), + ), + ) + ); +} + +function parseQuestionRecord(payload: unknown): QuestionRecord | null { + if (!isRecord(payload)) { + return null; + } + const id = readNonEmptyString(payload.id); + const createdAtMs = readTimestamp(payload.createdAtMs); + const expiresAtMs = readTimestamp(payload.expiresAtMs); + if (!id || createdAtMs === null || expiresAtMs === null || !Array.isArray(payload.questions)) { + return null; + } + if (payload.questions.length < 1 || payload.questions.length > 3) { + return null; + } + const questions = payload.questions.map(parseQuestion); + if (questions.some((question) => question === null)) { + return null; + } + const questionIds = new Set(questions.map((question) => question?.id)); + if (questionIds.size !== questions.length) { + return null; + } + const agentId = payload.agentId === undefined ? undefined : readNonEmptyString(payload.agentId); + const sessionKey = + payload.sessionKey === undefined ? undefined : readNonEmptyString(payload.sessionKey); + if ( + (payload.agentId !== undefined && !agentId) || + (payload.sessionKey !== undefined && !sessionKey) + ) { + return null; + } + const base = { + id, + questions: questions as Question[], + ...(agentId ? { agentId } : {}), + ...(sessionKey ? { sessionKey } : {}), + createdAtMs, + expiresAtMs, + }; + if (payload.status === "pending") { + return { ...base, status: "pending" }; + } + if (payload.status === "answered") { + const answers = parseQuestionAnswers(payload.answers); + return answers ? { ...base, status: "answered", answers } : null; + } + if (payload.status === "cancelled") { + return { ...base, status: "cancelled" }; + } + return payload.status === "expired" ? { ...base, status: "expired" } : null; +} + +function parseQuestionRequestedEvent(payload: unknown): QuestionRecord | null { + const record = parseQuestionRecord(payload); + return record?.status === "pending" ? record : null; +} + +function parseQuestionResolvedEvent(payload: unknown): QuestionResolvedEvent | null { + if (!isRecord(payload)) { + return null; + } + const id = readNonEmptyString(payload.id); + if (!id) { + return null; + } + if (payload.status === "answered") { + const answers = parseQuestionAnswers(payload.answers); + return answers ? { id, status: "answered", answers } : null; + } + if (payload.status === "cancelled" || payload.status === "expired") { + return { id, status: payload.status }; + } + return null; +} + +export function createQuestionPromptState(onChange: () => void): QuestionPromptState { + return { + client: null, + prompts: new Map(), + unmatchedResolutions: new Map(), + revision: 0, + tickTimer: null, + refreshRetryTimer: null, + onChange, + }; +} + +function scheduleTick(state: QuestionPromptState): void { + if ( + state.tickTimer || + ![...state.prompts.values()].some((prompt) => prompt.status === "pending") + ) { + return; + } + state.tickTimer = globalThis.setTimeout(() => { + state.tickTimer = null; + const now = Date.now(); + let changed = false; + for (const prompt of state.prompts.values()) { + if (prompt.status === "pending" && prompt.expiresAtMs <= now) { + prompt.status = "expired"; + prompt.locallyExpired = true; + prompt.submitting = false; + prompt.error = null; + prompt.revision = ++state.revision; + changed = true; + } + } + state.onChange(); + if (changed || [...state.prompts.values()].some((prompt) => prompt.status === "pending")) { + scheduleTick(state); + } + }, 1_000); +} + +function promptFromRecord( + state: QuestionPromptState, + record: QuestionRecord, + previous?: QuestionPrompt, +): QuestionPrompt { + const revision = ++state.revision; + return { + id: record.id, + questions: record.questions, + ...(record.agentId ? { agentId: record.agentId } : {}), + ...(record.sessionKey ? { sessionKey: record.sessionKey } : {}), + createdAtMs: record.createdAtMs, + expiresAtMs: record.expiresAtMs, + status: record.status, + ...(record.status === "answered" ? { answers: record.answers } : {}), + ...(previous?.submittedAnswers ? { submittedAnswers: previous.submittedAnswers } : {}), + answeredElsewhere: + record.status === "answered" + ? !(previous?.localResolutionConfirmed ?? false) && !(previous?.submitting ?? false) + : false, + localResolutionConfirmed: previous?.localResolutionConfirmed ?? false, + locallyExpired: false, + submitting: + record.status === "pending" || + (record.status === "answered" && !(previous?.localResolutionConfirmed ?? false)) + ? (previous?.submitting ?? false) + : false, + error: record.status === "pending" ? (previous?.error ?? null) : null, + drafts: previous?.drafts ?? new Map(), + revision, + }; +} + +function applyQuestionResolution( + state: QuestionPromptState, + prompt: QuestionPrompt, + resolved: QuestionResolvedEvent, +): void { + prompt.status = resolved.status; + prompt.answers = resolved.status === "answered" ? resolved.answers : undefined; + const matchesSubmittedAnswer = + resolved.status === "answered" && + questionAnswersEqual(prompt.submittedAnswers, resolved.answers); + prompt.answeredElsewhere = + resolved.status === "answered" && + !prompt.localResolutionConfirmed && + !matchesSubmittedAnswer && + !prompt.submitting; + prompt.locallyExpired = false; + if (resolved.status !== "answered" || prompt.localResolutionConfirmed) { + prompt.submitting = false; + } + prompt.error = null; + prompt.revision = ++state.revision; +} + +export function handleQuestionPromptEvent( + state: QuestionPromptState, + event: Pick, +): boolean { + if (event.event === "question.requested") { + const record = parseQuestionRequestedEvent(event.payload); + if (!record) { + return false; + } + const previous = state.prompts.get(record.id); + if (previous && previous.status !== "pending") { + return true; + } + const prompt = promptFromRecord(state, record, previous); + const unmatched = state.unmatchedResolutions.get(record.id); + if (unmatched) { + state.unmatchedResolutions.delete(record.id); + applyQuestionResolution(state, prompt, unmatched); + } + state.prompts.set(record.id, prompt); + scheduleTick(state); + state.onChange(); + return true; + } + if (event.event !== "question.resolved") { + return false; + } + const resolved = parseQuestionResolvedEvent(event.payload); + const prompt = resolved ? state.prompts.get(resolved.id) : undefined; + if (!resolved) { + return false; + } + if (!prompt) { + state.unmatchedResolutions.set(resolved.id, resolved); + state.revision += 1; + state.onChange(); + return true; + } + applyQuestionResolution(state, prompt, resolved); + state.onChange(); + return true; +} + +function parseQuestionListResult(value: unknown): QuestionRecord[] | null { + if (!isRecord(value) || !Array.isArray(value.questions)) { + return null; + } + const questions = value.questions.map(parseQuestionRequestedEvent); + return questions.some((question) => question === null) ? null : (questions as QuestionRecord[]); +} + +function parseQuestionGetResult(value: unknown): QuestionRecord | null { + return isRecord(value) ? parseQuestionRecord(value.question) : null; +} + +function isQuestionNotFoundError(error: unknown): boolean { + return ( + error instanceof Error && + error.name === "GatewayClientRequestError" && + isRecord((error as Error & { details?: unknown }).details) && + (error as Error & { details: Record }).details.reason === "QUESTION_NOT_FOUND" + ); +} + +function markResolvedElsewhere(state: QuestionPromptState, prompt: QuestionPrompt): void { + prompt.status = "answered"; + prompt.answers = undefined; + prompt.answeredElsewhere = true; + prompt.localResolutionConfirmed = false; + prompt.locallyExpired = false; + prompt.submitting = false; + prompt.error = null; + prompt.revision = ++state.revision; +} + +async function refreshPendingQuestions( + state: QuestionPromptState, + client: QuestionClient, + isCurrentClient: () => boolean = () => state.client === client, +): Promise { + const startedAtRevision = state.revision; + const listResult = await client.request("question.list", {}); + const records = parseQuestionListResult(listResult); + if (!records || !isCurrentClient()) { + return false; + } + const refreshedIds = new Set(records.map((record) => record.id)); + for (const record of records) { + const previous = state.prompts.get(record.id); + if (!previous || previous.revision <= startedAtRevision || previous.locallyExpired) { + const prompt = promptFromRecord(state, record, previous); + const unmatched = state.unmatchedResolutions.get(record.id); + if (unmatched) { + state.unmatchedResolutions.delete(record.id); + applyQuestionResolution(state, prompt, unmatched); + } + state.prompts.set(record.id, prompt); + } + } + scheduleTick(state); + state.onChange(); + const missing: Array<{ + id: string; + prompt: QuestionPrompt | undefined; + revision: number; + }> = [...state.prompts.values()] + .filter( + (prompt) => + (prompt.locallyExpired || + (prompt.status === "pending" && prompt.revision <= startedAtRevision)) && + !refreshedIds.has(prompt.id), + ) + .map((prompt) => ({ id: prompt.id, prompt, revision: prompt.revision })); + const missingIds = new Set(missing.map((candidate) => candidate.id)); + for (const id of state.unmatchedResolutions.keys()) { + if (!refreshedIds.has(id) && !missingIds.has(id)) { + missing.push({ id, prompt: undefined, revision: state.revision }); + missingIds.add(id); + } + } + const missingResults = await Promise.allSettled( + missing.map((candidate) => client.request("question.get", { id: candidate.id })), + ); + if (!isCurrentClient()) { + return false; + } + let complete = true; + for (const [index, candidate] of missing.entries()) { + const current = state.prompts.get(candidate.id); + if (candidate.prompt && current !== candidate.prompt) { + if (!current || current.status === "pending" || current.locallyExpired) { + complete = false; + } + continue; + } + if (candidate.prompt && current?.revision !== candidate.revision && !current?.locallyExpired) { + if (current?.status === "pending") { + complete = false; + } + continue; + } + if (!candidate.prompt && !state.unmatchedResolutions.has(candidate.id)) { + continue; + } + const missingResult = missingResults[index]; + if ( + current && + missingResult?.status === "rejected" && + isQuestionNotFoundError(missingResult.reason) + ) { + markResolvedElsewhere(state, current); + continue; + } + const record = + missingResult?.status === "fulfilled" ? parseQuestionGetResult(missingResult.value) : null; + if (record) { + const prompt = promptFromRecord(state, record, current); + const unmatched = state.unmatchedResolutions.get(candidate.id); + if (unmatched) { + state.unmatchedResolutions.delete(candidate.id); + applyQuestionResolution(state, prompt, unmatched); + } + state.prompts.set(candidate.id, prompt); + continue; + } + complete = false; + } + scheduleTick(state); + state.onChange(); + return complete; +} + +export function refreshPendingQuestionsWithRetry( + state: QuestionPromptState, + client: QuestionClient, + isCurrentClient: () => boolean = () => state.client === client, +): void { + let retryIndex = 0; + const run = async () => { + if (!isCurrentClient()) { + return; + } + let complete: boolean; + try { + complete = await refreshPendingQuestions(state, client, isCurrentClient); + } catch { + complete = false; + } + if (complete || !isCurrentClient()) { + return; + } + const delayMs = REFRESH_RETRY_DELAYS_MS[retryIndex]; + retryIndex = Math.min(retryIndex + 1, REFRESH_RETRY_DELAYS_MS.length - 1); + state.refreshRetryTimer = globalThis.setTimeout(() => { + state.refreshRetryTimer = null; + void run(); + }, delayMs); + }; + void run(); +} + +export function setQuestionPromptClient( + state: QuestionPromptState, + client: QuestionClient | null, +): void { + if (state.refreshRetryTimer) { + globalThis.clearTimeout(state.refreshRetryTimer); + state.refreshRetryTimer = null; + } + state.client = client; +} + +export function disposeQuestionPromptState(state: QuestionPromptState): void { + if (state.tickTimer) { + globalThis.clearTimeout(state.tickTimer); + state.tickTimer = null; + } + if (state.refreshRetryTimer) { + globalThis.clearTimeout(state.refreshRetryTimer); + state.refreshRetryTimer = null; + } + state.client = null; +} + +function buildAnswers(values: QuestionAnswerValues): QuestionAnswers { + return { + answers: Object.fromEntries( + Object.entries(values).map(([id, answers]) => [id, { answers: [...answers] }]), + ), + }; +} + +async function resolveQuestion( + client: QuestionClient, + id: string, + answers: QuestionAnswerValues, +): Promise { + await client.request("question.resolve", { id, answers: buildAnswers(answers) }); +} + +export async function submitQuestionPrompt( + state: QuestionPromptState, + id: string, + answers: QuestionAnswerValues, +): Promise { + const prompt = state.prompts.get(id); + const client = state.client; + if (!prompt || prompt.status !== "pending" || prompt.submitting) { + return; + } + if (!client) { + prompt.error = "Not connected. Try again after reconnecting."; + prompt.revision = ++state.revision; + state.onChange(); + return; + } + prompt.submitting = true; + prompt.submittedAnswers = buildAnswers(answers); + prompt.error = null; + prompt.revision = ++state.revision; + state.onChange(); + try { + await resolveQuestion(client, id, answers); + const current = state.prompts.get(id); + if (!current) { + return; + } + current.localResolutionConfirmed = true; + if (current.status === "answered") { + current.answeredElsewhere = false; + current.submitting = false; + } + current.revision = ++state.revision; + state.onChange(); + } catch (error) { + const current = state.prompts.get(id); + if (!current) { + return; + } + current.submitting = false; + if (current.status === "pending") { + current.error = error instanceof Error ? error.message : String(error); + current.revision = ++state.revision; + state.onChange(); + return; + } + if (current.status === "answered" && !current.localResolutionConfirmed) { + current.answeredElsewhere = !questionAnswersEqual(current.submittedAnswers, current.answers); + } + current.revision = ++state.revision; + state.onChange(); + } +} + +export function listQuestionPrompts(state: QuestionPromptState): QuestionPrompt[] { + return [...state.prompts.values()].toSorted( + (left, right) => left.createdAtMs - right.createdAtMs || left.id.localeCompare(right.id), + ); +} diff --git a/ui/src/e2e/question-flow.e2e.test.ts b/ui/src/e2e/question-flow.e2e.test.ts new file mode 100644 index 000000000000..e769fe5b1ccb --- /dev/null +++ b/ui/src/e2e/question-flow.e2e.test.ts @@ -0,0 +1,261 @@ +// Control UI E2E tests cover Gateway question cards through the mocked WebSocket. +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { chromium, type Browser, type BrowserContext, type Page } from "playwright"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, + type ControlUiE2eServer, + type MockGatewayControls, +} from "../test-helpers/control-ui-e2e.ts"; + +const chromiumExecutablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +const chromiumAvailable = canRunPlaywrightChromium(chromiumExecutablePath); +const allowMissingChromium = process.env.OPENCLAW_UI_E2E_ALLOW_MISSING_CHROMIUM === "1"; +const describeControlUiE2e = chromiumAvailable || !allowMissingChromium ? describe : describe.skip; +const captureUiProof = process.env.OPENCLAW_CAPTURE_UI_PROOF === "1"; +const proofDir = path.join(process.cwd(), ".artifacts", "control-ui-e2e", "question-flow"); + +let browser: Browser; +let context: BrowserContext | undefined; +let server: ControlUiE2eServer; + +function questionRecord( + id: string, + questions: Array<{ + id: string; + header: string; + question: string; + options: Array<{ label: string; description?: string }>; + multiSelect?: boolean; + isOther?: boolean; + }>, +) { + const createdAtMs = Date.now(); + return { + id, + questions, + agentId: "main", + sessionKey: "main", + createdAtMs, + expiresAtMs: createdAtMs + 15 * 60_000, + status: "pending" as const, + }; +} + +async function screenshot(page: Page, name: string) { + if (!captureUiProof) { + return; + } + await mkdir(proofDir, { recursive: true }); + await page.screenshot({ + animations: "disabled", + fullPage: true, + path: path.join(proofDir, name), + }); +} + +async function openQuestionPage() { + context = await browser.newContext({ + locale: "en-US", + serviceWorkers: "block", + viewport: { height: 900, width: 1440 }, + }); + const page = await context.newPage(); + const gateway = await installMockGateway(page, { + methodResponses: { + "question.list": { questions: [] }, + }, + sessionKey: "main", + }); + await page.goto(`${server.baseUrl}chat`); + await gateway.waitForRequest("question.list"); + return { gateway, page }; +} + +function cardFor(page: Page, prompt: string) { + return page.locator("openclaw-chat-question").filter({ hasText: prompt }); +} + +async function emitRequested( + gateway: MockGatewayControls, + record: ReturnType, +) { + await gateway.emitGatewayEvent("question.requested", record); +} + +describeControlUiE2e("Control UI Gateway question flow", () => { + beforeAll(async () => { + if (!chromiumAvailable) { + throw new Error(`Playwright Chromium is not available at ${chromiumExecutablePath}`); + } + server = await startControlUiE2eServer(); + browser = await chromium.launch({ executablePath: chromiumExecutablePath }); + }); + + afterEach(async () => { + await context?.close().catch(() => {}); + context = undefined; + }); + + afterAll(async () => { + await browser?.close().catch(() => {}); + await server?.close(); + }); + + it("renders and resolves a single-choice question in the active thread", async () => { + const { gateway, page } = await openQuestionPage(); + const request = questionRecord("question-deploy-target", [ + { + id: "deploy_target", + header: "Deploy", + question: "Where should I deploy?", + options: [ + { + label: "Staging (Recommended)", + description: "Validate the release before production.", + }, + { + label: "Production", + description: "Deploy directly to live users.", + }, + ], + isOther: true, + }, + ]); + + await emitRequested(gateway, request); + const card = cardFor(page, "Where should I deploy?"); + await card.waitFor(); + await expect.poll(() => card.getByText("Deploy", { exact: true }).count()).toBe(1); + await expect + .poll(() => card.getByText("Staging (Recommended)", { exact: true }).count()) + .toBe(1); + await expect.poll(() => card.getByText("Production", { exact: true }).count()).toBe(1); + await expect + .poll(() => card.getByRole("textbox", { name: "Your own answer for Deploy" }).count()) + .toBe(1); + await expect.poll(() => card.getByRole("button", { name: "Submit answer" }).count()).toBe(1); + await screenshot(page, "01-question-pending.png"); + + const staging = card.locator('input[type="radio"]').first(); + await staging.check(); + await card.getByRole("button", { name: "Submit answer" }).click(); + const resolveRequest = await gateway.waitForRequest("question.resolve"); + expect(resolveRequest.params).toEqual({ + id: request.id, + answers: { + answers: { + deploy_target: { answers: ["Staging (Recommended)"] }, + }, + }, + }); + + await gateway.emitGatewayEvent("question.resolved", { + id: request.id, + status: "answered", + answers: { + answers: { + deploy_target: { answers: ["Staging (Recommended)"] }, + }, + }, + }); + await card.getByText("Answered", { exact: true }).waitFor(); + await expect.poll(() => staging.isChecked()).toBe(true); + await expect.poll(() => staging.isDisabled()).toBe(true); + await screenshot(page, "02-question-answered.png"); + }); + + it("submits multi-select answers as an array", async () => { + const { gateway, page } = await openQuestionPage(); + const request = questionRecord("question-release-checks", [ + { + id: "release_checks", + header: "Checks", + question: "Which release checks should I run?", + options: [ + { label: "Tests", description: "Run focused automated tests." }, + { label: "Docs", description: "Verify documentation changes." }, + { label: "Metrics", description: "Inspect performance metrics." }, + { label: "Rollback", description: "Prepare a rollback plan." }, + ], + multiSelect: true, + }, + ]); + + await emitRequested(gateway, request); + const card = cardFor(page, "Which release checks should I run?"); + await card.waitFor(); + const options = card.locator('input[type="checkbox"]'); + await options.nth(0).check(); + await options.nth(2).check(); + await expect.poll(() => options.nth(0).isChecked()).toBe(true); + await expect.poll(() => options.nth(2).isChecked()).toBe(true); + await screenshot(page, "03-question-multiselect.png"); + + await card.getByRole("button", { name: "Submit answer" }).click(); + const resolveRequest = await gateway.waitForRequest("question.resolve"); + expect(resolveRequest.params).toEqual({ + id: request.id, + answers: { + answers: { + release_checks: { answers: ["Tests", "Metrics"] }, + }, + }, + }); + }); + + it("renders answered-elsewhere and expired terminal states", async () => { + const { gateway, page } = await openQuestionPage(); + const elsewhere = questionRecord("question-external-answer", [ + { + id: "approval_path", + header: "Approval", + question: "Who should approve the release?", + options: [{ label: "Maintainer" }, { label: "Release manager" }], + }, + ]); + const expired = questionRecord("question-expired-window", [ + { + id: "release_window", + header: "Window", + question: "When should the release start?", + options: [{ label: "Now" }, { label: "Tomorrow" }], + }, + ]); + + await emitRequested(gateway, elsewhere); + await emitRequested(gateway, expired); + const elsewhereCard = cardFor(page, "Who should approve the release?"); + const expiredCard = cardFor(page, "When should the release start?"); + await elsewhereCard.waitFor(); + await expiredCard.waitFor(); + + await gateway.emitGatewayEvent("question.resolved", { + id: elsewhere.id, + status: "answered", + answers: { + answers: { + approval_path: { answers: ["Release manager"] }, + }, + }, + }); + await elsewhereCard.getByText("Answered elsewhere", { exact: true }).waitFor(); + const elsewhereAnswer = elsewhereCard.locator('input[type="radio"]').nth(1); + await expect.poll(() => elsewhereAnswer.isChecked()).toBe(true); + await expect.poll(() => elsewhereAnswer.isDisabled()).toBe(true); + + await gateway.emitGatewayEvent("question.resolved", { + id: expired.id, + status: "expired", + }); + await expiredCard.getByText("Expired", { exact: true }).waitFor(); + await expect + .poll(() => expiredCard.locator('input[type="radio"]').first().isDisabled()) + .toBe(true); + await screenshot(page, "04-question-terminal-states.png"); + }); +}); diff --git a/ui/src/i18n/locales/en.ts b/ui/src/i18n/locales/en.ts index 3c087221ec88..a909b54d4e69 100644 --- a/ui/src/i18n/locales/en.ts +++ b/ui/src/i18n/locales/en.ts @@ -3695,6 +3695,17 @@ export const en: TranslationMap = { title: "Codex needs input", other: "Type another answer", submit: "Submit answer", + eyebrow: "Question", + cardLabel: "Question from the assistant", + timeRemaining: "Time remaining", + ownAnswer: "Your own answer…", + ownAnswerFor: "Your own answer for {header}", + submitting: "Submitting…", + submitFailed: "Could not submit: {error}", + answered: "Answered", + answeredElsewhere: "Answered elsewhere", + expired: "Expired", + cancelled: "Cancelled", }, messages: { activity: "Activity", diff --git a/ui/src/lib/chat/chat-types.ts b/ui/src/lib/chat/chat-types.ts index ac8c5b942281..77a740aa81eb 100644 --- a/ui/src/lib/chat/chat-types.ts +++ b/ui/src/lib/chat/chat-types.ts @@ -56,6 +56,7 @@ export type ChatItem = } | { kind: "stream"; key: string; text: string; startedAt: number; isStreaming: boolean } | { kind: "reading-indicator"; key: string; startedAt: number } + | { kind: "question"; key: string; questionId: string; startedAt: number; pending: boolean } | { kind: "plan"; key: string }; export type ChatStreamSegment = { diff --git a/ui/src/pages/chat/chat-pane.ts b/ui/src/pages/chat/chat-pane.ts index 4816b3e971ab..72101042284b 100644 --- a/ui/src/pages/chat/chat-pane.ts +++ b/ui/src/pages/chat/chat-pane.ts @@ -30,6 +30,16 @@ import { type ApplicationGatewaySnapshot, } from "../../app/context.ts"; import { hasOperatorAdminAccess, hasOperatorWriteAccess } from "../../app/operator-access.ts"; +import { + createQuestionPromptState, + disposeQuestionPromptState, + handleQuestionPromptEvent, + listQuestionPrompts, + refreshPendingQuestionsWithRetry, + setQuestionPromptClient, + submitQuestionPrompt, + type QuestionPrompt, +} from "../../app/question-prompt.ts"; import { BROWSER_ANNOTATION_EVENT, type BrowserAnnotationDraft, @@ -300,6 +310,11 @@ class ChatPane extends OpenClawLightDomElement { private readonly chatState = new ChatStateController(this); private readonly transcript = new ChatTranscriptController(this); + private readonly questionPromptState = createQuestionPromptState(() => { + this.questionPrompts = listQuestionPrompts(this.questionPromptState); + this.requestUpdate(); + }); + private questionPrompts: QuestionPrompt[] = []; private state: ChatPageHost | undefined; /* Infinity until the first ResizeObserver tick so an unmeasured pane keeps * the wide side-by-side layout instead of flashing the stacked one. */ @@ -1610,6 +1625,9 @@ class ChatPane extends OpenClawLightDomElement { chatState.addCleanup( this.context.gateway.subscribeEvents((event) => { const state = this.state; + if (state) { + handleQuestionPromptEvent(this.questionPromptState, event); + } if (state && !parseCatalogSessionKey(state.sessionKey)) { if (event.event === "task.suggestion" && event.payload) { this.handleTaskSuggestionEvent(event.payload as TaskSuggestionEvent); @@ -1686,6 +1704,7 @@ class ChatPane extends OpenClawLightDomElement { resetChatViewState(this.paneId); this.state = undefined; this.connectedClient = null; + disposeQuestionPromptState(this.questionPromptState); super.disconnectedCallback(); } @@ -1836,6 +1855,7 @@ class ChatPane extends OpenClawLightDomElement { markQueuedChatSendsWaitingForReconnect(state); } this.connectedClient = null; + setQuestionPromptClient(this.questionPromptState, null); state.realtimeTalkSession?.stop(); state.realtimeTalkSession = null; state.realtimeTalkActive = false; @@ -1876,6 +1896,8 @@ class ChatPane extends OpenClawLightDomElement { } }; this.connectedClient = startupClient; + setQuestionPromptClient(this.questionPromptState, startupClient); + refreshPendingQuestionsWithRetry(this.questionPromptState, startupClient, clientIsCurrent); this.headerWorktreePaths.clear(); this.headerBranches.clear(); this.headerPlatform = null; @@ -2279,6 +2301,13 @@ class ChatPane extends OpenClawLightDomElement { fallbackStatus: state.fallbackStatus, planStatus: state.planStatus, questionStatus: state.questionStatus, + gatewayQuestionPrompts: catalogKey ? [] : this.questionPrompts, + onGatewayQuestionChange: () => { + this.questionPrompts = [...this.questionPrompts]; + this.requestUpdate(); + }, + onGatewayQuestionSubmit: (id, answers) => + submitQuestionPrompt(this.questionPromptState, id, answers), messages: catalogKey ? this.catalogMessages : state.chatMessages, historyPagination: catalogKey || state.chatHistoryPagination?.hasMore || this.loadingOlder diff --git a/ui/src/pages/chat/chat-thread.question.test.ts b/ui/src/pages/chat/chat-thread.question.test.ts new file mode 100644 index 000000000000..26a023e60355 --- /dev/null +++ b/ui/src/pages/chat/chat-thread.question.test.ts @@ -0,0 +1,79 @@ +// Control UI tests cover question-card placement in live and terminal chat runs. +import { afterEach, describe, expect, it } from "vitest"; +import type { QuestionPrompt } from "../../app/question-prompt.ts"; +import { buildCachedChatItems, coalesceStreamRuns, resetChatThreadState } from "./chat-thread.ts"; + +function prompt(status: QuestionPrompt["status"]): QuestionPrompt { + return { + id: "question-1", + questions: [ + { + id: "format", + header: "Format", + question: "Which format?", + options: [{ label: "Compact" }, { label: "Detailed" }], + isOther: true, + }, + ], + sessionKey: "agent:main:main", + createdAtMs: 1_000, + expiresAtMs: 60_000, + status, + answeredElsewhere: false, + localResolutionConfirmed: false, + locallyExpired: false, + submitting: false, + error: null, + drafts: new Map(), + revision: 1, + }; +} + +function items(question: QuestionPrompt, runActive: boolean) { + return buildCachedChatItems({ + paneId: `pane-${question.status}`, + sessionKey: "agent:main:main", + messages: [], + toolMessages: [], + streamSegments: [], + stream: null, + streamStartedAt: null, + queue: [], + showToolCalls: true, + runWorking: runActive, + runActive, + planStatus: runActive + ? { steps: [{ step: "Wait for the answer", status: "in_progress" }] } + : null, + questionPrompts: [question], + }); +} + +afterEach(() => resetChatThreadState()); + +describe("question chat items", () => { + it("groups a pending question with the active run and plan", () => { + const result = coalesceStreamRuns(items(prompt("pending"), true)); + const run = result.find((item) => item.kind === "stream-run"); + + expect(run?.kind).toBe("stream-run"); + expect(run?.kind === "stream-run" ? run.parts.map((part) => part.kind) : []).toEqual([ + "question", + "reading-indicator", + "plan", + ]); + }); + + it("keeps a terminal question as a stable transcript item", () => { + const result = coalesceStreamRuns(items(prompt("expired"), false)); + + expect(result).toMatchObject([{ kind: "question", questionId: "question-1", pending: false }]); + }); + + it("omits questions belonging to another session", () => { + const other = prompt("pending"); + other.sessionKey = "agent:other:main"; + + expect(items(other, false)).toEqual([]); + }); +}); diff --git a/ui/src/pages/chat/chat-thread.ts b/ui/src/pages/chat/chat-thread.ts index c34d04914a1b..542b131ba651 100644 --- a/ui/src/pages/chat/chat-thread.ts +++ b/ui/src/pages/chat/chat-thread.ts @@ -5,6 +5,7 @@ import { isToolResultContentType, resolveToolUseId, } from "../../../../src/chat/tool-content.js"; +import type { QuestionPrompt } from "../../app/question-prompt.ts"; import type { ChatItem, MessageGroup, @@ -34,6 +35,7 @@ import { extractToolPreview, isToolCardError, } from "../../lib/chat/tool-cards.ts"; +import { areUiSessionKeysEquivalent } from "../../lib/sessions/session-key.ts"; import { normalizeLowercaseStringOrEmpty } from "../../lib/string-coerce.ts"; import { buildCompactionDividerItem, @@ -64,6 +66,7 @@ type BuildChatItemsProps = { /** True while the current session has an abortable live run. */ runActive?: boolean; planStatus?: PlanStatus | null; + questionPrompts?: readonly QuestionPrompt[]; /** True while chat history is loading (initial load or background reload). */ loading?: boolean; searchOpen?: boolean; @@ -82,7 +85,10 @@ type StreamRunRenderItem = { kind: "stream-run"; key: string; parts: Array< - Extract + Extract< + ChatItem, + { kind: "stream" } | { kind: "reading-indicator" } | { kind: "question" } | { kind: "plan" } + > >; }; @@ -1100,6 +1106,8 @@ function chatItemTimestamp(item: ChatItem): number | null { return item.timestamp; case "stream": return item.startedAt; + case "question": + return item.startedAt; case "reading-indicator": case "plan": return null; @@ -1479,6 +1487,20 @@ function buildChatItems(props: BuildChatItemsProps): Array 0) { items.push({ kind: "plan", key: `plan:${props.sessionKey}:active` }); } + for (const prompt of props.questionPrompts ?? []) { + // In-thread questions need an explicit owner. Routing an unscoped prompt to the active pane + // could expose it to the wrong session; ask_user supplies the originating session key. + if (!prompt.sessionKey || !areUiSessionKeysEquivalent(prompt.sessionKey, props.sessionKey)) { + continue; + } + items.push({ + kind: "question", + key: `question:${prompt.id}`, + questionId: prompt.id, + startedAt: prompt.createdAtMs, + pending: prompt.status === "pending", + }); + } return annotateToolTurnOutcome( groupMessages( @@ -1542,6 +1564,13 @@ function sameChatItem(previous: RenderChatItem, next: RenderChatItem): boolean { ); case "reading-indicator": return previous.kind === "reading-indicator" && previous.startedAt === next.startedAt; + case "question": + return ( + previous.kind === "question" && + previous.questionId === next.questionId && + previous.startedAt === next.startedAt && + previous.pending === next.pending + ); case "plan": return previous.kind === "plan"; } @@ -1645,6 +1674,7 @@ function sameChatItemsStructuralInput( previous.showToolCalls === next.showToolCalls && previous.runWorking === next.runWorking && previous.runActive === next.runActive && + previous.questionPrompts === next.questionPrompts && Boolean(previous.planStatus?.steps.length) === Boolean(next.planStatus?.steps.length) && previous.loading === next.loading && previous.searchOpen === next.searchOpen && @@ -1746,7 +1776,7 @@ export function coalesceStreamRuns( ): Array { const result: Array = []; let run: StreamRunRenderItem["parts"] = []; - // Contiguous in-flight stream, plan, and reading-indicator items render under one + // Contiguous in-flight stream, question, plan, and reading-indicator items render under one // assistant avatar; messages, groups, and dividers intentionally break the run. const flush = () => { const [first] = run; @@ -1756,7 +1786,12 @@ export function coalesceStreamRuns( } }; for (const item of items) { - if (item.kind === "stream" || item.kind === "reading-indicator" || item.kind === "plan") { + if ( + item.kind === "stream" || + item.kind === "reading-indicator" || + (item.kind === "question" && item.pending) || + item.kind === "plan" + ) { run.push(item); continue; } diff --git a/ui/src/pages/chat/chat-view.ts b/ui/src/pages/chat/chat-view.ts index 31bdffa44df1..0725f7315a95 100644 --- a/ui/src/pages/chat/chat-view.ts +++ b/ui/src/pages/chat/chat-view.ts @@ -8,6 +8,7 @@ import type { ControlUiSessionPullRequest, } from "../../../../src/gateway/control-ui-contract.js"; import type { SessionsListResult } from "../../api/types.ts"; +import type { QuestionPrompt } from "../../app/question-prompt.ts"; import type { ChatSendShortcut } from "../../app/settings.ts"; import { icons } from "../../components/icons.ts"; import { t } from "../../i18n/index.ts"; @@ -87,6 +88,9 @@ export type ChatProps = { fallbackStatus?: FallbackStatus | null; planStatus?: PlanStatus | null; questionStatus?: QuestionStatus | null; + gatewayQuestionPrompts?: readonly QuestionPrompt[]; + onGatewayQuestionChange?: () => void; + onGatewayQuestionSubmit?: (id: string, answers: Record) => void | Promise; messages: unknown[]; historyPagination?: { loading: boolean; @@ -285,6 +289,7 @@ export function renderChat(props: ChatProps) { runActive: Boolean(props.canAbort), runWorking: isChatRunWorking(props), planStatus: props.planStatus, + questionPrompts: props.gatewayQuestionPrompts, sessions: props.sessions, sessionHost: props.sessionHost, assistantName: props.assistantName, @@ -306,6 +311,8 @@ export function renderChat(props: ChatProps) { onOpenSessionCheckpoints: props.onOpenSessionCheckpoints, onAssistantAttachmentLoaded: props.onAssistantAttachmentLoaded, onRequestUpdate: requestUpdate, + onQuestionChange: props.onGatewayQuestionChange, + onQuestionSubmit: props.onGatewayQuestionSubmit, onChatScroll: props.onChatScroll, onHistoryIntent: props.onHistoryIntent, onDraftChange: props.onDraftChange, diff --git a/ui/src/pages/chat/components/chat-composer.ts b/ui/src/pages/chat/components/chat-composer.ts index eea929d9e853..aa47bff4bb95 100644 --- a/ui/src/pages/chat/components/chat-composer.ts +++ b/ui/src/pages/chat/components/chat-composer.ts @@ -54,12 +54,12 @@ import { renderChatAttachmentMenu, } from "./chat-attachments.ts"; import { renderChatPlanChecklist } from "./chat-plan-checklist.ts"; +import { createCodexQuestionCardProps } from "./chat-question-card.ts"; import { renderChatVoiceError, renderMicrophoneActivity, voiceStatusLabel, } from "./chat-voice-activity.ts"; -import "./chat-question-card.ts"; const COMPACTION_TOAST_DURATION_MS = 5000; const FALLBACK_TOAST_DURATION_MS = 8000; @@ -2233,16 +2233,15 @@ export function renderChatComposer(props: ChatComposerProps) {
${props.questionStatus ? html`, onRejected: () => void) => + .props=${createCodexQuestionCardProps(props.questionStatus, { + disabled: !props.connected || !props.onQuestionSubmit, + onSubmit: (answers, onRejected) => props.onQuestionSubmit?.( props.questionStatus!.actionToken, answers, onRejected, ), - }} + })} >` : nothing} ${renderChatPlanChecklist(props.planStatus, { diff --git a/ui/src/pages/chat/components/chat-message.ts b/ui/src/pages/chat/components/chat-message.ts index 0c7c10b75bc8..0104591d25c9 100644 --- a/ui/src/pages/chat/components/chat-message.ts +++ b/ui/src/pages/chat/components/chat-message.ts @@ -2,6 +2,7 @@ import { html, nothing } from "lit"; import { unsafeHTML } from "lit/directives/unsafe-html.js"; import { until } from "lit/directives/until.js"; +import type { QuestionPrompt } from "../../../app/question-prompt.ts"; import { resolveLocalUserName } from "../../../app/user-identity.ts"; import { renderCopyAsMarkdownButton } from "../../../components/copy-button.ts"; import { icons, type IconName } from "../../../components/icons.ts"; @@ -53,6 +54,7 @@ import { getSafeLocalStorage } from "../../../local-storage.ts"; import { renderChatAvatar } from "../chat-avatar.ts"; import type { PlanStatus } from "../tool-stream.ts"; import { renderChatPlanChecklist } from "./chat-plan-checklist.ts"; +import { renderChatQuestionCard } from "./chat-question-card.ts"; import type { SidebarContent } from "./chat-sidebar.ts"; import { isRunningToolCard, @@ -560,7 +562,7 @@ function extractTranscriptAttachments(message: unknown): AttachmentItem[] { /** A contiguous run of in-flight streaming items rendered under one assistant group. */ type StreamGroupPart = Extract< ChatItem, - { kind: "stream" } | { kind: "reading-indicator" } | { kind: "plan" } + { kind: "stream" } | { kind: "reading-indicator" } | { kind: "question" } | { kind: "plan" } >; type StreamGroupOptions = { @@ -570,8 +572,25 @@ type StreamGroupOptions = { authToken?: string | null; planStatus?: PlanStatus | null; planActive?: boolean; + questionPrompts?: ReadonlyMap; + onQuestionChange?: () => void; + onQuestionSubmit?: (id: string, answers: Record) => void | Promise; }; +function renderQuestionStreamPart( + part: Extract, + opts: StreamGroupOptions, +) { + const prompt = opts.questionPrompts?.get(part.questionId); + return prompt + ? renderChatQuestionCard(prompt, { + nowMs: Date.now(), + onChange: opts.onQuestionChange ?? (() => {}), + onSubmit: (answers) => opts.onQuestionSubmit?.(prompt.id, answers), + }) + : nothing; +} + // One assistant group per contiguous run of streaming items: a reply that // arrives as several stream segments renders under a single avatar/footer // instead of flashing a separate avatar+bubble per segment (#63956). @@ -600,21 +619,23 @@ export function renderStreamGroup(parts: StreamGroupPart[], opts: StreamGroupOpt ${parts.map((part) => part.kind === "reading-indicator" ? renderChatWorkingIndicator(part) - : part.kind === "plan" - ? renderChatPlanChecklist(opts.planStatus, { - active: opts.planActive === true, - variant: "card", - }) - : renderGroupedMessage( - { - role: "assistant", - content: [{ type: "text", text: part.text }], - timestamp: part.startedAt, - }, - part.key, - { isStreaming: part.isStreaming, showReasoning: false }, - onOpenSidebar, - ), + : part.kind === "question" + ? renderQuestionStreamPart(part, opts) + : part.kind === "plan" + ? renderChatPlanChecklist(opts.planStatus, { + active: opts.planActive === true, + variant: "card", + }) + : renderGroupedMessage( + { + role: "assistant", + content: [{ type: "text", text: part.text }], + timestamp: part.startedAt, + }, + part.key, + { isStreaming: part.isStreaming, showReasoning: false }, + onOpenSidebar, + ), )} ${footerStartedAt !== null ? html` diff --git a/ui/src/pages/chat/components/chat-question-card.test.ts b/ui/src/pages/chat/components/chat-question-card.test.ts index 9544cd0a90a5..666c31cee658 100644 --- a/ui/src/pages/chat/components/chat-question-card.test.ts +++ b/ui/src/pages/chat/components/chat-question-card.test.ts @@ -2,13 +2,68 @@ import { html, render } from "lit"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import "./chat-question-card.ts"; +import type { QuestionPrompt } from "../../../app/question-prompt.ts"; +import type { QuestionStatus } from "../tool-stream.ts"; +import { createCodexQuestionCardProps, renderChatQuestionCard } from "./chat-question-card.ts"; type ChatQuestionCardElement = HTMLElement & { updateComplete: Promise; }; -describe("native Codex question card", () => { +function codexStatus(overrides: Partial = {}): QuestionStatus { + return { + itemId: "item-1", + actionToken: "test-action-token", + questions: [ + { + id: "mode", + header: "Mode", + question: "Pick one", + isOther: false, + options: [{ label: "Fast" }, { label: " Deep ", description: "More reasoning" }], + }, + ], + ...overrides, + }; +} + +function gatewayPrompt(overrides: Partial = {}): QuestionPrompt { + return { + id: "question-1", + questions: [ + { + id: "format", + header: "Format", + question: "Which format should I use?", + options: [ + { label: "Compact", description: "Keep it brief" }, + { label: "Detailed", description: "Include rationale" }, + ], + isOther: true, + }, + ], + sessionKey: "agent:main:main", + createdAtMs: 1_000, + expiresAtMs: 62_000, + status: "pending", + answeredElsewhere: false, + localResolutionConfirmed: false, + locallyExpired: false, + submitting: false, + error: null, + drafts: new Map(), + revision: 1, + ...overrides, + }; +} + +async function cardIn(container: HTMLElement): Promise { + const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; + await card.updateComplete; + return card; +} + +describe("shared question card", () => { let container: HTMLDivElement; beforeEach(() => { @@ -20,177 +75,243 @@ describe("native Codex question card", () => { container.remove(); }); - it("submits a selected native option through the chat reply seam", async () => { - const onSubmit = vi.fn(); - render( - html``, - container, - ); - const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; - await card.updateComplete; - const options = container.querySelectorAll('input[type="radio"]'); - options[1]!.click(); - await card.updateComplete; - container.querySelector(".chat-question__submit")!.click(); - expect(onSubmit).toHaveBeenCalledWith({ mode: " Deep " }, expect.any(Function)); - await card.updateComplete; - expect(container.querySelector(".chat-question__submit")?.disabled).toBe( - true, - ); - }); + describe("Codex adapter", () => { + function draw( + status: QuestionStatus, + onSubmit: (answers: Record, onRejected: () => void) => void, + ) { + render( + html``, + container, + ); + } - it("renders a free-form answer field", async () => { - render( - html``, - container, - ); - const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; - await card.updateComplete; - expect(container.querySelector(".chat-question__other")?.type).toBe("text"); - }); + it("submits a selected native option through the string-answer seam", async () => { + const onSubmit = vi.fn(); + draw(codexStatus(), onSubmit); + const card = await cardIn(container); + const options = container.querySelectorAll('input[type="radio"]'); - it("re-enables submission when the scoped command is rejected", async () => { - let reject: (() => void) | undefined; - render( - html`, onRejected: () => void) => { - reject = onRejected; - }, - status: { - itemId: "item-retry", - actionToken: "test-action-token", - questions: [ - { - id: "mode", - header: "Mode", - question: "Pick one", - isOther: false, - options: [{ label: "Retry" }], - }, - ], - }, - }} - >`, - container, - ); - const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; - await card.updateComplete; - container.querySelector('input[type="radio"]')!.click(); - await card.updateComplete; - const submit = container.querySelector(".chat-question__submit")!; - submit.click(); - await card.updateComplete; - expect(submit.disabled).toBe(true); + options[1]!.click(); + await card.updateComplete; + container.querySelector(".chat-question__submit")!.click(); - reject?.(); - await card.updateComplete; - expect(submit.disabled).toBe(false); - }); - - it("clears free-form text when the request token changes", async () => { - const question = { - id: "other", - header: "Alternative", - question: "Type another answer", - isOther: true, - options: [] as Array<{ label: string }>, - }; - const props = (actionToken: string) => ({ - disabled: false, - onSubmit: vi.fn(), - status: { itemId: "reused-item", actionToken, questions: [question] }, + expect(onSubmit).toHaveBeenCalledWith({ mode: " Deep " }, expect.any(Function)); + await card.updateComplete; + expect(container.querySelector(".chat-question__submit")?.disabled).toBe( + true, + ); }); - render( - html``, - container, - ); - const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; - await card.updateComplete; - const input = container.querySelector(".chat-question__other")!; - input.value = "stale answer"; - input.dispatchEvent(new InputEvent("input", { bubbles: true })); - await card.updateComplete; - expect(input.value).toBe("stale answer"); - render( - html``, - container, - ); - await card.updateComplete; - expect(container.querySelector(".chat-question__other")?.value).toBe(""); + it("keeps the main submit label when the composer is disconnected", async () => { + render( + html``, + container, + ); + await cardIn(container); + + const submit = container.querySelector(".chat-question__submit"); + expect(submit?.textContent?.trim()).toBe("Submit answer"); + expect(submit?.disabled).toBe(true); + }); + + it("re-enables submission when the scoped command is rejected", async () => { + let reject: (() => void) | undefined; + draw(codexStatus(), (_answers, onRejected) => { + reject = onRejected; + }); + const card = await cardIn(container); + container.querySelector('input[type="radio"]')!.click(); + await card.updateComplete; + const submit = container.querySelector(".chat-question__submit")!; + submit.click(); + await card.updateComplete; + expect(submit.disabled).toBe(true); + + reject?.(); + await Promise.resolve(); + await card.updateComplete; + expect(submit.disabled).toBe(false); + }); + + it("clears free-form text when the request key changes", async () => { + const status = codexStatus({ + itemId: "reused-item", + questions: [ + { + id: "other", + header: "Alternative", + question: "Type another answer", + isOther: true, + options: [], + }, + ], + }); + draw(status, vi.fn()); + const card = await cardIn(container); + const input = container.querySelector(".chat-question__other")!; + input.value = "stale answer"; + input.dispatchEvent(new InputEvent("input", { bubbles: true })); + await card.updateComplete; + + // A new itemId changes the request key exactly like a new action token. + draw({ ...status, itemId: "item-second-request" }, vi.fn()); + await card.updateComplete; + expect(container.querySelector(".chat-question__other")?.value).toBe(""); + }); + + it("preserves free-form text that begins with an option label", async () => { + draw( + codexStatus({ + questions: [ + { + id: "reason", + header: "Decision", + question: "Continue?", + isOther: true, + options: [{ label: "No" }], + }, + ], + }), + vi.fn(), + ); + const card = await cardIn(container); + const input = container.querySelector(".chat-question__other")!; + input.value = "No, because the proof failed"; + input.dispatchEvent(new InputEvent("input", { bubbles: true })); + await card.updateComplete; + + expect(input.value).toBe("No, because the proof failed"); + }); }); - it("preserves free-form text that begins with an option label", async () => { - render( - html``, - container, - ); - const card = container.querySelector("openclaw-chat-question") as ChatQuestionCardElement; - await card.updateComplete; - const input = container.querySelector(".chat-question__other")!; - input.value = "No"; - input.dispatchEvent(new InputEvent("input", { bubbles: true })); - await card.updateComplete; - input.value = "No, because the proof failed"; - input.dispatchEvent(new InputEvent("input", { bubbles: true })); - await card.updateComplete; + describe("gateway adapter", () => { + async function draw( + prompt: QuestionPrompt, + onSubmit: (answers: Record) => void | Promise = vi.fn(), + ) { + const redraw = () => { + render( + renderChatQuestionCard(prompt, { + nowMs: 2_000, + onChange: redraw, + onSubmit, + }), + container, + ); + }; + redraw(); + await cardIn(container); + return onSubmit; + } - expect(input.value).toBe("No, because the proof failed"); + it("submits multiselect options and free text as arrays", async () => { + const prompt = gatewayPrompt({ + questions: [ + { + id: "extras", + header: "Extras", + question: "Which extras should I include?", + options: [{ label: "Tests" }, { label: "Docs" }], + multiSelect: true, + isOther: true, + }, + { + id: "target", + header: "Target", + question: "Where should I send it?", + options: [{ label: "Chat" }, { label: "File" }], + isOther: true, + }, + ], + }); + const onSubmit = await draw(prompt); + const card = await cardIn(container); + const checkboxes = container.querySelectorAll('input[type="checkbox"]'); + checkboxes[0]?.click(); + checkboxes[1]?.click(); + const targetInput = container.querySelectorAll(".chat-question__other")[1]!; + targetInput.value = "Issue comment"; + targetInput.dispatchEvent(new InputEvent("input", { bubbles: true })); + await card.updateComplete; + + container.querySelector(".chat-question__submit")?.click(); + + expect(onSubmit).toHaveBeenCalledWith({ + extras: ["Tests", "Docs"], + target: ["Issue comment"], + }); + }); + + it("renders countdown and answered-elsewhere state", async () => { + const prompt = gatewayPrompt(); + await draw(prompt); + expect(container.querySelector(".chat-question__countdown")?.textContent).toBe("1:00"); + + prompt.status = "answered"; + prompt.answeredElsewhere = true; + prompt.answers = { answers: { format: { answers: ["Detailed"] } } }; + await draw(prompt); + + expect(container.querySelector(".chat-question__status")?.textContent).toBe( + "Answered elsewhere", + ); + expect(container.querySelectorAll('input[type="radio"]')[1]?.checked).toBe( + true, + ); + expect(container.querySelector('input[type="radio"]')?.disabled).toBe(true); + }); + + it.each([ + ["expired", "Expired"], + ["cancelled", "Cancelled"], + ] as const)("renders %s terminal state", async (status, label) => { + await draw(gatewayPrompt({ status })); + + expect(container.querySelector(".chat-question__status")?.textContent).toBe(label); + expect(container.querySelector(".chat-question__submit")).toBeNull(); + }); + + it("shows resolve errors while leaving another attempt enabled", async () => { + const prompt = gatewayPrompt({ error: "gateway unavailable" }); + await draw(prompt); + + expect(container.querySelector(".chat-question__error")?.textContent).toContain( + "gateway unavailable", + ); + expect(container.querySelector(".chat-question__submit")?.disabled).toBe( + true, + ); + container.querySelector('input[type="radio"]')?.click(); + await cardIn(container); + expect(container.querySelector(".chat-question__submit")?.disabled).toBe( + false, + ); + }); + + it("clears the private submitted latch after a handled gateway rejection", async () => { + const prompt = gatewayPrompt(); + await draw(prompt, async () => { + prompt.error = "gateway unavailable"; + }); + const card = await cardIn(container); + container.querySelector('input[type="radio"]')?.click(); + await card.updateComplete; + + container.querySelector(".chat-question__submit")?.click(); + + await vi.waitFor(() => + expect(container.querySelector(".chat-question__submit")?.disabled).toBe( + false, + ), + ); + }); }); }); diff --git a/ui/src/pages/chat/components/chat-question-card.ts b/ui/src/pages/chat/components/chat-question-card.ts index 631875bcf592..20a2a773e819 100644 --- a/ui/src/pages/chat/components/chat-question-card.ts +++ b/ui/src/pages/chat/components/chat-question-card.ts @@ -1,58 +1,272 @@ +// Control UI chat module renders provider-neutral question cards. import { LitElement, html, nothing } from "lit"; import { property, state } from "lit/decorators.js"; +import type { QuestionPrompt } from "../../../app/question-prompt.ts"; import { t } from "../../../i18n/index.ts"; import type { QuestionStatus } from "../tool-stream.ts"; +type QuestionCardQuestion = { + id: string; + header: string; + question: string; + options: Array<{ label: string; description?: string }>; + multiSelect?: boolean; + isOther?: boolean; +}; + +type QuestionCardTerminalState = "answered" | "answered-elsewhere" | "expired" | "cancelled"; + +type QuestionCardViewModel = { + requestKey: string; + title: string; + questions: QuestionCardQuestion[]; + terminalState?: QuestionCardTerminalState; + disabled: boolean; + submitting?: boolean; + countdown?: string; + answersById?: Record; + error?: string | null; +}; + type QuestionCardProps = { - status: QuestionStatus; + model: QuestionCardViewModel; + onSubmit: (answersById: Record) => void | Promise; + onAnswersChange?: (answersById: Record) => void; + onDismissError?: () => void; +}; + +type CodexQuestionCardOptions = { disabled: boolean; onSubmit: (answers: Record, onRejected: () => void) => void; }; +type GatewayQuestionCardOptions = { + nowMs: number; + onChange: () => void; + onSubmit: (answers: Record) => void | Promise; +}; + +function formatRemaining(expiresAtMs: number, nowMs: number): string { + const seconds = Math.max(0, Math.ceil((expiresAtMs - nowMs) / 1_000)); + const minutes = Math.floor(seconds / 60); + return `${minutes}:${String(seconds % 60).padStart(2, "0")}`; +} + +function terminalStateForPrompt(prompt: QuestionPrompt): QuestionCardTerminalState | undefined { + if (prompt.status === "answered") { + return prompt.answeredElsewhere ? "answered-elsewhere" : "answered"; + } + return prompt.status === "pending" ? undefined : prompt.status; +} + +function promptDraftAnswers(prompt: QuestionPrompt): Record { + if (prompt.status === "answered") { + return Object.fromEntries( + prompt.questions.map((question) => [ + question.id, + prompt.answers?.answers[question.id]?.answers ?? [], + ]), + ); + } + return Object.fromEntries( + prompt.questions.map((question) => { + const draft = prompt.drafts.get(question.id); + return [ + question.id, + [...(draft?.selected ?? []), ...(draft?.freeText.trim() ? [draft.freeText.trim()] : [])], + ]; + }), + ); +} + +function updatePromptDrafts(prompt: QuestionPrompt, answersById: Record): void { + for (const question of prompt.questions) { + const values = answersById[question.id] ?? []; + const optionLabels = new Set(question.options.map((option) => option.label)); + prompt.drafts.set(question.id, { + selected: new Set(values.filter((value) => optionLabels.has(value))), + freeText: values.find((value) => !optionLabels.has(value)) ?? "", + }); + } +} + +export function createCodexQuestionCardProps( + status: QuestionStatus, + options: CodexQuestionCardOptions, +): QuestionCardProps { + return { + model: { + requestKey: `${status.itemId}:${status.actionToken}`, + title: t("chat.questions.title"), + questions: status.questions, + disabled: options.disabled, + }, + onSubmit: (answersById) => + new Promise((_resolve, reject) => { + const answers = Object.fromEntries( + Object.entries(answersById).map(([id, values]) => [id, values[0] ?? ""]), + ); + options.onSubmit(answers, () => reject(new Error("question submission rejected"))); + }), + }; +} + +export function renderChatQuestionCard( + prompt: QuestionPrompt, + options: GatewayQuestionCardOptions, +) { + const pending = prompt.status === "pending"; + const props: QuestionCardProps = { + model: { + requestKey: prompt.id, + title: t("chat.questions.eyebrow"), + questions: prompt.questions, + terminalState: terminalStateForPrompt(prompt), + disabled: !pending || prompt.submitting, + submitting: prompt.submitting, + countdown: pending ? formatRemaining(prompt.expiresAtMs, options.nowMs) : undefined, + answersById: promptDraftAnswers(prompt), + error: prompt.error, + }, + onAnswersChange: (answersById) => { + updatePromptDrafts(prompt, answersById); + options.onChange(); + }, + onSubmit: async (answersById) => { + await options.onSubmit(answersById); + if (prompt.status === "pending" && prompt.error) { + throw new Error(prompt.error); + } + }, + onDismissError: prompt.error + ? () => { + prompt.error = null; + options.onChange(); + } + : undefined, + }; + return html``; +} + +function answersSignature(answersById: Record): string { + return JSON.stringify( + Object.entries(answersById) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([id, values]) => [id, values]), + ); +} + class ChatQuestionCard extends LitElement { override createRenderRoot() { return this; } @property({ attribute: false }) props?: QuestionCardProps; - @state() private answers = new Map(); - @state() private freeFormAnswers = new Set(); + @state() private selectedById = new Map(); + @state() private freeTextById = new Map(); @state() private submitted = false; private requestKey: string | null = null; + private syncedAnswersSignature: string | null = null; override willUpdate() { - const status = this.props?.status; - const nextRequestKey = status ? `${status.itemId}:${status.actionToken}` : null; + const model = this.props?.model; + const nextRequestKey = model?.requestKey ?? null; if (nextRequestKey !== this.requestKey) { this.requestKey = nextRequestKey; - this.answers = new Map(); - this.freeFormAnswers = new Set(); + this.selectedById = new Map(); + this.freeTextById = new Map(); this.submitted = false; + this.syncedAnswersSignature = null; } - } - - private setAnswer(questionId: string, answer: string, source: "option" | "text"): void { - this.answers = new Map(this.answers).set(questionId, answer); - const freeFormAnswers = new Set(this.freeFormAnswers); - if (source === "text") { - freeFormAnswers.add(questionId); - } else { - freeFormAnswers.delete(questionId); - } - this.freeFormAnswers = freeFormAnswers; - } - - private submit(status: QuestionStatus): void { - if (!status.questions.every((question) => this.answers.get(question.id)?.trim())) { + if (!model?.answersById) { return; } - const answers = Object.fromEntries( - status.questions.map((question) => [question.id, this.answers.get(question.id)!]), + const signature = answersSignature(model.answersById); + if (signature === this.syncedAnswersSignature) { + return; + } + this.syncedAnswersSignature = signature; + const selectedById = new Map(); + const freeTextById = new Map(); + for (const question of model.questions) { + const optionLabels = new Set(question.options.map((option) => option.label)); + const values = model.answersById[question.id] ?? []; + selectedById.set( + question.id, + values.filter((value) => optionLabels.has(value)), + ); + const custom = values.filter((value) => !optionLabels.has(value)).join(", "); + if (custom) { + freeTextById.set(question.id, custom); + } + } + this.selectedById = selectedById; + this.freeTextById = freeTextById; + } + + private answerValues(question: QuestionCardQuestion): string[] { + const selected = this.selectedById.get(question.id) ?? []; + const freeText = this.freeTextById.get(question.id)?.trim(); + return [...selected, ...(freeText ? [freeText] : [])]; + } + + private buildAnswers(model: QuestionCardViewModel): Record { + return Object.fromEntries( + model.questions.map((question) => [question.id, this.answerValues(question)]), ); + } + + private answersChanged(model: QuestionCardViewModel): void { + const answersById = this.buildAnswers(model); + this.syncedAnswersSignature = answersSignature(answersById); + this.props?.onAnswersChange?.(answersById); + } + + private toggleOption( + model: QuestionCardViewModel, + question: QuestionCardQuestion, + label: string, + ) { + const selectedById = new Map(this.selectedById); + const current = selectedById.get(question.id) ?? []; + selectedById.set( + question.id, + question.multiSelect + ? current.includes(label) + ? current.filter((value) => value !== label) + : [...current, label] + : [label], + ); + this.selectedById = selectedById; + if (!question.multiSelect) { + const freeTextById = new Map(this.freeTextById); + freeTextById.delete(question.id); + this.freeTextById = freeTextById; + } + this.answersChanged(model); + } + + private setFreeText(model: QuestionCardViewModel, question: QuestionCardQuestion, value: string) { + this.freeTextById = new Map(this.freeTextById).set(question.id, value); + if (!question.multiSelect && value.trim()) { + this.selectedById = new Map(this.selectedById).set(question.id, []); + } + this.answersChanged(model); + } + + private async submit(model: QuestionCardViewModel): Promise { + if (!model.questions.every((question) => this.answerValues(question).length > 0)) { + return; + } + const requestKey = model.requestKey; this.submitted = true; - this.props?.onSubmit(answers, () => { - this.submitted = false; - }); + try { + await this.props?.onSubmit(this.buildAnswers(model)); + } catch { + if (this.requestKey === requestKey) { + this.submitted = false; + } + } } override render() { @@ -60,34 +274,42 @@ class ChatQuestionCard extends LitElement { if (!props) { return nothing; } - const complete = props.status.questions.every((question) => - Boolean(this.answers.get(question.id)?.trim()), - ); + const { model } = props; + const disabled = model.disabled || this.submitted || Boolean(model.terminalState); + const complete = model.questions.every((question) => this.answerValues(question).length > 0); return html` -
-
${t("chat.questions.title")}
- ${props.status.questions.map( +
+
+
${model.title}
+ ${model.countdown + ? html`${model.countdown}` + : nothing} +
+ ${model.questions.map( (question) => html` -
+
${question.header}
${question.question}
- ${question.options.map( - (option) => html` + ${question.options.map((option) => { + const selected = (this.selectedById.get(question.id) ?? []).includes(option.label); + return html` - `, - )} + `; + })} ${question.isOther || question.options.length === 0 ? html` - this.setAnswer( - question.id, - (event.target as HTMLInputElement).value, - "text", - )} + this.setFreeText(model, question, (event.target as HTMLInputElement).value)} + @keydown=${(event: KeyboardEvent) => { + if ( + event.key === "Enter" && + !event.isComposing && + event.keyCode !== 229 && + complete && + !disabled + ) { + event.preventDefault(); + void this.submit(model); + } + }} /> ` : nothing}
`, )} - +
+ ${model.terminalState + ? html`${t( + `chat.questions.${model.terminalState === "answered-elsewhere" ? "answeredElsewhere" : model.terminalState}`, + )}` + : nothing} + ${model.error + ? html` + ${t("chat.questions.submitFailed", { error: model.error })} + ${props.onDismissError + ? html`` + : nothing} + ` + : nothing} + ${model.terminalState + ? nothing + : html``} +
`; } diff --git a/ui/src/pages/chat/components/chat-thread.ts b/ui/src/pages/chat/components/chat-thread.ts index 5a0b0d887575..b28c6ef1e791 100644 --- a/ui/src/pages/chat/components/chat-thread.ts +++ b/ui/src/pages/chat/components/chat-thread.ts @@ -15,6 +15,7 @@ import { repeat } from "lit/directives/repeat.js"; import { styleMap } from "lit/directives/style-map.js"; import { classifySessionKind } from "../../../../../src/sessions/classify-session-kind.js"; import type { SessionsListResult } from "../../../api/types.ts"; +import type { QuestionPrompt } from "../../../app/question-prompt.ts"; import { resolveLocalUserName } from "../../../app/user-identity.ts"; import { icons } from "../../../components/icons.ts"; import "../../../components/tooltip.ts"; @@ -109,6 +110,7 @@ type ChatThreadProps = { /** True while the agent is visibly working (isChatRunWorking); shows the working spark. */ runWorking?: boolean; planStatus?: PlanStatus | null; + questionPrompts?: readonly QuestionPrompt[]; sessions: SessionsListResult | null; /** Host context resolving global-alias session keys (scope=global fleets). */ /** Includes assistantAgentId so bare-global welcome recents scope to the selected agent. */ @@ -132,6 +134,8 @@ type ChatThreadProps = { onOpenSessionCheckpoints?: () => void | Promise; onAssistantAttachmentLoaded?: () => void; onRequestUpdate?: () => void; + onQuestionChange?: () => void; + onQuestionSubmit?: (id: string, answers: Record) => void | Promise; onChatScroll?: (event: Event) => void; onHistoryIntent?: (event: Event) => void; onDraftChange: (next: string) => void; @@ -975,12 +979,16 @@ function renderChatThreadContents( runWorking: Boolean(props.runWorking), runActive: Boolean(props.runActive), planStatus: props.planStatus, + questionPrompts: props.questionPrompts, loading: props.loading, searchOpen: state.searchOpen, searchQuery: state.searchQuery, }); syncToolCardExpansionState(props.sessionKey, chatItems, Boolean(props.autoExpandToolCalls)); const expandedToolCards = getExpandedToolCards(props.sessionKey); + const questionPrompts = new Map( + (props.questionPrompts ?? []).map((prompt) => [prompt.id, prompt]), + ); const toggleToolCardExpanded = (toolCardId: string) => { expandedToolCards.set(toolCardId, !expandedToolCards.get(toolCardId)); requestUpdate(); @@ -1053,6 +1061,9 @@ function renderChatThreadContents( } if (item.kind === "stream-run") { return renderStreamGroup(item.parts, { + questionPrompts, + onQuestionChange: props.onQuestionChange, + onQuestionSubmit: props.onQuestionSubmit, planStatus: props.planStatus, planActive: Boolean(props.runActive), onOpenSidebar: props.onOpenSidebar, @@ -1077,6 +1088,13 @@ function renderChatThreadContents( if (item.kind === "group") { return renderGroupItem(item); } + if (item.kind === "question") { + return renderStreamGroup([item], { + questionPrompts, + onQuestionChange: props.onQuestionChange, + onQuestionSubmit: props.onQuestionSubmit, + }); + } return nothing; }); const collapsedItems = collapseCompletedTurnWork(coalesceStreamRuns(chatItems), { @@ -1123,6 +1141,7 @@ function renderChatThreadContents( Boolean(props.runActive), Boolean(props.runWorking), props.planStatus, + props.questionPrompts, Boolean(props.autoExpandToolCalls), props.assistantName, assistantIdentity.avatar, diff --git a/ui/src/styles/chat/question-card.css b/ui/src/styles/chat/question-card.css index 5b941f862401..4b05938e2c79 100644 --- a/ui/src/styles/chat/question-card.css +++ b/ui/src/styles/chat/question-card.css @@ -7,6 +7,13 @@ background: var(--bg-elevated); } +.chat-question__topline, +.chat-question__footer { + display: flex; + align-items: center; + gap: 8px; +} + .chat-question__title { color: var(--text-strong); font-size: 12px; @@ -15,6 +22,14 @@ text-transform: uppercase; } +.chat-question__countdown { + margin-left: auto; + color: var(--text-muted); + font-family: var(--mono); + font-size: 12px; + font-variant-numeric: tabular-nums; +} + .chat-question__field { display: grid; gap: 8px; @@ -50,10 +65,48 @@ color: var(--text-muted); } +.chat-question__option:has(input:checked) { + border-color: color-mix(in srgb, var(--accent) 55%, var(--border-subtle)); + background: color-mix(in srgb, var(--accent-subtle) 45%, var(--bg-elevated)); +} + +.chat-question__option:has(input:disabled) { + cursor: default; + opacity: 0.72; +} + +.chat-question__option:has(input:checked:disabled) { + opacity: 1; +} + .chat-question__other { width: 100%; } +.chat-question__footer { + min-height: 28px; +} + +.chat-question__status { + color: var(--text-muted); + font-size: 12px; +} + +.chat-question__error { + color: var(--danger); + font-size: 12px; +} + +.chat-question__error-dismiss { + margin-left: 4px; + padding: 0 2px; + border: 0; + background: transparent; + color: inherit; + cursor: pointer; +} + .chat-question__submit { + margin-left: auto; justify-self: end; }