From d23ee2f7022707b7edc0ee0cb63e9874b70dd4b1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:30:59 +0100 Subject: [PATCH 01/12] fix: hide bundled runtime npm windows --- CHANGELOG.md | 1 + scripts/postinstall-bundled-plugins.mjs | 1 + scripts/stage-bundled-plugin-runtime-deps.mjs | 8 +++- src/plugins/bundled-runtime-deps.test.ts | 37 ++++++++++++++++++- src/plugins/bundled-runtime-deps.ts | 2 + .../postinstall-bundled-plugins.test.ts | 1 + .../stage-bundled-plugin-runtime-deps.test.ts | 26 +++++++++++++ 7 files changed, 74 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11003bf1588..69385c0f3c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Docs: https://docs.openclaw.ai - Agents/bootstrap: refresh cached workspace bootstrap snapshots on long-lived main-session turns when `AGENTS.md`, `SOUL.md`, `MEMORY.md`, or `TOOLS.md` change on disk, while preserving unchanged snapshot identity through the workspace file cache. (#64871; related #43901, #26497, #28594, #30896) Thanks @aimqwest and @mikejuyoon. - macOS Gateway: detect installed-but-unloaded LaunchAgent split-brain states during status, doctor, and restart, and re-bootstrap launchd supervision before falling back to unmanaged listener restarts. Fixes #67335, #53475, and #71060; refs #58890, #60885, and #70801. Thanks @ze1tgeist88, @dafacto, and @vishutdhar. - Plugins/install: treat mirrored core logger dependencies as staged bundled runtime deps so packaged Gateway starts do not crash when the external plugin-runtime-deps root is missing `tslog`. Fixes #72228; supersedes #72493. Thanks @deepujain. +- Plugins/install: hide bundled runtime-dependency npm child windows on Windows across Gateway startup, postinstall, and packaged staging paths so Telegram/Anthropic dependency repair no longer flashes shell windows. Fixes #72315. Thanks @athuljayaram and @joshfeng. - Plugins/install: stage bundled plugin runtime dependencies before Gateway startup, drain update restarts, and materialize plugin-owned root chunks in external mirrors so staged deps resolve under native ESM. Fixes #72058; supersedes #72084. Thanks @amnesia106 and @drvoss. - TTS/SecretRef: resolve `messages.tts.providers.*.apiKey` from the active runtime snapshot so SecretRef-backed MiniMax and other TTS provider keys work in runtime reply/audio paths. Fixes #68690. Thanks @joshavant. - Gateway/install: surface systemd user-bus recovery hints during Linux service activation and retry via the machine user scope when `systemctl --user` reports no-medium bus failures. Fixes #39673; refs #44417 and #63561. Thanks @Arbor4, @myrsu, and @mssteuer. diff --git a/scripts/postinstall-bundled-plugins.mjs b/scripts/postinstall-bundled-plugins.mjs index c4917a0ca43..4d97e71dd5f 100644 --- a/scripts/postinstall-bundled-plugins.mjs +++ b/scripts/postinstall-bundled-plugins.mjs @@ -817,6 +817,7 @@ export function runBundledPluginPostinstall(params = {}) { encoding: "utf8", env: npmRunner.env ?? installEnv, stdio: "pipe", + windowsHide: true, shell: npmRunner.shell, windowsVerbatimArguments: npmRunner.windowsVerbatimArguments, }); diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index 56d44f1f44f..db8a5eafd7c 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -877,13 +877,15 @@ function runNpmInstall(params) { npm_config_save: "false", npm_config_yes: "true", }; - const result = spawnSync(params.npmRunner.command, params.npmRunner.args, { + const runSpawnSync = params.spawnSyncImpl ?? spawnSync; + const result = runSpawnSync(params.npmRunner.command, params.npmRunner.args, { cwd: params.cwd, encoding: "utf8", env: npmEnv, shell: params.npmRunner.shell, stdio: ["ignore", "pipe", "pipe"], timeout: params.timeoutMs ?? 5 * 60 * 1000, + windowsHide: true, windowsVerbatimArguments: params.npmRunner.windowsVerbatimArguments, }); if (result.status === 0) { @@ -1240,6 +1242,10 @@ export function stageBundledPluginRuntimeDeps(params = {}) { } } +export const __testing = { + runNpmInstall, +}; + if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { stageBundledPluginRuntimeDeps(); } diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index 857da9aa65b..12d5ce6e792 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -1,5 +1,6 @@ -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; +import { EventEmitter } from "node:events"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -27,9 +28,11 @@ import { vi.mock("node:child_process", async (importOriginal) => ({ ...(await importOriginal()), + spawn: vi.fn(), spawnSync: vi.fn(), })); +const spawnMock = vi.mocked(spawn); const spawnSyncMock = vi.mocked(spawnSync); const tempDirs: string[] = []; @@ -91,6 +94,7 @@ function statfsFixture(params: { afterEach(() => { vi.restoreAllMocks(); + spawnMock.mockReset(); spawnSyncMock.mockReset(); bundledRuntimeDepsActivityTesting.resetBundledRuntimeDepsInstallActivity(); for (const dir of tempDirs.splice(0)) { @@ -312,6 +316,7 @@ describe("installBundledRuntimeDeps", () => { ["C:\\node\\node_modules\\npm\\bin\\npm-cli.js", "install", "--ignore-scripts", "acpx@0.5.3"], expect.objectContaining({ cwd: installRoot, + windowsHide: true, env: expect.objectContaining({ npm_config_legacy_peer_deps: "true", npm_config_package_lock: "false", @@ -330,6 +335,36 @@ describe("installBundledRuntimeDeps", () => { ); }); + it("hides async npm child windows for startup repair installs", async () => { + const installRoot = makeTempDir(); + spawnMock.mockImplementation((_command, _args, options) => { + writeInstalledPackage(String(options?.cwd ?? ""), "acpx", "0.5.3"); + const child = new EventEmitter() as ReturnType; + Object.assign(child, { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + }); + queueMicrotask(() => child.emit("close", 0, null)); + return child; + }); + + await repairBundledRuntimeDepsInstallRootAsync({ + installRoot, + missingSpecs: ["acpx@0.5.3"], + installSpecs: ["acpx@0.5.3"], + env: {}, + }); + + expect(spawnMock).toHaveBeenCalledWith( + expect.any(String), + expect.any(Array), + expect.objectContaining({ + cwd: installRoot, + windowsHide: true, + }), + ); + }); + it("anchors non-isolated external install roots with a package manifest", () => { const parentRoot = makeTempDir(); const installRoot = path.join(parentRoot, ".openclaw", "plugin-runtime-deps", "openclaw-test"); diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index bd39ee6b7c1..6e8eff55cb0 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -1409,6 +1409,7 @@ async function spawnBundledRuntimeDepsInstall(params: { cwd: params.cwd, env: params.env, stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; @@ -1480,6 +1481,7 @@ export function installBundledRuntimeDeps(params: { encoding: "utf8", env: npmRunner.env ?? installEnv, stdio: "pipe", + windowsHide: true, }); if (result.status !== 0 || result.error) { throw new Error(formatBundledRuntimeDepsInstallError(result)); diff --git a/test/scripts/postinstall-bundled-plugins.test.ts b/test/scripts/postinstall-bundled-plugins.test.ts index d32542c8d2e..34107e7262f 100644 --- a/test/scripts/postinstall-bundled-plugins.test.ts +++ b/test/scripts/postinstall-bundled-plugins.test.ts @@ -79,6 +79,7 @@ describe("bundled plugin postinstall", () => { }, shell: false, stdio: "pipe", + windowsHide: true, windowsVerbatimArguments: undefined, }); } diff --git a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts index 378b83927a0..a03facfbf25 100644 --- a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts +++ b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + __testing as stageBundledPluginRuntimeDepsTesting, collectRuntimeDependencyInstallManifest, collectRuntimeDependencyInstallSpecs, stageBundledPluginRuntimeDeps, @@ -129,6 +130,31 @@ describe("stageBundledPluginRuntimeDeps", () => { }); }); + it("hides npm child windows during fallback runtime installs", () => { + const spawnSyncImpl = vi.fn(() => ({ status: 0, stderr: "", stdout: "" })); + + stageBundledPluginRuntimeDepsTesting.runNpmInstall({ + cwd: "C:\\openclaw\\dist\\extensions\\telegram\\.openclaw-install-stage", + npmRunner: { + command: "npm.cmd", + args: ["install", "--silent"], + env: { PATH: "C:\\node" }, + shell: false, + windowsVerbatimArguments: true, + }, + spawnSyncImpl, + }); + + expect(spawnSyncImpl).toHaveBeenCalledWith( + "npm.cmd", + ["install", "--silent"], + expect.objectContaining({ + windowsHide: true, + windowsVerbatimArguments: true, + }), + ); + }); + it("skips restaging when runtime deps stamp matches the sanitized manifest", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: { From 75c52b6c41adbf31a8c19741b8951941d5ba3770 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 27 Apr 2026 00:33:29 -0700 Subject: [PATCH 02/12] fix(ci): expose package deps to Telegram QA harness (#72680) * fix(ci): expose package deps to telegram QA harness * fix(ci): link QA package runtime deps * fix(agents): guard replay metadata in empty retries * fix(ci): keep plugin update smoke migration-stable --- .../qa-lab/src/gateway-rpc-client.test.ts | 2 +- extensions/qa-lab/src/gateway-rpc-client.ts | 2 +- scripts/e2e/npm-telegram-live-docker.sh | 21 ++++++ scripts/e2e/plugin-update-unchanged-docker.sh | 15 +---- src/agents/pi-embedded-runner/replay-state.ts | 8 ++- .../run.incomplete-turn.test.ts | 30 +++++++++ src/agents/pi-embedded-runner/run.ts | 14 ++-- .../pi-embedded-runner/run/incomplete-turn.ts | 66 ++++++++++++------- test/scripts/npm-telegram-live.test.ts | 14 ++++ .../plugin-update-unchanged-docker.test.ts | 20 ++++++ 10 files changed, 145 insertions(+), 47 deletions(-) create mode 100644 test/scripts/plugin-update-unchanged-docker.test.ts diff --git a/extensions/qa-lab/src/gateway-rpc-client.test.ts b/extensions/qa-lab/src/gateway-rpc-client.test.ts index 885d646de17..eab529a961f 100644 --- a/extensions/qa-lab/src/gateway-rpc-client.test.ts +++ b/extensions/qa-lab/src/gateway-rpc-client.test.ts @@ -10,7 +10,7 @@ const gatewayRpcMock = vi.hoisted(() => { }; }); -vi.mock("./runtime-api.js", () => ({ +vi.mock("openclaw/plugin-sdk/browser-node-runtime", () => ({ callGatewayFromCli: gatewayRpcMock.callGatewayFromCli, })); diff --git a/extensions/qa-lab/src/gateway-rpc-client.ts b/extensions/qa-lab/src/gateway-rpc-client.ts index 56b4889e538..e4b16d95c5e 100644 --- a/extensions/qa-lab/src/gateway-rpc-client.ts +++ b/extensions/qa-lab/src/gateway-rpc-client.ts @@ -1,6 +1,6 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime"; import { formatQaGatewayLogsForError } from "./gateway-log-redaction.js"; -import { callGatewayFromCli } from "./runtime-api.js"; type QaGatewayRpcRequestOptions = { expectFinal?: boolean; diff --git a/scripts/e2e/npm-telegram-live-docker.sh b/scripts/e2e/npm-telegram-live-docker.sh index 4b4ef890770..d2f13c392dc 100755 --- a/scripts/e2e/npm-telegram-live-docker.sh +++ b/scripts/e2e/npm-telegram-live-docker.sh @@ -279,6 +279,27 @@ for deps_dir in "$openclaw_package_dir/node_modules" /npm-global/lib/node_module done done +link_installed_package_dependency() { + local name="$1" + local source="/npm-global/lib/node_modules/openclaw/node_modules/$name" + local target="/app/node_modules/$name" + if [ ! -e "$source" ]; then + echo "Installed package dependency is missing: $name" >&2 + return 1 + fi + mkdir -p "$(dirname "$target")" + ln -sfn "$source" "$target" +} + +# QA Lab is intentionally mounted as harness source, so its package-local +# runtime imports must resolve from the installed package dependency tree. +for dependency in \ + @modelcontextprotocol/sdk \ + yaml \ + zod; do + link_installed_package_dependency "$dependency" +done + echo "Running installed-package onboarding recovery hot path..." OPENAI_API_KEY="${OPENAI_API_KEY:-sk-openclaw-npm-telegram-hotpath}" openclaw onboard --non-interactive --accept-risk \ --mode local \ diff --git a/scripts/e2e/plugin-update-unchanged-docker.sh b/scripts/e2e/plugin-update-unchanged-docker.sh index 25e2ca49e26..41918a65896 100755 --- a/scripts/e2e/plugin-update-unchanged-docker.sh +++ b/scripts/e2e/plugin-update-unchanged-docker.sh @@ -43,20 +43,7 @@ JSON if [ \"\$OPENCLAW_PACKAGE_ACCEPTANCE_LEGACY_COMPAT\" = \"1\" ]; then cat > \"\$HOME/.openclaw/openclaw.json\" <<'JSON' { - \"plugins\": { - \"installs\": { - \"lossless-claw\": { - \"source\": \"npm\", - \"spec\": \"@example/lossless-claw@0.9.0\", - \"installPath\": \"~/.openclaw/extensions/lossless-claw\", - \"resolvedName\": \"@example/lossless-claw\", - \"resolvedVersion\": \"0.9.0\", - \"resolvedSpec\": \"@example/lossless-claw@0.9.0\", - \"integrity\": \"sha512-same\", - \"shasum\": \"same\" - } - } - } + \"plugins\": {} } JSON else diff --git a/src/agents/pi-embedded-runner/replay-state.ts b/src/agents/pi-embedded-runner/replay-state.ts index 19e68d11821..26822c74b95 100644 --- a/src/agents/pi-embedded-runner/replay-state.ts +++ b/src/agents/pi-embedded-runner/replay-state.ts @@ -33,8 +33,14 @@ export function mergeEmbeddedRunReplayState( export function observeReplayMetadata( current: EmbeddedRunReplayState, - metadata: EmbeddedRunReplayMetadata, + metadata?: EmbeddedRunReplayMetadata | null, ): EmbeddedRunReplayState { + if (!metadata) { + return mergeEmbeddedRunReplayState(current, { + replayInvalid: true, + hadPotentialSideEffects: true, + }); + } return mergeEmbeddedRunReplayState(current, { replayInvalid: !metadata.replaySafe, hadPotentialSideEffects: metadata.hadPotentialSideEffects, diff --git a/src/agents/pi-embedded-runner/run.incomplete-turn.test.ts b/src/agents/pi-embedded-runner/run.incomplete-turn.test.ts index 0a9c0383368..c0f1f77589d 100644 --- a/src/agents/pi-embedded-runner/run.incomplete-turn.test.ts +++ b/src/agents/pi-embedded-runner/run.incomplete-turn.test.ts @@ -922,6 +922,13 @@ describe("runEmbeddedPiAgent incomplete-turn safety", () => { ).toBe("abandoned"); }); + it("treats missing replay metadata as replay-invalid", () => { + const attempt = makeAttemptResult(); + delete (attempt as Partial).replayMetadata; + + expect(resolveReplayInvalidFlag({ attempt })).toBe(true); + }); + it("detects reasoning-only GPT turns from signed thinking blocks", () => { const retryInstruction = resolveReasoningOnlyRetryInstruction({ provider: "openai", @@ -1073,6 +1080,29 @@ describe("runEmbeddedPiAgent incomplete-turn safety", () => { expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); }); + it("does not retry clean zero-token Ollama stop turns", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction({ + provider: "ollama", + modelId: "glm-5.1:cloud", + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + lastAssistant: { + role: "assistant", + stopReason: "stop", + provider: "ollama", + model: "glm-5.1:cloud", + content: [], + usage: { input: 100, output: 0, totalTokens: 100 }, + } as unknown as EmbeddedRunAttemptResult["lastAssistant"], + }), + }); + + expect(retryInstruction).toBeNull(); + }); + it("treats exact NO_REPLY as a deliberate silent assistant reply", () => { const incompleteTurnText = resolveIncompleteTurnPayloadText({ payloadCount: 0, diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 5f2bf0c83dd..be3135f4e40 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -191,10 +191,10 @@ function createEmptyAuthProfileStore(): AuthProfileStore { } function buildTraceToolSummary(params: { - toolMetas: Array<{ toolName: string; meta?: string }>; + toolMetas?: Array<{ toolName: string; meta?: string }>; hadFailure: boolean; }): ToolSummaryTrace | undefined { - if (params.toolMetas.length === 0) { + if (!params.toolMetas?.length) { return undefined; } const tools: string[] = []; @@ -208,7 +208,7 @@ function buildTraceToolSummary(params: { tools.push(toolName); } return { - calls: params.toolMetas.length, + calls: params.toolMetas?.length ?? 0, tools, failures: params.hadFailure ? 1 : 0, }; @@ -1067,8 +1067,8 @@ export async function runEmbeddedPiAgent( !attempt.didSendViaMessagingTool && !attempt.didSendDeterministicApprovalPrompt && !attempt.lastToolError && - attempt.toolMetas.length === 0 && - attempt.assistantTexts.length === 0; + (attempt.toolMetas?.length ?? 0) === 0 && + (attempt.assistantTexts?.length ?? 0) === 0; if (preflightRecovery?.handled) { log.info( `[context-overflow-precheck] early recovery route=${preflightRecovery.route} ` + @@ -2000,7 +2000,7 @@ export async function runEmbeddedPiAgent( nextPlanningOnlyRetryInstruction && planningOnlyRetryAttempts < maxPlanningOnlyRetryAttempts ) { - const planningOnlyText = attempt.assistantTexts.join("\n\n").trim(); + const planningOnlyText = (attempt.assistantTexts ?? []).join("\n\n").trim(); const planDetails = extractPlanningOnlyPlanDetails(planningOnlyText); if (planDetails) { emitAgentPlanEvent({ @@ -2222,7 +2222,7 @@ export async function runEmbeddedPiAgent( sessionLastAssistant?.stopReason === "error" && ((sessionLastAssistant?.usage as { output?: number } | undefined)?.output ?? 0) === 0 && (silentErrorContent?.length ?? 0) === 0 && - !attempt.replayMetadata.hadPotentialSideEffects && + (attempt.replayMetadata ? !attempt.replayMetadata.hadPotentialSideEffects : false) && emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES ) { emptyErrorRetries += 1; diff --git a/src/agents/pi-embedded-runner/run/incomplete-turn.ts b/src/agents/pi-embedded-runner/run/incomplete-turn.ts index 81ea308b3a0..112dfecd1c3 100644 --- a/src/agents/pi-embedded-runner/run/incomplete-turn.ts +++ b/src/agents/pi-embedded-runner/run/incomplete-turn.ts @@ -274,8 +274,12 @@ export function resolveIncompleteTurnPayloadText(params: { : "⚠️ Agent couldn't generate a response. Please try again."; } -function hasOnlySilentAssistantReply(assistantTexts: readonly string[]): boolean { - const nonEmptyTexts = assistantTexts.filter((text) => text.trim().length > 0); +function joinAssistantTexts(assistantTexts?: readonly string[]): string { + return (assistantTexts ?? []).join("\n\n").trim(); +} + +function hasOnlySilentAssistantReply(assistantTexts?: readonly string[]): boolean { + const nonEmptyTexts = (assistantTexts ?? []).filter((text) => text.trim().length > 0); return ( nonEmptyTexts.length > 0 && nonEmptyTexts.every((text) => isSilentReplyPayloadText(text, SILENT_REPLY_TOKEN)) @@ -342,12 +346,12 @@ export function resolveSilentToolResultReplyPayload(params: { params.payloadCount !== 0 || params.aborted || params.timedOut || - params.attempt.toolMetas.length === 0 || + (params.attempt.toolMetas?.length ?? 0) === 0 || params.attempt.clientToolCall || params.attempt.yieldDetected || params.attempt.didSendDeterministicApprovalPrompt || params.attempt.lastToolError || - params.attempt.messagesSnapshot.length === 0 + (params.attempt.messagesSnapshot?.length ?? 0) === 0 ) { return null; } @@ -411,7 +415,7 @@ function isEmptyResponseAssistantTurn(params: { if (params.payloadCount !== 0) { return false; } - if (params.attempt.assistantTexts.join("\n\n").trim().length > 0) { + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { return false; } const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; @@ -446,7 +450,7 @@ function isNonVisibleAssistantTurnEligibleForSilentReply(params: { if (params.payloadCount !== 0) { return false; } - if (params.attempt.assistantTexts.join("\n\n").trim().length > 0) { + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { return false; } const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; @@ -522,7 +526,7 @@ export function resolveReasoningOnlyRetryInstruction(params: { } const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - if (params.attempt.assistantTexts.join("\n\n").trim().length > 0) { + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { return null; } if (assistant?.stopReason === "error") { @@ -557,6 +561,16 @@ export function resolveEmptyResponseRetryInstruction(params: { return null; } + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null; + if ( + assistant?.stopReason === "stop" && + OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test( + normalizeLowercaseStringOrEmpty(params.provider ?? ""), + ) + ) { + return null; + } + if ( shouldApplyNonVisibleTurnRetryGuard({ provider: params.provider, @@ -566,9 +580,7 @@ export function resolveEmptyResponseRetryInstruction(params: { // Keep the generic zero-usage stop retry for providers that expose a // provider-neutral "nothing was generated" signal, even outside the // provider allowlist above. - isZeroUsageEmptyStopAssistantTurn( - params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null, - ) + isZeroUsageEmptyStopAssistantTurn(assistant) ) { return EMPTY_RESPONSE_RETRY_INSTRUCTION; } @@ -717,20 +729,28 @@ export function extractPlanningOnlyPlanDetails(text: string): PlanningOnlyPlanDe }; } -function countPlanOnlyToolMetas(toolMetas: PlanningOnlyAttempt["toolMetas"]): number { - return toolMetas.filter((entry) => entry.toolName === "update_plan").length; +function normalizePlanningToolMetas( + toolMetas?: PlanningOnlyAttempt["toolMetas"], +): PlanningOnlyAttempt["toolMetas"] { + return toolMetas ?? []; } -function countNonPlanToolCalls(toolMetas: PlanningOnlyAttempt["toolMetas"]): number { - return toolMetas.filter((entry) => entry.toolName !== "update_plan").length; +function countPlanOnlyToolMetas(toolMetas?: PlanningOnlyAttempt["toolMetas"]): number { + return normalizePlanningToolMetas(toolMetas).filter((entry) => entry.toolName === "update_plan") + .length; } -function hasNonPlanToolActivity(toolMetas: PlanningOnlyAttempt["toolMetas"]): boolean { - return toolMetas.some((entry) => entry.toolName !== "update_plan"); +function countNonPlanToolCalls(toolMetas?: PlanningOnlyAttempt["toolMetas"]): number { + return normalizePlanningToolMetas(toolMetas).filter((entry) => entry.toolName !== "update_plan") + .length; } -function hasSingleRetrySafeNonPlanTool(toolMetas: PlanningOnlyAttempt["toolMetas"]): boolean { - const nonPlanToolNames = toolMetas +function hasNonPlanToolActivity(toolMetas?: PlanningOnlyAttempt["toolMetas"]): boolean { + return normalizePlanningToolMetas(toolMetas).some((entry) => entry.toolName !== "update_plan"); +} + +function hasSingleRetrySafeNonPlanTool(toolMetas?: PlanningOnlyAttempt["toolMetas"]): boolean { + const nonPlanToolNames = normalizePlanningToolMetas(toolMetas) .map((entry) => normalizeLowercaseStringOrEmpty(entry.toolName)) .filter((toolName) => toolName && toolName !== "update_plan"); return ( @@ -746,14 +766,14 @@ function hasSingleRetrySafeNonPlanTool(toolMetas: PlanningOnlyAttempt["toolMetas * call path, which still counts as real multi-step progress. */ function isSingleActionThenNarrativePattern(params: { - toolMetas: PlanningOnlyAttempt["toolMetas"]; - assistantTexts: readonly string[]; + toolMetas?: PlanningOnlyAttempt["toolMetas"]; + assistantTexts?: readonly string[]; }): boolean { const nonPlanCount = countNonPlanToolCalls(params.toolMetas); if (nonPlanCount !== 1) { return false; } - const text = params.assistantTexts.join("\n\n").trim(); + const text = (params.assistantTexts ?? []).join("\n\n").trim(); if (!text || text.length > PLANNING_ONLY_MAX_VISIBLE_TEXT) { return false; } @@ -805,7 +825,7 @@ export function resolvePlanningOnlyRetryInstruction(params: { params.attempt.didSendViaMessagingTool || params.attempt.lastToolError || (hasNonPlanToolActivity(params.attempt.toolMetas) && !allowSingleActionRetryBypass) || - (params.attempt.itemLifecycle.startedCount > planOnlyToolMetaCount && + ((params.attempt.itemLifecycle?.startedCount ?? 0) > planOnlyToolMetaCount && !allowSingleActionRetryBypass) || resolveAttemptReplayMetadata(params.attempt).hadPotentialSideEffects ) { @@ -817,7 +837,7 @@ export function resolvePlanningOnlyRetryInstruction(params: { return null; } - const text = params.attempt.assistantTexts.join("\n\n").trim(); + const text = (params.attempt.assistantTexts ?? []).join("\n\n").trim(); if (!text || text.length > PLANNING_ONLY_MAX_VISIBLE_TEXT || text.includes("```")) { return null; } diff --git a/test/scripts/npm-telegram-live.test.ts b/test/scripts/npm-telegram-live.test.ts index e74ec6d727d..bd5bf18f18b 100644 --- a/test/scripts/npm-telegram-live.test.ts +++ b/test/scripts/npm-telegram-live.test.ts @@ -68,6 +68,20 @@ describe("package Telegram live Docker E2E", () => { expect(script).toContain('"./extensions/qa-channel/src/protocol.ts"'); }); + it("exposes installed package dependencies to the mounted QA harness", () => { + const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8"); + + expect(script).toContain("link_installed_package_dependency()"); + expect(script).toContain( + 'local source="/npm-global/lib/node_modules/openclaw/node_modules/$name"', + ); + expect(script).toContain('ln -sfn "$source" "$target"'); + expect(script).toContain("link_installed_package_dependency \"$dependency\""); + expect(script).toContain("@modelcontextprotocol/sdk"); + expect(script).toContain("yaml"); + expect(script).toContain("zod"); + }); + it("lets npm-specific credential aliases override shared QA env", () => { expect( __testing.resolveCredentialSource({ diff --git a/test/scripts/plugin-update-unchanged-docker.test.ts b/test/scripts/plugin-update-unchanged-docker.test.ts new file mode 100644 index 00000000000..848edde9e34 --- /dev/null +++ b/test/scripts/plugin-update-unchanged-docker.test.ts @@ -0,0 +1,20 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const PLUGIN_UPDATE_DOCKER_SCRIPT = "scripts/e2e/plugin-update-unchanged-docker.sh"; + +describe("plugin update unchanged Docker E2E", () => { + it("seeds current plugin install ledger state before checking config stability", () => { + const script = readFileSync(PLUGIN_UPDATE_DOCKER_SCRIPT, "utf8"); + const configSeedStart = script.indexOf('cat > \\"\\$HOME/.openclaw/openclaw.json\\"'); + const configSeedEnd = script.indexOf('cat > \\"\\$HOME/.openclaw/plugins/installs.json\\"'); + const configSeed = script.slice(configSeedStart, configSeedEnd); + + expect(configSeedStart).toBeGreaterThanOrEqual(0); + expect(configSeedEnd).toBeGreaterThan(configSeedStart); + expect(configSeed).toContain('\\"plugins\\": {}'); + expect(configSeed).not.toContain('\\"installs\\"'); + expect(script).toContain('\\"installRecords\\": {'); + expect(script).toContain('\\"lossless-claw\\": {'); + }); +}); From 1fc5b2b7032c12f64c2ce0011d52c89d4b72f456 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 27 Apr 2026 00:34:29 -0700 Subject: [PATCH 03/12] feat(migrations): add plugin-owned Hermes import * feat: add migration providers * feat: offer Hermes migration during onboarding * feat(hermes): map imported config surfaces * feat(onboard): require fresh migration imports * docs(cli): clarify Hermes import coverage * chore(migrations): rename Hermes importer package * chore(migrations): rewire Hermes importer id * fix(migrations): redact migration JSON details * fix(hermes): use provider runtime for config imports * test(hermes): cover missing source planning --------- Co-authored-by: Peter Steinberger --- .github/labeler.yml | 5 + CHANGELOG.md | 1 + .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/.i18n/glossary.zh-CN.json | 16 + docs/cli/migrate.md | 75 +++ docs/cli/onboard.md | 6 + docs/cli/setup.md | 7 +- docs/docs.json | 1 + docs/plugins/manifest.md | 2 + docs/plugins/sdk-subpaths.md | 14 +- extensions/migrate-hermes/apply.ts | 70 +++ extensions/migrate-hermes/config.test.ts | 224 +++++++++ extensions/migrate-hermes/config.ts | 434 ++++++++++++++++ .../migrate-hermes/files-and-skills.test.ts | 204 ++++++++ extensions/migrate-hermes/helpers.ts | 134 +++++ extensions/migrate-hermes/index.ts | 11 + extensions/migrate-hermes/items.ts | 113 +++++ extensions/migrate-hermes/model.apply.test.ts | 188 +++++++ extensions/migrate-hermes/model.plan.test.ts | 107 ++++ extensions/migrate-hermes/model.ts | 87 ++++ .../migrate-hermes/openclaw.plugin.json | 13 + extensions/migrate-hermes/package.json | 27 + extensions/migrate-hermes/plan.ts | 162 ++++++ .../provider.secret-failure.test.ts | 99 ++++ extensions/migrate-hermes/provider.test.ts | 129 +++++ extensions/migrate-hermes/provider.ts | 35 ++ extensions/migrate-hermes/secrets.test.ts | 159 ++++++ extensions/migrate-hermes/secrets.ts | 118 +++++ extensions/migrate-hermes/skills.ts | 70 +++ extensions/migrate-hermes/source.ts | 74 +++ extensions/migrate-hermes/targets.ts | 30 ++ .../migrate-hermes/test/provider-helpers.ts | 65 +++ .../src/bot-native-commands.registry.test.ts | 1 + package.json | 8 + pnpm-lock.yaml | 13 + scripts/lib/plugin-sdk-entrypoints.json | 2 + src/agents/agent-scope.test.ts | 54 ++ src/agents/agent-scope.ts | 38 ++ src/cli/command-catalog.ts | 1 + src/cli/program/command-registry-core.ts | 5 + src/cli/program/core-command-descriptors.ts | 5 + src/cli/program/register.migrate.ts | 117 +++++ src/cli/program/register.onboard.test.ts | 22 + src/cli/program/register.onboard.ts | 10 +- src/cli/program/register.setup.test.ts | 21 + src/cli/program/register.setup.ts | 9 + src/commands/migrate.test.ts | 471 ++++++++++++++++++ src/commands/migrate.ts | 162 ++++++ src/commands/migrate/apply.ts | 86 ++++ src/commands/migrate/context.ts | 51 ++ src/commands/migrate/output.ts | 103 ++++ src/commands/migrate/providers.ts | 38 ++ src/commands/migrate/types.ts | 21 + src/commands/onboard-types.ts | 5 +- src/flows/channel-setup.test.ts | 1 + src/gateway/server-plugins.test.ts | 1 + src/gateway/test-helpers.plugin-registry.ts | 1 + src/plugin-sdk/agent-runtime.ts | 1 + src/plugin-sdk/migration-runtime.test.ts | 123 +++++ src/plugin-sdk/migration-runtime.ts | 186 +++++++ src/plugin-sdk/migration.ts | 153 ++++++ src/plugin-sdk/plugin-entry.ts | 14 + src/plugin-sdk/provider-auth.ts | 1 + src/plugins/api-builder.ts | 3 + .../bundled-capability-metadata.test.ts | 8 + .../bundled-capability-runtime.test.ts | 23 +- src/plugins/bundled-capability-runtime.ts | 11 + src/plugins/captured-registration.ts | 7 + .../inventory/bundled-capability-metadata.ts | 5 + .../contracts/registry.contract.test.ts | 16 + src/plugins/contracts/registry.retry.test.ts | 8 + src/plugins/contracts/registry.ts | 6 + src/plugins/hooks.test-helpers.ts | 1 + .../installed-plugin-index-record-builder.ts | 1 + src/plugins/installed-plugin-index.test.ts | 37 ++ src/plugins/loader.ts | 4 + src/plugins/manifest-contract-runtime.ts | 53 ++ src/plugins/manifest-registry.ts | 3 +- src/plugins/manifest.ts | 3 + .../migration-provider-runtime.test.ts | 214 ++++++++ src/plugins/migration-provider-runtime.ts | 117 +++++ src/plugins/plugin-registry-snapshot.ts | 7 +- src/plugins/plugin-registry.test.ts | 36 ++ src/plugins/registry-empty.ts | 1 + src/plugins/registry-types.ts | 5 + src/plugins/registry.ts | 13 + src/plugins/status.test-helpers.ts | 2 + src/plugins/status.ts | 1 + src/plugins/types.ts | 96 ++++ src/test-utils/channel-plugins.ts | 1 + src/trajectory/metadata.test.ts | 1 + src/wizard/setup.migration-import.test.ts | 61 +++ src/wizard/setup.migration-import.ts | 304 +++++++++++ src/wizard/setup.ts | 43 +- test/helpers/plugins/plugin-api.ts | 1 + test/setup-openclaw-runtime.ts | 1 + 96 files changed, 5477 insertions(+), 24 deletions(-) create mode 100644 docs/cli/migrate.md create mode 100644 extensions/migrate-hermes/apply.ts create mode 100644 extensions/migrate-hermes/config.test.ts create mode 100644 extensions/migrate-hermes/config.ts create mode 100644 extensions/migrate-hermes/files-and-skills.test.ts create mode 100644 extensions/migrate-hermes/helpers.ts create mode 100644 extensions/migrate-hermes/index.ts create mode 100644 extensions/migrate-hermes/items.ts create mode 100644 extensions/migrate-hermes/model.apply.test.ts create mode 100644 extensions/migrate-hermes/model.plan.test.ts create mode 100644 extensions/migrate-hermes/model.ts create mode 100644 extensions/migrate-hermes/openclaw.plugin.json create mode 100644 extensions/migrate-hermes/package.json create mode 100644 extensions/migrate-hermes/plan.ts create mode 100644 extensions/migrate-hermes/provider.secret-failure.test.ts create mode 100644 extensions/migrate-hermes/provider.test.ts create mode 100644 extensions/migrate-hermes/provider.ts create mode 100644 extensions/migrate-hermes/secrets.test.ts create mode 100644 extensions/migrate-hermes/secrets.ts create mode 100644 extensions/migrate-hermes/skills.ts create mode 100644 extensions/migrate-hermes/source.ts create mode 100644 extensions/migrate-hermes/targets.ts create mode 100644 extensions/migrate-hermes/test/provider-helpers.ts create mode 100644 src/cli/program/register.migrate.ts create mode 100644 src/commands/migrate.test.ts create mode 100644 src/commands/migrate.ts create mode 100644 src/commands/migrate/apply.ts create mode 100644 src/commands/migrate/context.ts create mode 100644 src/commands/migrate/output.ts create mode 100644 src/commands/migrate/providers.ts create mode 100644 src/commands/migrate/types.ts create mode 100644 src/plugin-sdk/migration-runtime.test.ts create mode 100644 src/plugin-sdk/migration-runtime.ts create mode 100644 src/plugin-sdk/migration.ts create mode 100644 src/plugins/manifest-contract-runtime.ts create mode 100644 src/plugins/migration-provider-runtime.test.ts create mode 100644 src/plugins/migration-provider-runtime.ts create mode 100644 src/wizard/setup.migration-import.test.ts create mode 100644 src/wizard/setup.migration-import.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index f2391091284..84e4f084753 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -35,6 +35,11 @@ - any-glob-to-any-file: - "extensions/google-meet/**" - "docs/plugins/google-meet.md" +"plugin: migrate-hermes": + - changed-files: + - any-glob-to-any-file: + - "extensions/migrate-hermes/**" + - "docs/cli/migrate.md" "plugin: bonjour": - changed-files: - any-glob-to-any-file: diff --git a/CHANGELOG.md b/CHANGELOG.md index 69385c0f3c5..fc966249f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Docs: https://docs.openclaw.ai - Control UI: polish the quick settings dashboard grid so common cards align across desktop, tablet, and mobile layouts without wasting horizontal space. Thanks @BunsDev. - Matrix/E2EE: add `openclaw matrix encryption setup` to enable Matrix encryption, bootstrap recovery, and print verification status from one setup flow. Thanks @gumadeiras. - Agents/compaction: add an opt-in `agents.defaults.compaction.maxActiveTranscriptBytes` preflight trigger that runs normal local compaction when the active JSONL grows too large, requiring transcript rotation so successful compaction moves future turns onto a smaller successor file instead of raw byte-splitting history. Thanks @vincentkoc. +- CLI/migration: add `openclaw migrate` with plan, dry-run, JSON, pre-migration backup, onboarding detection, archive-only report copies, and a bundled Hermes importer for configuration, memory/plugin hints, model providers, MCP servers, skills, and supported credentials. Thanks @NousResearch. ### Fixes diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 2d76d9b4abf..e5b23fb4e6d 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -2a3fb85feb7420de8b166a695c3693dcc1eaa7a7f31de0dd139da856f10b2085 plugin-sdk-api-baseline.json -6bdb96f7f92c34d7ae698784c0073343c34fb4274ab7eeded49acebb81056074 plugin-sdk-api-baseline.jsonl +8371f19a19ceeae4eb20fbfe8e68e51f6f54f42c487d7d5c75f214ab1ba0922a plugin-sdk-api-baseline.json +a5f5e15e75f8cf27ebaa1302cfe0488974edd53121279c1e90705bc531a4761a plugin-sdk-api-baseline.jsonl diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 3a16f990b14..f48391bc82f 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -435,6 +435,22 @@ "source": "Setup", "target": "设置" }, + { + "source": "Migrate", + "target": "迁移" + }, + { + "source": "Migration", + "target": "迁移" + }, + { + "source": "Hermes", + "target": "Hermes" + }, + { + "source": "Archive-only", + "target": "仅归档" + }, { "source": "Channel Plugin SDK", "target": "渠道插件 SDK" diff --git a/docs/cli/migrate.md b/docs/cli/migrate.md new file mode 100644 index 00000000000..635dc6313cb --- /dev/null +++ b/docs/cli/migrate.md @@ -0,0 +1,75 @@ +--- +summary: "CLI reference for importing state from another agent system" +read_when: + - You want to migrate from Hermes or another agent system into OpenClaw + - You are adding a plugin-owned migration provider +title: "Migrate" +--- + +# `openclaw migrate` + +Import state from another agent system through a plugin-owned migration provider. + +```bash +openclaw migrate list +openclaw migrate hermes --dry-run +openclaw migrate hermes +openclaw migrate apply hermes --yes +openclaw migrate apply hermes --include-secrets --yes +openclaw onboard --flow import +openclaw onboard --import-from hermes --import-source ~/.hermes +``` + +## Safety model + +`openclaw migrate` is preview-first. The provider returns an itemized plan before anything changes, including conflicts, skipped items, and sensitive items. JSON plans, apply output, and migration reports redact nested secret-looking keys such as API keys, tokens, authorization headers, cookies, and passwords. + +`openclaw migrate apply ` previews the plan and prompts before changing state unless `--yes` is set. In non-interactive mode, apply requires `--yes`. With `--json` and no `--yes`, apply prints the JSON plan and does not mutate state. + +Apply creates and verifies an OpenClaw backup before applying the migration. If no local OpenClaw state exists yet, the backup step is skipped and the migration can continue. To skip a backup when state exists, pass both `--no-backup` and `--force`. + +Apply mode refuses to continue when the plan has conflicts. Review the plan, then rerun with `--overwrite` if replacing existing targets is intentional. Providers may still write item-level backups for overwritten files in the migration report directory. + +Secrets are never imported by default. Use `--include-secrets` to import supported credentials. + +## Hermes + +The bundled Hermes provider detects Hermes state at `~/.hermes` by default. Use `--from ` when Hermes lives elsewhere. + +The Hermes migration can import: + +- default model configuration from `config.yaml` +- configured model providers and custom OpenAI-compatible endpoints from `providers` and `custom_providers` +- MCP server definitions from `mcp_servers` or `mcp.servers` +- `SOUL.md` and `AGENTS.md` into the OpenClaw agent workspace +- `memories/MEMORY.md` and `memories/USER.md` by appending them to workspace memory files +- memory config defaults for OpenClaw file memory, plus archive/manual-review items for external memory providers such as Honcho +- skills with a `SKILL.md` file from `skills//` +- per-skill config values from `skills.config` +- supported API keys from `.env`, only with `--include-secrets` + +Archive-only Hermes state is copied into the migration report for manual review, but it is not loaded into live OpenClaw config or credentials. This preserves opaque or unsafe state such as `plugins/`, `sessions/`, `logs/`, `cron/`, `mcp-tokens/`, `auth.json`, and `state.db` without pretending OpenClaw can execute or trust it automatically. + +Supported Hermes `.env` keys include `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `GOOGLE_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY`, `XAI_API_KEY`, `MISTRAL_API_KEY`, and `DEEPSEEK_API_KEY`. + +After applying a migration, run: + +```bash +openclaw doctor +``` + +## Plugin contract + +Migration sources are plugins. A plugin declares its provider ids in `openclaw.plugin.json`: + +```json +{ + "contracts": { + "migrationProviders": ["hermes"] + } +} +``` + +At runtime the plugin calls `api.registerMigrationProvider(...)`. The provider implements `detect`, `plan`, and `apply`; core owns CLI orchestration, backup policy, prompts, JSON output, and conflict preflight. Core passes the reviewed plan into `apply(ctx, plan)`, and providers may rebuild the plan only when that argument is absent for compatibility. Provider plugins can use `openclaw/plugin-sdk/migration` for item construction and summary counts, plus `openclaw/plugin-sdk/migration-runtime` for conflict-aware file copies, archive-only report copies, and migration reports. + +Onboarding can also offer migration when a provider detects a known source. `openclaw onboard --flow import` and `openclaw setup --wizard --import-from hermes` use the same plugin migration provider and still show a preview before applying. Onboarding imports require a fresh OpenClaw setup; reset config, credentials, sessions, and the workspace first if you already have local state. Backup plus overwrite or merge imports are feature-gated for existing setups. diff --git a/docs/cli/onboard.md b/docs/cli/onboard.md index 8494367a293..e0c715034f2 100644 --- a/docs/cli/onboard.md +++ b/docs/cli/onboard.md @@ -36,10 +36,14 @@ openclaw onboard openclaw onboard --modern openclaw onboard --flow quickstart openclaw onboard --flow manual +openclaw onboard --flow import +openclaw onboard --import-from hermes --import-source ~/.hermes openclaw onboard --skip-bootstrap openclaw onboard --mode remote --remote-url wss://gateway-host:18789 ``` +`--flow import` uses plugin-owned migration providers such as Hermes. It only runs against a fresh OpenClaw setup; if existing config, credentials, sessions, or workspace memory/identity files are present, reset or choose a fresh setup before importing. + `--modern` starts the Crestodian conversational onboarding preview. Without `--modern`, `openclaw onboard` keeps the classic onboarding flow. @@ -176,6 +180,7 @@ openclaw onboard --non-interactive \ - `quickstart`: minimal prompts, auto-generates a gateway token. - `manual`: full prompts for port, bind, and auth (alias of `advanced`). + - `import`: runs a detected migration provider, previews the plan, then applies after confirmation. When an auth choice implies a preferred provider, onboarding prefilters the default-model and allowlist pickers to that provider. For Volcengine and BytePlus, this also matches the coding-plan variants (`volcengine-plan/*`, `byteplus-plan/*`). @@ -194,6 +199,7 @@ openclaw onboard --non-interactive \ - Local onboarding DM scope behavior: [CLI setup reference](/start/wizard-cli-reference#outputs-and-internals). - Fastest first chat: `openclaw dashboard` (Control UI, no channel setup). - Custom provider: connect any OpenAI or Anthropic compatible endpoint, including hosted providers not listed. Use Unknown to auto-detect. + - If Hermes state is detected, onboarding offers a migration flow. Use [Migrate](/cli/migrate) for dry-run plans, overwrite mode, reports, and exact mappings. diff --git a/docs/cli/setup.md b/docs/cli/setup.md index 2a0d6fc50d4..b1fbfcc2338 100644 --- a/docs/cli/setup.md +++ b/docs/cli/setup.md @@ -21,6 +21,7 @@ Related: openclaw setup openclaw setup --workspace ~/.openclaw/workspace openclaw setup --wizard +openclaw setup --wizard --import-from hermes --import-source ~/.hermes openclaw setup --non-interactive --mode remote --remote-url wss://gateway-host:18789 --remote-token ``` @@ -30,6 +31,9 @@ openclaw setup --non-interactive --mode remote --remote-url wss://gateway-host:1 - `--wizard`: run onboarding - `--non-interactive`: run onboarding without prompts - `--mode `: onboarding mode +- `--import-from `: migration provider to run during onboarding +- `--import-source `: source agent home for `--import-from` +- `--import-secrets`: import supported secrets during onboarding migration - `--remote-url `: remote Gateway WebSocket URL - `--remote-token `: remote Gateway token @@ -42,7 +46,8 @@ openclaw setup --wizard Notes: - Plain `openclaw setup` initializes config + workspace without the full onboarding flow. -- Onboarding auto-runs when any onboarding flags are present (`--wizard`, `--non-interactive`, `--mode`, `--remote-url`, `--remote-token`). +- Onboarding auto-runs when any onboarding flags are present (`--wizard`, `--non-interactive`, `--mode`, `--import-from`, `--import-source`, `--import-secrets`, `--remote-url`, `--remote-token`). +- If Hermes state is detected, interactive onboarding can offer migration automatically. Import onboarding requires a fresh setup; use [Migrate](/cli/migrate) for dry-run plans, backups, and overwrite mode outside onboarding. ## Related diff --git a/docs/docs.json b/docs/docs.json index 11e5a8d93e5..27e58e0c0c0 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1545,6 +1545,7 @@ "cli/gateway", "cli/health", "cli/logs", + "cli/migrate", "cli/onboard", "cli/reset", "cli/secrets", diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index 481c3896fe8..afac720bd5e 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -429,6 +429,7 @@ read without importing the plugin runtime. "videoGenerationProviders": ["qwen"], "webFetchProviders": ["firecrawl"], "webSearchProviders": ["gemini"], + "migrationProviders": ["hermes"], "tools": ["firecrawl_search", "firecrawl_scrape"] } } @@ -450,6 +451,7 @@ Each list is optional: | `videoGenerationProviders` | `string[]` | Video-generation provider ids this plugin owns. | | `webFetchProviders` | `string[]` | Web-fetch provider ids this plugin owns. | | `webSearchProviders` | `string[]` | Web-search provider ids this plugin owns. | +| `migrationProviders` | `string[]` | Import provider ids this plugin owns for `openclaw migrate`. | | `tools` | `string[]` | Agent tool names this plugin owns for bundled contract checks. | `contracts.embeddedExtensionFactories` is retained for bundled Codex diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 19c1256f6fe..e8b497f66b3 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -16,12 +16,14 @@ For the plugin authoring guide, see [Plugin SDK overview](/plugins/sdk-overview) ## Plugin entry -| Subpath | Key exports | -| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `plugin-sdk/plugin-entry` | `definePluginEntry` | -| `plugin-sdk/core` | `defineChannelPluginEntry`, `createChatChannelPlugin`, `createChannelPluginBase`, `defineSetupPluginEntry`, `buildChannelConfigSchema` | -| `plugin-sdk/config-schema` | `OpenClawSchema` | -| `plugin-sdk/provider-entry` | `defineSingleProviderPluginEntry` | +| Subpath | Key exports | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| `plugin-sdk/plugin-entry` | `definePluginEntry` | +| `plugin-sdk/core` | `defineChannelPluginEntry`, `createChatChannelPlugin`, `createChannelPluginBase`, `defineSetupPluginEntry`, `buildChannelConfigSchema` | +| `plugin-sdk/config-schema` | `OpenClawSchema` | +| `plugin-sdk/provider-entry` | `defineSingleProviderPluginEntry` | +| `plugin-sdk/migration` | Migration provider item helpers such as `createMigrationItem`, reason constants, item status markers, redaction helpers, and `summarizeMigrationItems` | +| `plugin-sdk/migration-runtime` | Runtime migration helpers such as `copyMigrationFileItem` and `writeMigrationReport` | diff --git a/extensions/migrate-hermes/apply.ts b/extensions/migrate-hermes/apply.ts new file mode 100644 index 00000000000..1af8852b4fd --- /dev/null +++ b/extensions/migrate-hermes/apply.ts @@ -0,0 +1,70 @@ +import path from "node:path"; +import { summarizeMigrationItems } from "openclaw/plugin-sdk/migration"; +import { + archiveMigrationItem, + copyMigrationFileItem, + writeMigrationReport, +} from "openclaw/plugin-sdk/migration-runtime"; +import type { + MigrationApplyResult, + MigrationItem, + MigrationPlan, + MigrationProviderContext, +} from "openclaw/plugin-sdk/plugin-entry"; +import { applyConfigItem, applyManualItem } from "./config.js"; +import { appendItem } from "./helpers.js"; +import { applyModelItem } from "./model.js"; +import { buildHermesPlan } from "./plan.js"; +import { applySecretItem } from "./secrets.js"; +import { resolveTargets } from "./targets.js"; + +export async function applyHermesPlan(params: { + ctx: MigrationProviderContext; + plan?: MigrationPlan; + runtime?: MigrationProviderContext["runtime"]; +}): Promise { + const plan = params.plan ?? (await buildHermesPlan(params.ctx)); + const reportDir = params.ctx.reportDir ?? path.join(params.ctx.stateDir, "migration", "hermes"); + const targets = resolveTargets(params.ctx); + const items: MigrationItem[] = []; + for (const item of plan.items) { + if (item.status !== "planned") { + items.push(item); + continue; + } + if (item.id === "config:default-model") { + items.push( + await applyModelItem( + { ...params.ctx, runtime: params.ctx.runtime ?? params.runtime }, + item, + ), + ); + } else if (item.kind === "config") { + items.push( + await applyConfigItem( + { ...params.ctx, runtime: params.ctx.runtime ?? params.runtime }, + item, + ), + ); + } else if (item.kind === "manual") { + items.push(applyManualItem(item)); + } else if (item.action === "archive") { + items.push(await archiveMigrationItem(item, reportDir)); + } else if (item.kind === "secret") { + items.push(await applySecretItem(params.ctx, item, targets)); + } else if (item.action === "append") { + items.push(await appendItem(item)); + } else { + items.push(await copyMigrationFileItem(item, reportDir, { overwrite: params.ctx.overwrite })); + } + } + const result: MigrationApplyResult = { + ...plan, + items, + summary: summarizeMigrationItems(items), + backupPath: params.ctx.backupPath, + reportDir, + }; + await writeMigrationReport(result, { title: "Hermes Migration Report" }); + return result; +} diff --git a/extensions/migrate-hermes/config.test.ts b/extensions/migrate-hermes/config.test.ts new file mode 100644 index 00000000000..4d8de242592 --- /dev/null +++ b/extensions/migrate-hermes/config.test.ts @@ -0,0 +1,224 @@ +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +function makeConfigRuntime(config: Record) { + return { + config: { + loadConfig: () => config, + writeConfigFile: async (next: Record) => { + Object.keys(config).forEach((key) => { + delete config[key]; + }); + Object.assign(config, next); + return next; + }, + }, + } as never; +} + +describe("Hermes migration config mapping", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("plans provider, MCP, skill, and memory plugin config as plugin-owned items", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + [ + "model:", + " provider: openai", + " model: gpt-5.4", + "providers:", + " openai:", + " base_url: https://api.openai.example/v1", + " api_key_env: OPENAI_API_KEY", + " models: [gpt-5.4]", + "custom_providers:", + " - name: local-llm", + " base_url: http://127.0.0.1:11434/v1", + " models: [local-model]", + "memory:", + " provider: honcho", + " honcho:", + " project: hermes", + "skills:", + " config:", + " ship-it:", + " mode: fast", + "mcp_servers:", + " time:", + " command: npx", + " args: ['-y', 'mcp-server-time']", + "", + ].join("\n"), + ); + await writeFile(path.join(source, "memories", "MEMORY.md"), "memory line\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:memory-plugin:honcho", + kind: "config", + action: "merge", + target: "plugins.entries.honcho", + }), + expect.objectContaining({ + id: "manual:memory-provider:honcho", + kind: "manual", + status: "skipped", + }), + expect.objectContaining({ + id: "config:model-providers", + details: expect.objectContaining({ + value: expect.objectContaining({ + openai: expect.objectContaining({ + baseUrl: "https://api.openai.example/v1", + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }), + "local-llm": expect.objectContaining({ + baseUrl: "http://127.0.0.1:11434/v1", + }), + }), + }), + }), + expect.objectContaining({ + id: "config:mcp-servers", + details: expect.objectContaining({ + value: { + time: { + command: "npx", + args: ["-y", "mcp-server-time"], + }, + }, + }), + }), + expect.objectContaining({ + id: "config:skill-entries", + details: expect.objectContaining({ + value: { + "ship-it": { + config: { + mode: "fast", + }, + }, + }, + }), + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([expect.stringContaining("manual review")]), + ); + }); + + it("applies mapped config items through the migration runtime config writer", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const config: Record = { + agents: { defaults: { workspace: workspaceDir } }, + }; + await writeFile( + path.join(source, "config.yaml"), + [ + "providers:", + " openai:", + " api_key_env: OPENAI_API_KEY", + " models: [gpt-5.4]", + "mcp_servers:", + " time:", + " command: npx", + "skills:", + " config:", + " ship-it:", + " mode: fast", + "", + ].join("\n"), + ); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + runtime: makeConfigRuntime(config), + }), + ); + + expect(result.summary.errors).toBe(0); + expect(config).toMatchObject({ + models: { + providers: { + openai: { + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }, + }, + }, + mcp: { + servers: { + time: { + command: "npx", + }, + }, + }, + skills: { + entries: { + "ship-it": { + config: { + mode: "fast", + }, + }, + }, + }, + }); + }); + + it("uses the provider runtime for CLI-applied config items", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const config: Record = { + agents: { defaults: { workspace: workspaceDir } }, + }; + await writeFile( + path.join(source, "config.yaml"), + [ + "mcp_servers:", + " time:", + " command: npx", + " env:", + " OPENAI_API_KEY: short-dev-key", + "", + ].join("\n"), + ); + + const provider = buildHermesMigrationProvider({ runtime: makeConfigRuntime(config) }); + const result = await provider.apply(makeContext({ source, stateDir, workspaceDir })); + + expect(result.summary.errors).toBe(0); + expect(config).toMatchObject({ + mcp: { + servers: { + time: { + command: "npx", + env: { + OPENAI_API_KEY: "short-dev-key", + }, + }, + }, + }, + }); + }); +}); diff --git a/extensions/migrate-hermes/config.ts b/extensions/migrate-hermes/config.ts new file mode 100644 index 00000000000..82170ee5718 --- /dev/null +++ b/extensions/migrate-hermes/config.ts @@ -0,0 +1,434 @@ +import { + createMigrationItem, + markMigrationItemConflict, + markMigrationItemError, + markMigrationItemSkipped, + MIGRATION_REASON_TARGET_EXISTS, +} from "openclaw/plugin-sdk/migration"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { childRecord, isRecord, readString, readStringArray } from "./helpers.js"; + +type HermesProviderConfig = { + id: string; + baseUrl?: string; + apiKeyEnv?: string; + models: string[]; +}; + +type ConfigPatchDetails = { + path: string[]; + value: unknown; +}; + +const CONFIG_RUNTIME_UNAVAILABLE = "config runtime unavailable"; +const MISSING_CONFIG_PATCH = "missing config patch"; + +function envKeyForProvider(providerId: string): string { + return `${providerId.toUpperCase().replaceAll(/[^A-Z0-9]/gu, "_")}_API_KEY`; +} + +function splitProviderModel(modelRef: string | undefined): { provider?: string; model?: string } { + if (!modelRef) { + return {}; + } + const slash = modelRef.indexOf("/"); + if (slash > 0 && slash < modelRef.length - 1) { + return { provider: modelRef.slice(0, slash), model: modelRef.slice(slash + 1) }; + } + return { model: modelRef }; +} + +function modelDefinition(modelId: string, baseUrl?: string): Record { + return { + id: modelId, + name: modelId, + api: baseUrl ? "openai-completions" : "openai-responses", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8192, + ...(baseUrl ? { baseUrl } : {}), + metadataSource: "models-add", + }; +} + +function providerConfig(entry: HermesProviderConfig): Record { + const models = entry.models.length > 0 ? entry.models : [`${entry.id}/default`]; + return { + baseUrl: entry.baseUrl ?? "", + ...(entry.apiKeyEnv + ? { apiKey: { source: "env", provider: "default", id: entry.apiKeyEnv } } + : {}), + api: "openai-completions", + models: models.map((modelId) => modelDefinition(modelId, entry.baseUrl)), + }; +} + +export function collectHermesProviders( + config: Record, + modelRef?: string, +): HermesProviderConfig[] { + const collected: HermesProviderConfig[] = []; + for (const [id, raw] of Object.entries(childRecord(config, "providers"))) { + if (!isRecord(raw)) { + continue; + } + const baseUrl = + readString(raw.base_url) ?? + readString(raw.baseUrl) ?? + readString(raw.url) ?? + readString(raw.api); + const apiKeyEnv = + readString(raw.api_key_env) ?? + readString(raw.apiKeyEnv) ?? + readString(raw.env) ?? + envKeyForProvider(id); + const models = [ + ...readStringArray(raw.models), + ...Object.keys(childRecord(raw, "models")), + readString(raw.model), + ].filter((value): value is string => Boolean(value)); + collected.push({ id, baseUrl, apiKeyEnv, models: [...new Set(models)] }); + } + + const customProviders = config.custom_providers; + if (Array.isArray(customProviders)) { + for (const raw of customProviders) { + if (!isRecord(raw)) { + continue; + } + const id = readString(raw.name) ?? readString(raw.id); + if (!id) { + continue; + } + const baseUrl = readString(raw.base_url) ?? readString(raw.baseUrl) ?? readString(raw.url); + const apiKeyEnv = readString(raw.api_key_env) ?? readString(raw.apiKeyEnv); + const models = [ + ...readStringArray(raw.models), + ...Object.keys(childRecord(raw, "models")), + readString(raw.model), + ].filter((value): value is string => Boolean(value)); + collected.push({ id, baseUrl, apiKeyEnv, models: [...new Set(models)] }); + } + } + + const defaultRef = splitProviderModel(modelRef); + if (defaultRef.provider && !collected.some((entry) => entry.id === defaultRef.provider)) { + collected.push({ + id: defaultRef.provider, + apiKeyEnv: envKeyForProvider(defaultRef.provider), + models: defaultRef.model ? [defaultRef.model] : [], + }); + } + return collected; +} + +function mapMcpServers(raw: unknown): Record | undefined { + if (!isRecord(raw)) { + return undefined; + } + const mapped: Record = {}; + for (const [name, value] of Object.entries(raw)) { + if (!isRecord(value)) { + continue; + } + const next: Record = {}; + for (const key of [ + "command", + "args", + "env", + "cwd", + "workingDirectory", + "url", + "transport", + "headers", + "connectionTimeoutMs", + ]) { + if (value[key] !== undefined) { + next[key] = value[key]; + } + } + if (Object.keys(next).length > 0) { + mapped[name] = next; + } + } + return Object.keys(mapped).length > 0 ? mapped : undefined; +} + +function mapSkillEntries(config: Record): Record | undefined { + const entries: Record = {}; + for (const [skillKey, value] of Object.entries( + childRecord(childRecord(config, "skills"), "config"), + )) { + if (isRecord(value)) { + entries[skillKey] = { config: value }; + } + } + return Object.keys(entries).length > 0 ? entries : undefined; +} + +function readPath(root: Record, path: readonly string[]): unknown { + let current: unknown = root; + for (const segment of path) { + if (!isRecord(current)) { + return undefined; + } + current = current[segment]; + } + return current; +} + +function mergeValue(left: unknown, right: unknown): unknown { + if (!isRecord(left) || !isRecord(right)) { + return structuredClone(right); + } + const next: Record = { ...left }; + for (const [key, value] of Object.entries(right)) { + next[key] = mergeValue(next[key], value); + } + return next; +} + +function writePath(root: Record, path: readonly string[], value: unknown): void { + let current = root; + for (const segment of path.slice(0, -1)) { + const existing = current[segment]; + if (!isRecord(existing)) { + current[segment] = {}; + } + current = current[segment] as Record; + } + const leaf = path.at(-1); + if (!leaf) { + return; + } + current[leaf] = mergeValue(current[leaf], value); +} + +function hasPatchConflict( + config: MigrationProviderContext["config"], + path: readonly string[], + value: unknown, +): boolean { + if (!isRecord(value)) { + return readPath(config as Record, path) !== undefined; + } + const existing = readPath(config as Record, path); + if (!isRecord(existing)) { + return false; + } + return Object.keys(value).some((key) => existing[key] !== undefined); +} + +function createConfigPatchItem(params: { + id: string; + target: string; + path: string[]; + value: unknown; + message: string; + conflict?: boolean; +}): MigrationItem { + return createMigrationItem({ + id: params.id, + kind: "config", + action: "merge", + target: params.target, + status: params.conflict ? "conflict" : "planned", + reason: params.conflict ? MIGRATION_REASON_TARGET_EXISTS : undefined, + message: params.message, + details: { path: params.path, value: params.value }, + }); +} + +function createManualItem(params: { + id: string; + source: string; + message: string; + recommendation: string; +}): MigrationItem { + return createMigrationItem({ + id: params.id, + kind: "manual", + action: "manual", + source: params.source, + status: "skipped", + message: params.message, + reason: params.recommendation, + }); +} + +export function buildConfigItems(params: { + ctx: MigrationProviderContext; + config: Record; + modelRef?: string; + hasMemoryFiles?: boolean; +}): MigrationItem[] { + const items: MigrationItem[] = []; + const memory = childRecord(params.config, "memory"); + const memoryProvider = readString(memory.provider); + + if (params.hasMemoryFiles || memoryProvider) { + items.push( + createConfigPatchItem({ + id: "config:memory", + target: "memory", + path: ["memory"], + value: { backend: "builtin" }, + message: "Use OpenClaw built-in file memory for imported Hermes memory files.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["memory"], { backend: true }), + }), + ); + items.push( + createConfigPatchItem({ + id: "config:memory-plugin-slot", + target: "plugins.slots", + path: ["plugins", "slots"], + value: { memory: "memory-core" }, + message: "Select the default OpenClaw memory plugin for imported file memory.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["plugins", "slots"], { memory: true }), + }), + ); + } + + if (memoryProvider === "honcho") { + const value = { + honcho: { + enabled: true, + config: childRecord(memory, "honcho"), + }, + }; + items.push( + createConfigPatchItem({ + id: "config:memory-plugin:honcho", + target: "plugins.entries.honcho", + path: ["plugins", "entries"], + value, + message: "Preserve Hermes Honcho memory settings as a plugin entry for manual activation.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["plugins", "entries"], value), + }), + ); + items.push( + createManualItem({ + id: "manual:memory-provider:honcho", + source: "config.yaml:memory.provider", + message: + "Hermes used Honcho memory. OpenClaw keeps built-in memory selected until the matching plugin is installed and reviewed.", + recommendation: + "Install or review the Honcho memory plugin before selecting it for plugins.slots.memory.", + }), + ); + } else if (memoryProvider && !["builtin", "file", "files"].includes(memoryProvider)) { + items.push( + createManualItem({ + id: `manual:memory-provider:${memoryProvider}`, + source: "config.yaml:memory.provider", + message: `Hermes memory provider "${memoryProvider}" does not have a known OpenClaw mapping.`, + recommendation: "Install or configure an equivalent OpenClaw memory plugin manually.", + }), + ); + } + + const providers = collectHermesProviders(params.config, params.modelRef); + if (providers.length > 0) { + const value = Object.fromEntries(providers.map((entry) => [entry.id, providerConfig(entry)])); + items.push( + createConfigPatchItem({ + id: "config:model-providers", + target: "models.providers", + path: ["models", "providers"], + value, + message: "Import Hermes provider and custom endpoint config.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["models", "providers"], value), + }), + ); + } + + const mcpConfig = params.config.mcp; + const rawMcpServers = + params.config.mcp_servers ?? + (isRecord(mcpConfig) && isRecord(mcpConfig.servers) ? mcpConfig.servers : mcpConfig); + const mcpServers = mapMcpServers(rawMcpServers); + if (mcpServers) { + items.push( + createConfigPatchItem({ + id: "config:mcp-servers", + target: "mcp.servers", + path: ["mcp", "servers"], + value: mcpServers, + message: "Import Hermes MCP server definitions.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["mcp", "servers"], mcpServers), + }), + ); + } + + const skillEntries = mapSkillEntries(params.config); + if (skillEntries) { + items.push( + createConfigPatchItem({ + id: "config:skill-entries", + target: "skills.entries", + path: ["skills", "entries"], + value: skillEntries, + message: "Import Hermes skill config values.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["skills", "entries"], skillEntries), + }), + ); + } + + return items; +} + +function readConfigPatchDetails(item: MigrationItem): ConfigPatchDetails | undefined { + const path = item.details?.path; + if ( + !Array.isArray(path) || + !path.every((segment): segment is string => typeof segment === "string") + ) { + return undefined; + } + return { path, value: item.details?.value }; +} + +export async function applyConfigItem( + ctx: MigrationProviderContext, + item: MigrationItem, +): Promise { + if (item.status !== "planned") { + return item; + } + const details = readConfigPatchDetails(item); + if (!details) { + return markMigrationItemError(item, MISSING_CONFIG_PATCH); + } + if (!ctx.runtime?.config.writeConfigFile) { + return markMigrationItemError(item, CONFIG_RUNTIME_UNAVAILABLE); + } + try { + const nextConfig = structuredClone(ctx.runtime.config.loadConfig?.() ?? ctx.config); + if (!ctx.overwrite && hasPatchConflict(nextConfig, details.path, details.value)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + writePath(nextConfig as Record, details.path, details.value); + await ctx.runtime.config.writeConfigFile(nextConfig); + return { ...item, status: "migrated" }; + } catch (err) { + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export function applyManualItem(item: MigrationItem): MigrationItem { + return markMigrationItemSkipped(item, item.reason ?? "manual follow-up required"); +} diff --git a/extensions/migrate-hermes/files-and-skills.test.ts b/extensions/migrate-hermes/files-and-skills.test.ts new file mode 100644 index 00000000000..8f8f0d4bb43 --- /dev/null +++ b/extensions/migrate-hermes/files-and-skills.test.ts @@ -0,0 +1,204 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { MIGRATION_REASON_TARGET_EXISTS } from "openclaw/plugin-sdk/migration"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration file and skill items", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + function configRuntime(config: Record) { + return { + config: { + loadConfig: () => config, + writeConfigFile: async (next: Record) => { + Object.keys(config).forEach((key) => { + delete config[key]; + }); + Object.assign(config, next); + return next; + }, + }, + } as never; + } + + it("reports normalized skill-name collisions instead of overwriting during apply", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(source, "skills", "ship-it", "SKILL.md"), "# ship-it\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + const skillItems = plan.items.filter((item) => item.kind === "skill"); + + expect(skillItems).toHaveLength(2); + expect(skillItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "skill:ship-it", + status: "conflict", + reason: 'multiple Hermes skill directories normalize to "ship-it"', + target: path.join(workspaceDir, "skills", "ship-it"), + }), + ]), + ); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + overwrite: true, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.summary.conflicts).toBe(2); + await expect(fs.access(path.join(workspaceDir, "skills", "ship-it"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("reports late-created copy targets as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, "AGENTS.md"), "# Hermes agents\n"); + + const provider = buildHermesMigrationProvider(); + const ctx = makeContext({ source, stateDir, workspaceDir, reportDir }); + const plan = await provider.plan(ctx); + await writeFile(path.join(workspaceDir, "AGENTS.md"), "# Late agents\n"); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "workspace:AGENTS.md", + status: "conflict", + reason: MIGRATION_REASON_TARGET_EXISTS, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + expect(await fs.readFile(path.join(workspaceDir, "AGENTS.md"), "utf8")).toBe("# Late agents\n"); + }); + + it("applies files, appended memories, item backups, reports, and opt-in API keys", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile(path.join(source, "AGENTS.md"), "# Hermes agents\n"); + await writeFile(path.join(source, "memories", "MEMORY.md"), "memory line\n"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(workspaceDir, "AGENTS.md"), "# Existing agents\n"); + + const provider = buildHermesMigrationProvider(); + const config: Record = {}; + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + includeSecrets: true, + overwrite: true, + reportDir, + runtime: configRuntime(config), + }), + ); + + expect(result.summary.errors).toBe(0); + expect(result.summary.conflicts).toBe(0); + expect(await fs.readFile(path.join(workspaceDir, "AGENTS.md"), "utf8")).toBe( + "# Hermes agents\n", + ); + expect( + await fs.readFile(path.join(workspaceDir, "skills", "ship-it", "SKILL.md"), "utf8"), + ).toBe("# Ship It\n"); + await expect(fs.access(path.join(reportDir, "summary.md"))).resolves.toBeUndefined(); + expect(await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf8")).toContain( + "Imported from Hermes", + ); + const copiedAgentsItem = result.items.find((item) => item.id === "workspace:AGENTS.md"); + expect(copiedAgentsItem?.details?.backupPath).toEqual(expect.stringContaining("AGENTS.md")); + const authStore = JSON.parse( + await fs.readFile( + path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), + "utf8", + ), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]).toMatchObject({ + provider: "openai", + key: "sk-hermes", + }); + }); + + it("archives unsupported Hermes state into the report without importing it", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, "logs", "session.log"), "log line\n"); + await writeFile(path.join(source, "auth.json"), '{"token":"opaque"}\n'); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir, reportDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "archive:logs", + kind: "archive", + action: "archive", + status: "planned", + }), + expect.objectContaining({ + id: "archive:auth.json", + kind: "archive", + action: "archive", + status: "planned", + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([expect.stringContaining("archive-only")]), + ); + + const result = await provider.apply(makeContext({ source, stateDir, workspaceDir, reportDir })); + + expect(result.summary.errors).toBe(0); + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "archive:logs", + status: "migrated", + target: path.join(reportDir, "archive", "logs"), + }), + expect.objectContaining({ + id: "archive:auth.json", + status: "migrated", + target: path.join(reportDir, "archive", "auth.json"), + }), + ]), + ); + expect(await fs.readFile(path.join(reportDir, "archive", "logs", "session.log"), "utf8")).toBe( + "log line\n", + ); + await expect(fs.access(path.join(workspaceDir, "logs", "session.log"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); +}); diff --git a/extensions/migrate-hermes/helpers.ts b/extensions/migrate-hermes/helpers.ts new file mode 100644 index 00000000000..ad11ab6d7c0 --- /dev/null +++ b/extensions/migrate-hermes/helpers.ts @@ -0,0 +1,134 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + markMigrationItemError, + MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, +} from "openclaw/plugin-sdk/migration"; +import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { parse as parseYaml } from "yaml"; + +export function resolveHomePath(input: string): string { + if (input === "~") { + return os.homedir(); + } + if (input.startsWith("~/")) { + return path.join(os.homedir(), input.slice(2)); + } + return path.resolve(input); +} + +export async function exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +export async function isDirectory(dirPath: string): Promise { + try { + return (await fs.stat(dirPath)).isDirectory(); + } catch { + return false; + } +} + +export function sanitizeName(name: string): string { + return name + .trim() + .toLowerCase() + .replaceAll(/[^a-z0-9._-]+/g, "-") + .replaceAll(/^-+|-+$/g, ""); +} + +export async function readText(filePath: string | undefined): Promise { + if (!filePath) { + return undefined; + } + try { + return await fs.readFile(filePath, "utf8"); + } catch { + return undefined; + } +} + +export function parseEnv(content: string | undefined): Record { + const env: Record = {}; + if (!content) { + return env; + } + for (const line of content.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u.exec(trimmed); + if (!match) { + continue; + } + const key = match[1]; + let value = match[2] ?? ""; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[key] = value; + } + return env; +} + +export function parseHermesConfig(content: string | undefined): Record { + if (!content) { + return {}; + } + try { + const parsed = parseYaml(content); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +export function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +export function childRecord( + root: Record | undefined, + key: string, +): Record { + const value = root?.[key]; + return isRecord(value) ? value : {}; +} + +export function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +export function readStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.filter((entry): entry is string => typeof entry === "string" && entry.trim() !== ""); +} + +export async function appendItem(item: MigrationItem): Promise { + if (!item.source || !item.target) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const content = await fs.readFile(item.source, "utf8"); + const header = `\n\n\n\n`; + await fs.mkdir(path.dirname(item.target), { recursive: true }); + await fs.appendFile(item.target, `${header}${content.trimEnd()}\n`, "utf8"); + return { ...item, status: "migrated" }; + } catch (err) { + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} diff --git a/extensions/migrate-hermes/index.ts b/extensions/migrate-hermes/index.ts new file mode 100644 index 00000000000..ff87eba7bb5 --- /dev/null +++ b/extensions/migrate-hermes/index.ts @@ -0,0 +1,11 @@ +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { buildHermesMigrationProvider } from "./provider.js"; + +export default definePluginEntry({ + id: "migrate-hermes", + name: "Hermes Migration", + description: "Imports Hermes state into OpenClaw.", + register(api) { + api.registerMigrationProvider(buildHermesMigrationProvider({ runtime: api.runtime })); + }, +}); diff --git a/extensions/migrate-hermes/items.ts b/extensions/migrate-hermes/items.ts new file mode 100644 index 00000000000..598c7cb6de2 --- /dev/null +++ b/extensions/migrate-hermes/items.ts @@ -0,0 +1,113 @@ +import type { MigrationItem } from "openclaw/plugin-sdk/migration"; +import { + createMigrationItem, + markMigrationItemConflict, + markMigrationItemError, + markMigrationItemSkipped, +} from "openclaw/plugin-sdk/migration"; +import { readString } from "./helpers.js"; + +export type HermesModelDetails = { + model: string; +}; + +export type HermesSecretDetails = { + envVar: string; + provider: string; + profileId: string; +}; + +export type HermesModelItem = MigrationItem & { + id: "config:default-model"; + kind: "config"; + action: "skip" | "update"; + details: HermesModelDetails; +}; + +export type HermesSecretItem = MigrationItem & { + kind: "secret"; + action: "skip" | "create"; + details: HermesSecretDetails; +}; + +export const HERMES_REASON_ALREADY_CONFIGURED = "already configured"; +export const HERMES_REASON_DEFAULT_MODEL_CONFIGURED = "default model already configured"; +export const HERMES_REASON_INCLUDE_SECRETS = "use --include-secrets to import"; +export const HERMES_REASON_AUTH_PROFILE_EXISTS = "auth profile exists"; +export const HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE = "config runtime unavailable"; +export const HERMES_REASON_MISSING_SECRET_METADATA = "missing secret metadata"; +export const HERMES_REASON_SECRET_NO_LONGER_PRESENT = "secret no longer present"; +export const HERMES_REASON_AUTH_PROFILE_WRITE_FAILED = "failed to write auth profile"; + +export function createHermesModelItem(params: { + model: string; + currentModel?: string; + overwrite?: boolean; +}): HermesModelItem { + const alreadyConfigured = params.currentModel === params.model; + const conflict = Boolean(params.currentModel && !params.overwrite && !alreadyConfigured); + return createMigrationItem({ + id: "config:default-model", + kind: "config", + action: alreadyConfigured ? "skip" : "update", + target: "agents.defaults.model", + status: alreadyConfigured ? "skipped" : conflict ? "conflict" : "planned", + reason: alreadyConfigured + ? HERMES_REASON_ALREADY_CONFIGURED + : conflict + ? HERMES_REASON_DEFAULT_MODEL_CONFIGURED + : undefined, + details: { model: params.model }, + }) as HermesModelItem; +} + +export function readHermesModelDetails(item: MigrationItem): HermesModelDetails | undefined { + const model = readString(item.details?.model); + return model ? { model } : undefined; +} + +export function createHermesSecretItem(params: { + id: string; + source?: string; + target: string; + includeSecrets?: boolean; + existsAlready?: boolean; + details: HermesSecretDetails; +}): HermesSecretItem { + const skipped = !params.includeSecrets; + const conflict = Boolean(params.existsAlready && !skipped); + return createMigrationItem({ + id: params.id, + kind: "secret", + action: skipped ? "skip" : "create", + source: params.source, + target: params.target, + status: skipped ? "skipped" : conflict ? "conflict" : "planned", + sensitive: true, + reason: skipped + ? HERMES_REASON_INCLUDE_SECRETS + : conflict + ? HERMES_REASON_AUTH_PROFILE_EXISTS + : undefined, + details: params.details, + }) as HermesSecretItem; +} + +export function readHermesSecretDetails(item: MigrationItem): HermesSecretDetails | undefined { + const envVar = readString(item.details?.envVar); + const provider = readString(item.details?.provider); + const profileId = readString(item.details?.profileId); + return envVar && provider && profileId ? { envVar, provider, profileId } : undefined; +} + +export function hermesItemConflict(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemConflict(item, reason); +} + +export function hermesItemError(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemError(item, reason); +} + +export function hermesItemSkipped(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemSkipped(item, reason); +} diff --git a/extensions/migrate-hermes/model.apply.test.ts b/extensions/migrate-hermes/model.apply.test.ts new file mode 100644 index 00000000000..9e8edf20d9c --- /dev/null +++ b/extensions/migrate-hermes/model.apply.test.ts @@ -0,0 +1,188 @@ +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_DEFAULT_MODEL_CONFIGURED } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration model apply", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("updates only the primary model when applying over object-form model config", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const existingConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + timeoutMs: 120_000, + }, + }, + }, + } as OpenClawConfig; + let writtenConfig: OpenClawConfig | undefined; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => existingConfig, + writeConfigFile: async (next: OpenClawConfig) => { + writtenConfig = next; + }, + }, + } as never, + }); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + overwrite: true, + model: existingConfig.agents?.defaults?.model, + reportDir, + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "migrated", + }), + ]), + ); + expect(writtenConfig?.agents?.defaults?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + timeoutMs: 120_000, + }); + }); + + it("updates the default-agent model override when applying with overwrite", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const existingConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: { + primary: "google/gemini-3-pro", + fallbacks: ["openai/gpt-5.4"], + }, + }, + list: [ + { + id: "main", + default: true, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }, + }, + ], + }, + } as OpenClawConfig; + let writtenConfig: OpenClawConfig | undefined; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => existingConfig, + writeConfigFile: async (next: OpenClawConfig) => { + writtenConfig = next; + }, + }, + } as never, + }); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + config: existingConfig, + overwrite: true, + reportDir, + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "migrated", + }), + ]), + ); + expect(writtenConfig?.agents?.list?.[0]?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }); + expect(writtenConfig?.agents?.defaults?.model).toEqual(existingConfig.agents?.defaults?.model); + }); + + it("reports late-created default models as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const lateConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: "anthropic/claude-sonnet-4.6", + }, + }, + } as OpenClawConfig; + let writeCalled = false; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => lateConfig, + writeConfigFile: async () => { + writeCalled = true; + }, + }, + } as never, + }); + const ctx = makeContext({ source, stateDir, workspaceDir, reportDir }); + const plan = await provider.plan(ctx); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + expect(writeCalled).toBe(false); + }); +}); diff --git a/extensions/migrate-hermes/model.plan.test.ts b/extensions/migrate-hermes/model.plan.test.ts new file mode 100644 index 00000000000..8a68cabc6fd --- /dev/null +++ b/extensions/migrate-hermes/model.plan.test.ts @@ -0,0 +1,107 @@ +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_DEFAULT_MODEL_CONFIGURED } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration model planning", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("preserves the provider for top-level string model refs", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, "config.yaml"), "provider: openai\nmodel: gpt-5.4\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + details: { model: "openai/gpt-5.4" }, + status: "planned", + }), + ]), + ); + }); + + it("treats existing object-form default model primaries as conflicts", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openai/gpt-5.4"], + timeoutMs: 120_000, + }, + }), + ); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + }); + + it("treats default-agent model overrides as conflicts", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const config = { + agents: { + defaults: { + workspace: workspaceDir, + model: "openai/gpt-5.4", + }, + list: [ + { + id: "main", + default: true, + model: "anthropic/claude-sonnet-4.6", + }, + ], + }, + } as OpenClawConfig; + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir, config })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + }); +}); diff --git a/extensions/migrate-hermes/model.ts b/extensions/migrate-hermes/model.ts new file mode 100644 index 00000000000..7dc8356a9dd --- /dev/null +++ b/extensions/migrate-hermes/model.ts @@ -0,0 +1,87 @@ +import { + resolveAgentEffectiveModelPrimary, + resolveDefaultAgentId, + setAgentEffectiveModelPrimary, +} from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { readString } from "./helpers.js"; +import { + HERMES_REASON_ALREADY_CONFIGURED, + HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE, + HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + hermesItemConflict, + hermesItemError, + hermesItemSkipped, + readHermesModelDetails, +} from "./items.js"; + +export function resolveHermesModelRef(config: Record): string | undefined { + const model = config.model; + if (typeof model === "string" && model.trim()) { + const rawModel = model.trim(); + const provider = readString(config.provider); + if (provider && !rawModel.includes("/")) { + return `${provider}/${rawModel}`; + } + return rawModel; + } + if (model && typeof model === "object" && !Array.isArray(model)) { + const modelRecord = model as Record; + const rawModel = readString(modelRecord.default) ?? readString(modelRecord.model); + const provider = readString(modelRecord.provider); + if (rawModel && provider && !rawModel.includes("/")) { + return `${provider}/${rawModel}`; + } + return rawModel; + } + const rootModel = readString(config.default_model) ?? readString(config.model_name); + const rootProvider = readString(config.provider); + if (rootModel && rootProvider && !rootModel.includes("/")) { + return `${rootProvider}/${rootModel}`; + } + return rootModel; +} + +function resolveDefaultAgentModelState(config: MigrationProviderContext["config"]): { + agentId: string; + effectivePrimary?: string; +} { + const agentId = resolveDefaultAgentId(config); + const effectivePrimary = resolveAgentEffectiveModelPrimary(config, agentId); + return { + agentId, + effectivePrimary, + }; +} + +export function resolveCurrentModelRef(ctx: MigrationProviderContext): string | undefined { + return resolveDefaultAgentModelState(ctx.config).effectivePrimary; +} + +export async function applyModelItem( + ctx: MigrationProviderContext, + item: MigrationItem, +): Promise { + const details = readHermesModelDetails(item); + if (!details || item.status !== "planned") { + return item; + } + try { + if (!ctx.runtime?.config.writeConfigFile) { + return hermesItemError(item, HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE); + } + const nextConfig = structuredClone(ctx.runtime?.config.loadConfig?.() ?? ctx.config); + const currentState = resolveDefaultAgentModelState(nextConfig); + if (currentState.effectivePrimary === details.model) { + return hermesItemSkipped(item, HERMES_REASON_ALREADY_CONFIGURED); + } + if (currentState.effectivePrimary && !ctx.overwrite) { + return hermesItemConflict(item, HERMES_REASON_DEFAULT_MODEL_CONFIGURED); + } + setAgentEffectiveModelPrimary(nextConfig, currentState.agentId, details.model); + await ctx.runtime.config.writeConfigFile(nextConfig); + return { ...item, status: "migrated" }; + } catch (err) { + return hermesItemError(item, err instanceof Error ? err.message : String(err)); + } +} diff --git a/extensions/migrate-hermes/openclaw.plugin.json b/extensions/migrate-hermes/openclaw.plugin.json new file mode 100644 index 00000000000..0b848a4f627 --- /dev/null +++ b/extensions/migrate-hermes/openclaw.plugin.json @@ -0,0 +1,13 @@ +{ + "id": "migrate-hermes", + "name": "Hermes Migration", + "description": "Imports Hermes configuration, memories, skills, and supported credentials into OpenClaw.", + "contracts": { + "migrationProviders": ["hermes"] + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/migrate-hermes/package.json b/extensions/migrate-hermes/package.json new file mode 100644 index 00000000000..cc7485db619 --- /dev/null +++ b/extensions/migrate-hermes/package.json @@ -0,0 +1,27 @@ +{ + "name": "@openclaw/migrate-hermes", + "version": "2026.4.25", + "private": true, + "description": "Hermes to OpenClaw migration provider", + "type": "module", + "dependencies": { + "yaml": "^2.8.3" + }, + "devDependencies": { + "@openclaw/plugin-sdk": "workspace:*", + "openclaw": "workspace:*" + }, + "peerDependencies": { + "openclaw": ">=2026.4.25" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/extensions/migrate-hermes/plan.ts b/extensions/migrate-hermes/plan.ts new file mode 100644 index 00000000000..9930cd8021c --- /dev/null +++ b/extensions/migrate-hermes/plan.ts @@ -0,0 +1,162 @@ +import path from "node:path"; +import { + createMigrationItem, + MIGRATION_REASON_TARGET_EXISTS, + summarizeMigrationItems, +} from "openclaw/plugin-sdk/migration"; +import type { + MigrationItem, + MigrationPlan, + MigrationProviderContext, +} from "openclaw/plugin-sdk/plugin-entry"; +import { buildConfigItems } from "./config.js"; +import { exists, parseHermesConfig, readText } from "./helpers.js"; +import { createHermesModelItem } from "./items.js"; +import { resolveCurrentModelRef, resolveHermesModelRef } from "./model.js"; +import { buildSecretItems } from "./secrets.js"; +import { buildSkillItems } from "./skills.js"; +import { discoverHermesSource, hasHermesSource } from "./source.js"; +import { resolveTargets } from "./targets.js"; + +async function addFileItem(params: { + items: MigrationItem[]; + id: string; + source?: string; + target: string; + kind?: MigrationItem["kind"]; + action?: MigrationItem["action"]; + overwrite?: boolean; +}): Promise { + if (!params.source) { + return; + } + const targetExists = await exists(params.target); + params.items.push( + createMigrationItem({ + id: params.id, + kind: params.kind ?? "file", + action: params.action ?? "copy", + source: params.source, + target: params.target, + status: targetExists && !params.overwrite ? "conflict" : "planned", + reason: targetExists && !params.overwrite ? MIGRATION_REASON_TARGET_EXISTS : undefined, + }), + ); +} + +export async function buildHermesPlan(ctx: MigrationProviderContext): Promise { + const source = await discoverHermesSource(ctx.source); + if (!hasHermesSource(source)) { + throw new Error( + `Hermes state was not found at ${source.root}. Pass --from if it lives elsewhere.`, + ); + } + const targets = resolveTargets(ctx); + const config = parseHermesConfig(await readText(source.configPath)); + const modelRef = resolveHermesModelRef(config); + const items: MigrationItem[] = []; + + if (modelRef) { + const currentModel = resolveCurrentModelRef(ctx); + items.push( + createHermesModelItem({ + model: modelRef, + currentModel, + overwrite: ctx.overwrite, + }), + ); + } + items.push( + ...buildConfigItems({ + ctx, + config, + modelRef, + hasMemoryFiles: Boolean(source.memoryPath || source.userPath), + }), + ); + + await addFileItem({ + items, + id: "workspace:SOUL.md", + kind: "workspace", + source: source.soulPath, + target: path.join(targets.workspaceDir, "SOUL.md"), + overwrite: ctx.overwrite, + }); + await addFileItem({ + items, + id: "workspace:AGENTS.md", + kind: "workspace", + source: source.agentsPath, + target: path.join(targets.workspaceDir, "AGENTS.md"), + overwrite: ctx.overwrite, + }); + if (source.memoryPath) { + items.push( + createMigrationItem({ + id: "memory:MEMORY.md", + kind: "memory", + action: "append", + source: source.memoryPath, + target: path.join(targets.workspaceDir, "MEMORY.md"), + }), + ); + } + if (source.userPath) { + items.push( + createMigrationItem({ + id: "memory:USER.md", + kind: "memory", + action: "append", + source: source.userPath, + target: path.join(targets.workspaceDir, "USER.md"), + }), + ); + } + items.push(...(await buildSkillItems({ source, targets, overwrite: ctx.overwrite }))); + items.push(...(await buildSecretItems({ ctx, source, targets }))); + for (const archivePath of source.archivePaths) { + items.push( + createMigrationItem({ + id: archivePath.id, + kind: "archive", + action: "archive", + source: archivePath.path, + message: + "Archived in the migration report for manual review; not imported into live config.", + details: { archiveRelativePath: archivePath.relativePath }, + }), + ); + } + + const warnings = [ + ...(!ctx.includeSecrets && items.some((item) => item.kind === "secret") + ? [ + "Secrets were detected but skipped. Re-run with --include-secrets to import supported API keys.", + ] + : []), + ...(items.some((item) => item.status === "conflict") + ? [ + "Conflicts were found. Re-run with --overwrite to replace conflicting targets after item-level backups.", + ] + : []), + ...(source.archivePaths.length > 0 + ? [ + "Some Hermes files are archive-only. They will be copied into the migration report for manual review, not loaded into OpenClaw.", + ] + : []), + ...(items.some((item) => item.kind === "manual") + ? ["Some Hermes settings require manual review before they can be activated safely."] + : []), + ]; + return { + providerId: "hermes", + source: source.root, + target: targets.workspaceDir, + summary: summarizeMigrationItems(items), + items, + warnings, + nextSteps: ["Run openclaw doctor after applying the migration."], + metadata: { agentDir: targets.agentDir }, + }; +} diff --git a/extensions/migrate-hermes/provider.secret-failure.test.ts b/extensions/migrate-hermes/provider.secret-failure.test.ts new file mode 100644 index 00000000000..b11640ca480 --- /dev/null +++ b/extensions/migrate-hermes/provider.secret-failure.test.ts @@ -0,0 +1,99 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HERMES_REASON_AUTH_PROFILE_WRITE_FAILED } from "./items.js"; + +const mocks = vi.hoisted(() => ({ + updateAuthProfileStoreWithLock: vi.fn(async () => null), +})); + +vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ + updateAuthProfileStoreWithLock: mocks.updateAuthProfileStoreWithLock, +})); + +const { buildHermesMigrationProvider } = await import("./provider.js"); + +const tempRoots = new Set(); +const logger = { + info() {}, + warn() {}, + error() {}, + debug() {}, +}; + +async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-hermes-secret-failure-")); + tempRoots.add(root); + return root; +} + +async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +function makeContext(params: { + source: string; + stateDir: string; + workspaceDir: string; + reportDir: string; +}): MigrationProviderContext { + return { + config: { + agents: { + defaults: { + workspace: params.workspaceDir, + }, + }, + } as OpenClawConfig, + stateDir: params.stateDir, + source: params.source, + includeSecrets: true, + overwrite: true, + reportDir: params.reportDir, + logger, + }; +} + +describe("Hermes migration provider secret write failures", () => { + afterEach(async () => { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); + mocks.updateAuthProfileStoreWithLock.mockClear(); + }); + + it("reports an error when a secret auth-profile write fails", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + status: "error", + reason: HERMES_REASON_AUTH_PROFILE_WRITE_FAILED, + }), + ]), + ); + expect(result.summary.errors).toBe(1); + expect(result.summary.migrated).toBe(0); + }); +}); diff --git a/extensions/migrate-hermes/provider.test.ts b/extensions/migrate-hermes/provider.test.ts new file mode 100644 index 00000000000..2819a5ec136 --- /dev/null +++ b/extensions/migrate-hermes/provider.test.ts @@ -0,0 +1,129 @@ +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createCapturedPluginRegistration } from "../../src/plugins/captured-registration.js"; +import pluginEntry from "./index.js"; +import { HERMES_REASON_INCLUDE_SECRETS } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration provider", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("registers the Hermes migration provider through the plugin entry", () => { + const captured = createCapturedPluginRegistration(); + pluginEntry.register(captured.api); + expect(captured.migrationProviders.map((provider) => provider.id)).toEqual(["hermes"]); + }); + + it("detects Hermes sources supported by planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + await writeFile(path.join(source, "SOUL.md"), "# Hermes soul\n"); + + const provider = buildHermesMigrationProvider(); + const detected = await provider.detect?.( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ); + + expect(detected).toMatchObject({ + found: true, + source, + confidence: "high", + }); + }); + + it("detects archive-only Hermes sources", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + await writeFile(path.join(source, "logs", "run.log"), "log line\n"); + + const provider = buildHermesMigrationProvider(); + const detected = await provider.detect?.( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ); + + expect(detected).toMatchObject({ + found: true, + source, + confidence: "high", + }); + }); + + it("rejects missing Hermes sources before planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "missing-hermes"); + + const provider = buildHermesMigrationProvider(); + + await expect( + provider.plan( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ), + ).rejects.toThrow(`Hermes state was not found at ${source}`); + }); + + it("plans model, workspace, memory, skill, and secret items without importing secrets by default", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile(path.join(source, "SOUL.md"), "# Hermes soul\n"); + await writeFile(path.join(source, "memories", "MEMORY.md"), "remember this\n"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(workspaceDir, "SOUL.md"), "# Existing soul\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + model: "anthropic/claude-sonnet-4.6", + }), + ); + + expect(plan.summary).toMatchObject({ total: 8, conflicts: 2, sensitive: 1 }); + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "config:default-model", status: "conflict" }), + expect.objectContaining({ id: "config:memory", status: "planned" }), + expect.objectContaining({ id: "config:memory-plugin-slot", status: "planned" }), + expect.objectContaining({ id: "config:model-providers", status: "planned" }), + expect.objectContaining({ id: "workspace:SOUL.md", status: "conflict" }), + expect.objectContaining({ id: "memory:MEMORY.md", action: "append", status: "planned" }), + expect.objectContaining({ id: "skill:ship-it", status: "planned" }), + expect.objectContaining({ + id: "secret:openai", + sensitive: true, + status: "skipped", + reason: HERMES_REASON_INCLUDE_SECRETS, + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([ + expect.stringContaining("Secrets were detected but skipped"), + expect.stringContaining("Conflicts were found"), + ]), + ); + }); +}); diff --git a/extensions/migrate-hermes/provider.ts b/extensions/migrate-hermes/provider.ts new file mode 100644 index 00000000000..212cf237122 --- /dev/null +++ b/extensions/migrate-hermes/provider.ts @@ -0,0 +1,35 @@ +import type { + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, +} from "openclaw/plugin-sdk/plugin-entry"; +import { applyHermesPlan } from "./apply.js"; +import { buildHermesPlan } from "./plan.js"; +import { discoverHermesSource, hasHermesSource } from "./source.js"; + +export function buildHermesMigrationProvider( + params: { + runtime?: MigrationProviderContext["runtime"]; + } = {}, +): MigrationProviderPlugin { + return { + id: "hermes", + label: "Hermes", + description: "Import Hermes config, memories, skills, and supported credentials.", + async detect(ctx) { + const source = await discoverHermesSource(ctx.source); + const found = hasHermesSource(source); + return { + found, + source: source.root, + label: "Hermes", + confidence: found ? "high" : "low", + message: found ? "Hermes state found." : "Hermes state not found.", + }; + }, + plan: buildHermesPlan, + async apply(ctx, plan?: MigrationPlan) { + return await applyHermesPlan({ ctx, plan, runtime: params.runtime }); + }, + }; +} diff --git a/extensions/migrate-hermes/secrets.test.ts b/extensions/migrate-hermes/secrets.test.ts new file mode 100644 index 00000000000..5f4bca0fb43 --- /dev/null +++ b/extensions/migrate-hermes/secrets.test.ts @@ -0,0 +1,159 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_AUTH_PROFILE_EXISTS } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration secret items", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("uses configured agentDir for secret planning and imports without runtime helpers", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const customAgentDir = path.join(root, "custom-agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + const config = { + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [ + { + id: "custom", + default: true, + agentDir: customAgentDir, + }, + ], + }, + } as OpenClawConfig; + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + config, + includeSecrets: true, + }), + ); + + expect(plan.metadata?.agentDir).toBe(customAgentDir); + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + target: `${customAgentDir}/auth-profiles.json#openai:hermes-import`, + status: "planned", + }), + ]), + ); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + config, + includeSecrets: true, + overwrite: true, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.summary.errors).toBe(0); + const authStore = JSON.parse( + await fs.readFile(path.join(customAgentDir, "auth-profiles.json"), "utf8"), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]).toMatchObject({ + provider: "openai", + key: "sk-hermes", + }); + await expect( + fs.access(path.join(stateDir, "agents", "custom", "agent", "auth-profiles.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("keeps secret conflict checks read-only during planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile( + path.join(agentDir, "auth.json"), + JSON.stringify({ + openai: { type: "api_key", provider: "openai", key: "legacy-main-key" }, + }), + ); + + const provider = buildHermesMigrationProvider(); + await provider.plan(makeContext({ source, stateDir, workspaceDir, includeSecrets: true })); + + await expect(fs.access(path.join(agentDir, "auth.json"))).resolves.toBeUndefined(); + await expect(fs.access(path.join(agentDir, "auth-profiles.json"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("reports late-created auth profiles as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + + const provider = buildHermesMigrationProvider(); + const ctx = makeContext({ + source, + stateDir, + workspaceDir, + includeSecrets: true, + reportDir, + }); + const plan = await provider.plan(ctx); + await writeFile( + path.join(agentDir, "auth-profiles.json"), + JSON.stringify( + { + version: 1, + profiles: { + "openai:hermes-import": { + type: "api_key", + provider: "openai", + key: "sk-late", + }, + }, + }, + null, + 2, + ), + ); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + status: "conflict", + reason: HERMES_REASON_AUTH_PROFILE_EXISTS, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + const authStore = JSON.parse( + await fs.readFile(path.join(agentDir, "auth-profiles.json"), "utf8"), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]?.key).toBe("sk-late"); + }); +}); diff --git a/extensions/migrate-hermes/secrets.ts b/extensions/migrate-hermes/secrets.ts new file mode 100644 index 00000000000..0ecf876b1b1 --- /dev/null +++ b/extensions/migrate-hermes/secrets.ts @@ -0,0 +1,118 @@ +import { loadAuthProfileStoreWithoutExternalProfiles } from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { updateAuthProfileStoreWithLock } from "openclaw/plugin-sdk/provider-auth"; +import { parseEnv, readText } from "./helpers.js"; +import { + createHermesSecretItem, + HERMES_REASON_AUTH_PROFILE_EXISTS, + HERMES_REASON_AUTH_PROFILE_WRITE_FAILED, + HERMES_REASON_MISSING_SECRET_METADATA, + HERMES_REASON_SECRET_NO_LONGER_PRESENT, + hermesItemConflict, + hermesItemError, + hermesItemSkipped, + readHermesSecretDetails, +} from "./items.js"; +import type { HermesSource } from "./source.js"; +import type { PlannedTargets } from "./targets.js"; + +type SecretMapping = { + envVar: string; + provider: string; + profileId: string; +}; + +const SECRET_MAPPINGS: readonly SecretMapping[] = [ + { envVar: "OPENAI_API_KEY", provider: "openai", profileId: "openai:hermes-import" }, + { envVar: "ANTHROPIC_API_KEY", provider: "anthropic", profileId: "anthropic:hermes-import" }, + { envVar: "OPENROUTER_API_KEY", provider: "openrouter", profileId: "openrouter:hermes-import" }, + { envVar: "GOOGLE_API_KEY", provider: "google", profileId: "google:hermes-import" }, + { envVar: "GEMINI_API_KEY", provider: "google", profileId: "google:hermes-import" }, + { envVar: "GROQ_API_KEY", provider: "groq", profileId: "groq:hermes-import" }, + { envVar: "XAI_API_KEY", provider: "xai", profileId: "xai:hermes-import" }, + { envVar: "MISTRAL_API_KEY", provider: "mistral", profileId: "mistral:hermes-import" }, + { envVar: "DEEPSEEK_API_KEY", provider: "deepseek", profileId: "deepseek:hermes-import" }, +] as const; + +export async function buildSecretItems(params: { + ctx: MigrationProviderContext; + source: HermesSource; + targets: PlannedTargets; +}): Promise { + const env = parseEnv(await readText(params.source.envPath)); + const store = loadAuthProfileStoreWithoutExternalProfiles(params.targets.agentDir); + const seenProfiles = new Set(); + const items: MigrationItem[] = []; + for (const mapping of SECRET_MAPPINGS) { + const value = env[mapping.envVar]?.trim(); + if (!value || seenProfiles.has(mapping.profileId)) { + continue; + } + seenProfiles.add(mapping.profileId); + const existsAlready = Boolean(store.profiles[mapping.profileId]); + items.push( + createHermesSecretItem({ + id: `secret:${mapping.provider}`, + source: params.source.envPath, + target: `${params.targets.agentDir}/auth-profiles.json#${mapping.profileId}`, + includeSecrets: params.ctx.includeSecrets, + existsAlready: existsAlready && !params.ctx.overwrite, + details: { + envVar: mapping.envVar, + provider: mapping.provider, + profileId: mapping.profileId, + }, + }), + ); + } + return items; +} + +export async function applySecretItem( + ctx: MigrationProviderContext, + item: MigrationItem, + targets: PlannedTargets, +): Promise { + if (item.status !== "planned") { + return item; + } + const details = readHermesSecretDetails(item); + const source = item.source; + if (!details || !source) { + return hermesItemError(item, HERMES_REASON_MISSING_SECRET_METADATA); + } + const env = parseEnv(await readText(source)); + const key = env[details.envVar]?.trim(); + if (!key) { + return hermesItemSkipped(item, HERMES_REASON_SECRET_NO_LONGER_PRESENT); + } + let conflicted = false; + let wrote = false; + const store = await updateAuthProfileStoreWithLock({ + agentDir: targets.agentDir, + updater: (freshStore) => { + if (!ctx.overwrite && freshStore.profiles[details.profileId]) { + conflicted = true; + return false; + } + freshStore.profiles[details.profileId] = { + type: "api_key", + provider: details.provider, + key, + displayName: "Hermes import", + }; + wrote = true; + return true; + }, + }); + if (conflicted) { + return hermesItemConflict(item, HERMES_REASON_AUTH_PROFILE_EXISTS); + } + if (!store?.profiles[details.profileId]) { + return hermesItemError(item, HERMES_REASON_AUTH_PROFILE_WRITE_FAILED); + } + if (!wrote && !ctx.overwrite) { + return hermesItemConflict(item, HERMES_REASON_AUTH_PROFILE_EXISTS); + } + return { ...item, status: "migrated" }; +} diff --git a/extensions/migrate-hermes/skills.ts b/extensions/migrate-hermes/skills.ts new file mode 100644 index 00000000000..16b0e30bf7f --- /dev/null +++ b/extensions/migrate-hermes/skills.ts @@ -0,0 +1,70 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { createMigrationItem, MIGRATION_REASON_TARGET_EXISTS } from "openclaw/plugin-sdk/migration"; +import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { exists, sanitizeName } from "./helpers.js"; +import type { HermesSource } from "./source.js"; +import type { PlannedTargets } from "./targets.js"; + +type PlannedSkill = { + name: string; + source: string; + target: string; +}; + +export async function buildSkillItems(params: { + source: HermesSource; + targets: PlannedTargets; + overwrite?: boolean; +}): Promise { + if (!params.source.skillsDir) { + return []; + } + const entries = await fs + .readdir(params.source.skillsDir, { withFileTypes: true }) + .catch(() => []); + const plannedSkills: PlannedSkill[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const name = sanitizeName(entry.name); + if (!name) { + continue; + } + const source = path.join(params.source.skillsDir, entry.name); + if (!(await exists(path.join(source, "SKILL.md")))) { + continue; + } + plannedSkills.push({ + name, + source, + target: path.join(params.targets.workspaceDir, "skills", name), + }); + } + const counts = new Map(); + for (const skill of plannedSkills) { + counts.set(skill.name, (counts.get(skill.name) ?? 0) + 1); + } + const items: MigrationItem[] = []; + for (const skill of plannedSkills) { + const collides = (counts.get(skill.name) ?? 0) > 1; + const targetExists = await exists(skill.target); + items.push( + createMigrationItem({ + id: `skill:${skill.name}`, + kind: "skill", + action: "copy", + source: skill.source, + target: skill.target, + status: collides ? "conflict" : targetExists && !params.overwrite ? "conflict" : "planned", + reason: collides + ? `multiple Hermes skill directories normalize to "${skill.name}"` + : targetExists && !params.overwrite + ? MIGRATION_REASON_TARGET_EXISTS + : undefined, + }), + ); + } + return items; +} diff --git a/extensions/migrate-hermes/source.ts b/extensions/migrate-hermes/source.ts new file mode 100644 index 00000000000..83d5de8a65d --- /dev/null +++ b/extensions/migrate-hermes/source.ts @@ -0,0 +1,74 @@ +import path from "node:path"; +import { exists, isDirectory, resolveHomePath } from "./helpers.js"; + +export type HermesSource = { + root: string; + configPath?: string; + envPath?: string; + soulPath?: string; + agentsPath?: string; + memoryPath?: string; + userPath?: string; + skillsDir?: string; + archivePaths: HermesArchivePath[]; +}; + +export type HermesArchivePath = { + id: string; + path: string; + relativePath: string; +}; + +const HERMES_ARCHIVE_DIRS = ["plugins", "sessions", "logs", "cron", "mcp-tokens"] as const; +const HERMES_ARCHIVE_FILES = ["auth.json", "state.db"] as const; + +export async function discoverHermesSource(input?: string): Promise { + const root = resolveHomePath(input?.trim() || "~/.hermes"); + const archivePaths: HermesArchivePath[] = []; + for (const dir of HERMES_ARCHIVE_DIRS) { + const candidate = path.join(root, dir); + if (await isDirectory(candidate)) { + archivePaths.push({ id: `archive:${dir}`, path: candidate, relativePath: dir }); + } + } + for (const file of HERMES_ARCHIVE_FILES) { + const candidate = path.join(root, file); + if (await exists(candidate)) { + archivePaths.push({ id: `archive:${file}`, path: candidate, relativePath: file }); + } + } + return { + root, + archivePaths, + ...((await exists(path.join(root, "config.yaml"))) + ? { configPath: path.join(root, "config.yaml") } + : {}), + ...((await exists(path.join(root, ".env"))) ? { envPath: path.join(root, ".env") } : {}), + ...((await exists(path.join(root, "SOUL.md"))) ? { soulPath: path.join(root, "SOUL.md") } : {}), + ...((await exists(path.join(root, "AGENTS.md"))) + ? { agentsPath: path.join(root, "AGENTS.md") } + : {}), + ...((await exists(path.join(root, "memories", "MEMORY.md"))) + ? { memoryPath: path.join(root, "memories", "MEMORY.md") } + : {}), + ...((await exists(path.join(root, "memories", "USER.md"))) + ? { userPath: path.join(root, "memories", "USER.md") } + : {}), + ...((await isDirectory(path.join(root, "skills"))) + ? { skillsDir: path.join(root, "skills") } + : {}), + }; +} + +export function hasHermesSource(source: HermesSource): boolean { + return Boolean( + source.configPath || + source.envPath || + source.soulPath || + source.agentsPath || + source.memoryPath || + source.userPath || + source.skillsDir || + source.archivePaths.length > 0, + ); +} diff --git a/extensions/migrate-hermes/targets.ts b/extensions/migrate-hermes/targets.ts new file mode 100644 index 00000000000..c0e1923f821 --- /dev/null +++ b/extensions/migrate-hermes/targets.ts @@ -0,0 +1,30 @@ +import path from "node:path"; +import { + resolveAgentConfig, + resolveAgentWorkspaceDir, + resolveDefaultAgentId, +} from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { resolveHomePath } from "./helpers.js"; + +export type PlannedTargets = { + workspaceDir: string; + stateDir: string; + agentDir: string; +}; + +export function resolveTargets(ctx: MigrationProviderContext): PlannedTargets { + const cfg = ctx.config; + const agentId = resolveDefaultAgentId(cfg); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const configuredAgentDir = resolveAgentConfig(cfg, agentId)?.agentDir?.trim(); + const agentDir = + ctx.runtime?.agent?.resolveAgentDir(cfg, agentId) ?? + (configuredAgentDir ? resolveHomePath(configuredAgentDir) : undefined) ?? + path.join(ctx.stateDir, "agents", agentId, "agent"); + return { + workspaceDir, + stateDir: ctx.stateDir, + agentDir, + }; +} diff --git a/extensions/migrate-hermes/test/provider-helpers.ts b/extensions/migrate-hermes/test/provider-helpers.ts new file mode 100644 index 00000000000..ef79994a517 --- /dev/null +++ b/extensions/migrate-hermes/test/provider-helpers.ts @@ -0,0 +1,65 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; + +const tempRoots = new Set(); + +export const logger = { + info() {}, + warn() {}, + error() {}, + debug() {}, +}; + +export async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migrate-hermes-")); + tempRoots.add(root); + return root; +} + +export async function cleanupTempRoots() { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); +} + +export async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +export function makeContext(params: { + source: string; + stateDir: string; + workspaceDir: string; + config?: OpenClawConfig; + includeSecrets?: boolean; + overwrite?: boolean; + model?: NonNullable["defaults"]>["model"]; + reportDir?: string; + runtime?: MigrationProviderContext["runtime"]; +}): MigrationProviderContext { + const config = + params.config ?? + ({ + agents: { + defaults: { + workspace: params.workspaceDir, + ...(params.model !== undefined ? { model: params.model } : {}), + }, + }, + } as OpenClawConfig); + return { + config, + stateDir: params.stateDir, + source: params.source, + includeSecrets: params.includeSecrets, + overwrite: params.overwrite, + reportDir: params.reportDir, + runtime: params.runtime, + logger, + }; +} diff --git a/extensions/telegram/src/bot-native-commands.registry.test.ts b/extensions/telegram/src/bot-native-commands.registry.test.ts index acf93f67d30..9cb7abfde05 100644 --- a/extensions/telegram/src/bot-native-commands.registry.test.ts +++ b/extensions/telegram/src/bot-native-commands.registry.test.ts @@ -60,6 +60,7 @@ function createTelegramPluginRegistry() { videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], gatewayHandlers: {}, httpRoutes: [], cliRegistrars: [], diff --git a/package.json b/package.json index e1f45690c76..fd73a7e1fb7 100644 --- a/package.json +++ b/package.json @@ -442,6 +442,14 @@ "types": "./dist/plugin-sdk/logging-core.d.ts", "default": "./dist/plugin-sdk/logging-core.js" }, + "./plugin-sdk/migration": { + "types": "./dist/plugin-sdk/migration.d.ts", + "default": "./dist/plugin-sdk/migration.js" + }, + "./plugin-sdk/migration-runtime": { + "types": "./dist/plugin-sdk/migration-runtime.d.ts", + "default": "./dist/plugin-sdk/migration-runtime.js" + }, "./plugin-sdk/markdown-table-runtime": { "types": "./dist/plugin-sdk/markdown-table-runtime.d.ts", "default": "./dist/plugin-sdk/markdown-table-runtime.js" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52af352ac8f..8fa376a3b50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -877,6 +877,19 @@ importers: specifier: workspace:* version: link:../.. + extensions/migrate-hermes: + dependencies: + yaml: + specifier: ^2.8.3 + version: 2.8.3 + devDependencies: + '@openclaw/plugin-sdk': + specifier: workspace:* + version: link:../../packages/plugin-sdk + openclaw: + specifier: workspace:* + version: link:../.. + extensions/microsoft: dependencies: node-edge-tts: diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index ee0371c4f72..1522d709929 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -94,6 +94,8 @@ "testing", "temp-path", "logging-core", + "migration", + "migration-runtime", "markdown-table-runtime", "account-helpers", "account-core", diff --git a/src/agents/agent-scope.test.ts b/src/agents/agent-scope.test.ts index ea2825a9c60..ad349027db8 100644 --- a/src/agents/agent-scope.test.ts +++ b/src/agents/agent-scope.test.ts @@ -18,6 +18,7 @@ import { resolveAgentWorkspaceDir, resolveAgentIdByWorkspacePath, resolveAgentIdsByWorkspacePath, + setAgentEffectiveModelPrimary, } from "./agent-scope.js"; afterEach(() => { @@ -267,6 +268,59 @@ describe("resolveAgentConfig", () => { ).toEqual([]); }); + it("updates the effective model primary at the winning config layer", () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }, + }, + list: [ + { + id: "linus", + default: true, + model: { + primary: "anthropic/claude-sonnet-4-6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }, + }, + ], + }, + }; + + expect(setAgentEffectiveModelPrimary(cfg, "linus", "google/gemini-3-pro")).toBe("agent"); + expect(cfg.agents?.list?.[0]?.model).toEqual({ + primary: "google/gemini-3-pro", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }); + expect(cfg.agents?.defaults?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }); + + const inheritedCfg: OpenClawConfig = { + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }, + }, + list: [{ id: "main", default: true }], + }, + }; + + expect(setAgentEffectiveModelPrimary(inheritedCfg, "main", "google/gemini-3-pro")).toBe( + "defaults", + ); + expect(inheritedCfg.agents?.defaults?.model).toEqual({ + primary: "google/gemini-3-pro", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }); + }); + it("resolves fallback agent id from explicit agent id first", () => { expect( resolveFallbackAgentId({ diff --git a/src/agents/agent-scope.ts b/src/agents/agent-scope.ts index 19bd7ca1240..ee1cc6a44d6 100644 --- a/src/agents/agent-scope.ts +++ b/src/agents/agent-scope.ts @@ -2,6 +2,8 @@ import fs from "node:fs"; import path from "node:path"; import { resolveAgentModelFallbackValues } from "../config/model-input.js"; import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js"; +import type { AgentModelConfig } from "../config/types.agents-shared.js"; +import type { AgentConfig } from "../config/types.agents.js"; import type { OpenClawConfig } from "../config/types.js"; import { normalizeAgentId, @@ -108,6 +110,42 @@ export function resolveAgentEffectiveModelPrimary( ); } +function findMutableAgentEntry(cfg: OpenClawConfig, agentId: string): AgentConfig | undefined { + const id = normalizeAgentId(agentId); + return cfg.agents?.list?.find((entry) => normalizeAgentId(entry?.id) === id); +} + +function updateAgentModelPrimary( + existing: AgentModelConfig | undefined, + primary: string, +): AgentModelConfig { + if (existing && typeof existing === "object" && !Array.isArray(existing)) { + return { ...existing, primary }; + } + return primary; +} + +export type AgentModelPrimaryWriteTarget = "agent" | "defaults"; + +export function setAgentEffectiveModelPrimary( + cfg: OpenClawConfig, + agentId: string, + primary: string, +): AgentModelPrimaryWriteTarget { + const id = normalizeAgentId(agentId); + if (resolveAgentExplicitModelPrimary(cfg, id)) { + const entry = findMutableAgentEntry(cfg, id); + if (entry) { + entry.model = updateAgentModelPrimary(entry.model, primary); + return "agent"; + } + } + cfg.agents ??= {}; + cfg.agents.defaults ??= {}; + cfg.agents.defaults.model = updateAgentModelPrimary(cfg.agents.defaults.model, primary); + return "defaults"; +} + // Backward-compatible alias. Prefer explicit/effective helpers at new call sites. export function resolveAgentModelPrimary(cfg: OpenClawConfig, agentId: string): string | undefined { return resolveAgentExplicitModelPrimary(cfg, agentId); diff --git a/src/cli/command-catalog.ts b/src/cli/command-catalog.ts index a970430dc6c..4b0a5c0e6cb 100644 --- a/src/cli/command-catalog.ts +++ b/src/cli/command-catalog.ts @@ -67,6 +67,7 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ policy: { loadPlugins: "never" }, }, { commandPath: ["configure"], policy: { bypassConfigGuard: true, loadPlugins: "never" } }, + { commandPath: ["migrate"], policy: { bypassConfigGuard: true, loadPlugins: "never" } }, { commandPath: ["status"], policy: { diff --git a/src/cli/program/command-registry-core.ts b/src/cli/program/command-registry-core.ts index 6604387b096..98591862f9b 100644 --- a/src/cli/program/command-registry-core.ts +++ b/src/cli/program/command-registry-core.ts @@ -81,6 +81,11 @@ const coreEntrySpecs: readonly CommandGroupDescriptorSpec< loadModule: () => import("./register.backup.js"), exportName: "registerBackupCommand", }, + { + commandNames: ["migrate"], + loadModule: () => import("./register.migrate.js"), + exportName: "registerMigrateCommand", + }, { commandNames: ["doctor", "dashboard", "reset", "uninstall"], loadModule: () => import("./register.maintenance.js"), diff --git a/src/cli/program/core-command-descriptors.ts b/src/cli/program/core-command-descriptors.ts index 42f916bbe5a..fd996b86bef 100644 --- a/src/cli/program/core-command-descriptors.ts +++ b/src/cli/program/core-command-descriptors.ts @@ -35,6 +35,11 @@ const coreCliCommandCatalog = defineCommandDescriptorCatalog([ description: "Create and verify local backup archives for OpenClaw state", hasSubcommands: true, }, + { + name: "migrate", + description: "Import state from another agent system", + hasSubcommands: true, + }, { name: "doctor", description: "Health checks + quick fixes for the gateway and channels", diff --git a/src/cli/program/register.migrate.ts b/src/cli/program/register.migrate.ts new file mode 100644 index 00000000000..18e9f709a27 --- /dev/null +++ b/src/cli/program/register.migrate.ts @@ -0,0 +1,117 @@ +import type { Command } from "commander"; +import { + migrateApplyCommand, + migrateDefaultCommand, + migrateListCommand, + migratePlanCommand, +} from "../../commands/migrate.js"; +import { defaultRuntime } from "../../runtime.js"; +import { theme } from "../../terminal/theme.js"; +import { runCommandWithRuntime } from "../cli-utils.js"; +import { formatHelpExamples } from "../help-format.js"; + +function addMigrationOptions(command: Command): Command { + return command + .option("--from ", "Source directory to migrate from") + .option("--include-secrets", "Import supported credentials and secrets", false) + .option("--overwrite", "Overwrite conflicting target files after item-level backups", false) + .option("--json", "Output JSON", false); +} + +export function registerMigrateCommand(program: Command) { + const migrate = program + .command("migrate") + .description("Import state from another agent system") + .argument("[provider]", "Migration provider id, for example hermes") + .option("--from ", "Source directory to migrate from") + .option("--include-secrets", "Import supported credentials and secrets", false) + .option("--overwrite", "Overwrite conflicting target files after item-level backups", false) + .option("--dry-run", "Preview only; do not apply changes", false) + .option("--yes", "Apply without prompting after preview", false) + .option("--backup-output ", "Pre-migration backup archive path or directory") + .option("--no-backup", "Skip the pre-migration OpenClaw backup") + .option("--force", "Allow dangerous options such as --no-backup", false) + .option("--json", "Output JSON", false) + .addHelpText( + "after", + () => + `\n${theme.heading("Examples:")}\n${formatHelpExamples([ + ["openclaw migrate list", "Show available migration providers."], + ["openclaw migrate hermes", "Preview Hermes migration, then prompt before applying."], + ["openclaw migrate hermes --dry-run", "Preview Hermes migration only."], + [ + "openclaw migrate apply hermes --yes", + "Apply Hermes migration non-interactively after writing a verified backup.", + ], + [ + "openclaw migrate apply hermes --include-secrets --yes", + "Include supported credentials in the migration.", + ], + ])}`, + ) + .action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateDefaultCommand(defaultRuntime, { + provider: provider as string | undefined, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + dryRun: Boolean(opts.dryRun), + yes: Boolean(opts.yes), + backupOutput: opts.backupOutput as string | undefined, + noBackup: opts.backup === false, + force: Boolean(opts.force), + json: Boolean(opts.json), + }); + }); + }); + + migrate + .command("list") + .description("List migration providers") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateListCommand(defaultRuntime, { json: Boolean(opts.json) }); + }); + }); + + addMigrationOptions( + migrate + .command("plan ") + .description("Preview a migration without changing OpenClaw state"), + ).action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migratePlanCommand(defaultRuntime, { + provider: provider as string, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + json: Boolean(opts.json), + }); + }); + }); + + addMigrationOptions( + migrate.command("apply ").description("Apply a migration after a verified backup"), + ) + .option("--yes", "Apply without prompting", false) + .option("--backup-output ", "Pre-migration backup archive path or directory") + .option("--no-backup", "Skip the pre-migration OpenClaw backup") + .option("--force", "Allow dangerous options such as --no-backup", false) + .action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateApplyCommand(defaultRuntime, { + provider: provider as string, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + yes: Boolean(opts.yes), + backupOutput: opts.backupOutput as string | undefined, + noBackup: opts.backup === false, + force: Boolean(opts.force), + json: Boolean(opts.json), + }); + }); + }); +} diff --git a/src/cli/program/register.onboard.test.ts b/src/cli/program/register.onboard.test.ts index cefc608339e..c2275717e4a 100644 --- a/src/cli/program/register.onboard.test.ts +++ b/src/cli/program/register.onboard.test.ts @@ -181,6 +181,28 @@ describe("registerOnboardCommand", () => { ); }); + it("forwards onboarding migration flags", async () => { + await runCli([ + "onboard", + "--flow", + "import", + "--import-from", + "hermes", + "--import-source", + "/tmp/hermes", + "--import-secrets", + ]); + expect(setupWizardCommandMock).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "import", + importFrom: "hermes", + importSource: "/tmp/hermes", + importSecrets: true, + }), + runtime, + ); + }); + it("reports errors via runtime on setup wizard command failures", async () => { setupWizardCommandMock.mockRejectedValueOnce(new Error("setup failed")); diff --git a/src/cli/program/register.onboard.ts b/src/cli/program/register.onboard.ts index 75cffcd1b43..9f032358e07 100644 --- a/src/cli/program/register.onboard.ts +++ b/src/cli/program/register.onboard.ts @@ -111,7 +111,7 @@ export function registerOnboardCommand(program: Command) { "Acknowledge that agents are powerful and full system access is risky (required for --non-interactive)", false, ) - .option("--flow ", "Onboard flow: quickstart|advanced|manual") + .option("--flow ", "Onboard flow: quickstart|advanced|manual|import") .option("--mode ", "Onboard mode: local|remote") .option("--auth-choice ", `Auth: ${AUTH_CHOICE_HELP}`) .option( @@ -168,6 +168,9 @@ export function registerOnboardCommand(program: Command) { .option("--skip-health", "Skip health check") .option("--skip-ui", "Skip Control UI/TUI prompts") .option("--node-manager ", "Node manager for skills: npm|pnpm|bun") + .option("--import-from ", "Migration provider to run during onboarding") + .option("--import-source ", "Source agent home for --import-from") + .option("--import-secrets", "Import supported secrets during onboarding migration", false) .option("--json", "Output JSON summary", false); command.action(async (opts, commandRuntime) => { @@ -195,7 +198,7 @@ export function registerOnboardCommand(program: Command) { workspace: opts.workspace as string | undefined, nonInteractive: Boolean(opts.nonInteractive), acceptRisk: Boolean(opts.acceptRisk), - flow: opts.flow as "quickstart" | "advanced" | "manual" | undefined, + flow: opts.flow as "quickstart" | "advanced" | "manual" | "import" | undefined, mode: opts.mode as "local" | "remote" | undefined, authChoice: opts.authChoice as AuthChoice | undefined, tokenProvider: opts.tokenProvider as string | undefined, @@ -235,6 +238,9 @@ export function registerOnboardCommand(program: Command) { skipHealth: Boolean(opts.skipHealth), skipUi: Boolean(opts.skipUi), nodeManager: opts.nodeManager as NodeManagerChoice | undefined, + importFrom: opts.importFrom as string | undefined, + importSource: opts.importSource as string | undefined, + importSecrets: Boolean(opts.importSecrets), json: Boolean(opts.json), }, defaultRuntime, diff --git a/src/cli/program/register.setup.test.ts b/src/cli/program/register.setup.test.ts index c8b7ceacacf..a293ea35f51 100644 --- a/src/cli/program/register.setup.test.ts +++ b/src/cli/program/register.setup.test.ts @@ -79,6 +79,27 @@ describe("registerSetupCommand", () => { expect(setupCommandMock).not.toHaveBeenCalled(); }); + it("runs setup wizard command for migration import flags", async () => { + await runCli([ + "setup", + "--import-from", + "hermes", + "--import-source", + "/tmp/hermes", + "--import-secrets", + ]); + + expect(setupWizardCommandMock).toHaveBeenCalledWith( + expect.objectContaining({ + importFrom: "hermes", + importSource: "/tmp/hermes", + importSecrets: true, + }), + runtime, + ); + expect(setupCommandMock).not.toHaveBeenCalled(); + }); + it("reports setup errors through runtime", async () => { setupCommandMock.mockRejectedValueOnce(new Error("setup failed")); diff --git a/src/cli/program/register.setup.ts b/src/cli/program/register.setup.ts index 84b698c863c..0dc980189a0 100644 --- a/src/cli/program/register.setup.ts +++ b/src/cli/program/register.setup.ts @@ -23,6 +23,9 @@ export function registerSetupCommand(program: Command) { .option("--wizard", "Run interactive onboarding", false) .option("--non-interactive", "Run onboarding without prompts", false) .option("--mode ", "Onboard mode: local|remote") + .option("--import-from ", "Migration provider to run during onboarding") + .option("--import-source ", "Source agent home for --import-from") + .option("--import-secrets", "Import supported secrets during onboarding migration", false) .option("--remote-url ", "Remote Gateway WebSocket URL") .option("--remote-token ", "Remote Gateway token (optional)") .action(async (opts, command) => { @@ -31,6 +34,9 @@ export function registerSetupCommand(program: Command) { "wizard", "nonInteractive", "mode", + "importFrom", + "importSource", + "importSecrets", "remoteUrl", "remoteToken", ]); @@ -40,6 +46,9 @@ export function registerSetupCommand(program: Command) { workspace: opts.workspace as string | undefined, nonInteractive: Boolean(opts.nonInteractive), mode: opts.mode as "local" | "remote" | undefined, + importFrom: opts.importFrom as string | undefined, + importSource: opts.importSource as string | undefined, + importSecrets: Boolean(opts.importSecrets), remoteUrl: opts.remoteUrl as string | undefined, remoteToken: opts.remoteToken as string | undefined, }, diff --git a/src/commands/migrate.test.ts b/src/commands/migrate.test.ts new file mode 100644 index 00000000000..078c22f84ac --- /dev/null +++ b/src/commands/migrate.test.ts @@ -0,0 +1,471 @@ +import fs from "node:fs/promises"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MigrationApplyResult, MigrationPlan } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; + +const mocks = vi.hoisted(() => ({ + backupCreateCommand: vi.fn(), + promptYesNo: vi.fn(), + provider: { + id: "hermes", + label: "Hermes", + plan: vi.fn(), + apply: vi.fn(), + }, +})); + +vi.mock("../config/config.js", () => ({ + loadConfig: () => ({}), +})); + +vi.mock("../config/paths.js", () => ({ + resolveStateDir: () => "/tmp/openclaw-migrate-command-test", +})); + +vi.mock("../cli/prompt.js", () => ({ + promptYesNo: mocks.promptYesNo, +})); + +vi.mock("../plugins/migration-provider-runtime.js", () => ({ + resolvePluginMigrationProvider: () => mocks.provider, + resolvePluginMigrationProviders: () => [mocks.provider], +})); + +vi.mock("./backup.js", () => ({ + backupCreateCommand: mocks.backupCreateCommand, +})); + +const { migrateApplyCommand, migrateDefaultCommand } = await import("./migrate.js"); + +function plan(overrides: Partial = {}): MigrationPlan { + return { + providerId: "hermes", + source: "/tmp/hermes", + summary: { + total: 1, + planned: 1, + migrated: 0, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [{ id: "workspace:AGENTS.md", kind: "workspace", action: "copy", status: "planned" }], + ...overrides, + }; +} + +const runtime: RuntimeEnv = { + log: vi.fn(), + error: vi.fn(), + exit(code) { + throw new Error(`exit ${code}`); + }, +}; + +describe("migrateApplyCommand", () => { + const originalIsTty = process.stdin.isTTY; + + beforeEach(async () => { + await fs.rm("/tmp/openclaw-migrate-command-test", { force: true, recursive: true }); + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: false, + }); + mocks.provider.plan.mockReset(); + mocks.provider.apply.mockReset(); + mocks.promptYesNo.mockReset(); + mocks.backupCreateCommand.mockReset(); + mocks.backupCreateCommand.mockResolvedValue({ archivePath: "/tmp/openclaw-backup.tgz" }); + }); + + afterEach(async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: originalIsTty, + }); + await fs.rm("/tmp/openclaw-migrate-command-test", { force: true, recursive: true }); + vi.clearAllMocks(); + }); + + it("requires explicit force before skipping the pre-migration backup", async () => { + await expect( + migrateApplyCommand(runtime, { provider: "hermes", yes: true, noBackup: true }), + ).rejects.toThrow("--no-backup requires --force"); + expect(mocks.provider.plan).not.toHaveBeenCalled(); + }); + + it("requires --yes in non-interactive apply mode", async () => { + await expect(migrateApplyCommand(runtime, { provider: "hermes" })).rejects.toThrow( + "requires --yes", + ); + expect(mocks.provider.plan).not.toHaveBeenCalled(); + }); + + it("previews and prompts before interactive apply without --yes", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + mocks.promptYesNo.mockResolvedValue(true); + + await migrateApplyCommand(runtime, { provider: "hermes" }); + + expect(mocks.provider.plan).toHaveBeenCalledTimes(1); + expect(mocks.promptYesNo).toHaveBeenCalledWith("Apply this migration now?", false); + expect(mocks.backupCreateCommand).toHaveBeenCalled(); + expect(mocks.provider.apply).toHaveBeenCalledWith(expect.any(Object), planned); + }); + + it("does not apply when interactive apply confirmation is declined", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan(); + mocks.provider.plan.mockResolvedValue(planned); + mocks.promptYesNo.mockResolvedValue(false); + + const result = await migrateApplyCommand(runtime, { provider: "hermes", overwrite: true }); + + expect(result).toBe(planned); + expect(mocks.promptYesNo).toHaveBeenCalledWith("Apply this migration now?", false); + expect(runtime.log).toHaveBeenCalledWith("Migration cancelled."); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("prints a JSON plan without applying when interactive apply uses --json without --yes", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan({ + items: [ + { + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "planned", + details: { + value: { + time: { + env: { OPENAI_API_KEY: "short-dev-key", SAFE_FLAG: "visible" }, + headers: { Authorization: "Bearer short-dev-key" }, + }, + }, + }, + }, + ], + }); + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + + const result = await migrateApplyCommand(jsonRuntime, { + provider: "hermes", + json: true, + }); + + expect(result).toBe(planned); + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { planned: 1 }, + items: [ + { + details: { + value: { + time: { + env: { OPENAI_API_KEY: "[redacted]", SAFE_FLAG: "visible" }, + headers: { Authorization: "[redacted]" }, + }, + }, + }, + }, + ], + }); + expect(logs[0]).not.toContain("short-dev-key"); + expect(mocks.promptYesNo).not.toHaveBeenCalled(); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("does not create a backup or apply when the preflight plan has conflicts", async () => { + mocks.provider.plan.mockResolvedValue( + plan({ + summary: { + total: 1, + planned: 0, + migrated: 0, + skipped: 0, + conflicts: 1, + errors: 0, + sensitive: 0, + }, + items: [ + { + id: "workspace:SOUL.md", + kind: "workspace", + action: "copy", + status: "conflict", + }, + ], + }), + ); + + await expect(migrateApplyCommand(runtime, { provider: "hermes", yes: true })).rejects.toThrow( + "Migration has 1 conflict", + ); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("creates a verified backup before applying a conflict-free migration", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + const result = await migrateApplyCommand(runtime, { provider: "hermes", yes: true }); + + expect(mocks.backupCreateCommand).toHaveBeenCalledWith( + expect.objectContaining({ log: expect.any(Function) }), + { output: undefined, verify: true }, + ); + expect(mocks.provider.apply).toHaveBeenCalledWith( + expect.objectContaining({ + backupPath: "/tmp/openclaw-backup.tgz", + reportDir: expect.stringContaining("/migration/hermes/"), + }), + planned, + ); + expect(result.backupPath).toBe("/tmp/openclaw-backup.tgz"); + }); + + it("prints only the final result for root apply in JSON mode", async () => { + const planned = plan({ + items: [ + { + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "planned", + details: { + value: { + time: { + env: { OPENAI_API_KEY: "short-dev-key" }, + headers: { "x-api-key": "another-short-dev-key" }, + }, + }, + }, + }, + ], + }); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await migrateDefaultCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + backupPath: "/tmp/openclaw-backup.tgz", + items: [ + { + details: { + value: { + time: { + env: { OPENAI_API_KEY: "[redacted]" }, + headers: { "x-api-key": "[redacted]" }, + }, + }, + }, + }, + ], + }); + expect(logs[0]).not.toContain("short-dev-key"); + expect(logs[0]).not.toContain("another-short-dev-key"); + expect(logs[0]).not.toContain("Migration plan"); + }); + + it("keeps provider info logs off stdout in JSON mode", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + const logs: string[] = []; + const errors: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + error(message) { + errors.push(String(message)); + }, + }; + mocks.provider.plan.mockImplementation(async (ctx) => { + ctx.logger.info("provider planning"); + return planned; + }); + mocks.provider.apply.mockImplementation(async (ctx) => { + ctx.logger.info("provider applying"); + return applied; + }); + + await migrateDefaultCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ providerId: "hermes" }); + expect(errors).toEqual(["provider planning", "provider applying"]); + }); + + it("applies the already-reviewed default plan instead of planning again", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await migrateDefaultCommand(runtime, { provider: "hermes", yes: true }); + + expect(mocks.provider.plan).toHaveBeenCalledTimes(1); + expect(mocks.provider.apply).toHaveBeenCalledWith(expect.any(Object), planned); + }); + + it("fails after writing JSON output when apply reports item errors", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { + ...planned.summary, + planned: 0, + errors: 1, + }, + items: planned.items.map((item) => ({ + ...item, + status: "error", + reason: "copy failed", + })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await expect( + migrateApplyCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }), + ).rejects.toThrow("Migration finished with 1 error"); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { errors: 1 }, + reportDir: expect.stringContaining("/migration/hermes/"), + }); + }); + + it("fails after writing JSON output when apply reports late conflicts", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { + ...planned.summary, + planned: 0, + conflicts: 1, + }, + items: planned.items.map((item) => ({ + ...item, + status: "conflict", + reason: "target exists", + })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await expect( + migrateApplyCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }), + ).rejects.toThrow("Migration finished with 1 conflict"); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { conflicts: 1 }, + reportDir: expect.stringContaining("/migration/hermes/"), + }); + }); + + it("prints the dry-run plan in JSON mode even when --yes is set", async () => { + const planned = plan(); + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + + await migrateDefaultCommand(jsonRuntime, { + provider: "hermes", + yes: true, + dryRun: true, + json: true, + }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { planned: 1 }, + }); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts new file mode 100644 index 00000000000..dc104de837f --- /dev/null +++ b/src/commands/migrate.ts @@ -0,0 +1,162 @@ +import { promptYesNo } from "../cli/prompt.js"; +import { loadConfig } from "../config/config.js"; +import { redactMigrationPlan } from "../plugin-sdk/migration.js"; +import { resolvePluginMigrationProviders } from "../plugins/migration-provider-runtime.js"; +import type { MigrationApplyResult, MigrationPlan } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { writeRuntimeJson } from "../runtime.js"; +import { runMigrationApply } from "./migrate/apply.js"; +import { formatMigrationPlan } from "./migrate/output.js"; +import { createMigrationPlan, resolveMigrationProvider } from "./migrate/providers.js"; +import type { + MigrateApplyOptions, + MigrateCommonOptions, + MigrateDefaultOptions, +} from "./migrate/types.js"; + +export type { MigrateApplyOptions, MigrateCommonOptions, MigrateDefaultOptions }; + +export async function migrateListCommand(runtime: RuntimeEnv, opts: { json?: boolean } = {}) { + const providers = resolvePluginMigrationProviders({ cfg: loadConfig() }).map((provider) => ({ + id: provider.id, + label: provider.label, + description: provider.description, + })); + if (opts.json) { + writeRuntimeJson(runtime, { providers }); + return; + } + if (providers.length === 0) { + runtime.log("No migration providers found."); + return; + } + runtime.log( + providers + .map((provider) => + provider.description + ? `${provider.id}\t${provider.label} - ${provider.description}` + : `${provider.id}\t${provider.label}`, + ) + .join("\n"), + ); +} + +export async function migratePlanCommand( + runtime: RuntimeEnv, + opts: MigrateCommonOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + throw new Error("Migration provider is required."); + } + const plan = await createMigrationPlan(runtime, { ...opts, provider: providerId }); + if (opts.json) { + writeRuntimeJson(runtime, redactMigrationPlan(plan)); + } else { + runtime.log(formatMigrationPlan(plan).join("\n")); + } + return plan; +} + +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions & { yes: true }, +): Promise; +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, +): Promise; +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + throw new Error("Migration provider is required."); + } + if (opts.noBackup && !opts.force) { + throw new Error("--no-backup requires --force."); + } + if (!opts.yes && !process.stdin.isTTY) { + throw new Error("openclaw migrate apply requires --yes in non-interactive mode."); + } + const provider = resolveMigrationProvider(providerId); + if (!opts.yes) { + const plan = await migratePlanCommand(runtime, { + ...opts, + provider: providerId, + json: opts.json, + }); + if (opts.json) { + return plan; + } + const ok = await promptYesNo("Apply this migration now?", false); + if (!ok) { + runtime.log("Migration cancelled."); + return plan; + } + return await runMigrationApply({ + runtime, + opts: { ...opts, provider: providerId, yes: true, preflightPlan: plan }, + providerId, + provider, + }); + } + return await runMigrationApply({ runtime, opts, providerId, provider }); +} + +export async function migrateDefaultCommand( + runtime: RuntimeEnv, + opts: MigrateDefaultOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + await migrateListCommand(runtime, { json: opts.json }); + return { + providerId: "list", + source: "", + summary: { + total: 0, + planned: 0, + migrated: 0, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [], + }; + } + const plan = + opts.json && opts.yes && !opts.dryRun + ? await createMigrationPlan(runtime, { ...opts, provider: providerId }) + : await migratePlanCommand(runtime, { + ...opts, + provider: providerId, + json: opts.json && (opts.dryRun || !opts.yes), + }); + if (opts.dryRun) { + return plan; + } + if (opts.json && !opts.yes) { + return plan; + } + if (!opts.yes) { + if (!process.stdin.isTTY) { + runtime.log("Re-run with --yes to apply this migration non-interactively."); + return plan; + } + const ok = await promptYesNo("Apply this migration now?", false); + if (!ok) { + runtime.log("Migration cancelled."); + return plan; + } + } + return await migrateApplyCommand(runtime, { + ...opts, + provider: providerId, + yes: true, + json: opts.json, + preflightPlan: plan, + }); +} diff --git a/src/commands/migrate/apply.ts b/src/commands/migrate/apply.ts new file mode 100644 index 00000000000..bc88412fd6f --- /dev/null +++ b/src/commands/migrate/apply.ts @@ -0,0 +1,86 @@ +import fs from "node:fs/promises"; +import { resolveStateDir } from "../../config/paths.js"; +import type { MigrationApplyResult, MigrationProviderPlugin } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { backupCreateCommand } from "../backup.js"; +import { buildMigrationContext, buildMigrationReportDir } from "./context.js"; +import { assertApplySucceeded, assertConflictFreePlan, writeApplyResult } from "./output.js"; +import type { MigrateApplyOptions } from "./types.js"; + +function shouldTreatMissingBackupAsEmptyState(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("No local OpenClaw state was found to back up") || + message.includes("No OpenClaw config file was found to back up") + ); +} + +export async function createPreMigrationBackup(opts: { + output?: string; +}): Promise { + try { + const result = await backupCreateCommand( + { + log() {}, + error() {}, + exit(code) { + throw new Error(`backup exited with ${code}`); + }, + }, + { + output: opts.output, + verify: true, + }, + ); + return result.archivePath; + } catch (err) { + if (shouldTreatMissingBackupAsEmptyState(err)) { + return undefined; + } + throw err; + } +} + +export async function runMigrationApply(params: { + runtime: RuntimeEnv; + opts: MigrateApplyOptions; + providerId: string; + provider: MigrationProviderPlugin; +}): Promise { + const preflightPlan = + params.opts.preflightPlan ?? + (await params.provider.plan( + buildMigrationContext({ + source: params.opts.source, + includeSecrets: params.opts.includeSecrets, + overwrite: params.opts.overwrite, + runtime: params.runtime, + json: params.opts.json, + }), + )); + assertConflictFreePlan(preflightPlan, params.providerId); + const stateDir = resolveStateDir(); + const reportDir = buildMigrationReportDir(params.providerId, stateDir); + const backupPath = params.opts.noBackup + ? undefined + : await createPreMigrationBackup({ output: params.opts.backupOutput }); + await fs.mkdir(reportDir, { recursive: true }); + const ctx = buildMigrationContext({ + source: params.opts.source, + includeSecrets: params.opts.includeSecrets, + overwrite: params.opts.overwrite, + runtime: params.runtime, + backupPath, + reportDir, + json: params.opts.json, + }); + const result = await params.provider.apply(ctx, preflightPlan); + const withBackup = { + ...result, + backupPath: result.backupPath ?? backupPath, + reportDir: result.reportDir ?? reportDir, + }; + writeApplyResult(params.runtime, params.opts, withBackup); + assertApplySucceeded(withBackup); + return withBackup; +} diff --git a/src/commands/migrate/context.ts b/src/commands/migrate/context.ts new file mode 100644 index 00000000000..b51dca7a3c6 --- /dev/null +++ b/src/commands/migrate/context.ts @@ -0,0 +1,51 @@ +import path from "node:path"; +import { loadConfig } from "../../config/config.js"; +import { resolveStateDir } from "../../config/paths.js"; +import type { MigrationProviderContext } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; + +export function createMigrationLogger(runtime: RuntimeEnv, opts: { json?: boolean } = {}) { + const info = opts.json ? runtime.error : runtime.log; + return { + debug: (message: string) => { + if (process.env.OPENCLAW_VERBOSE === "1") { + info(message); + } + }, + info: (message: string) => info(message), + warn: (message: string) => runtime.error(message), + error: (message: string) => runtime.error(message), + }; +} + +export function buildMigrationReportDir( + providerId: string, + stateDir: string, + nowMs = Date.now(), +): string { + const stamp = new Date(nowMs).toISOString().replaceAll(":", "-"); + return path.join(stateDir, "migration", providerId, stamp); +} + +export function buildMigrationContext(params: { + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + backupPath?: string; + runtime: RuntimeEnv; + reportDir?: string; + json?: boolean; +}): MigrationProviderContext { + const config = loadConfig(); + const stateDir = resolveStateDir(); + return { + config, + stateDir, + source: params.source, + includeSecrets: Boolean(params.includeSecrets), + overwrite: Boolean(params.overwrite), + backupPath: params.backupPath, + reportDir: params.reportDir, + logger: createMigrationLogger(params.runtime, { json: params.json }), + }; +} diff --git a/src/commands/migrate/output.ts b/src/commands/migrate/output.ts new file mode 100644 index 00000000000..03415e82bdc --- /dev/null +++ b/src/commands/migrate/output.ts @@ -0,0 +1,103 @@ +import type { MigrationApplyResult, MigrationItem, MigrationPlan } from "../../plugins/types.js"; +import { redactMigrationPlan } from "../../plugin-sdk/migration.js"; +import { writeRuntimeJson } from "../../runtime.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { theme } from "../../terminal/theme.js"; +import type { MigrateApplyOptions } from "./types.js"; + +export function formatCount(value: number, label: string): string { + return `${value} ${label}${value === 1 ? "" : "s"}`; +} + +export function formatMigrationPlan(plan: MigrationPlan): string[] { + const lines = [ + `${theme.heading("Migration plan:")} ${plan.providerId}`, + `Source: ${plan.source}`, + ]; + if (plan.target) { + lines.push(`Target: ${plan.target}`); + } + lines.push( + [ + formatCount(plan.summary.total, "item"), + formatCount(plan.summary.conflicts, "conflict"), + formatCount(plan.summary.sensitive, "sensitive item"), + ].join(", "), + ); + if (plan.warnings && plan.warnings.length > 0) { + lines.push(""); + lines.push(theme.warn("Warnings:")); + for (const warning of plan.warnings) { + lines.push(`- ${warning}`); + } + } + const visibleItems = plan.items.slice(0, 25); + if (visibleItems.length > 0) { + lines.push(""); + lines.push(theme.heading("Items:")); + for (const item of visibleItems) { + lines.push(formatMigrationItem(item)); + } + if (plan.items.length > visibleItems.length) { + lines.push(`- ... ${plan.items.length - visibleItems.length} more`); + } + } + if (plan.nextSteps && plan.nextSteps.length > 0) { + lines.push(""); + lines.push(theme.heading("Next:")); + for (const step of plan.nextSteps) { + lines.push(`- ${step}`); + } + } + return lines; +} + +export function formatMigrationItem(item: MigrationItem): string { + const target = item.target ? ` -> ${item.target}` : ""; + const message = item.message ? ` (${item.message})` : item.reason ? ` (${item.reason})` : ""; + const sensitive = item.sensitive ? " [sensitive]" : ""; + return `- ${item.status}: ${item.kind}/${item.action} ${item.id}${target}${sensitive}${message}`; +} + +export function assertConflictFreePlan(plan: MigrationPlan, providerId: string): void { + if (plan.summary.conflicts > 0) { + throw new Error( + `Migration has ${formatCount(plan.summary.conflicts, "conflict")}. Re-run with --overwrite after reviewing openclaw migrate plan ${providerId}.`, + ); + } +} + +export function writeApplyResult( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, + result: MigrationApplyResult, +): void { + if (opts.json) { + writeRuntimeJson(runtime, redactMigrationPlan(result)); + return; + } + runtime.log(formatMigrationPlan(result).join("\n")); + if (result.backupPath) { + runtime.log(`Backup: ${result.backupPath}`); + } else if (!opts.noBackup) { + runtime.log("Backup: skipped (no existing OpenClaw state found)"); + } + if (result.reportDir) { + runtime.log(`Report: ${result.reportDir}`); + } +} + +export function assertApplySucceeded(result: MigrationApplyResult): void { + if (result.summary.errors === 0 && result.summary.conflicts === 0) { + return; + } + const reportHint = result.reportDir ? ` See report: ${result.reportDir}.` : ""; + if (result.summary.errors > 0) { + throw new Error( + `Migration finished with ${formatCount(result.summary.errors, "error")}.${reportHint}`, + ); + } + throw new Error( + `Migration finished with ${formatCount(result.summary.conflicts, "conflict")}.${reportHint}`, + ); +} diff --git a/src/commands/migrate/providers.ts b/src/commands/migrate/providers.ts new file mode 100644 index 00000000000..ed2012c62cc --- /dev/null +++ b/src/commands/migrate/providers.ts @@ -0,0 +1,38 @@ +import { loadConfig } from "../../config/config.js"; +import { + resolvePluginMigrationProvider, + resolvePluginMigrationProviders, +} from "../../plugins/migration-provider-runtime.js"; +import type { MigrationPlan, MigrationProviderPlugin } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { buildMigrationContext } from "./context.js"; +import type { MigrateCommonOptions } from "./types.js"; + +export function resolveMigrationProvider(providerId: string): MigrationProviderPlugin { + const config = loadConfig(); + const provider = resolvePluginMigrationProvider({ providerId, cfg: config }); + if (!provider) { + const available = resolvePluginMigrationProviders({ cfg: config }).map((entry) => entry.id); + const suffix = + available.length > 0 + ? ` Available providers: ${available.join(", ")}.` + : " No providers found."; + throw new Error(`Unknown migration provider "${providerId}".${suffix}`); + } + return provider; +} + +export async function createMigrationPlan( + runtime: RuntimeEnv, + opts: MigrateCommonOptions & { provider: string }, +): Promise { + const provider = resolveMigrationProvider(opts.provider); + const ctx = buildMigrationContext({ + source: opts.source, + includeSecrets: opts.includeSecrets, + overwrite: opts.overwrite, + runtime, + json: opts.json, + }); + return await provider.plan(ctx); +} diff --git a/src/commands/migrate/types.ts b/src/commands/migrate/types.ts new file mode 100644 index 00000000000..6b9bbfcf29c --- /dev/null +++ b/src/commands/migrate/types.ts @@ -0,0 +1,21 @@ +import type { MigrationPlan } from "../../plugins/types.js"; + +export type MigrateCommonOptions = { + provider?: string; + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + json?: boolean; +}; + +export type MigrateApplyOptions = MigrateCommonOptions & { + yes?: boolean; + noBackup?: boolean; + force?: boolean; + backupOutput?: string; + preflightPlan?: MigrationPlan; +}; + +export type MigrateDefaultOptions = MigrateApplyOptions & { + dryRun?: boolean; +}; diff --git a/src/commands/onboard-types.ts b/src/commands/onboard-types.ts index 91672cea0d8..c1ee73a8ea6 100644 --- a/src/commands/onboard-types.ts +++ b/src/commands/onboard-types.ts @@ -35,7 +35,7 @@ type OnboardDynamicProviderOptions = { export type OnboardOptions = OnboardDynamicProviderOptions & { mode?: OnboardMode; /** "manual" is an alias for "advanced". */ - flow?: "quickstart" | "advanced" | "manual"; + flow?: "quickstart" | "advanced" | "manual" | "import"; workspace?: string; nonInteractive?: boolean; /** Required for non-interactive setup; skips the interactive risk prompt when true. */ @@ -83,5 +83,8 @@ export type OnboardOptions = OnboardDynamicProviderOptions & { nodeManager?: NodeManagerChoice; remoteUrl?: string; remoteToken?: string; + importFrom?: string; + importSource?: string; + importSecrets?: boolean; json?: boolean; }; diff --git a/src/flows/channel-setup.test.ts b/src/flows/channel-setup.test.ts index 6f958836ad6..dcbc98c2663 100644 --- a/src/flows/channel-setup.test.ts +++ b/src/flows/channel-setup.test.ts @@ -53,6 +53,7 @@ function makePluginRegistry(overrides: Partial = {}): PluginRegi authRequirements: [], webSearchProviders: [], webFetchProviders: [], + migrationProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], videoGenerationProviders: [], diff --git a/src/gateway/server-plugins.test.ts b/src/gateway/server-plugins.test.ts index 5fdcb06d898..00e50107428 100644 --- a/src/gateway/server-plugins.test.ts +++ b/src/gateway/server-plugins.test.ts @@ -87,6 +87,7 @@ const createRegistry = (diagnostics: PluginDiagnostic[]): PluginRegistry => ({ videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], memoryEmbeddingProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], diff --git a/src/gateway/test-helpers.plugin-registry.ts b/src/gateway/test-helpers.plugin-registry.ts index 8aa03fd6f2d..d78e6cf91fd 100644 --- a/src/gateway/test-helpers.plugin-registry.ts +++ b/src/gateway/test-helpers.plugin-registry.ts @@ -22,6 +22,7 @@ function createStubPluginRegistry(): PluginRegistry { musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugin-sdk/agent-runtime.ts b/src/plugin-sdk/agent-runtime.ts index 13ed82bb22b..6781f9718be 100644 --- a/src/plugin-sdk/agent-runtime.ts +++ b/src/plugin-sdk/agent-runtime.ts @@ -38,6 +38,7 @@ export { suggestOAuthProfileIdForLegacyDefault, clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, + loadAuthProfileStoreWithoutExternalProfiles, loadAuthProfileStoreForSecretsRuntime, loadAuthProfileStoreForRuntime, replaceRuntimeAuthProfileStoreSnapshots, diff --git a/src/plugin-sdk/migration-runtime.test.ts b/src/plugin-sdk/migration-runtime.test.ts new file mode 100644 index 00000000000..ed7d648b62e --- /dev/null +++ b/src/plugin-sdk/migration-runtime.test.ts @@ -0,0 +1,123 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { copyMigrationFileItem, writeMigrationReport } from "./migration-runtime.js"; +import { createMigrationItem } from "./migration.js"; + +async function writeFile(filePath: string, contents: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents, "utf8"); +} + +describe("copyMigrationFileItem", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("uses unique backup paths for same-basename targets in the same millisecond", async () => { + vi.spyOn(Date, "now").mockReturnValue(123); + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migration-runtime-")); + const reportDir = path.join(root, "report"); + const sourceOne = path.join(root, "source-one", "AGENTS.md"); + const sourceTwo = path.join(root, "source-two", "AGENTS.md"); + const targetOne = path.join(root, "target-one", "AGENTS.md"); + const targetTwo = path.join(root, "target-two", "AGENTS.md"); + + await writeFile(sourceOne, "new one"); + await writeFile(sourceTwo, "new two"); + await writeFile(targetOne, "old one"); + await writeFile(targetTwo, "old two"); + + const first = await copyMigrationFileItem( + createMigrationItem({ + id: "first", + kind: "file", + action: "copy", + source: sourceOne, + target: targetOne, + }), + reportDir, + { overwrite: true }, + ); + const second = await copyMigrationFileItem( + createMigrationItem({ + id: "second", + kind: "file", + action: "copy", + source: sourceTwo, + target: targetTwo, + }), + reportDir, + { overwrite: true }, + ); + + expect(first.status).toBe("migrated"); + expect(second.status).toBe("migrated"); + const firstBackup = first.details?.backupPath; + const secondBackup = second.details?.backupPath; + expect(firstBackup).toEqual(expect.stringContaining("AGENTS.md")); + expect(secondBackup).toEqual(expect.stringContaining("AGENTS.md")); + expect(firstBackup).not.toBe(secondBackup); + await expect(fs.readFile(firstBackup as string, "utf8")).resolves.toBe("old one"); + await expect(fs.readFile(secondBackup as string, "utf8")).resolves.toBe("old two"); + }); +}); + +describe("writeMigrationReport", () => { + it("redacts nested secret-looking config values in JSON reports", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migration-report-")); + const reportDir = path.join(root, "report"); + + await writeMigrationReport({ + providerId: "hermes", + source: path.join(root, "hermes"), + summary: { + total: 1, + planned: 0, + migrated: 1, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [ + createMigrationItem({ + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "migrated", + details: { + value: { + mcp: { + env: { + OPENAI_API_KEY: "short-dev-key", + SAFE_FLAG: "visible", + }, + headers: { + Authorization: "Bearer short-dev-key", + "x-api-key": "another-short-dev-key", + }, + }, + }, + }, + }), + ], + reportDir, + }); + + const report = await fs.readFile(path.join(reportDir, "report.json"), "utf8"); + expect(report).not.toContain("short-dev-key"); + expect(report).not.toContain("another-short-dev-key"); + expect(JSON.parse(report).items[0].details.value.mcp).toEqual({ + env: { + OPENAI_API_KEY: "[redacted]", + SAFE_FLAG: "visible", + }, + headers: { + Authorization: "[redacted]", + "x-api-key": "[redacted]", + }, + }); + }); +}); diff --git a/src/plugin-sdk/migration-runtime.ts b/src/plugin-sdk/migration-runtime.ts new file mode 100644 index 00000000000..e8f97988c72 --- /dev/null +++ b/src/plugin-sdk/migration-runtime.ts @@ -0,0 +1,186 @@ +// Runtime helpers for migration providers that need filesystem side effects. + +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MigrationApplyResult, MigrationItem } from "../plugins/types.js"; +import { + MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, + MIGRATION_REASON_TARGET_EXISTS, + markMigrationItemConflict, + markMigrationItemError, + redactMigrationPlan, +} from "./migration.js"; + +export type { MigrationApplyResult, MigrationItem } from "../plugins/types.js"; + +async function exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function backupExistingMigrationTarget( + target: string, + reportDir: string, +): Promise { + if (!(await exists(target))) { + return undefined; + } + const backupRoot = path.join(reportDir, "item-backups"); + await fs.mkdir(backupRoot, { recursive: true }); + const targetHash = crypto + .createHash("sha256") + .update(path.resolve(target)) + .digest("hex") + .slice(0, 12); + const backupDir = await fs.mkdtemp( + path.join(backupRoot, `${Date.now()}-${targetHash}-${path.basename(target)}-`), + ); + const backupPath = path.join(backupDir, path.basename(target)); + await fs.cp(target, backupPath, { recursive: true, force: true }); + return backupPath; +} + +function isFileAlreadyExistsError(err: unknown): boolean { + return Boolean( + err && + typeof err === "object" && + "code" in err && + ((err as { code?: unknown }).code === "ERR_FS_CP_EEXIST" || + (err as { code?: unknown }).code === "EEXIST"), + ); +} + +function readArchiveRelativePath(item: MigrationItem): string { + const detailPath = item.details?.archiveRelativePath; + const raw = typeof detailPath === "string" && detailPath.trim() ? detailPath : undefined; + const fallback = item.source ? path.basename(item.source) : item.id; + const normalized = path + .normalize(raw ?? fallback) + .split(path.sep) + .filter((part) => part && part !== "." && part !== "..") + .join(path.sep); + return normalized || "item"; +} + +async function resolveUniqueArchivePath( + archiveRoot: string, + relativePath: string, +): Promise { + const parsed = path.parse(relativePath); + let candidate = path.join(archiveRoot, relativePath); + let index = 2; + while (await exists(candidate)) { + const filename = `${parsed.name}-${index}${parsed.ext}`; + candidate = path.join(archiveRoot, parsed.dir, filename); + index += 1; + } + return candidate; +} + +export async function archiveMigrationItem( + item: MigrationItem, + reportDir: string, +): Promise { + if (!item.source) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const sourceStat = await fs.lstat(item.source); + if (sourceStat.isSymbolicLink()) { + return markMigrationItemError(item, "archive source is a symlink"); + } + const archiveRoot = path.join(reportDir, "archive"); + const relativePath = readArchiveRelativePath(item); + const archivePath = await resolveUniqueArchivePath(archiveRoot, relativePath); + await fs.mkdir(path.dirname(archivePath), { recursive: true }); + await fs.cp(item.source, archivePath, { + recursive: true, + force: false, + errorOnExist: true, + verbatimSymlinks: true, + }); + return { + ...item, + status: "migrated", + target: archivePath, + details: { ...item.details, archivePath, archiveRelativePath: relativePath }, + }; + } catch (err) { + if (isFileAlreadyExistsError(err)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export async function copyMigrationFileItem( + item: MigrationItem, + reportDir: string, + opts: { overwrite?: boolean } = {}, +): Promise { + if (!item.source || !item.target) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const targetExists = await exists(item.target); + if (targetExists && !opts.overwrite) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + const backupPath = opts.overwrite + ? await backupExistingMigrationTarget(item.target, reportDir) + : undefined; + await fs.mkdir(path.dirname(item.target), { recursive: true }); + await fs.cp(item.source, item.target, { + recursive: true, + force: Boolean(opts.overwrite), + errorOnExist: !opts.overwrite, + }); + return { + ...item, + status: "migrated", + details: { ...item.details, ...(backupPath ? { backupPath } : {}) }, + }; + } catch (err) { + if (isFileAlreadyExistsError(err)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export async function writeMigrationReport( + result: MigrationApplyResult, + opts: { title?: string } = {}, +): Promise { + if (!result.reportDir) { + return; + } + await fs.mkdir(result.reportDir, { recursive: true }); + await fs.writeFile( + path.join(result.reportDir, "report.json"), + `${JSON.stringify(redactMigrationPlan(result), null, 2)}\n`, + "utf8", + ); + const lines = [ + `# ${opts.title ?? "Migration Report"}`, + "", + `Source: ${result.source}`, + result.target ? `Target: ${result.target}` : undefined, + result.backupPath ? `Backup: ${result.backupPath}` : undefined, + "", + `Migrated: ${result.summary.migrated}`, + `Skipped: ${result.summary.skipped}`, + `Conflicts: ${result.summary.conflicts}`, + `Errors: ${result.summary.errors}`, + "", + ...result.items.map( + (item) => `- ${item.status}: ${item.id}${item.reason ? ` (${item.reason})` : ""}`, + ), + ].filter((line): line is string => typeof line === "string"); + await fs.writeFile(path.join(result.reportDir, "summary.md"), `${lines.join("\n")}\n`, "utf8"); +} diff --git a/src/plugin-sdk/migration.ts b/src/plugin-sdk/migration.ts new file mode 100644 index 00000000000..2e68eb8bb7e --- /dev/null +++ b/src/plugin-sdk/migration.ts @@ -0,0 +1,153 @@ +// Shared migration-provider helpers for plan/apply item bookkeeping. + +import type { + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, +} from "../plugins/types.js"; + +export type { + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, +}; + +export const MIGRATION_REASON_MISSING_SOURCE_OR_TARGET = "missing source or target"; +export const MIGRATION_REASON_TARGET_EXISTS = "target exists"; + +export function createMigrationItem( + params: Omit & { status?: MigrationItem["status"] }, +): MigrationItem { + return { + ...params, + status: params.status ?? "planned", + }; +} + +export function markMigrationItemConflict(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "conflict", reason }; +} + +export function markMigrationItemError(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "error", reason }; +} + +export function markMigrationItemSkipped(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "skipped", reason }; +} + +export function summarizeMigrationItems(items: readonly MigrationItem[]): MigrationSummary { + return { + total: items.length, + planned: items.filter((item) => item.status === "planned").length, + migrated: items.filter((item) => item.status === "migrated").length, + skipped: items.filter((item) => item.status === "skipped").length, + conflicts: items.filter((item) => item.status === "conflict").length, + errors: items.filter((item) => item.status === "error").length, + sensitive: items.filter((item) => item.sensitive).length, + }; +} + +const REDACTED_MIGRATION_VALUE = "[redacted]"; +const SECRET_KEY_MARKERS = [ + "accesstoken", + "apikey", + "authorization", + "bearertoken", + "clientsecret", + "cookie", + "credential", + "password", + "privatekey", + "refreshtoken", + "secret", +] as const; + +const SECRET_VALUE_PATTERNS = [ + /\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, + /\bsk-[A-Za-z0-9_-]{8,}\b/gu, + /\bgh[pousr]_[A-Za-z0-9_]{16,}\b/gu, + /\bxox[abprs]-[A-Za-z0-9-]{8,}\b/gu, + /\bAIza[0-9A-Za-z_-]{12,}\b/gu, +] as const; + +function normalizeSecretKey(key: string): string { + return key.toLowerCase().replaceAll(/[^a-z0-9]/gu, ""); +} + +function isSecretKey(key: string): boolean { + const normalized = normalizeSecretKey(key); + if (normalized === "token" || normalized.endsWith("token")) { + return true; + } + if (normalized === "auth" || normalized === "authorization") { + return true; + } + return SECRET_KEY_MARKERS.some((marker) => normalized.includes(marker)); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isSecretReferenceLike(value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + return ( + value.source === "env" && + typeof value.id === "string" && + (value.provider === undefined || typeof value.provider === "string") + ); +} + +function redactString(value: string): string { + let next = value; + for (const pattern of SECRET_VALUE_PATTERNS) { + next = next.replace(pattern, REDACTED_MIGRATION_VALUE); + } + return next; +} + +function redactMigrationValueInternal(value: unknown, seen: WeakSet): unknown { + if (typeof value === "string") { + return redactString(value); + } + if (Array.isArray(value)) { + return value.map((entry) => redactMigrationValueInternal(entry, seen)); + } + if (!value || typeof value !== "object") { + return value; + } + if (seen.has(value)) { + return REDACTED_MIGRATION_VALUE; + } + seen.add(value); + const next: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (isSecretKey(key) && !isSecretReferenceLike(entry)) { + next[key] = REDACTED_MIGRATION_VALUE; + continue; + } + next[key] = redactMigrationValueInternal(entry, seen); + } + return next; +} + +export function redactMigrationValue(value: unknown): unknown { + return redactMigrationValueInternal(value, new WeakSet()); +} + +export function redactMigrationItem(item: MigrationItem): MigrationItem { + return redactMigrationValue(item) as MigrationItem; +} + +export function redactMigrationPlan(plan: T): T { + return redactMigrationValue(plan) as T; +} diff --git a/src/plugin-sdk/plugin-entry.ts b/src/plugin-sdk/plugin-entry.ts index 8242c8f33bf..53b1a6d6943 100644 --- a/src/plugin-sdk/plugin-entry.ts +++ b/src/plugin-sdk/plugin-entry.ts @@ -5,6 +5,13 @@ import type { AnyAgentTool, AgentHarness, MediaUnderstandingProviderPlugin, + MigrationApplyResult, + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, OpenClawPluginApi, OpenClawPluginCommandDefinition, OpenClawPluginConfigSchema, @@ -80,6 +87,13 @@ export type { AnyAgentTool, AgentHarness, MediaUnderstandingProviderPlugin, + MigrationApplyResult, + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, OpenClawPluginApi, OpenClawPluginNodeHostCommand, OpenClawPluginReloadRegistration, diff --git a/src/plugin-sdk/provider-auth.ts b/src/plugin-sdk/provider-auth.ts index 8ef12652f83..e647a457c8d 100644 --- a/src/plugin-sdk/provider-auth.ts +++ b/src/plugin-sdk/provider-auth.ts @@ -19,6 +19,7 @@ export { CLAUDE_CLI_PROFILE_ID, CODEX_CLI_PROFILE_ID } from "../agents/auth-prof export { ensureAuthProfileStore, ensureAuthProfileStoreForLocalUpdate, + updateAuthProfileStoreWithLock, } from "../agents/auth-profiles/store.js"; export { listProfilesForProvider, diff --git a/src/plugins/api-builder.ts b/src/plugins/api-builder.ts index 84d6d827ad5..d9ce38a5e0c 100644 --- a/src/plugins/api-builder.ts +++ b/src/plugins/api-builder.ts @@ -32,6 +32,7 @@ export type BuildPluginApiParams = { | "registerCliBackend" | "registerTextTransforms" | "registerConfigMigration" + | "registerMigrationProvider" | "registerAutoEnableProbe" | "registerProvider" | "registerSpeechProvider" @@ -80,6 +81,7 @@ const noopRegisterGatewayDiscoveryService: OpenClawPluginApi["registerGatewayDis const noopRegisterCliBackend: OpenClawPluginApi["registerCliBackend"] = () => {}; const noopRegisterTextTransforms: OpenClawPluginApi["registerTextTransforms"] = () => {}; const noopRegisterConfigMigration: OpenClawPluginApi["registerConfigMigration"] = () => {}; +const noopRegisterMigrationProvider: OpenClawPluginApi["registerMigrationProvider"] = () => {}; const noopRegisterAutoEnableProbe: OpenClawPluginApi["registerAutoEnableProbe"] = () => {}; const noopRegisterProvider: OpenClawPluginApi["registerProvider"] = () => {}; const noopRegisterSpeechProvider: OpenClawPluginApi["registerSpeechProvider"] = () => {}; @@ -151,6 +153,7 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi registerCliBackend: handlers.registerCliBackend ?? noopRegisterCliBackend, registerTextTransforms: handlers.registerTextTransforms ?? noopRegisterTextTransforms, registerConfigMigration: handlers.registerConfigMigration ?? noopRegisterConfigMigration, + registerMigrationProvider: handlers.registerMigrationProvider ?? noopRegisterMigrationProvider, registerAutoEnableProbe: handlers.registerAutoEnableProbe ?? noopRegisterAutoEnableProbe, registerProvider: handlers.registerProvider ?? noopRegisterProvider, registerSpeechProvider: handlers.registerSpeechProvider ?? noopRegisterSpeechProvider, diff --git a/src/plugins/bundled-capability-metadata.test.ts b/src/plugins/bundled-capability-metadata.test.ts index 1a5cff4a428..4f09e565a15 100644 --- a/src/plugins/bundled-capability-metadata.test.ts +++ b/src/plugins/bundled-capability-metadata.test.ts @@ -45,6 +45,14 @@ describe("bundled capability metadata", () => { .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)); expect(BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS).toEqual(expected); + expect(BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + pluginId: "migrate-hermes", + migrationProviderIds: ["hermes"], + }), + ]), + ); }); it("keeps lightweight alias maps aligned with bundled plugin manifests", () => { diff --git a/src/plugins/bundled-capability-runtime.test.ts b/src/plugins/bundled-capability-runtime.test.ts index 67ee7b334b5..42adae5a6fe 100644 --- a/src/plugins/bundled-capability-runtime.test.ts +++ b/src/plugins/bundled-capability-runtime.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { buildVitestCapabilityShimAliasMap } from "./bundled-capability-runtime.js"; +import { + buildVitestCapabilityShimAliasMap, + loadBundledCapabilityRuntimeRegistry, +} from "./bundled-capability-runtime.js"; describe("buildVitestCapabilityShimAliasMap", () => { it("keeps scoped and unscoped capability shim aliases aligned", () => { @@ -22,3 +25,21 @@ describe("buildVitestCapabilityShimAliasMap", () => { ); }); }); + +describe("loadBundledCapabilityRuntimeRegistry", () => { + it("captures bundled migration providers", () => { + const registry = loadBundledCapabilityRuntimeRegistry({ + pluginIds: ["migrate-hermes"], + pluginSdkResolution: "dist", + }); + + const record = registry.plugins.find((entry) => entry.id === "migrate-hermes"); + expect(record?.migrationProviderIds).toEqual(["hermes"]); + expect( + registry.migrationProviders.map((entry) => ({ + pluginId: entry.pluginId, + providerId: entry.provider.id, + })), + ).toEqual([{ pluginId: "migrate-hermes", providerId: "hermes" }]); + }); +}); diff --git a/src/plugins/bundled-capability-runtime.ts b/src/plugins/bundled-capability-runtime.ts index d8f9e5c8c63..90c918f23ed 100644 --- a/src/plugins/bundled-capability-runtime.ts +++ b/src/plugins/bundled-capability-runtime.ts @@ -158,6 +158,7 @@ function createCapabilityPluginRecord(params: { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], gatewayMethods: [], @@ -335,6 +336,7 @@ export function loadBundledCapabilityRuntimeRegistry(params: { ); record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id)); record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id)); + record.migrationProviderIds.push(...captured.migrationProviders.map((entry) => entry.id)); record.memoryEmbeddingProviderIds.push( ...captured.memoryEmbeddingProviders.map((entry) => entry.id), ); @@ -449,6 +451,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: { rootDir: record.rootDir, })), ); + registry.migrationProviders.push( + ...captured.migrationProviders.map((provider) => ({ + pluginId: record.id, + pluginName: record.name, + provider, + source: record.source, + rootDir: record.rootDir, + })), + ); registry.memoryEmbeddingProviders.push( ...captured.memoryEmbeddingProviders.map((provider) => ({ pluginId: record.id, diff --git a/src/plugins/captured-registration.ts b/src/plugins/captured-registration.ts index 7bbba7dbbca..4fb6a13d40c 100644 --- a/src/plugins/captured-registration.ts +++ b/src/plugins/captured-registration.ts @@ -16,6 +16,7 @@ import type { OpenClawPluginApi, ImageGenerationProviderPlugin, MediaUnderstandingProviderPlugin, + MigrationProviderPlugin, MusicGenerationProviderPlugin, OpenClawPluginCliCommandDescriptor, OpenClawPluginCliRegistrar, @@ -53,6 +54,7 @@ export type CapturedPluginRegistration = { musicGenerationProviders: MusicGenerationProviderPlugin[]; webFetchProviders: WebFetchProviderPlugin[]; webSearchProviders: WebSearchProviderPlugin[]; + migrationProviders: MigrationProviderPlugin[]; memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[]; tools: AnyAgentTool[]; }; @@ -77,6 +79,7 @@ export function createCapturedPluginRegistration(params?: { const musicGenerationProviders: MusicGenerationProviderPlugin[] = []; const webFetchProviders: WebFetchProviderPlugin[] = []; const webSearchProviders: WebSearchProviderPlugin[] = []; + const migrationProviders: MigrationProviderPlugin[] = []; const memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[] = []; const tools: AnyAgentTool[] = []; const noopLogger = { @@ -103,6 +106,7 @@ export function createCapturedPluginRegistration(params?: { musicGenerationProviders, webFetchProviders, webSearchProviders, + migrationProviders, memoryEmbeddingProviders, tools, api: buildPluginApi({ @@ -194,6 +198,9 @@ export function createCapturedPluginRegistration(params?: { registerWebSearchProvider(provider: WebSearchProviderPlugin) { webSearchProviders.push(provider); }, + registerMigrationProvider(provider: MigrationProviderPlugin) { + migrationProviders.push(provider); + }, registerMemoryEmbeddingProvider(adapter: MemoryEmbeddingProviderAdapter) { memoryEmbeddingProviders.push(adapter); }, diff --git a/src/plugins/contracts/inventory/bundled-capability-metadata.ts b/src/plugins/contracts/inventory/bundled-capability-metadata.ts index 403908f5a9b..39d22fb8800 100644 --- a/src/plugins/contracts/inventory/bundled-capability-metadata.ts +++ b/src/plugins/contracts/inventory/bundled-capability-metadata.ts @@ -28,6 +28,7 @@ export type BundledPluginContractSnapshot = { webContentExtractorIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; toolNames: string[]; }; @@ -164,6 +165,9 @@ export function buildBundledPluginContractSnapshot( webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders, (value) => value.trim(), ), + migrationProviderIds: uniqueStrings(manifest.contracts?.migrationProviders, (value) => + value.trim(), + ), toolNames: uniqueStrings(manifest.contracts?.tools, (value) => value.trim()), }; } @@ -185,6 +189,7 @@ export function hasBundledPluginContractSnapshotCapabilities( entry.webContentExtractorIds.length > 0 || entry.webFetchProviderIds.length > 0 || entry.webSearchProviderIds.length > 0 || + entry.migrationProviderIds.length > 0 || entry.toolNames.length > 0 ); } diff --git a/src/plugins/contracts/registry.contract.test.ts b/src/plugins/contracts/registry.contract.test.ts index 6a3573748ff..b3decb14734 100644 --- a/src/plugins/contracts/registry.contract.test.ts +++ b/src/plugins/contracts/registry.contract.test.ts @@ -22,6 +22,7 @@ describe("plugin contract registry", () => { speechProviders?: unknown[]; realtimeTranscriptionProviders?: unknown[]; realtimeVoiceProviders?: unknown[]; + migrationProviders?: unknown[]; }; }) => boolean; }) { @@ -38,6 +39,7 @@ describe("plugin contract registry", () => { speechProviders?: unknown[]; realtimeTranscriptionProviders?: unknown[]; realtimeVoiceProviders?: unknown[]; + migrationProviders?: unknown[]; }; }) => boolean, ) { @@ -65,6 +67,10 @@ describe("plugin contract registry", () => { name: "does not duplicate bundled web search provider ids", ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.webSearchProviderIds), }, + { + name: "does not duplicate bundled migration provider ids", + ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.migrationProviderIds), + }, { name: "does not duplicate bundled media provider ids", ids: () => @@ -200,4 +206,14 @@ describe("plugin contract registry", () => { ), ).toEqual(bundledWebSearchPluginIds); }); + + it("covers every bundled migration provider plugin discovered from manifests", () => { + expectRegistryPluginIds({ + actualPluginIds: pluginRegistrationContractRegistry + .filter((entry) => entry.migrationProviderIds.length > 0) + .map((entry) => entry.pluginId), + predicate: (plugin) => + plugin.origin === "bundled" && (plugin.contracts?.migrationProviders?.length ?? 0) > 0, + }); + }); }); diff --git a/src/plugins/contracts/registry.retry.test.ts b/src/plugins/contracts/registry.retry.test.ts index 4a3244ad635..6c1c7b1be38 100644 --- a/src/plugins/contracts/registry.retry.test.ts +++ b/src/plugins/contracts/registry.retry.test.ts @@ -8,6 +8,7 @@ type MockPluginRecord = { providerIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; }; type MockRuntimeRegistry = { @@ -52,6 +53,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [{ pluginId: "arcee", message: "transient arcee load failure" }], }), @@ -64,6 +66,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: ["arcee"], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, providers: [ { @@ -106,6 +109,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [{ pluginId: "searxng", message: "transient searxng load failure" }], }), @@ -118,6 +122,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: ["searxng"], + migrationProviderIds: [], }, webSearchProviders: [ { @@ -170,6 +175,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: ["byteplus"], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, providers: [ { @@ -311,6 +317,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [ { pluginId: "firecrawl", message: "transient firecrawl fetch load failure" }, @@ -325,6 +332,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: ["firecrawl"], webSearchProviderIds: ["firecrawl"], + migrationProviderIds: [], }, webFetchProviders: [ { diff --git a/src/plugins/contracts/registry.ts b/src/plugins/contracts/registry.ts index 0875e6cdd15..a9f094eb26b 100644 --- a/src/plugins/contracts/registry.ts +++ b/src/plugins/contracts/registry.ts @@ -69,6 +69,7 @@ type ManifestContractKey = | "webContentExtractors" | "webFetchProviders" | "webSearchProviders" + | "migrationProviders" | "tools"; type ManifestRegistryContractKey = "webFetchProviders" | "webSearchProviders"; @@ -102,6 +103,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { webContentExtractorIds: [...entry.webContentExtractorIds], webFetchProviderIds: [...entry.webFetchProviderIds], webSearchProviderIds: [...entry.webSearchProviderIds], + migrationProviderIds: [...entry.migrationProviderIds], toolNames: [...entry.toolNames], })); } @@ -122,6 +124,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { (plugin.contracts?.webContentExtractors?.length ?? 0) > 0 || (plugin.contracts?.webFetchProviders?.length ?? 0) > 0 || (plugin.contracts?.webSearchProviders?.length ?? 0) > 0 || + (plugin.contracts?.migrationProviders?.length ?? 0) > 0 || (plugin.contracts?.tools?.length ?? 0) > 0), ) .map((plugin) => ({ @@ -144,6 +147,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { webContentExtractorIds: uniqueStrings(plugin.contracts?.webContentExtractors ?? []), webFetchProviderIds: uniqueStrings(plugin.contracts?.webFetchProviders ?? []), webSearchProviderIds: uniqueStrings(plugin.contracts?.webSearchProviders ?? []), + migrationProviderIds: uniqueStrings(plugin.contracts?.migrationProviders ?? []), toolNames: uniqueStrings(plugin.contracts?.tools ?? []), })); } @@ -204,6 +208,8 @@ function resolveBundledManifestPluginIdsForContract(contract: ManifestContractKe return entry.webFetchProviderIds.length > 0; case "webSearchProviders": return entry.webSearchProviderIds.length > 0; + case "migrationProviders": + return entry.migrationProviderIds.length > 0; case "tools": return entry.toolNames.length > 0; } diff --git a/src/plugins/hooks.test-helpers.ts b/src/plugins/hooks.test-helpers.ts index 12ea7b8bcb3..3ea07cab615 100644 --- a/src/plugins/hooks.test-helpers.ts +++ b/src/plugins/hooks.test-helpers.ts @@ -42,6 +42,7 @@ export function createMockPluginRegistry( musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/installed-plugin-index-record-builder.ts b/src/plugins/installed-plugin-index-record-builder.ts index d63c83d0a06..c91cb11713a 100644 --- a/src/plugins/installed-plugin-index-record-builder.ts +++ b/src/plugins/installed-plugin-index-record-builder.ts @@ -43,6 +43,7 @@ function hasRuntimeContractSurface(record: PluginManifestRecord): boolean { record.contracts?.webContentExtractors?.length || record.contracts?.webFetchProviders?.length || record.contracts?.webSearchProviders?.length || + record.contracts?.migrationProviders?.length || record.contracts?.memoryEmbeddingProviders?.length || hasKind(record.kind, "memory"), ); diff --git a/src/plugins/installed-plugin-index.test.ts b/src/plugins/installed-plugin-index.test.ts index 9d2dff291e2..decff32735a 100644 --- a/src/plugins/installed-plugin-index.test.ts +++ b/src/plugins/installed-plugin-index.test.ts @@ -226,6 +226,43 @@ describe("installed plugin index", () => { expect(index.plugins[0]?.installRecordHash).toBeUndefined(); }); + it("does not classify migration-provider-only plugins as gateway startup sidecars", () => { + const rootDir = makeTempDir(); + writeRuntimeEntry(rootDir); + writePackageJson(rootDir, { + name: "@vendor/migration-plugin", + version: "1.0.0", + }); + writePluginManifest(rootDir, { + id: "migration-plugin", + name: "Migration Plugin", + enabledByDefault: true, + configSchema: { type: "object" }, + contracts: { + migrationProviders: ["legacy-import"], + }, + }); + + const index = loadInstalledPluginIndex({ + candidates: [ + createPluginCandidate({ + rootDir, + packageName: "@vendor/migration-plugin", + packageVersion: "1.0.0", + }), + ], + env: hermeticEnv(), + }); + + expect(index.plugins[0]).toMatchObject({ + pluginId: "migration-plugin", + enabledByDefault: true, + startup: { + sidecar: false, + }, + }); + }); + it("keeps bundle format metadata needed for manifest reconstruction", () => { const rootDir = makeTempDir(); fs.mkdirSync(path.join(rootDir, ".claude-plugin"), { recursive: true }); diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 5f9788fc910..d78f25948b9 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -291,6 +291,7 @@ type PluginRegistrySnapshot = { musicGenerationProviders: PluginRegistry["musicGenerationProviders"]; webFetchProviders: PluginRegistry["webFetchProviders"]; webSearchProviders: PluginRegistry["webSearchProviders"]; + migrationProviders: PluginRegistry["migrationProviders"]; codexAppServerExtensionFactories: PluginRegistry["codexAppServerExtensionFactories"]; agentToolResultMiddlewares: PluginRegistry["agentToolResultMiddlewares"]; memoryEmbeddingProviders: PluginRegistry["memoryEmbeddingProviders"]; @@ -329,6 +330,7 @@ function snapshotPluginRegistry(registry: PluginRegistry): PluginRegistrySnapsho musicGenerationProviders: [...registry.musicGenerationProviders], webFetchProviders: [...registry.webFetchProviders], webSearchProviders: [...registry.webSearchProviders], + migrationProviders: [...registry.migrationProviders], codexAppServerExtensionFactories: [...registry.codexAppServerExtensionFactories], agentToolResultMiddlewares: [...registry.agentToolResultMiddlewares], memoryEmbeddingProviders: [...registry.memoryEmbeddingProviders], @@ -366,6 +368,7 @@ function restorePluginRegistry(registry: PluginRegistry, snapshot: PluginRegistr registry.musicGenerationProviders = snapshot.arrays.musicGenerationProviders; registry.webFetchProviders = snapshot.arrays.webFetchProviders; registry.webSearchProviders = snapshot.arrays.webSearchProviders; + registry.migrationProviders = snapshot.arrays.migrationProviders; registry.codexAppServerExtensionFactories = snapshot.arrays.codexAppServerExtensionFactories; registry.agentToolResultMiddlewares = snapshot.arrays.agentToolResultMiddlewares; registry.memoryEmbeddingProviders = snapshot.arrays.memoryEmbeddingProviders; @@ -1830,6 +1833,7 @@ function createPluginRecord(params: { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], contextEngineIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], diff --git a/src/plugins/manifest-contract-runtime.ts b/src/plugins/manifest-contract-runtime.ts new file mode 100644 index 00000000000..ed21db7edf3 --- /dev/null +++ b/src/plugins/manifest-contract-runtime.ts @@ -0,0 +1,53 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginManifestContractListKey } from "./manifest-registry.js"; +import { loadPluginManifestRegistryForPluginRegistry } from "./plugin-registry.js"; + +export type ManifestContractRuntimePluginResolution = { + pluginIds: string[]; + bundledCompatPluginIds: string[]; +}; + +const DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS = { + preferPersisted: false, +} as const; + +function hasManifestContractValue( + plugin: ReturnType["plugins"][number], + contract: PluginManifestContractListKey, + value?: string, +): boolean { + const values = plugin.contracts?.[contract] ?? []; + return values.length > 0 && (!value || values.includes(value)); +} + +export function resolveManifestContractRuntimePluginResolution(params: { + cfg?: OpenClawConfig; + contract: PluginManifestContractListKey; + value?: string; +}): ManifestContractRuntimePluginResolution { + const allContractPlugins = loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env: process.env, + includeDisabled: true, + ...DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS, + }).plugins.filter((plugin) => hasManifestContractValue(plugin, params.contract, params.value)); + const bundledCompatPluginIds = allContractPlugins + .filter((plugin) => plugin.origin === "bundled") + .map((plugin) => plugin.id); + const enabledPluginIds = new Set( + loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env: process.env, + ...DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS, + }).plugins.map((plugin) => plugin.id), + ); + const pluginIds = allContractPlugins + .filter((plugin) => plugin.origin === "bundled" || enabledPluginIds.has(plugin.id)) + .map((plugin) => plugin.id); + return { + pluginIds: [...new Set(pluginIds)].toSorted((left, right) => left.localeCompare(right)), + bundledCompatPluginIds: [...new Set(bundledCompatPluginIds)].toSorted((left, right) => + left.localeCompare(right), + ), + }; +} diff --git a/src/plugins/manifest-registry.ts b/src/plugins/manifest-registry.ts index 51ecf2d88d7..633267b44be 100644 --- a/src/plugins/manifest-registry.ts +++ b/src/plugins/manifest-registry.ts @@ -79,7 +79,8 @@ export type PluginManifestContractListKey = | "memoryEmbeddingProviders" | "webContentExtractors" | "webFetchProviders" - | "webSearchProviders"; + | "webSearchProviders" + | "migrationProviders"; type SeenIdEntry = { candidate: PluginCandidate; diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 52be7ccb505..bfb13022957 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -294,6 +294,7 @@ export type PluginManifestContracts = { webContentExtractors?: string[]; webFetchProviders?: string[]; webSearchProviders?: string[]; + migrationProviders?: string[]; tools?: string[]; }; @@ -488,6 +489,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u const webContentExtractors = normalizeTrimmedStringList(value.webContentExtractors); const webFetchProviders = normalizeTrimmedStringList(value.webFetchProviders); const webSearchProviders = normalizeTrimmedStringList(value.webSearchProviders); + const migrationProviders = normalizeTrimmedStringList(value.migrationProviders); const tools = normalizeTrimmedStringList(value.tools); const contracts = { ...(embeddedExtensionFactories.length > 0 ? { embeddedExtensionFactories } : {}), @@ -505,6 +507,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u ...(webContentExtractors.length > 0 ? { webContentExtractors } : {}), ...(webFetchProviders.length > 0 ? { webFetchProviders } : {}), ...(webSearchProviders.length > 0 ? { webSearchProviders } : {}), + ...(migrationProviders.length > 0 ? { migrationProviders } : {}), ...(tools.length > 0 ? { tools } : {}), } satisfies PluginManifestContracts; diff --git a/src/plugins/migration-provider-runtime.test.ts b/src/plugins/migration-provider-runtime.test.ts new file mode 100644 index 00000000000..4d51c1788f4 --- /dev/null +++ b/src/plugins/migration-provider-runtime.test.ts @@ -0,0 +1,214 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginRegistry } from "./registry-types.js"; +import { createEmptyPluginRegistry } from "./registry.js"; + +type MockManifestRegistry = { + plugins: Array>; + diagnostics: unknown[]; +}; + +function createEmptyMockManifestRegistry(): MockManifestRegistry { + return { plugins: [], diagnostics: [] }; +} + +const mocks = vi.hoisted(() => ({ + resolveRuntimePluginRegistry: vi.fn<(params?: unknown) => PluginRegistry | undefined>( + () => undefined, + ), + loadPluginManifestRegistry: vi.fn<(params?: Record) => MockManifestRegistry>( + () => createEmptyMockManifestRegistry(), + ), + withBundledPluginAllowlistCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), + withBundledPluginEnablementCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), + withBundledPluginVitestCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), +})); + +vi.mock("./loader.js", () => ({ + resolveRuntimePluginRegistry: mocks.resolveRuntimePluginRegistry, +})); + +vi.mock("./plugin-registry.js", () => ({ + loadPluginManifestRegistryForPluginRegistry: mocks.loadPluginManifestRegistry, +})); + +vi.mock("./bundled-compat.js", () => ({ + withBundledPluginAllowlistCompat: mocks.withBundledPluginAllowlistCompat, + withBundledPluginEnablementCompat: mocks.withBundledPluginEnablementCompat, + withBundledPluginVitestCompat: mocks.withBundledPluginVitestCompat, +})); + +let resolvePluginMigrationProvider: typeof import("./migration-provider-runtime.js").resolvePluginMigrationProvider; +let resolvePluginMigrationProviders: typeof import("./migration-provider-runtime.js").resolvePluginMigrationProviders; + +function createMigrationProvider(id: string) { + return { + id, + label: id, + plan: vi.fn(), + apply: vi.fn(), + }; +} + +describe("migration provider runtime", () => { + beforeEach(async () => { + vi.clearAllMocks(); + mocks.resolveRuntimePluginRegistry.mockReturnValue(createEmptyPluginRegistry()); + mocks.loadPluginManifestRegistry.mockReturnValue(createEmptyMockManifestRegistry()); + const runtime = await import("./migration-provider-runtime.js"); + resolvePluginMigrationProvider = runtime.resolvePluginMigrationProvider; + resolvePluginMigrationProviders = runtime.resolvePluginMigrationProviders; + }); + + it("loads configured external migration-provider plugins from manifest contracts", () => { + const cfg = { + plugins: { entries: { "external-migration": { enabled: true } } }, + } as OpenClawConfig; + const provider = createMigrationProvider("external-import"); + const active = createEmptyPluginRegistry(); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "external-migration", + pluginName: "External Migration", + source: "test", + provider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => ({ + diagnostics: [], + plugins: params?.includeDisabled + ? [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + { + id: "disabled-external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ] + : [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ], + })); + + const resolved = resolvePluginMigrationProvider({ providerId: "external-import", cfg }); + + expect(resolved).toBe(provider); + expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({ + config: cfg, + env: process.env, + includeDisabled: true, + preferPersisted: false, + }); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith(); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ + config: cfg, + onlyPluginIds: ["external-migration"], + activate: false, + }); + }); + + it("derives a fresh manifest registry so newly bundled migration providers are discoverable", () => { + const provider = createMigrationProvider("hermes"); + const active = createEmptyPluginRegistry(); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "migrate-hermes", + pluginName: "Hermes Migration", + source: "test", + provider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => { + if (params?.preferPersisted !== false) { + return createEmptyMockManifestRegistry(); + } + return { + diagnostics: [], + plugins: [ + { + id: "migrate-hermes", + origin: "bundled", + contracts: { migrationProviders: ["hermes"] }, + }, + ], + }; + }); + + const resolved = resolvePluginMigrationProvider({ providerId: "hermes" }); + + expect(resolved).toBe(provider); + expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({ + config: undefined, + env: process.env, + includeDisabled: true, + preferPersisted: false, + }); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ + onlyPluginIds: ["migrate-hermes"], + activate: false, + }); + }); + + it("lists configured external migration providers alongside active providers", () => { + const activeProvider = createMigrationProvider("active-import"); + const externalProvider = createMigrationProvider("external-import"); + const active = createEmptyPluginRegistry(); + active.migrationProviders.push({ + pluginId: "active-migration", + pluginName: "Active Migration", + source: "test", + provider: activeProvider, + } as never); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "external-migration", + pluginName: "External Migration", + source: "test", + provider: externalProvider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => ({ + diagnostics: [], + plugins: params?.includeDisabled + ? [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ] + : [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ], + })); + + expect(resolvePluginMigrationProviders().map((provider) => provider.id)).toEqual([ + "active-import", + "external-import", + ]); + }); +}); diff --git a/src/plugins/migration-provider-runtime.ts b/src/plugins/migration-provider-runtime.ts new file mode 100644 index 00000000000..2601f975248 --- /dev/null +++ b/src/plugins/migration-provider-runtime.ts @@ -0,0 +1,117 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { + withBundledPluginAllowlistCompat, + withBundledPluginEnablementCompat, + withBundledPluginVitestCompat, +} from "./bundled-compat.js"; +import { resolveRuntimePluginRegistry } from "./loader.js"; +import { resolveManifestContractRuntimePluginResolution } from "./manifest-contract-runtime.js"; +import type { MigrationProviderPlugin } from "./types.js"; + +function resolveMigrationProviderConfig(params: { + cfg?: OpenClawConfig; + bundledCompatPluginIds: string[]; +}): OpenClawConfig | undefined { + const allowlistCompat = withBundledPluginAllowlistCompat({ + config: params.cfg, + pluginIds: params.bundledCompatPluginIds, + }); + const enablementCompat = withBundledPluginEnablementCompat({ + config: allowlistCompat, + pluginIds: params.bundledCompatPluginIds, + }); + return withBundledPluginVitestCompat({ + config: enablementCompat, + pluginIds: params.bundledCompatPluginIds, + env: process.env, + }); +} + +function findMigrationProviderById( + entries: ReadonlyArray<{ provider: MigrationProviderPlugin }>, + providerId: string, +): MigrationProviderPlugin | undefined { + return entries.find((entry) => entry.provider.id === providerId)?.provider; +} + +function resolveMigrationProviderRegistry(params: { + cfg?: OpenClawConfig; + pluginIds: string[]; + bundledCompatPluginIds: string[]; +}) { + const compatConfig = resolveMigrationProviderConfig({ + cfg: params.cfg, + bundledCompatPluginIds: params.bundledCompatPluginIds, + }); + return resolveRuntimePluginRegistry({ + ...(compatConfig === undefined ? {} : { config: compatConfig }), + onlyPluginIds: params.pluginIds, + activate: false, + }); +} + +function mergeMigrationProviders( + left: ReadonlyArray<{ provider: MigrationProviderPlugin }>, + right: ReadonlyArray<{ provider: MigrationProviderPlugin }>, +): MigrationProviderPlugin[] { + const merged = new Map(); + for (const entry of [...left, ...right]) { + if (!merged.has(entry.provider.id)) { + merged.set(entry.provider.id, entry.provider); + } + } + return [...merged.values()].toSorted((a, b) => a.id.localeCompare(b.id)); +} + +export function resolvePluginMigrationProvider(params: { + providerId: string; + cfg?: OpenClawConfig; +}): MigrationProviderPlugin | undefined { + const activeRegistry = resolveRuntimePluginRegistry(); + const activeProvider = findMigrationProviderById( + activeRegistry?.migrationProviders ?? [], + params.providerId, + ); + if (activeProvider) { + return activeProvider; + } + + const resolution = resolveManifestContractRuntimePluginResolution({ + cfg: params.cfg, + contract: "migrationProviders", + value: params.providerId, + }); + const pluginIds = resolution.pluginIds; + if (pluginIds.length === 0) { + return undefined; + } + const registry = resolveMigrationProviderRegistry({ + cfg: params.cfg, + pluginIds, + bundledCompatPluginIds: resolution.bundledCompatPluginIds, + }); + return findMigrationProviderById(registry?.migrationProviders ?? [], params.providerId); +} + +export function resolvePluginMigrationProviders( + params: { + cfg?: OpenClawConfig; + } = {}, +): MigrationProviderPlugin[] { + const activeRegistry = resolveRuntimePluginRegistry(); + const activeProviders = activeRegistry?.migrationProviders ?? []; + const resolution = resolveManifestContractRuntimePluginResolution({ + cfg: params.cfg, + contract: "migrationProviders", + }); + const pluginIds = resolution.pluginIds; + if (pluginIds.length === 0) { + return mergeMigrationProviders(activeProviders, []); + } + const registry = resolveMigrationProviderRegistry({ + cfg: params.cfg, + pluginIds, + bundledCompatPluginIds: resolution.bundledCompatPluginIds, + }); + return mergeMigrationProviders(activeProviders, registry?.migrationProviders ?? []); +} diff --git a/src/plugins/plugin-registry-snapshot.ts b/src/plugins/plugin-registry-snapshot.ts index 778cc1747ef..af24b9f5dc3 100644 --- a/src/plugins/plugin-registry-snapshot.ts +++ b/src/plugins/plugin-registry-snapshot.ts @@ -76,10 +76,11 @@ export function loadPluginRegistrySnapshotWithMetadata( const disabledByCaller = params.preferPersisted === false; const disabledByEnv = hasEnvFlag(env, DISABLE_PERSISTED_PLUGIN_REGISTRY_ENV); const persistedReadsEnabled = !disabledByCaller && !disabledByEnv; + const persistedInstallRecordReadsEnabled = !disabledByEnv; let persistedIndex: InstalledPluginIndex | null = null; - if (persistedReadsEnabled) { + if (persistedInstallRecordReadsEnabled) { persistedIndex = readPersistedInstalledPluginIndexSync(params); - if (persistedIndex) { + if (persistedReadsEnabled && persistedIndex) { if ( params.config && persistedIndex.policyHash !== resolveInstalledPluginIndexPolicyHash(params.config) @@ -97,7 +98,7 @@ export function loadPluginRegistrySnapshotWithMetadata( diagnostics, }; } - } else { + } else if (persistedReadsEnabled) { diagnostics.push({ level: "info", code: "persisted-registry-missing", diff --git a/src/plugins/plugin-registry.test.ts b/src/plugins/plugin-registry.test.ts index 6dda1b905a4..0078effb693 100644 --- a/src/plugins/plugin-registry.test.ts +++ b/src/plugins/plugin-registry.test.ts @@ -388,6 +388,42 @@ describe("plugin registry facade", () => { ]); }); + it("derives a fresh registry without dropping persisted install records", async () => { + const stateDir = makeTempDir(); + const rootDir = makeTempDir(); + const candidate = createCandidate(rootDir); + await writePersistedInstalledPluginIndex( + createIndex("persisted", { + installRecords: { + persisted: { + source: "npm", + spec: "persisted-plugin@1.0.0", + installPath: path.join(stateDir, "plugins", "persisted"), + }, + }, + }), + { stateDir }, + ); + + const result = loadPluginRegistrySnapshotWithMetadata({ + stateDir, + candidates: [candidate], + env: hermeticEnv(), + preferPersisted: false, + }); + + expect(result.source).toBe("derived"); + expect(listPluginRecords({ index: result.snapshot }).map((plugin) => plugin.pluginId)).toEqual([ + "demo", + ]); + expect(result.snapshot.installRecords).toMatchObject({ + persisted: { + source: "npm", + spec: "persisted-plugin@1.0.0", + }, + }); + }); + it("exposes explicit persisted registry inspect and refresh operations", async () => { const stateDir = makeTempDir(); const pluginDir = path.join(stateDir, "plugins", "demo"); diff --git a/src/plugins/registry-empty.ts b/src/plugins/registry-empty.ts index 96d77148d28..ace8481395e 100644 --- a/src/plugins/registry-empty.ts +++ b/src/plugins/registry-empty.ts @@ -20,6 +20,7 @@ export function createEmptyPluginRegistry(): PluginRegistry { musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/registry-types.ts b/src/plugins/registry-types.ts index f77e805a37f..f1421ff9ef3 100644 --- a/src/plugins/registry-types.ts +++ b/src/plugins/registry-types.ts @@ -43,6 +43,7 @@ import type { PluginLogger, PluginOrigin, PluginTextTransformRegistration, + MigrationProviderPlugin, ProviderPlugin, RealtimeTranscriptionProviderPlugin, RealtimeVoiceProviderPlugin, @@ -149,6 +150,8 @@ export type PluginWebFetchProviderRegistration = PluginOwnedProviderRegistration; export type PluginWebSearchProviderRegistration = PluginOwnedProviderRegistration; +export type PluginMigrationProviderRegistration = + PluginOwnedProviderRegistration; export type PluginMemoryEmbeddingProviderRegistration = PluginOwnedProviderRegistration; export type PluginCodexAppServerExtensionFactoryRegistration = { @@ -279,6 +282,7 @@ export type PluginRecord = { musicGenerationProviderIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; contextEngineIds?: string[]; memoryEmbeddingProviderIds: string[]; agentHarnessIds: string[]; @@ -315,6 +319,7 @@ export type PluginRegistry = { musicGenerationProviders: PluginMusicGenerationProviderRegistration[]; webFetchProviders: PluginWebFetchProviderRegistration[]; webSearchProviders: PluginWebSearchProviderRegistration[]; + migrationProviders: PluginMigrationProviderRegistration[]; codexAppServerExtensionFactories: PluginCodexAppServerExtensionFactoryRegistration[]; agentToolResultMiddlewares: PluginAgentToolResultMiddlewareRegistration[]; memoryEmbeddingProviders: PluginMemoryEmbeddingProviderRegistration[]; diff --git a/src/plugins/registry.ts b/src/plugins/registry.ts index 2cc1fea101c..ac528582f9a 100644 --- a/src/plugins/registry.ts +++ b/src/plugins/registry.ts @@ -113,6 +113,7 @@ import type { OpenClawPluginReloadRegistration, OpenClawPluginSecurityAuditCollector, MediaUnderstandingProviderPlugin, + MigrationProviderPlugin, OpenClawPluginService, OpenClawPluginToolContext, OpenClawPluginToolFactory, @@ -1016,6 +1017,16 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { }); }; + const registerMigrationProvider = (record: PluginRecord, provider: MigrationProviderPlugin) => { + registerUniqueProviderLike({ + record, + provider, + kindLabel: "migration provider", + registrations: registry.migrationProviders, + ownedIds: record.migrationProviderIds, + }); + }; + const registerCli = ( record: PluginRecord, registrar: OpenClawPluginCliRegistrar, @@ -1487,6 +1498,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerMusicGenerationProvider(record, provider), registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider), registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider), + registerMigrationProvider: (provider) => registerMigrationProvider(record, provider), registerGatewayMethod: (method, handler, opts) => registerGatewayMethod(record, method, handler, opts), registerService: (service) => registerService(record, service), @@ -1764,6 +1776,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerVideoGenerationProvider, registerMusicGenerationProvider, registerWebSearchProvider, + registerMigrationProvider, registerGatewayMethod, registerCli, registerReload, diff --git a/src/plugins/status.test-helpers.ts b/src/plugins/status.test-helpers.ts index d063049f53e..acf64cb91ee 100644 --- a/src/plugins/status.test-helpers.ts +++ b/src/plugins/status.test-helpers.ts @@ -61,6 +61,7 @@ export function createPluginRecord( musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], contextEngineIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], @@ -131,6 +132,7 @@ export function createPluginLoadResult( musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/status.ts b/src/plugins/status.ts index 848622b90f7..74214837364 100644 --- a/src/plugins/status.ts +++ b/src/plugins/status.ts @@ -192,6 +192,7 @@ function buildPluginRecordFromInstalledIndex( musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], gatewayMethods: [], diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 22b242b8876..dbcefa6142f 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -2049,6 +2049,100 @@ export type PluginConfigMigration = (config: OpenClawConfig) => | null | undefined; +export type MigrationItemStatus = "planned" | "migrated" | "skipped" | "conflict" | "error"; +export type MigrationItemKind = + | "config" + | "secret" + | "memory" + | "skill" + | "workspace" + | "session" + | "file" + | "archive" + | "manual"; +export type MigrationItemAction = + | "copy" + | "create" + | "update" + | "merge" + | "append" + | "archive" + | "skip" + | "manual"; + +export type MigrationItem = { + id: string; + kind: MigrationItemKind | (string & {}); + action: MigrationItemAction | (string & {}); + status: MigrationItemStatus; + source?: string; + target?: string; + message?: string; + reason?: string; + sensitive?: boolean; + details?: Record; +}; + +export type MigrationSummary = { + total: number; + planned: number; + migrated: number; + skipped: number; + conflicts: number; + errors: number; + sensitive: number; +}; + +export type MigrationDetection = { + found: boolean; + source?: string; + label?: string; + confidence?: "low" | "medium" | "high"; + message?: string; +}; + +export type MigrationPlan = { + providerId: string; + source: string; + target?: string; + summary: MigrationSummary; + items: MigrationItem[]; + warnings?: string[]; + nextSteps?: string[]; + metadata?: Record; +}; + +export type MigrationApplyResult = MigrationPlan & { + backupPath?: string; + reportDir?: string; +}; + +export type MigrationProviderContext = { + config: OpenClawConfig; + runtime?: PluginRuntime; + logger: PluginLogger; + stateDir: string; + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + backupPath?: string; + reportDir?: string; + signal?: AbortSignal; +}; + +/** Migration source implemented by a plugin and orchestrated by `openclaw migrate`. */ +export type MigrationProviderPlugin = { + id: string; + label: string; + description?: string; + detect?: (ctx: MigrationProviderContext) => MigrationDetection | Promise; + plan: (ctx: MigrationProviderContext) => MigrationPlan | Promise; + apply: ( + ctx: MigrationProviderContext, + plan?: MigrationPlan, + ) => MigrationApplyResult | Promise; +}; + export type PluginSetupAutoEnableContext = { config: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -2128,6 +2222,8 @@ export type OpenClawPluginApi = { registerTextTransforms: (transforms: PluginTextTransformRegistration) => void; /** Register a lightweight config migration that can run before plugin runtime loads. */ registerConfigMigration: (migrate: PluginConfigMigration) => void; + /** Register an importer for `openclaw migrate` (migration capability). */ + registerMigrationProvider: (provider: MigrationProviderPlugin) => void; /** Register a lightweight config probe that can auto-enable this plugin generically. */ registerAutoEnableProbe: (probe: PluginSetupAutoEnableProbe) => void; /** Register a native model/provider plugin (text inference capability). */ diff --git a/src/test-utils/channel-plugins.ts b/src/test-utils/channel-plugins.ts index 53062f220f9..b68c42d3864 100644 --- a/src/test-utils/channel-plugins.ts +++ b/src/test-utils/channel-plugins.ts @@ -35,6 +35,7 @@ export const createTestRegistry = (channels: TestChannelRegistration[] = []): Pl musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/trajectory/metadata.test.ts b/src/trajectory/metadata.test.ts index ee1d7854516..a1894fe784b 100644 --- a/src/trajectory/metadata.test.ts +++ b/src/trajectory/metadata.test.ts @@ -96,6 +96,7 @@ describe("trajectory metadata", () => { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: ["pi"], gatewayMethods: [], diff --git a/src/wizard/setup.migration-import.test.ts b/src/wizard/setup.migration-import.test.ts new file mode 100644 index 00000000000..83d541d7d40 --- /dev/null +++ b/src/wizard/setup.migration-import.test.ts @@ -0,0 +1,61 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectSetupMigrationFreshness } from "./setup.migration-import.js"; + +const tempRoots = new Set(); + +async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-setup-migration-")); + tempRoots.add(root); + return root; +} + +async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +describe("setup migration import freshness", () => { + afterEach(async () => { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); + }); + + it("allows empty config and empty target directories", async () => { + const root = await makeTempRoot(); + const result = await inspectSetupMigrationFreshness({ + baseConfig: {}, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }); + + expect(result).toEqual({ fresh: true, reasons: [] }); + }); + + it("rejects existing config, workspace files, and state", async () => { + const root = await makeTempRoot(); + const stateDir = path.join(root, "state"); + const workspaceDir = path.join(root, "workspace"); + await writeFile(path.join(workspaceDir, "MEMORY.md"), "existing memory\n"); + await writeFile(path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), "{}\n"); + + const result = await inspectSetupMigrationFreshness({ + baseConfig: { gateway: { port: 3131 } }, + stateDir, + workspaceDir, + }); + + expect(result.fresh).toBe(false); + expect(result.reasons).toEqual( + expect.arrayContaining([ + "existing config values are loaded", + "workspace MEMORY.md exists", + "state agents/ exists", + ]), + ); + }); +}); diff --git a/src/wizard/setup.migration-import.ts b/src/wizard/setup.migration-import.ts new file mode 100644 index 00000000000..150cce46da3 --- /dev/null +++ b/src/wizard/setup.migration-import.ts @@ -0,0 +1,304 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OnboardOptions } from "../commands/onboard-types.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import type { MigrationProviderPlugin } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { resolveUserPath } from "../utils.js"; +import { WizardCancelledError, type WizardPrompter } from "./prompts.js"; + +export type SetupMigrationDetection = { + providerId: string; + label: string; + source?: string; + message?: string; +}; + +const MEANINGFUL_CONFIG_IGNORED_KEYS = new Set(["$schema", "meta"]); +const MEANINGFUL_WORKSPACE_ENTRIES = [ + "AGENTS.md", + "SOUL.md", + "USER.md", + "IDENTITY.md", + "MEMORY.md", + "skills", +] as const; +const MEANINGFUL_STATE_ENTRIES = ["credentials", "sessions", "agents"] as const; + +async function exists(candidate: string): Promise { + try { + await fs.access(candidate); + return true; + } catch { + return false; + } +} + +async function hasDirectoryEntries(candidate: string): Promise { + try { + return (await fs.readdir(candidate)).length > 0; + } catch { + return false; + } +} + +function hasMeaningfulConfig(config: OpenClawConfig): boolean { + return Object.keys(config as Record).some( + (key) => !MEANINGFUL_CONFIG_IGNORED_KEYS.has(key), + ); +} + +export async function inspectSetupMigrationFreshness(params: { + baseConfig: OpenClawConfig; + stateDir: string; + workspaceDir: string; +}): Promise<{ fresh: boolean; reasons: string[] }> { + const reasons: string[] = []; + if (hasMeaningfulConfig(params.baseConfig)) { + reasons.push("existing config values are loaded"); + } + for (const entry of MEANINGFUL_WORKSPACE_ENTRIES) { + if (await exists(path.join(params.workspaceDir, entry))) { + reasons.push(`workspace ${entry} exists`); + } + } + for (const entry of MEANINGFUL_STATE_ENTRIES) { + if (await hasDirectoryEntries(path.join(params.stateDir, entry))) { + reasons.push(`state ${entry}/ exists`); + } + } + return { fresh: reasons.length === 0, reasons }; +} + +function assertFreshSetupMigrationTarget(freshness: { + fresh: boolean; + reasons: readonly string[]; +}): void { + if (freshness.fresh || process.env.OPENCLAW_MIGRATION_EXISTING_IMPORT === "1") { + return; + } + throw new Error( + [ + "Migration import during onboarding requires a fresh OpenClaw setup.", + "Create a fresh setup or reset config, credentials, sessions, and workspace before importing.", + "Backup plus overwrite/merge imports are feature-gated for now.", + "Existing setup:", + ...freshness.reasons.map((reason) => `- ${reason}`), + ].join("\n"), + ); +} + +export async function detectSetupMigrationSources(params: { + config: OpenClawConfig; + runtime: RuntimeEnv; +}): Promise { + const [{ resolvePluginMigrationProviders }, { createMigrationLogger }, { resolveStateDir }] = + await Promise.all([ + import("../plugins/migration-provider-runtime.js"), + import("../commands/migrate/context.js"), + import("../config/paths.js"), + ]); + const stateDir = resolveStateDir(); + const logger = createMigrationLogger(params.runtime); + const detections: SetupMigrationDetection[] = []; + for (const provider of resolvePluginMigrationProviders({ cfg: params.config })) { + if (!provider.detect) { + continue; + } + try { + const detection = await provider.detect({ + config: params.config, + stateDir, + logger, + }); + if (detection.found) { + detections.push({ + providerId: provider.id, + label: detection.label ?? provider.label, + ...(detection.source ? { source: detection.source } : {}), + ...(detection.message ? { message: detection.message } : {}), + }); + } + } catch (error) { + logger.debug?.( + `Migration provider ${provider.id} detection failed: ${formatErrorMessage(error)}`, + ); + } + } + return detections; +} + +function resolveImportSourceDefault(params: { + providerId: string; + detections: readonly SetupMigrationDetection[]; +}): string { + const detected = params.detections.find( + (detection) => detection.providerId === params.providerId, + ); + if (detected?.source) { + return detected.source; + } + return params.providerId === "hermes" ? "~/.hermes" : ""; +} + +async function selectSetupMigrationProvider(params: { + opts: OnboardOptions; + baseConfig: OpenClawConfig; + detections: readonly SetupMigrationDetection[]; + prompter: WizardPrompter; +}): Promise<{ + provider: MigrationProviderPlugin; + providerId: string; +}> { + const { resolvePluginMigrationProvider, resolvePluginMigrationProviders } = + await import("../plugins/migration-provider-runtime.js"); + const providers = resolvePluginMigrationProviders({ cfg: params.baseConfig }); + if (providers.length === 0) { + throw new Error("No migration providers found."); + } + const providerById = new Map(providers.map((provider) => [provider.id, provider])); + const providerId = + params.opts.importFrom?.trim() || + (await params.prompter.select({ + message: "Migration source", + options: [ + ...params.detections.map((detection) => ({ + value: detection.providerId, + label: detection.label, + ...(detection.source || detection.message + ? { hint: detection.source ?? detection.message } + : {}), + })), + ...providers + .filter( + (provider) => + !params.detections.some((detection) => detection.providerId === provider.id), + ) + .map((provider) => ({ + value: provider.id, + label: provider.label, + hint: provider.description ?? "Enter a source path next", + })), + ], + initialValue: params.detections[0]?.providerId ?? providers[0]?.id, + })); + const provider = + providerById.get(providerId) ?? + resolvePluginMigrationProvider({ providerId, cfg: params.baseConfig }); + if (!provider) { + throw new Error(`Unknown migration provider "${providerId}".`); + } + return { provider, providerId }; +} + +export async function runSetupMigrationImport(params: { + opts: OnboardOptions; + baseConfig: OpenClawConfig; + detections: readonly SetupMigrationDetection[]; + prompter: WizardPrompter; + runtime: RuntimeEnv; + writeConfigFile: (config: OpenClawConfig) => Promise; +}): Promise { + const [ + { applyLocalSetupWorkspaceConfig, applySkipBootstrapConfig }, + { createMigrationLogger, buildMigrationReportDir }, + { createPreMigrationBackup }, + { assertApplySucceeded, assertConflictFreePlan, formatMigrationPlan }, + { resolveStateDir }, + onboardHelpers, + ] = await Promise.all([ + import("../commands/onboard-config.js"), + import("../commands/migrate/context.js"), + import("../commands/migrate/apply.js"), + import("../commands/migrate/output.js"), + import("../config/paths.js"), + import("../commands/onboard-helpers.js"), + ]); + const { provider, providerId } = await selectSetupMigrationProvider({ + opts: params.opts, + baseConfig: params.baseConfig, + detections: params.detections, + prompter: params.prompter, + }); + const sourceDefault = resolveImportSourceDefault({ providerId, detections: params.detections }); + const sourceDir = + params.opts.importSource?.trim() || + sourceDefault || + (params.opts.nonInteractive + ? (() => { + throw new Error("--import-source is required for non-interactive migration import."); + })() + : await params.prompter.text({ + message: "Source agent home", + initialValue: providerId === "hermes" ? "~/.hermes" : undefined, + })); + const workspaceInput = + params.opts.workspace ?? + (params.opts.nonInteractive + ? (params.baseConfig.agents?.defaults?.workspace ?? onboardHelpers.DEFAULT_WORKSPACE) + : await params.prompter.text({ + message: "Target workspace directory", + initialValue: + params.baseConfig.agents?.defaults?.workspace ?? onboardHelpers.DEFAULT_WORKSPACE, + })); + const workspaceDir = resolveUserPath(workspaceInput.trim() || onboardHelpers.DEFAULT_WORKSPACE); + let targetConfig = applyLocalSetupWorkspaceConfig(params.baseConfig, workspaceDir); + if (params.opts.skipBootstrap) { + targetConfig = applySkipBootstrapConfig(targetConfig); + } + + const stateDir = resolveStateDir(); + assertFreshSetupMigrationTarget( + await inspectSetupMigrationFreshness({ + baseConfig: params.baseConfig, + stateDir, + workspaceDir, + }), + ); + const ctx = { + config: targetConfig, + stateDir, + source: sourceDir, + includeSecrets: Boolean(params.opts.importSecrets), + overwrite: false, + logger: createMigrationLogger(params.runtime), + }; + const plan = await provider.plan(ctx); + await params.prompter.note(formatMigrationPlan(plan).join("\n"), "Migration preview"); + assertConflictFreePlan(plan, providerId); + + const confirmed = + params.opts.nonInteractive === true + ? true + : await params.prompter.confirm({ + message: "Apply this migration now?", + initialValue: false, + }); + if (!confirmed) { + throw new WizardCancelledError("migration cancelled"); + } + + const reportDir = buildMigrationReportDir(providerId, stateDir); + const backupPath = await createPreMigrationBackup({}); + targetConfig = onboardHelpers.applyWizardMetadata(targetConfig, { + command: "onboard", + mode: "local", + }); + targetConfig = await params.writeConfigFile(targetConfig); + const applyCtx = { + ...ctx, + config: targetConfig, + ...(backupPath ? { backupPath } : {}), + reportDir, + }; + const result = await provider.apply(applyCtx, plan); + const withReport = { + ...result, + ...((result.backupPath ?? backupPath) ? { backupPath: result.backupPath ?? backupPath } : {}), + reportDir: result.reportDir ?? reportDir, + }; + assertApplySucceeded(withReport); + await params.prompter.note(formatMigrationPlan(withReport).join("\n"), "Migration applied"); + await params.prompter.outro("Migration complete. Run `openclaw doctor` next."); +} diff --git a/src/wizard/setup.ts b/src/wizard/setup.ts index 48761819632..eb8288e500d 100644 --- a/src/wizard/setup.ts +++ b/src/wizard/setup.ts @@ -20,6 +20,7 @@ import type { RuntimeEnv } from "../runtime.js"; import { defaultRuntime } from "../runtime.js"; import { resolveUserPath } from "../utils.js"; import { WizardCancelledError, type WizardPrompter } from "./prompts.js"; +import { detectSetupMigrationSources, runSetupMigrationImport } from "./setup.migration-import.js"; import { resolveSetupSecretInputString } from "./setup.secret-input.js"; import { SECURITY_CONFIRM_MESSAGE, @@ -28,6 +29,8 @@ import { } from "./setup.security-note.js"; import type { QuickstartGatewayDefaults, WizardFlow } from "./setup.types.js"; +type SetupFlowChoice = WizardFlow | "import"; + type AuthChoiceModule = typeof import("../commands/auth-choice.js"); type ConfigLoggingModule = typeof import("../config/logging.js"); type ModelPickerModule = typeof import("../commands/model-picker.js"); @@ -229,28 +232,41 @@ export async function runSetupWizard( const quickstartHint = `Configure details later via ${formatCliCommand("openclaw configure")}.`; const manualHint = "Configure port, network, Tailscale, and auth options."; + const migrationDetections = await detectSetupMigrationSources({ config: baseConfig, runtime }); + const firstMigrationDetection = migrationDetections[0]; + const importOption = firstMigrationDetection + ? { + value: "import" as const, + label: `Import from ${firstMigrationDetection.label}`, + ...(firstMigrationDetection.source ? { hint: firstMigrationDetection.source } : {}), + } + : undefined; const explicitFlowRaw = opts.flow?.trim(); const normalizedExplicitFlow = explicitFlowRaw === "manual" ? "advanced" : explicitFlowRaw; if ( normalizedExplicitFlow && normalizedExplicitFlow !== "quickstart" && - normalizedExplicitFlow !== "advanced" + normalizedExplicitFlow !== "advanced" && + normalizedExplicitFlow !== "import" ) { - runtime.error("Invalid --flow (use quickstart, manual, or advanced)."); + runtime.error("Invalid --flow (use quickstart, manual, advanced, or import)."); runtime.exit(1); return; } - const explicitFlow: WizardFlow | undefined = - normalizedExplicitFlow === "quickstart" || normalizedExplicitFlow === "advanced" + const explicitFlow: SetupFlowChoice | undefined = + normalizedExplicitFlow === "quickstart" || + normalizedExplicitFlow === "advanced" || + normalizedExplicitFlow === "import" ? normalizedExplicitFlow : undefined; - let flow: WizardFlow = + let flow: SetupFlowChoice = explicitFlow ?? (await prompter.select({ message: "Setup mode", options: [ { value: "quickstart", label: "QuickStart", hint: quickstartHint }, { value: "advanced", label: "Manual", hint: manualHint }, + ...(importOption ? [importOption] : []), ], initialValue: "quickstart", })); @@ -300,6 +316,19 @@ export async function runSetupWizard( } } + if (opts.importFrom || flow === "import") { + await runSetupMigrationImport({ + opts, + baseConfig, + detections: migrationDetections, + prompter, + runtime, + writeConfigFile: writeWizardConfigFile, + }); + return; + } + const wizardFlow: WizardFlow = flow; + const quickstartGateway: QuickstartGatewayDefaults = (() => { const hasExisting = typeof baseConfig.gateway?.port === "number" || @@ -669,7 +698,7 @@ export async function runSetupWizard( const { configureGatewayForSetup } = await import("./setup.gateway-config.js"); const gateway = await configureGatewayForSetup({ - flow, + flow: wizardFlow, baseConfig, nextConfig, localPort, @@ -746,7 +775,7 @@ export async function runSetupWizard( const { finalizeSetupWizard } = await import("./setup.finalize.js"); const { launchedTui } = await finalizeSetupWizard({ - flow, + flow: wizardFlow, opts, baseConfig, nextConfig, diff --git a/test/helpers/plugins/plugin-api.ts b/test/helpers/plugins/plugin-api.ts index e8fed03cf37..68db5993506 100644 --- a/test/helpers/plugins/plugin-api.ts +++ b/test/helpers/plugins/plugin-api.ts @@ -25,6 +25,7 @@ export function createTestPluginApi(api: TestPluginApiInput = {}): OpenClawPlugi registerNodeHostCommand() {}, registerSecurityAuditCollector() {}, registerConfigMigration() {}, + registerMigrationProvider() {}, registerAutoEnableProbe() {}, registerProvider() {}, registerSpeechProvider() {}, diff --git a/test/setup-openclaw-runtime.ts b/test/setup-openclaw-runtime.ts index a1783104daf..e7c7e5e248a 100644 --- a/test/setup-openclaw-runtime.ts +++ b/test/setup-openclaw-runtime.ts @@ -153,6 +153,7 @@ function createTestRegistryForSetup( videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], memoryEmbeddingProviders: [], gatewayHandlers: {}, gatewayMethodScopes: {}, From 98e7242b536f678c1b40bfe11f6131e809516673 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:34:57 +0100 Subject: [PATCH 04/12] test: split ui unit tests from generic lane --- docs/help/testing.md | 2 +- test/vitest-projects-config.test.ts | 12 ++++++------ test/vitest-scoped-config.test.ts | 4 ++-- test/vitest-unit-config.test.ts | 7 +++++++ test/vitest-unit-paths.test.ts | 8 ++++---- test/vitest/vitest.config.ts | 1 + test/vitest/vitest.unit-paths.mjs | 9 --------- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/help/testing.md b/docs/help/testing.md index 62020aeca2f..4480345a8b9 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -443,7 +443,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - Command: `pnpm test` - Config: untargeted runs use the `vitest.full-*.config.ts` shard set and may expand multi-project shards into per-project configs for parallel scheduling -- Files: core/unit inventories under `src/**/*.test.ts`, `packages/**/*.test.ts`, `test/**/*.test.ts`, and the whitelisted `ui` node tests covered by `vitest.unit.config.ts` +- Files: core/unit inventories under `src/**/*.test.ts`, `packages/**/*.test.ts`, and `test/**/*.test.ts`; UI unit tests run in the dedicated `unit-ui` shard - Scope: - Pure unit tests - In-process integration tests (gateway auth, routing, tooling, parsing, config) diff --git a/test/vitest-projects-config.test.ts b/test/vitest-projects-config.test.ts index 3d84a922659..d3d6a405e48 100644 --- a/test/vitest-projects-config.test.ts +++ b/test/vitest-projects-config.test.ts @@ -140,21 +140,21 @@ describe("projects vitest config", () => { ]); }); - it("keeps the root ui lane aligned with the isolated jsdom setup", () => { + it("keeps the root ui lane aligned with the shared jsdom setup", () => { const config = createUiVitestConfig(); expect(config.test.environment).toBe("jsdom"); - expect(config.test.isolate).toBe(true); - expect(config.test.runner).toBeUndefined(); + expect(config.test.isolate).toBe(false); + expect(normalizeConfigPath(config.test.runner)).toBe("test/non-isolated-runner.ts"); const setupFiles = normalizeConfigPaths(config.test.setupFiles); expect(setupFiles).not.toContain("test/setup-openclaw-runtime.ts"); expect(setupFiles).toContain("ui/src/test-helpers/lit-warnings.setup.ts"); expect(config.test.deps?.optimizer?.web?.enabled).toBe(true); }); - it("keeps the unit-ui shard aligned with the isolated jsdom setup", () => { + it("keeps the unit-ui shard aligned with the shared jsdom setup", () => { expect(unitUiConfig.test?.environment).toBe("jsdom"); - expect(unitUiConfig.test?.isolate).toBe(true); - expect(unitUiConfig.test?.runner).toBeUndefined(); + expect(unitUiConfig.test?.isolate).toBe(false); + expect(normalizeConfigPath(unitUiConfig.test?.runner)).toBe("test/non-isolated-runner.ts"); const setupFiles = normalizeConfigPaths(unitUiConfig.test?.setupFiles); expect(setupFiles).not.toContain("test/setup-openclaw-runtime.ts"); expect(setupFiles).toContain("ui/src/test-helpers/lit-warnings.setup.ts"); diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 9cee32e5ea2..f3c9d65f637 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -327,8 +327,8 @@ describe("scoped vitest configs", () => { } expect(defaultUiConfig.test?.pool).toBe("threads"); - expect(defaultUiConfig.test?.isolate).toBe(true); - expect(defaultUiConfig.test?.runner).toBeUndefined(); + expect(defaultUiConfig.test?.isolate).toBe(false); + expect(normalizeConfigPath(defaultUiConfig.test?.runner)).toBe("test/non-isolated-runner.ts"); }); it("keeps the process lane off the openclaw runtime setup", () => { diff --git a/test/vitest-unit-config.test.ts b/test/vitest-unit-config.test.ts index 99dba3abec1..e86c9fcfd62 100644 --- a/test/vitest-unit-config.test.ts +++ b/test/vitest-unit-config.test.ts @@ -78,6 +78,13 @@ describe("unit vitest config", () => { it("keeps acp and ui tests out of the generic unit lane", () => { const unitConfig = createUnitVitestConfig({}); expect(unitConfig.test?.exclude).toEqual(expect.arrayContaining(["extensions/**", "test/**"])); + expect(unitConfig.test?.include).not.toEqual( + expect.arrayContaining([ + "ui/src/ui/app-chat.test.ts", + "ui/src/ui/chat/**/*.test.ts", + "ui/src/ui/views/chat.test.ts", + ]), + ); }); it("narrows the active include list to CLI file filters when present", () => { diff --git a/test/vitest-unit-paths.test.ts b/test/vitest-unit-paths.test.ts index 3e6d459725d..d09874eee53 100644 --- a/test/vitest-unit-paths.test.ts +++ b/test/vitest-unit-paths.test.ts @@ -3,8 +3,8 @@ import { bundledPluginFile } from "./helpers/bundled-plugin-paths.js"; import { isUnitConfigTestFile } from "./vitest/vitest.unit-paths.mjs"; describe("isUnitConfigTestFile", () => { - it("accepts unit-config src tests", () => { - expect(isUnitConfigTestFile("ui/src/ui/views/channels.test.ts")).toBe(true); + it("accepts unit-config package tests", () => { + expect(isUnitConfigTestFile("packages/plugin-package-contract/src/index.test.ts")).toBe(true); }); it("rejects files excluded from the unit config", () => { @@ -31,8 +31,8 @@ describe("isUnitConfigTestFile", () => { expect(isUnitConfigTestFile("test/extension-test-boundary.test.ts")).toBe(false); expect(isUnitConfigTestFile("src/agents/pi-embedded-runner.test.ts")).toBe(false); expect(isUnitConfigTestFile("src/commands/onboard.test.ts")).toBe(false); - expect(isUnitConfigTestFile("ui/src/ui/views/channels.test.ts")).toBe(true); - expect(isUnitConfigTestFile("ui/src/ui/views/chat.test.ts")).toBe(true); + expect(isUnitConfigTestFile("ui/src/ui/views/channels.test.ts")).toBe(false); + expect(isUnitConfigTestFile("ui/src/ui/views/chat.test.ts")).toBe(false); expect(isUnitConfigTestFile("ui/src/ui/views/other.test.ts")).toBe(false); expect(isUnitConfigTestFile("src/infra/git-commit.live.test.ts")).toBe(false); expect(isUnitConfigTestFile("src/infra/git-commit.e2e.test.ts")).toBe(false); diff --git a/test/vitest/vitest.config.ts b/test/vitest/vitest.config.ts index 4a922d7d326..dffe2ddb031 100644 --- a/test/vitest/vitest.config.ts +++ b/test/vitest/vitest.config.ts @@ -11,6 +11,7 @@ export { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers, resolveLocalVit export const rootVitestProjects = [ "test/vitest/vitest.unit.config.ts", + "test/vitest/vitest.unit-ui.config.ts", "test/vitest/vitest.infra.config.ts", "test/vitest/vitest.boundary.config.ts", "test/vitest/vitest.contracts-channel-surface.config.ts", diff --git a/test/vitest/vitest.unit-paths.mjs b/test/vitest/vitest.unit-paths.mjs index 1b005c2905e..2287afee3d7 100644 --- a/test/vitest/vitest.unit-paths.mjs +++ b/test/vitest/vitest.unit-paths.mjs @@ -5,15 +5,6 @@ export const unitTestIncludePatterns = [ "src/**/*.test.ts", "packages/**/*.test.ts", "test/**/*.test.ts", - "ui/src/ui/app-chat.test.ts", - "ui/src/ui/chat/**/*.test.ts", - "ui/src/ui/views/agents-utils.test.ts", - "ui/src/ui/views/channels.test.ts", - "ui/src/ui/views/chat.test.ts", - "ui/src/ui/views/dreams.test.ts", - "ui/src/ui/views/usage-render-details.test.ts", - "ui/src/ui/controllers/agents.test.ts", - "ui/src/ui/controllers/chat.test.ts", ]; export const boundaryTestFiles = [ From f6db86f9a0ee2d5e6078caa1478273774e6ebd35 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:35:36 +0100 Subject: [PATCH 05/12] fix: normalize lazy service override imports --- CHANGELOG.md | 1 + src/plugins/import-specifier.test.ts | 37 +++++++++++++++++++++++++ src/plugins/import-specifier.ts | 22 +++++++++++++++ src/plugins/lazy-service-module.test.ts | 36 +++++++++++++++++++++++- src/plugins/lazy-service-module.ts | 12 ++++++-- src/plugins/loader.ts | 27 +----------------- 6 files changed, 106 insertions(+), 29 deletions(-) create mode 100644 src/plugins/import-specifier.test.ts create mode 100644 src/plugins/import-specifier.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fc966249f3b..c0fbaac3ff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Docs: https://docs.openclaw.ai - macOS Gateway: detect installed-but-unloaded LaunchAgent split-brain states during status, doctor, and restart, and re-bootstrap launchd supervision before falling back to unmanaged listener restarts. Fixes #67335, #53475, and #71060; refs #58890, #60885, and #70801. Thanks @ze1tgeist88, @dafacto, and @vishutdhar. - Plugins/install: treat mirrored core logger dependencies as staged bundled runtime deps so packaged Gateway starts do not crash when the external plugin-runtime-deps root is missing `tslog`. Fixes #72228; supersedes #72493. Thanks @deepujain. - Plugins/install: hide bundled runtime-dependency npm child windows on Windows across Gateway startup, postinstall, and packaged staging paths so Telegram/Anthropic dependency repair no longer flashes shell windows. Fixes #72315. Thanks @athuljayaram and @joshfeng. +- Plugins/Windows: normalize lazy plugin service override imports before Node ESM loading so drive-letter browser-control module paths no longer fail with `ERR_UNSUPPORTED_ESM_URL_SCHEME`. Fixes #72573; supersedes #72599 and #72582. Thanks @llzzww316, @feineryonah-byte, and @WuKongAI-CMU. - Plugins/install: stage bundled plugin runtime dependencies before Gateway startup, drain update restarts, and materialize plugin-owned root chunks in external mirrors so staged deps resolve under native ESM. Fixes #72058; supersedes #72084. Thanks @amnesia106 and @drvoss. - TTS/SecretRef: resolve `messages.tts.providers.*.apiKey` from the active runtime snapshot so SecretRef-backed MiniMax and other TTS provider keys work in runtime reply/audio paths. Fixes #68690. Thanks @joshavant. - Gateway/install: surface systemd user-bus recovery hints during Linux service activation and retry via the machine user scope when `systemctl --user` reports no-medium bus failures. Fixes #39673; refs #44417 and #63561. Thanks @Arbor4, @myrsu, and @mssteuer. diff --git a/src/plugins/import-specifier.test.ts b/src/plugins/import-specifier.test.ts new file mode 100644 index 00000000000..c154326c8d2 --- /dev/null +++ b/src/plugins/import-specifier.test.ts @@ -0,0 +1,37 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { toSafeImportPath } from "./import-specifier.js"; + +describe("toSafeImportPath", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("converts Windows absolute import specifiers to file URLs", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + + expect(toSafeImportPath("C:\\Users\\alice\\plugin\\index.mjs")).toBe( + "file:///C:/Users/alice/plugin/index.mjs", + ); + expect(toSafeImportPath("\\\\server\\share\\plugin\\index.mjs")).toBe( + "file://server/share/plugin/index.mjs", + ); + }); + + it("preserves import specifiers that Node can already resolve", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + + expect(toSafeImportPath("file:///C:/Users/alice/plugin/index.mjs")).toBe( + "file:///C:/Users/alice/plugin/index.mjs", + ); + expect(toSafeImportPath("./relative/index.mjs")).toBe("./relative/index.mjs"); + expect(toSafeImportPath("@openclaw/plugin")).toBe("@openclaw/plugin"); + }); + + it("does not rewrite non-Windows paths", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + + expect(toSafeImportPath("C:\\Users\\alice\\plugin\\index.mjs")).toBe( + "C:\\Users\\alice\\plugin\\index.mjs", + ); + }); +}); diff --git a/src/plugins/import-specifier.ts b/src/plugins/import-specifier.ts new file mode 100644 index 00000000000..c7433c1a0e9 --- /dev/null +++ b/src/plugins/import-specifier.ts @@ -0,0 +1,22 @@ +import path from "node:path"; + +/** + * On Windows, Node's ESM loader requires absolute paths to be expressed as + * file:// URLs. Raw drive-letter paths like C:\... are parsed as URL schemes. + */ +export function toSafeImportPath(specifier: string): string { + if (process.platform !== "win32") { + return specifier; + } + if (specifier.startsWith("file://")) { + return specifier; + } + if (path.win32.isAbsolute(specifier)) { + const normalizedSpecifier = specifier.replaceAll("\\", "/"); + if (normalizedSpecifier.startsWith("//")) { + return new URL(`file:${encodeURI(normalizedSpecifier)}`).href; + } + return new URL(`file:///${encodeURI(normalizedSpecifier)}`).href; + } + return specifier; +} diff --git a/src/plugins/lazy-service-module.test.ts b/src/plugins/lazy-service-module.test.ts index 90ea5ece26d..62669785f83 100644 --- a/src/plugins/lazy-service-module.test.ts +++ b/src/plugins/lazy-service-module.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { startLazyPluginServiceModule } from "./lazy-service-module.js"; +import { defaultLoadOverrideModule, startLazyPluginServiceModule } from "./lazy-service-module.js"; function createAsyncHookMock() { return vi.fn(async () => {}); @@ -89,6 +89,40 @@ describe("startLazyPluginServiceModule", () => { expect(start).toHaveBeenCalledTimes(1); }); + it("normalizes Windows absolute paths in the default override loader", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const start = createAsyncHookMock(); + const importModule = vi.fn(async () => ({ startOverride: start })); + + try { + await defaultLoadOverrideModule("C:\\Users\\alice\\browser-service.mjs", importModule); + } finally { + platformSpy.mockRestore(); + } + + expect(importModule).toHaveBeenCalledWith("file:///C:/Users/alice/browser-service.mjs"); + }); + + it("leaves caller-supplied override loaders responsible for their own specifiers", async () => { + process.env.OPENCLAW_LAZY_SERVICE_OVERRIDE = "C:\\Users\\alice\\browser-service.mjs"; + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const start = createAsyncHookMock(); + const loadOverrideModule = vi.fn(async () => ({ startOverride: start })); + + try { + await expectLifecycleStarted({ + overrideEnvVar: "OPENCLAW_LAZY_SERVICE_OVERRIDE", + loadOverrideModule, + startExportNames: ["startOverride"], + }); + } finally { + platformSpy.mockRestore(); + } + + expect(loadOverrideModule).toHaveBeenCalledWith("C:\\Users\\alice\\browser-service.mjs"); + expect(start).toHaveBeenCalledTimes(1); + }); + it("validates the override specifier before loading it", async () => { process.env.OPENCLAW_LAZY_SERVICE_OVERRIDE = "virtual:service"; const loadOverrideModule = vi.fn(async () => ({ startOverride: createAsyncHookMock() })); diff --git a/src/plugins/lazy-service-module.ts b/src/plugins/lazy-service-module.ts index 3fa7e11dc28..120e5f58a62 100644 --- a/src/plugins/lazy-service-module.ts +++ b/src/plugins/lazy-service-module.ts @@ -1,4 +1,5 @@ import { isTruthyEnvValue } from "../infra/env.js"; +import { toSafeImportPath } from "./import-specifier.js"; type LazyServiceModule = Record; @@ -17,6 +18,14 @@ function resolveExport(mod: LazyServiceModule, names: string[]): T | null { return null; } +export async function defaultLoadOverrideModule( + specifier: string, + importModule: (specifier: string) => Promise = async (source: string) => + await import(source), +): Promise { + return importModule(toSafeImportPath(specifier)); +} + export async function startLazyPluginServiceModule(params: { skipEnvVar?: string; overrideEnvVar?: string; @@ -33,8 +42,7 @@ export async function startLazyPluginServiceModule(params: { const overrideEnvVar = params.overrideEnvVar?.trim(); const override = overrideEnvVar ? process.env[overrideEnvVar]?.trim() : undefined; - const loadOverrideModule = - params.loadOverrideModule ?? (async (specifier: string) => await import(specifier)); + const loadOverrideModule = params.loadOverrideModule ?? defaultLoadOverrideModule; const validatedOverride = override && params.validateOverrideSpecifier ? params.validateOverrideSpecifier(override) diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index d78f25948b9..9258271f927 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -66,6 +66,7 @@ import { } from "./config-state.js"; import { discoverOpenClawPlugins } from "./discovery.js"; import { getGlobalHookRunner, initializeGlobalHookRunner } from "./hook-runner-global.js"; +import { toSafeImportPath } from "./import-specifier.js"; import { loadInstalledPluginIndexInstallRecordsSync } from "./installed-plugin-index-records.js"; import { clearPluginInteractiveHandlers, @@ -429,32 +430,6 @@ function runPluginRegisterSync( } } -/** - * On Windows, the Node.js ESM loader requires absolute paths to be expressed - * as file:// URLs (e.g. file:///C:/Users/...). Raw drive-letter paths like - * C:\... are rejected with ERR_UNSUPPORTED_ESM_URL_SCHEME because the loader - * mistakes the drive letter for an unknown URL scheme. - * - * This helper converts Windows absolute import specifiers to file:// URLs and - * leaves everything else unchanged. - */ -function toSafeImportPath(specifier: string): string { - if (process.platform !== "win32") { - return specifier; - } - if (specifier.startsWith("file://")) { - return specifier; - } - if (path.win32.isAbsolute(specifier)) { - const normalizedSpecifier = specifier.replaceAll("\\", "/"); - if (normalizedSpecifier.startsWith("//")) { - return new URL(`file:${encodeURI(normalizedSpecifier)}`).href; - } - return new URL(`file:///${encodeURI(normalizedSpecifier)}`).href; - } - return specifier; -} - type RuntimeDependencyPackageJson = { dependencies?: Record; optionalDependencies?: Record; From d5e6abcb3d22c5fb96869e916059163928dba718 Mon Sep 17 00:00:00 2001 From: BsnizND Date: Mon, 27 Apr 2026 00:36:59 -0700 Subject: [PATCH 06/12] Add Google Meet realtime consult agentId (#72381) Remote proof: - CI run 24982271745 passed on 6122e13c9f897a34a28a82ff466b225be24424c8. - Blacksmith Testbox tbx_01kq6vwehcszjfpp52f0pb3v1q passed focused Google Meet formatting, docs/link checks, realtime consult runtime tests, Google Meet tests, extension test typecheck, the core-unit-fast-support shard, and the core support boundary shard. Thanks @BsnizND. Co-authored-by: BSnizND <199837910+BsnizND@users.noreply.github.com> --- CHANGELOG.md | 1 + docs/plugins/google-meet.md | 7 +++++++ extensions/google-meet/index.test.ts | 16 +++++++++++++++- extensions/google-meet/index.ts | 5 +++++ extensions/google-meet/openclaw.plugin.json | 9 +++++++++ extensions/google-meet/src/agent-consult.ts | 6 +++++- extensions/google-meet/src/config.ts | 2 ++ 7 files changed, 44 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0fbaac3ff9..75cbc28efe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai - Voice Call: allow SecretRef-backed Twilio auth tokens and call-specific OpenAI/ElevenLabs TTS API keys through the plugin config surface. Fixes #68690. Thanks @joshavant. - Google Meet: clean stale chrome-node realtime audio bridges by URL before rejoining, expose active node bridge inspection, and tolerate transient node input pull failures instead of dropping the Meet session. Fixes #72371. (#72372) Thanks @BsnizND. - Google Meet: clear queued Gemini Live playback when realtime interruptions arrive, restart Chrome command-pair audio output after clears, and expose Google Live interruption/VAD config knobs for Meet and Voice Call realtime bridges. Fixes #72523. (#72524) Thanks @BsnizND. +- Google Meet: add `realtime.agentId` so live meeting consults can target a named OpenClaw agent instead of always using `main`. (#72381) Thanks @BsnizND. - Matrix/E2EE: stabilize recovery and broken-device QA flows while avoiding Matrix device-cleanup sync races that could leave shutdown-time crypto work running. Thanks @gumadeiras. - Cron: treat isolated run-level agent failures as job errors even when no reply payload is produced, synthesizing a safe error payload so model/provider failures increment error counters and trigger failure notifications instead of clearing as successful. Fixes #43604; carries forward #43631. Thanks @SPFAdvisors. - Cron: preserve exact `NO_REPLY` tool results from isolated jobs with empty final assistant turns as quiet successes instead of surfacing incomplete-turn errors. Fixes #68452; carries forward #68453. Thanks @anyech. diff --git a/docs/plugins/google-meet.md b/docs/plugins/google-meet.md index a2b4d1d5997..68d53f496ec 100644 --- a/docs/plugins/google-meet.md +++ b/docs/plugins/google-meet.md @@ -897,6 +897,8 @@ Defaults: `openclaw_agent_consult` for deeper answers - `realtime.introMessage`: short spoken readiness check when the realtime bridge connects; set it to `""` to join silently +- `realtime.agentId`: optional OpenClaw agent id for + `openclaw_agent_consult`; defaults to `main` Optional overrides: @@ -915,6 +917,7 @@ Optional overrides: }, realtime: { provider: "google", + agentId: "jay", toolPolicy: "owner", introMessage: "Say exactly: I'm here.", providers: { @@ -1001,6 +1004,10 @@ meeting transcript context and returns a concise spoken answer to the realtime voice session. The voice model can then speak that answer back into the meeting. It uses the same shared realtime consult tool as Voice Call. +By default, consults run against the `main` agent. Set `realtime.agentId` when a +Meet lane should consult a dedicated OpenClaw agent workspace, model defaults, +tool policy, memory, and session history. + `realtime.toolPolicy` controls the consult run: - `safe-read-only`: expose the consult tool and limit the regular agent to diff --git a/extensions/google-meet/index.test.ts b/extensions/google-meet/index.test.ts index b15a4cf394a..fb6a164cf4a 100644 --- a/extensions/google-meet/index.test.ts +++ b/extensions/google-meet/index.test.ts @@ -287,6 +287,16 @@ describe("google-meet plugin", () => { expect(resolveGoogleMeetConfig({}).realtime.instructions).toContain("openclaw_agent_consult"); }); + it("resolves the realtime consult agent id", () => { + expect( + resolveGoogleMeetConfig({ + realtime: { + agentId: " jay ", + }, + }).realtime.agentId, + ).toBe("jay"); + }); + it("uses env fallbacks for OAuth, preview, and default meeting values", () => { expect( resolveGoogleMeetConfigWithEnv( @@ -1976,7 +1986,7 @@ describe("google-meet plugin", () => { const handle = await startCommandRealtimeAudioBridge({ config: resolveGoogleMeetConfig({ - realtime: { provider: "openai", model: "gpt-realtime" }, + realtime: { provider: "openai", model: "gpt-realtime", agentId: "jay" }, }), fullConfig: {} as never, runtime: runtime as never, @@ -2041,10 +2051,14 @@ describe("google-meet plugin", () => { expect(runtime.agent.runEmbeddedPiAgent).toHaveBeenCalledWith( expect.objectContaining({ messageProvider: "google-meet", + agentId: "jay", + sessionKey: "agent:jay:google-meet:meet-1", + sandboxSessionKey: "agent:jay:google-meet:meet-1", thinkLevel: "high", toolsAllow: ["read", "web_search", "web_fetch", "x_search", "memory_search", "memory_get"], }), ); + expect(sessionStore).toHaveProperty("agent:jay:google-meet:meet-1"); await handle.stop(); expect(bridge.close).toHaveBeenCalled(); diff --git a/extensions/google-meet/index.ts b/extensions/google-meet/index.ts index aeeacda25c6..9e81567b725 100644 --- a/extensions/google-meet/index.ts +++ b/extensions/google-meet/index.ts @@ -120,6 +120,11 @@ const googleMeetConfigSchema = { label: "Realtime Intro Message", help: "Spoken once when the realtime bridge is ready. Set to an empty string to join silently.", }, + "realtime.agentId": { + label: "Realtime Consult Agent", + help: 'OpenClaw agent id used by openclaw_agent_consult. Defaults to "main".', + advanced: true, + }, "realtime.toolPolicy": { label: "Realtime Tool Policy", help: "Safe read-only tools are available by default; owner requests can unlock broader tools.", diff --git a/extensions/google-meet/openclaw.plugin.json b/extensions/google-meet/openclaw.plugin.json index 8c23a4df121..d5a308f9165 100644 --- a/extensions/google-meet/openclaw.plugin.json +++ b/extensions/google-meet/openclaw.plugin.json @@ -129,6 +129,11 @@ "label": "Realtime Intro Message", "help": "Spoken once when the realtime bridge is ready. Set to an empty string to join silently." }, + "realtime.agentId": { + "label": "Realtime Consult Agent", + "help": "OpenClaw agent id used by openclaw_agent_consult. Defaults to \"main\".", + "advanced": true + }, "realtime.toolPolicy": { "label": "Realtime Tool Policy", "help": "Safe read-only tools are available by default; owner requests can unlock broader tools.", @@ -353,6 +358,10 @@ "type": "string", "default": "Say exactly: I'm here and listening." }, + "agentId": { + "type": "string", + "description": "OpenClaw agent id used by openclaw_agent_consult. Defaults to \"main\"." + }, "toolPolicy": { "type": "string", "enum": ["safe-read-only", "owner", "none"], diff --git a/extensions/google-meet/src/agent-consult.ts b/extensions/google-meet/src/agent-consult.ts index 36bd1e3c2df..80b3e78b023 100644 --- a/extensions/google-meet/src/agent-consult.ts +++ b/extensions/google-meet/src/agent-consult.ts @@ -8,6 +8,7 @@ import { resolveRealtimeVoiceAgentConsultToolsAllow, type RealtimeVoiceTool, } from "openclaw/plugin-sdk/realtime-voice"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; import type { GoogleMeetConfig, GoogleMeetToolPolicy } from "./config.js"; export const GOOGLE_MEET_AGENT_CONSULT_TOOL_NAME = REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME; @@ -26,11 +27,14 @@ export async function consultOpenClawAgentForGoogleMeet(params: { args: unknown; transcript: Array<{ role: "user" | "assistant"; text: string }>; }): Promise<{ text: string }> { + const agentId = normalizeAgentId(params.config.realtime.agentId); + const sessionKey = `agent:${agentId}:google-meet:${params.meetingSessionId}`; return await consultRealtimeVoiceAgent({ cfg: params.fullConfig, agentRuntime: params.runtime.agent, logger: params.logger, - sessionKey: `google-meet:${params.meetingSessionId}`, + agentId, + sessionKey, messageProvider: "google-meet", lane: "google-meet", runIdPrefix: `google-meet:${params.meetingSessionId}`, diff --git a/extensions/google-meet/src/config.ts b/extensions/google-meet/src/config.ts index 92efb86382f..4e650a56e95 100644 --- a/extensions/google-meet/src/config.ts +++ b/extensions/google-meet/src/config.ts @@ -57,6 +57,7 @@ export type GoogleMeetConfig = { model?: string; instructions?: string; introMessage?: string; + agentId?: string; toolPolicy: GoogleMeetToolPolicy; providers: Record>; }; @@ -361,6 +362,7 @@ export function resolveGoogleMeetConfigWithEnv( introMessage: normalizeOptionalString(realtime.introMessage) ?? DEFAULT_GOOGLE_MEET_CONFIG.realtime.introMessage, + agentId: normalizeOptionalString(realtime.agentId), toolPolicy: resolveRealtimeVoiceAgentConsultToolPolicy( realtime.toolPolicy, DEFAULT_GOOGLE_MEET_CONFIG.realtime.toolPolicy, From da55212c6e2ad3fa730b1708ba8d0f893e23e4de Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:38:42 +0100 Subject: [PATCH 07/12] fix(lmstudio): promote bracketed tool calls --- CHANGELOG.md | 1 + docs/gateway/local-models.md | 5 + .../lmstudio/src/plain-text-tool-calls.ts | 167 +++++++++++++ extensions/lmstudio/src/stream.test.ts | 113 ++++++++- extensions/lmstudio/src/stream.ts | 219 +++++++++++++++++- .../text/assistant-visible-text.test.ts | 26 +++ src/shared/text/assistant-visible-text.ts | 2 + .../text/plain-text-tool-call-blocks.ts | 211 +++++++++++++++++ 8 files changed, 740 insertions(+), 4 deletions(-) create mode 100644 extensions/lmstudio/src/plain-text-tool-calls.ts create mode 100644 src/shared/text/plain-text-tool-call-blocks.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cbc28efe2..eaf48b30b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Docs: https://docs.openclaw.ai - CLI/update: keep the automatic post-update completion refresh on the core-command tree so it no longer stages bundled plugin runtime deps before the Gateway restart path, avoiding `.24` update hangs and 1006 disconnect cascades. Fixes #72665. Thanks @sakalaboator and @He-Pin. - Agents/Bedrock: stop heartbeat runs from persisting blank user transcript turns and repair existing blank user text messages before replay, preventing AWS Bedrock `ContentBlock` blank-text validation failures. Fixes #72640 and #72622. Thanks @goldzulu. +- Agents/LM Studio: promote standalone bracketed local-model tool requests into registered tool calls and hide unsupported bracket blocks from visible replies, so MemPalace MCP lookups do not print raw `[tool]` JSON scaffolding in chat. Fixes #66178. Thanks @detroit357. - Agents/LM Studio: strip prior-turn Gemma 4 reasoning from OpenAI-compatible replay while preserving active tool-call continuation reasoning. Fixes #68704. Thanks @chip-snomo and @Kailigithub. - LM Studio: allow interactive onboarding to leave the API key blank for unauthenticated local servers, using local synthetic auth while clearing stale LM Studio auth profiles. Fixes #66937. Thanks @olamedia. - Process/Windows: decode command stdout and stderr from raw bytes with console-codepage awareness, while preserving valid UTF-8 output and multibyte characters split across chunks. Fixes #50519. Thanks @iready, @kevinten10, @zhangyongjie1997, @knightplat-blip, @heiqishi666, and @slepybear. diff --git a/docs/gateway/local-models.md b/docs/gateway/local-models.md index 338b70f2296..2967a39ed50 100644 --- a/docs/gateway/local-models.md +++ b/docs/gateway/local-models.md @@ -164,6 +164,11 @@ Compatibility notes for stricter OpenAI-compatible backends: structured content-part arrays. Set `models.providers..models[].compat.requiresStringContent: true` for those endpoints. +- Some local models emit standalone bracketed tool requests as text, such as + `[tool_name]` followed by JSON and `[END_TOOL_REQUEST]`. OpenClaw promotes + those into real tool calls only when the name exactly matches a registered + tool for the turn; otherwise the block is treated as unsupported text and is + hidden from user-visible replies. - Some smaller or stricter local backends are unstable with OpenClaw's full agent-runtime prompt shape, especially when tool schemas are included. If the backend works for tiny direct `/v1/chat/completions` calls but fails on normal diff --git a/extensions/lmstudio/src/plain-text-tool-calls.ts b/extensions/lmstudio/src/plain-text-tool-calls.ts new file mode 100644 index 00000000000..124a38e407c --- /dev/null +++ b/extensions/lmstudio/src/plain-text-tool-calls.ts @@ -0,0 +1,167 @@ +import { randomUUID } from "node:crypto"; + +export type LmstudioPlainTextToolCallBlock = { + arguments: Record; + name: string; +}; + +const END_TOOL_REQUEST = "[END_TOOL_REQUEST]"; +const MAX_PAYLOAD_CHARS = 256_000; + +function isToolNameChar(char: string | undefined): boolean { + return Boolean(char && /[A-Za-z0-9_-]/.test(char)); +} + +function skipHorizontalWhitespace(text: string, start: number): number { + let index = start; + while (index < text.length && (text[index] === " " || text[index] === "\t")) { + index += 1; + } + return index; +} + +function skipWhitespace(text: string, start: number): number { + let index = start; + while (index < text.length && /\s/.test(text[index] ?? "")) { + index += 1; + } + return index; +} + +function consumeLineBreak(text: string, start: number): number | null { + if (text[start] === "\r") { + return text[start + 1] === "\n" ? start + 2 : start + 1; + } + if (text[start] === "\n") { + return start + 1; + } + return null; +} + +function parseOpening(text: string, start: number): { end: number; name: string } | null { + if (text[start] !== "[") { + return null; + } + let cursor = start + 1; + const nameStart = cursor; + while (isToolNameChar(text[cursor])) { + cursor += 1; + } + if (cursor === nameStart || text[cursor] !== "]") { + return null; + } + const name = text.slice(nameStart, cursor); + cursor += 1; + cursor = skipHorizontalWhitespace(text, cursor); + const afterLineBreak = consumeLineBreak(text, cursor); + if (afterLineBreak === null) { + return null; + } + return { end: afterLineBreak, name }; +} + +function consumeJsonObject( + text: string, + start: number, +): { end: number; value: Record } | null { + const cursor = skipWhitespace(text, start); + if (text[cursor] !== "{") { + return null; + } + let depth = 0; + let inString = false; + let escaped = false; + for (let index = cursor; index < text.length; index += 1) { + if (index + 1 - cursor > MAX_PAYLOAD_CHARS) { + return null; + } + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + try { + const parsed = JSON.parse(text.slice(cursor, index + 1)) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + return { end: index + 1, value: parsed as Record }; + } catch { + return null; + } + } + } + } + return null; +} + +function parseClosing(text: string, start: number, name: string): number | null { + const cursor = skipWhitespace(text, start); + if (text.startsWith(END_TOOL_REQUEST, cursor)) { + return cursor + END_TOOL_REQUEST.length; + } + const namedClosing = `[/${name}]`; + if (text.startsWith(namedClosing, cursor)) { + return cursor + namedClosing.length; + } + return null; +} + +function parseBlockAt( + text: string, + start: number, + allowedToolNames: Set, +): { block: LmstudioPlainTextToolCallBlock; end: number } | null { + const opening = parseOpening(text, start); + if (!opening || !allowedToolNames.has(opening.name)) { + return null; + } + const payload = consumeJsonObject(text, opening.end); + if (!payload) { + return null; + } + const end = parseClosing(text, payload.end, opening.name); + if (end === null) { + return null; + } + return { + block: { arguments: payload.value, name: opening.name }, + end, + }; +} + +export function parseLmstudioPlainTextToolCalls( + text: string, + allowedToolNames: Set, +): LmstudioPlainTextToolCallBlock[] | null { + const blocks: LmstudioPlainTextToolCallBlock[] = []; + let cursor = skipWhitespace(text, 0); + while (cursor < text.length) { + const parsed = parseBlockAt(text, cursor, allowedToolNames); + if (!parsed) { + return null; + } + blocks.push(parsed.block); + cursor = skipWhitespace(text, parsed.end); + } + return blocks.length > 0 ? blocks : null; +} + +export function createLmstudioSyntheticToolCallId(): string { + return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`; +} diff --git a/extensions/lmstudio/src/stream.test.ts b/extensions/lmstudio/src/stream.test.ts index 7c61a65721f..a82e7c0b730 100644 --- a/extensions/lmstudio/src/stream.test.ts +++ b/extensions/lmstudio/src/stream.test.ts @@ -28,7 +28,7 @@ vi.mock("./runtime.js", async (importOriginal) => { }; }); -type StreamEvent = { type: string }; +type StreamEvent = { type: string } & Record; async function collectEvents(stream: ReturnType): Promise { const resolved = stream instanceof Promise ? await stream : stream; @@ -50,6 +50,19 @@ function buildDoneStreamFn(): StreamFn { }); } +function buildEventStreamFn(events: unknown[]): StreamFn { + return vi.fn((_model, _context, _options) => { + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => { + for (const event of events) { + stream.push(event as never); + } + stream.end(); + }); + return stream; + }); +} + function createWrappedLmstudioStream( baseStream: StreamFn, params?: { baseUrl?: string }, @@ -75,6 +88,7 @@ function runWrappedLmstudioStream( wrapped: StreamFn, model: Record, options?: Record, + context?: Record, ) { return wrapped( { @@ -83,7 +97,7 @@ function runWrappedLmstudioStream( id: "lmstudio/qwen3-8b-instruct", ...model, } as never, - { messages: [] } as never, + { messages: [], ...context } as never, options as never, ); } @@ -400,4 +414,99 @@ describe("lmstudio stream wrapper", () => { undefined, ); }); + + it("promotes standalone bracketed local-model tool text to a structured tool call", async () => { + const rawToolText = [ + "[mempalace_mempalace_search]", + '{"query":"codename","wing":"personal","room":"identities"}', + "[END_TOOL_REQUEST]", + ].join("\n"); + const baseStream = buildEventStreamFn([ + { type: "start", partial: { content: [] } }, + { type: "text_start", contentIndex: 0, partial: { content: [{ type: "text", text: "" }] } }, + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { type: "text_end", contentIndex: 0, content: rawToolText }, + { + type: "done", + reason: "stop", + message: { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }, + }, + ]); + const wrapped = createWrappedLmstudioStream(baseStream); + const events = await collectEvents( + runWrappedLmstudioStream(wrapped, {}, undefined, { + tools: [ + { + name: "mempalace_mempalace_search", + description: "Search MemPalace", + parameters: { type: "object", properties: {} }, + }, + ], + }), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "done", + ]); + expect(events.some((event) => event.type === "text_delta")).toBe(false); + const done = events.find((event) => event.type === "done") as { + message?: { content?: Array>; stopReason?: string }; + reason?: string; + }; + expect(done.reason).toBe("toolUse"); + expect(done.message?.stopReason).toBe("toolUse"); + expect(done.message?.content?.[0]).toMatchObject({ + type: "toolCall", + name: "mempalace_mempalace_search", + arguments: { query: "codename", wing: "personal", room: "identities" }, + }); + expect(String(done.message?.content?.[0]?.id)).toMatch(/^call_[a-f0-9]{24}$/); + }); + + it("passes through bracketed text when the tool is not registered", async () => { + const rawToolText = [ + "[mempalace_mempalace_search]", + '{"query":"codename"}', + "[/mempalace_mempalace_search]", + ].join("\n"); + const baseStream = buildEventStreamFn([ + { type: "start", partial: { content: [] } }, + { type: "text_start", contentIndex: 0, partial: { content: [{ type: "text", text: "" }] } }, + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { type: "text_end", contentIndex: 0, content: rawToolText }, + { + type: "done", + reason: "stop", + message: { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }, + }, + ]); + const wrapped = createWrappedLmstudioStream(baseStream); + const events = await collectEvents( + runWrappedLmstudioStream(wrapped, {}, undefined, { + tools: [{ name: "read", description: "Read", parameters: { type: "object" } }], + }), + ); + + expect(events.map((event) => event.type)).toEqual([ + "start", + "text_start", + "text_delta", + "text_end", + "done", + ]); + expect(events.find((event) => event.type === "text_delta")).toMatchObject({ + delta: rawToolText, + }); + }); }); diff --git a/extensions/lmstudio/src/stream.ts b/extensions/lmstudio/src/stream.ts index e0c70b7ca94..7631117e5a2 100644 --- a/extensions/lmstudio/src/stream.ts +++ b/extensions/lmstudio/src/stream.ts @@ -1,17 +1,22 @@ import type { StreamFn } from "@mariozechner/pi-agent-core"; -import { streamSimple } from "@mariozechner/pi-ai"; +import { createAssistantMessageEventStream, streamSimple } from "@mariozechner/pi-ai"; import { createSubsystemLogger } from "openclaw/plugin-sdk/logging-core"; import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry"; import { ssrfPolicyFromHttpBaseUrlAllowedHostname } from "openclaw/plugin-sdk/ssrf-runtime"; import { LMSTUDIO_PROVIDER_ID } from "./defaults.js"; import { ensureLmstudioModelLoaded } from "./models.fetch.js"; import { resolveLmstudioInferenceBase } from "./models.js"; +import { + createLmstudioSyntheticToolCallId, + parseLmstudioPlainTextToolCalls, +} from "./plain-text-tool-calls.js"; import { resolveLmstudioProviderHeaders, resolveLmstudioRuntimeApiKey } from "./runtime.js"; const log = createSubsystemLogger("extensions/lmstudio/stream"); type StreamOptions = Parameters[2]; type StreamModel = Parameters[0]; +type StreamContext = Parameters[1]; const preloadInFlight = new Map>(); @@ -112,6 +117,215 @@ function resolveModelHeaders(model: StreamModel): Record | undef return model.headers; } +function toRecord(value: unknown): Record | undefined { + return value && typeof value === "object" ? (value as Record) : undefined; +} + +function resolveContextToolNames(context: StreamContext): Set { + const tools = (context as { tools?: unknown }).tools; + if (!Array.isArray(tools)) { + return new Set(); + } + const names = tools + .map((tool) => { + const record = toRecord(tool); + return typeof record?.name === "string" && record.name.trim() ? record.name : undefined; + }) + .filter((name): name is string => Boolean(name)); + return new Set(names); +} + +function couldStillBePlainTextToolCall(text: string): boolean { + if (text.length > 256_000) { + return false; + } + const trimmed = text.trimStart(); + return trimmed.length === 0 || trimmed.startsWith("["); +} + +function createLmstudioToolCallBlock(parsed: { + arguments: Record; + name: string; +}): Record { + return { + type: "toolCall", + id: createLmstudioSyntheticToolCallId(), + name: parsed.name, + arguments: parsed.arguments, + partialArgs: JSON.stringify(parsed.arguments), + }; +} + +function promoteLmstudioPlainTextToolCalls( + message: unknown, + toolNames: Set, +): Record | undefined { + const messageRecord = toRecord(message); + if (!messageRecord) { + return undefined; + } + if (!Array.isArray(messageRecord.content)) { + if (typeof messageRecord.content !== "string" || !messageRecord.content.trim()) { + return undefined; + } + const parsed = parseLmstudioPlainTextToolCalls(messageRecord.content, toolNames); + if (!parsed) { + return undefined; + } + return { + ...messageRecord, + content: parsed.map(createLmstudioToolCallBlock), + stopReason: "toolUse", + }; + } + if ( + messageRecord.content.some((block) => toRecord(block)?.type === "toolCall") || + messageRecord.content.length === 0 + ) { + return undefined; + } + + let promoted = false; + const nextContent: Array> = []; + for (const block of messageRecord.content) { + const blockRecord = toRecord(block); + if (!blockRecord) { + return undefined; + } + if (blockRecord.type !== "text") { + nextContent.push(blockRecord); + continue; + } + const text = typeof blockRecord.text === "string" ? blockRecord.text : ""; + if (!text.trim()) { + continue; + } + const parsed = parseLmstudioPlainTextToolCalls(text, toolNames); + if (!parsed) { + return undefined; + } + nextContent.push(...parsed.map(createLmstudioToolCallBlock)); + promoted = true; + } + + if (!promoted) { + return undefined; + } + return { + ...messageRecord, + content: nextContent, + stopReason: "toolUse", + }; +} + +function emitPromotedToolCallEvents( + stream: { push(event: unknown): void }, + message: Record, +): void { + const content = Array.isArray(message.content) ? message.content : []; + content.forEach((block, contentIndex) => { + const record = toRecord(block); + if (record?.type !== "toolCall") { + return; + } + stream.push({ type: "toolcall_start", contentIndex, partial: message }); + stream.push({ + type: "toolcall_delta", + contentIndex, + delta: typeof record.partialArgs === "string" ? record.partialArgs : "{}", + partial: message, + }); + }); +} + +function wrapLmstudioPlainTextToolCalls( + source: ReturnType, + context: StreamContext, +): ReturnType { + const toolNames = resolveContextToolNames(context); + if (toolNames.size === 0) { + return source; + } + const output = createAssistantMessageEventStream(); + const stream = output as unknown as { push(event: unknown): void; end(): void }; + + void (async () => { + const bufferedTextEvents: unknown[] = []; + let bufferedText = ""; + let ended = false; + const endStream = () => { + if (!ended) { + ended = true; + stream.end(); + } + }; + const flushBufferedTextEvents = () => { + for (const event of bufferedTextEvents.splice(0)) { + stream.push(event); + } + bufferedText = ""; + }; + + try { + for await (const event of source as AsyncIterable) { + const record = toRecord(event); + const type = typeof record?.type === "string" ? record.type : ""; + + if (type === "text_start" || type === "text_delta" || type === "text_end") { + bufferedTextEvents.push(event); + if (typeof record?.delta === "string") { + bufferedText += record.delta; + } else if (typeof record?.content === "string" && !bufferedText) { + bufferedText = record.content; + } + if (!couldStillBePlainTextToolCall(bufferedText)) { + flushBufferedTextEvents(); + } + continue; + } + + if (type === "done") { + const promotedMessage = promoteLmstudioPlainTextToolCalls(record?.message, toolNames); + if (promotedMessage) { + bufferedTextEvents.splice(0); + bufferedText = ""; + emitPromotedToolCallEvents(stream, promotedMessage); + stream.push({ ...record, reason: "toolUse", message: promotedMessage }); + } else { + flushBufferedTextEvents(); + stream.push(event); + } + endStream(); + return; + } + + flushBufferedTextEvents(); + stream.push(event); + if (type === "error") { + endStream(); + return; + } + } + flushBufferedTextEvents(); + } catch (error) { + stream.push({ + type: "error", + reason: "error", + error: { + role: "assistant", + content: [], + stopReason: "error", + errorMessage: error instanceof Error ? error.message : String(error), + }, + }); + } finally { + endStream(); + } + })(); + + return output as ReturnType; +} + function createPreloadKey(params: { baseUrl: string; modelKey: string; @@ -248,7 +462,8 @@ export function wrapLmstudioInferencePreload(ctx: ProviderWrapStreamFnContext): }, }; const stream = underlying(modelWithUsageCompat, context, options); - return stream instanceof Promise ? await stream : stream; + const resolvedStream = stream instanceof Promise ? await stream : stream; + return wrapLmstudioPlainTextToolCalls(resolvedStream, context); })(); }; } diff --git a/src/shared/text/assistant-visible-text.test.ts b/src/shared/text/assistant-visible-text.test.ts index b599b83f03b..b27181710ce 100644 --- a/src/shared/text/assistant-visible-text.test.ts +++ b/src/shared/text/assistant-visible-text.test.ts @@ -152,6 +152,32 @@ describe("stripAssistantInternalScaffolding", () => { ); }); + it("strips standalone bracketed local-model tool blocks", () => { + expectVisibleText( + [ + "Let me check.", + "[mempalace_mempalace_search]", + '{"query":"codename","wing":"personal","room":"identities"}', + "[END_TOOL_REQUEST]", + "Done.", + ].join("\n"), + "Let me check.\n\nDone.", + ); + }); + + it("strips bracketed local-model tool blocks with named closing tags", () => { + expectVisibleText( + [ + "Before", + "[mempalace_mempalace_search]", + '{"query":"codename","limit":1}', + "[/mempalace_mempalace_search]", + "After", + ].join("\n"), + "Before\n\nAfter", + ); + }); + it("strips Qwen-style with nested XML", () => { expectVisibleText( "prefix\n/home/user\nsuffix", diff --git a/src/shared/text/assistant-visible-text.ts b/src/shared/text/assistant-visible-text.ts index fe63abbc8e7..26eb77f9d68 100644 --- a/src/shared/text/assistant-visible-text.ts +++ b/src/shared/text/assistant-visible-text.ts @@ -1,6 +1,7 @@ import { normalizeLowercaseStringOrEmpty } from "../string-coerce.js"; import { findCodeRegions, isInsideCode } from "./code-regions.js"; import { stripModelSpecialTokens } from "./model-special-tokens.js"; +import { stripPlainTextToolCallBlocks } from "./plain-text-tool-call-blocks.js"; import { stripReasoningTagsFromText, type ReasoningTagMode, @@ -586,6 +587,7 @@ function applyAssistantVisibleTextStagePipeline( cleaned = stripModelSpecialTokens(cleaned); cleaned = stripRelevantMemoriesTags(cleaned); cleaned = stripToolCallXmlTags(cleaned); + cleaned = stripPlainTextToolCallBlocks(cleaned); if (!options.preserveDowngradedToolText) { cleaned = stripDowngradedToolCallText(cleaned); } diff --git a/src/shared/text/plain-text-tool-call-blocks.ts b/src/shared/text/plain-text-tool-call-blocks.ts new file mode 100644 index 00000000000..9d84037034a --- /dev/null +++ b/src/shared/text/plain-text-tool-call-blocks.ts @@ -0,0 +1,211 @@ +export type PlainTextToolCallBlock = { + arguments: Record; + end: number; + name: string; + raw: string; + start: number; +}; + +type ParseOptions = { + allowedToolNames?: Iterable; + maxPayloadBytes?: number; +}; + +const DEFAULT_MAX_PAYLOAD_BYTES = 256_000; +const END_TOOL_REQUEST = "[END_TOOL_REQUEST]"; + +function isToolNameChar(char: string | undefined): boolean { + return Boolean(char && /[A-Za-z0-9_-]/.test(char)); +} + +function skipHorizontalWhitespace(text: string, start: number): number { + let index = start; + while (index < text.length && (text[index] === " " || text[index] === "\t")) { + index += 1; + } + return index; +} + +function skipWhitespace(text: string, start: number): number { + let index = start; + while (index < text.length && /\s/.test(text[index] ?? "")) { + index += 1; + } + return index; +} + +function consumeLineBreak(text: string, start: number): number | null { + if (text[start] === "\r") { + return text[start + 1] === "\n" ? start + 2 : start + 1; + } + if (text[start] === "\n") { + return start + 1; + } + return null; +} + +function parseOpening(text: string, start: number): { end: number; name: string } | null { + if (text[start] !== "[") { + return null; + } + let cursor = start + 1; + const nameStart = cursor; + while (isToolNameChar(text[cursor])) { + cursor += 1; + } + if (cursor === nameStart || text[cursor] !== "]") { + return null; + } + const name = text.slice(nameStart, cursor); + cursor += 1; + cursor = skipHorizontalWhitespace(text, cursor); + const afterLineBreak = consumeLineBreak(text, cursor); + if (afterLineBreak === null) { + return null; + } + return { end: afterLineBreak, name }; +} + +function consumeJsonObject( + text: string, + start: number, + maxPayloadBytes: number, +): { end: number; value: Record } | null { + let cursor = skipWhitespace(text, start); + if (text[cursor] !== "{") { + return null; + } + let depth = 0; + let inString = false; + let escaped = false; + for (let index = cursor; index < text.length; index += 1) { + const char = text[index]; + if (index + 1 - cursor > maxPayloadBytes) { + return null; + } + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + if (depth === 0) { + const rawJson = text.slice(cursor, index + 1); + try { + const parsed = JSON.parse(rawJson) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + return { end: index + 1, value: parsed as Record }; + } catch { + return null; + } + } + } + } + return null; +} + +function parseClosing(text: string, start: number, name: string): number | null { + let cursor = skipWhitespace(text, start); + if (text.startsWith(END_TOOL_REQUEST, cursor)) { + return cursor + END_TOOL_REQUEST.length; + } + const namedClosing = `[/${name}]`; + if (text.startsWith(namedClosing, cursor)) { + return cursor + namedClosing.length; + } + return null; +} + +function parseBlockAt( + text: string, + start: number, + options?: ParseOptions, +): PlainTextToolCallBlock | null { + const opening = parseOpening(text, start); + if (!opening) { + return null; + } + const allowedToolNames = options?.allowedToolNames + ? new Set(options.allowedToolNames) + : undefined; + if (allowedToolNames && !allowedToolNames.has(opening.name)) { + return null; + } + const payload = consumeJsonObject( + text, + opening.end, + options?.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES, + ); + if (!payload) { + return null; + } + const end = parseClosing(text, payload.end, opening.name); + if (end === null) { + return null; + } + return { + arguments: payload.value, + end, + name: opening.name, + raw: text.slice(start, end), + start, + }; +} + +export function parseStandalonePlainTextToolCallBlocks( + text: string, + options?: ParseOptions, +): PlainTextToolCallBlock[] | null { + const blocks: PlainTextToolCallBlock[] = []; + let cursor = skipWhitespace(text, 0); + while (cursor < text.length) { + const block = parseBlockAt(text, cursor, options); + if (!block) { + return null; + } + blocks.push(block); + cursor = skipWhitespace(text, block.end); + } + return blocks.length > 0 ? blocks : null; +} + +export function stripPlainTextToolCallBlocks(text: string): string { + if (!text || !/\[[A-Za-z0-9_-]+\]/.test(text)) { + return text; + } + let result = ""; + let cursor = 0; + let index = 0; + while (index < text.length) { + const lineStart = index === 0 || text[index - 1] === "\n"; + if (!lineStart) { + index += 1; + continue; + } + const blockStart = skipHorizontalWhitespace(text, index); + const block = parseBlockAt(text, blockStart); + if (!block) { + index += 1; + continue; + } + result += text.slice(cursor, index); + cursor = block.end; + index = block.end; + } + result += text.slice(cursor); + return result; +} From 4514a731700a21005b5a6f802c71e1c583055fa0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:39:30 +0100 Subject: [PATCH 08/12] fix: preserve live runtime deps temp dirs --- CHANGELOG.md | 1 + scripts/stage-bundled-plugin-runtime-deps.mjs | 54 ++++++++++++++- .../stage-bundled-plugin-runtime-deps.test.ts | 67 +++++++++++++++++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf48b30b0e..ad42f520df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Docs: https://docs.openclaw.ai - Agents/bootstrap: refresh cached workspace bootstrap snapshots on long-lived main-session turns when `AGENTS.md`, `SOUL.md`, `MEMORY.md`, or `TOOLS.md` change on disk, while preserving unchanged snapshot identity through the workspace file cache. (#64871; related #43901, #26497, #28594, #30896) Thanks @aimqwest and @mikejuyoon. - macOS Gateway: detect installed-but-unloaded LaunchAgent split-brain states during status, doctor, and restart, and re-bootstrap launchd supervision before falling back to unmanaged listener restarts. Fixes #67335, #53475, and #71060; refs #58890, #60885, and #70801. Thanks @ze1tgeist88, @dafacto, and @vishutdhar. - Plugins/install: treat mirrored core logger dependencies as staged bundled runtime deps so packaged Gateway starts do not crash when the external plugin-runtime-deps root is missing `tslog`. Fixes #72228; supersedes #72493. Thanks @deepujain. +- Build/plugins: preserve active bundled runtime-dependency staging temp directories owned by live build processes so overlapping postbuild runs no longer delete each other's staged deps mid-prune. Supersedes #72220. Thanks @VACInc. - Plugins/install: hide bundled runtime-dependency npm child windows on Windows across Gateway startup, postinstall, and packaged staging paths so Telegram/Anthropic dependency repair no longer flashes shell windows. Fixes #72315. Thanks @athuljayaram and @joshfeng. - Plugins/Windows: normalize lazy plugin service override imports before Node ESM loading so drive-letter browser-control module paths no longer fail with `ERR_UNSUPPORTED_ESM_URL_SCHEME`. Fixes #72573; supersedes #72599 and #72582. Thanks @llzzww316, @feineryonah-byte, and @WuKongAI-CMU. - Plugins/install: stage bundled plugin runtime dependencies before Gateway startup, drain update restarts, and materialize plugin-owned root chunks in external mirrors so staged deps resolve under native ESM. Fixes #72058; supersedes #72084. Thanks @amnesia106 and @drvoss. diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index db8a5eafd7c..df5bdc8fa5a 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -8,6 +8,7 @@ import { resolveNpmRunner } from "./npm-runner.mjs"; const TRANSIENT_TEMP_REMOVE_ERROR_CODES = new Set(["EBUSY", "ENOTEMPTY", "EPERM"]); const TEMP_REMOVE_RETRY_DELAYS_MS = [10, 25, 50]; +const TEMP_OWNER_FILE = "owner.json"; function readJson(filePath) { return JSON.parse(fs.readFileSync(filePath, "utf8")); @@ -48,13 +49,26 @@ function makeTempDir(parentDir, prefix) { return fs.mkdtempSync(path.join(parentDir, prefix)); } +function writeRuntimeDepsTempOwner(tempDir) { + writeJson(path.join(tempDir, TEMP_OWNER_FILE), { + pid: process.pid, + createdAtMs: Date.now(), + }); +} + +function makeOwnedTempDir(parentDir, prefix) { + const tempDir = makeTempDir(parentDir, prefix); + writeRuntimeDepsTempOwner(tempDir); + return tempDir; +} + function sanitizeTempPrefixSegment(value) { const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-"); return normalized.length > 0 ? normalized : "plugin"; } function makePluginOwnedTempDir(pluginDir, label) { - return makeTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`); + return makeOwnedTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`); } function assertPathIsNotSymlink(targetPath, label) { @@ -84,6 +98,7 @@ function replaceDirAtomically(targetPath, sourcePath) { try { if (fs.existsSync(targetPath)) { fs.renameSync(targetPath, backupPath); + writeRuntimeDepsTempOwner(backupPath); movedExistingTarget = true; } fs.renameSync(sourcePath, targetPath); @@ -100,7 +115,7 @@ function writeJsonAtomically(targetPath, value) { assertPathIsNotSymlink(targetPath, "write runtime deps stamp"); const targetParentDir = path.dirname(targetPath); fs.mkdirSync(targetParentDir, { recursive: true }); - const tempDir = makeTempDir( + const tempDir = makeOwnedTempDir( targetParentDir, `.openclaw-runtime-deps-stamp-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, ); @@ -954,6 +969,35 @@ function readRuntimeDepsStamp(stampPath) { } } +function readRuntimeDepsTempOwner(tempDir) { + try { + const owner = readJson(path.join(tempDir, TEMP_OWNER_FILE)); + return owner && typeof owner === "object" ? owner : null; + } catch { + return null; + } +} + +function isLiveProcess(pid) { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +function shouldRemoveRuntimeDepsTempDir(tempDir) { + const owner = readRuntimeDepsTempOwner(tempDir); + if (!owner || typeof owner.pid !== "number") { + return true; + } + return !isLiveProcess(owner.pid); +} + function removeStaleRuntimeDepsTempDirs(pluginDir) { if (!fs.existsSync(pluginDir)) { return; @@ -961,6 +1005,9 @@ function removeStaleRuntimeDepsTempDirs(pluginDir) { for (const entry of fs.readdirSync(pluginDir, { withFileTypes: true })) { if (entry.name.startsWith(".openclaw-runtime-deps-")) { const targetPath = path.join(pluginDir, entry.name); + if (!shouldRemoveRuntimeDepsTempDir(targetPath)) { + continue; + } for (let attempt = 0; attempt <= TEMP_REMOVE_RETRY_DELAYS_MS.length; attempt += 1) { try { removePathIfExists(targetPath); @@ -1243,7 +1290,10 @@ export function stageBundledPluginRuntimeDeps(params = {}) { } export const __testing = { + removeStaleRuntimeDepsTempDirs, + replaceDirAtomically, runNpmInstall, + writeRuntimeDepsTempOwner, }; if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { diff --git a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts index a03facfbf25..fa543db54d1 100644 --- a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts +++ b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts @@ -327,6 +327,73 @@ describe("stageBundledPluginRuntimeDeps", () => { ); }); + it("keeps runtime deps temp dirs owned by a live build process", () => { + const { pluginDir, repoRoot } = createBundledPluginFixture({ + packageJson: { + name: "@openclaw/fixture-plugin", + version: "1.0.0", + dependencies: { "left-pad": "1.3.0" }, + openclaw: { bundle: { stageRuntimeDependencies: true } }, + }, + }); + const activeTempDir = path.join(pluginDir, ".openclaw-runtime-deps-stage-active"); + fs.mkdirSync(activeTempDir, { recursive: true }); + stageBundledPluginRuntimeDepsTesting.writeRuntimeDepsTempOwner(activeTempDir); + fs.writeFileSync(path.join(activeTempDir, "marker.txt"), "active\n", "utf8"); + + stageBundledPluginRuntimeDeps({ + cwd: repoRoot, + installPluginRuntimeDepsImpl: ({ fingerprint, stampPath }: RuntimeDepsStampParams) => { + const nodeModulesDir = path.join(pluginDir, "node_modules"); + fs.mkdirSync(nodeModulesDir, { recursive: true }); + fs.writeFileSync(path.join(nodeModulesDir, "marker.txt"), "installed\n", "utf8"); + writeRuntimeDepsStamp(stampPath, fingerprint); + }, + }); + + expect(fs.readFileSync(path.join(activeTempDir, "marker.txt"), "utf8")).toBe("active\n"); + expect(fs.readFileSync(path.join(pluginDir, "node_modules", "marker.txt"), "utf8")).toBe( + "installed\n", + ); + }); + + it("restores atomically replaced dirs when concurrent cleanup runs during rename failure", () => { + const parentDir = createTempDir("openclaw-runtime-deps-replace-"); + const targetPath = path.join(parentDir, "node_modules"); + const sourcePath = path.join(parentDir, "source-node_modules"); + fs.mkdirSync(targetPath, { recursive: true }); + fs.writeFileSync(path.join(targetPath, "marker.txt"), "original\n", "utf8"); + fs.mkdirSync(sourcePath, { recursive: true }); + fs.writeFileSync(path.join(sourcePath, "marker.txt"), "replacement\n", "utf8"); + + const realRenameSync = fs.renameSync.bind(fs); + let backupPath: string | null = null; + vi.spyOn(fs, "renameSync").mockImplementation((oldPath, newPath) => { + const oldPathString = String(oldPath); + const newPathString = String(newPath); + if ( + oldPathString === targetPath && + path.basename(newPathString).startsWith(".openclaw-runtime-deps-backup-") + ) { + backupPath = newPathString; + return realRenameSync(oldPath, newPath); + } + if (oldPathString === sourcePath && newPathString === targetPath) { + expect(backupPath).not.toBeNull(); + stageBundledPluginRuntimeDepsTesting.removeStaleRuntimeDepsTempDirs(parentDir); + expect(fs.existsSync(path.join(backupPath ?? "", "marker.txt"))).toBe(true); + throw new Error("rename failed after backup"); + } + return realRenameSync(oldPath, newPath); + }); + + expect(() => + stageBundledPluginRuntimeDepsTesting.replaceDirAtomically(targetPath, sourcePath), + ).toThrow("rename failed after backup"); + + expect(fs.readFileSync(path.join(targetPath, "marker.txt"), "utf8")).toBe("original\n"); + }); + it("restages when installed root runtime dependency contents change", () => { const { pluginDir, repoRoot } = createBundledPluginFixture({ packageJson: { From 15e634d50c93429dd7a276c39bfe368ebc7b1525 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:35:16 +0100 Subject: [PATCH 09/12] fix(plugins): normalize windows override imports --- docs/ci.md | 4 +- scripts/openclaw-cross-os-release-checks.ts | 118 ++++++++++++++++++ src/plugins/import-specifier.test.ts | 3 + src/plugins/import-specifier.ts | 7 +- src/plugins/lazy-service-module.test.ts | 6 +- src/plugins/loader.test.ts | 3 + .../openclaw-cross-os-release-checks.test.ts | 15 +++ 7 files changed, 148 insertions(+), 8 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index 154e6b7787a..ff4dec6c8c1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -93,7 +93,9 @@ GitHub-native replacement for most Parallels package/update validation, with Telegram proving the same package artifact through the QA live transport. Cross-OS release checks still cover OS-specific onboarding, installer, and platform behavior; package/update product validation should start with Package -Acceptance. +Acceptance. The Windows packaged and installer fresh lanes also verify that an +installed package can import a browser-control override from a raw absolute +Windows path. Package Acceptance has a bounded legacy-compatibility window for already published packages through `2026.4.25`, including `2026.4.25-beta.*`. Those diff --git a/scripts/openclaw-cross-os-release-checks.ts b/scripts/openclaw-cross-os-release-checks.ts index a3efeea12a1..bd9013f0687 100644 --- a/scripts/openclaw-cross-os-release-checks.ts +++ b/scripts/openclaw-cross-os-release-checks.ts @@ -566,6 +566,17 @@ async function runFreshLane(params) { logPath: join(params.logsDir, "fresh-install.log"), }); + let browserOverrideImportStatus = "skipped"; + if (shouldRunWindowsInstalledBrowserOverrideImportSmoke()) { + logLanePhase(lane, "windows-browser-override-import"); + browserOverrideImportStatus = await runInstalledBrowserOverrideImportSmoke({ + lane, + env, + prefixDir: lane.prefixDir, + logPath: join(params.logsDir, "fresh-windows-browser-override-import.log"), + }); + } + logLanePhase(lane, "onboard"); await runOnboard({ lane, @@ -617,6 +628,7 @@ async function runFreshLane(params) { installedCommit: installed.commit, dashboardStatus: "pass", gatewayPort: lane.gatewayPort, + browserOverrideImportStatus, agentOutput: trimForSummary(agent.stdout), }; } finally { @@ -795,6 +807,17 @@ async function runInstallerFreshSuite(params) { const installed = readInstalledMetadataFromCliPath(freshShell.cliPath); verifyInstalledCandidate(installed, params.build); + let browserOverrideImportStatus = "skipped"; + if (shouldRunWindowsInstalledBrowserOverrideImportSmoke()) { + logLanePhase(lane, "windows-browser-override-import"); + browserOverrideImportStatus = await runInstalledBrowserOverrideImportSmoke({ + lane, + env, + prefixDir: resolveInstalledPrefixDirFromCliPath(freshShell.cliPath), + logPath: join(params.logsDir, "installer-fresh-windows-browser-override-import.log"), + }); + } + logLanePhase(lane, "onboard"); await runOnboardWithInstalledCli({ lane, @@ -900,6 +923,7 @@ async function runInstallerFreshSuite(params) { installedCommit: installed.commit, gatewayPort: lane.gatewayPort, dashboardStatus: "pass", + browserOverrideImportStatus, discordStatus, agentOutput: trimForSummary(agent.stdout), }; @@ -2200,6 +2224,100 @@ async function runBundledPluginPostinstall(params) { }); } +export function shouldRunWindowsInstalledBrowserOverrideImportSmoke(platform = process.platform) { + return platform === "win32"; +} + +export function buildInstalledBrowserOverrideImportProbeScript() { + return ` +import { existsSync } from "node:fs"; +import { startLazyPluginServiceModule } from "openclaw/plugin-sdk/browser-node-runtime"; + +const startedPath = process.env.OPENCLAW_BROWSER_OVERRIDE_STARTED_PATH; +const stoppedPath = process.env.OPENCLAW_BROWSER_OVERRIDE_STOPPED_PATH; + +if (!process.env.OPENCLAW_BROWSER_CONTROL_MODULE) { + throw new Error("Missing OPENCLAW_BROWSER_CONTROL_MODULE."); +} +if (!startedPath || !stoppedPath) { + throw new Error("Missing browser override sentinel path env."); +} + +const handle = await startLazyPluginServiceModule({ + overrideEnvVar: "OPENCLAW_BROWSER_CONTROL_MODULE", + validateOverrideSpecifier: (specifier) => specifier, + loadDefaultModule: async () => { + throw new Error("Default browser control service should not load during override probe."); + }, + startExportNames: ["startBrowserControlService"], + stopExportNames: ["stopBrowserControlService"], +}); + +if (!handle) { + throw new Error("Browser control override probe did not return a service handle."); +} +if (!existsSync(startedPath)) { + throw new Error("Browser control override start sentinel was not written."); +} + +await handle.stop(); + +if (!existsSync(stoppedPath)) { + throw new Error("Browser control override stop sentinel was not written."); +} + +console.log("windows browser override import OK"); +`.trim(); +} + +function buildBrowserOverrideProbeServiceModule() { + return ` +import { writeFileSync } from "node:fs"; + +export async function startBrowserControlService() { + writeFileSync(process.env.OPENCLAW_BROWSER_OVERRIDE_STARTED_PATH, "started\\n", "utf8"); +} + +export async function stopBrowserControlService() { + writeFileSync(process.env.OPENCLAW_BROWSER_OVERRIDE_STOPPED_PATH, "stopped\\n", "utf8"); +} +`.trim(); +} + +async function runInstalledBrowserOverrideImportSmoke(params) { + if (!shouldRunWindowsInstalledBrowserOverrideImportSmoke()) { + return "skipped"; + } + + const probeDir = join(params.lane.rootDir, "browser override import probe"); + mkdirSync(probeDir, { recursive: true }); + const overridePath = join(probeDir, "browser override #module.mjs"); + const probePath = join(probeDir, "run browser override probe.mjs"); + const startedPath = join(probeDir, "started.txt"); + const stoppedPath = join(probeDir, "stopped.txt"); + + writeFileSync(overridePath, `${buildBrowserOverrideProbeServiceModule()}\n`, "utf8"); + writeFileSync(probePath, `${buildInstalledBrowserOverrideImportProbeScript()}\n`, "utf8"); + + await runCommand(process.execPath, [probePath], { + cwd: installedPackageRoot(params.prefixDir), + env: { + ...params.env, + OPENCLAW_BROWSER_CONTROL_MODULE: overridePath, + OPENCLAW_BROWSER_OVERRIDE_STARTED_PATH: startedPath, + OPENCLAW_BROWSER_OVERRIDE_STOPPED_PATH: stoppedPath, + }, + logPath: params.logPath, + timeoutMs: 60_000, + }); + + if (!existsSync(startedPath) || !existsSync(stoppedPath)) { + throw new Error("Browser control override import probe did not write both sentinels."); + } + + return "pass"; +} + function ensureLocalNpmShim(lane) { const shimPath = npmShimPath(lane.prefixDir); if (existsSync(shimPath)) { diff --git a/src/plugins/import-specifier.test.ts b/src/plugins/import-specifier.test.ts index c154326c8d2..ebd4e8117b5 100644 --- a/src/plugins/import-specifier.test.ts +++ b/src/plugins/import-specifier.test.ts @@ -12,6 +12,9 @@ describe("toSafeImportPath", () => { expect(toSafeImportPath("C:\\Users\\alice\\plugin\\index.mjs")).toBe( "file:///C:/Users/alice/plugin/index.mjs", ); + expect(toSafeImportPath("C:\\Users\\alice\\plugin folder\\x#y.mjs")).toBe( + "file:///C:/Users/alice/plugin%20folder/x%23y.mjs", + ); expect(toSafeImportPath("\\\\server\\share\\plugin\\index.mjs")).toBe( "file://server/share/plugin/index.mjs", ); diff --git a/src/plugins/import-specifier.ts b/src/plugins/import-specifier.ts index c7433c1a0e9..ceed20819d2 100644 --- a/src/plugins/import-specifier.ts +++ b/src/plugins/import-specifier.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { pathToFileURL } from "node:url"; /** * On Windows, Node's ESM loader requires absolute paths to be expressed as @@ -12,11 +13,7 @@ export function toSafeImportPath(specifier: string): string { return specifier; } if (path.win32.isAbsolute(specifier)) { - const normalizedSpecifier = specifier.replaceAll("\\", "/"); - if (normalizedSpecifier.startsWith("//")) { - return new URL(`file:${encodeURI(normalizedSpecifier)}`).href; - } - return new URL(`file:///${encodeURI(normalizedSpecifier)}`).href; + return pathToFileURL(specifier, { windows: true }).href; } return specifier; } diff --git a/src/plugins/lazy-service-module.test.ts b/src/plugins/lazy-service-module.test.ts index 62669785f83..2922e553f2d 100644 --- a/src/plugins/lazy-service-module.test.ts +++ b/src/plugins/lazy-service-module.test.ts @@ -95,12 +95,14 @@ describe("startLazyPluginServiceModule", () => { const importModule = vi.fn(async () => ({ startOverride: start })); try { - await defaultLoadOverrideModule("C:\\Users\\alice\\browser-service.mjs", importModule); + await defaultLoadOverrideModule("C:\\Users\\alice\\plugin folder\\x#y.mjs", importModule); } finally { platformSpy.mockRestore(); } - expect(importModule).toHaveBeenCalledWith("file:///C:/Users/alice/browser-service.mjs"); + expect(importModule).toHaveBeenCalledWith( + "file:///C:/Users/alice/plugin%20folder/x%23y.mjs", + ); }); it("leaves caller-supplied override loaders responsible for their own specifiers", async () => { diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 11f3fb39246..1e0b3a8e925 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -7338,6 +7338,9 @@ export const runtimeValue = helperValue;`, expect(__testing.toSafeImportPath("C:\\Users\\alice\\plugin\\index.mjs")).toBe( "file:///C:/Users/alice/plugin/index.mjs", ); + expect(__testing.toSafeImportPath("C:\\Users\\alice\\plugin folder\\x#y.mjs")).toBe( + "file:///C:/Users/alice/plugin%20folder/x%23y.mjs", + ); expect(__testing.toSafeImportPath("\\\\server\\share\\plugin\\index.mjs")).toBe( "file://server/share/plugin/index.mjs", ); diff --git a/test/scripts/openclaw-cross-os-release-checks.test.ts b/test/scripts/openclaw-cross-os-release-checks.test.ts index 5ca2474e1e2..a940509ebf5 100644 --- a/test/scripts/openclaw-cross-os-release-checks.test.ts +++ b/test/scripts/openclaw-cross-os-release-checks.test.ts @@ -8,6 +8,7 @@ import { agentOutputHasExpectedOkMarker, buildWindowsDevUpdateToolchainCheckScript, buildWindowsFreshShellVersionCheckScript, + buildInstalledBrowserOverrideImportProbeScript, buildWindowsPathBootstrapScript, canConnectToLoopbackPort, buildDiscordSmokeGuildsConfig, @@ -35,6 +36,7 @@ import { resolveRunnerMatrix, resolveStaticFileContentType, shouldExerciseManagedGatewayLifecycleAfterInstall, + shouldRunWindowsInstalledBrowserOverrideImportSmoke, shouldSkipInstallerDaemonHealthCheck, shouldStopManagedGatewayBeforeManualFallback, shouldRunMainChannelDevUpdate, @@ -289,6 +291,19 @@ describe("scripts/openclaw-cross-os-release-checks", () => { expect(shouldSkipInstallerDaemonHealthCheck("linux")).toBe(false); }); + it("runs the installed browser override import smoke only on native Windows", () => { + expect(shouldRunWindowsInstalledBrowserOverrideImportSmoke("win32")).toBe(true); + expect(shouldRunWindowsInstalledBrowserOverrideImportSmoke("darwin")).toBe(false); + expect(shouldRunWindowsInstalledBrowserOverrideImportSmoke("linux")).toBe(false); + + const script = buildInstalledBrowserOverrideImportProbeScript(); + expect(script).toContain('from "openclaw/plugin-sdk/browser-node-runtime"'); + expect(script).toContain('overrideEnvVar: "OPENCLAW_BROWSER_CONTROL_MODULE"'); + expect(script).toContain("startBrowserControlService"); + expect(script).toContain("stopBrowserControlService"); + expect(script).toContain("Browser control override start sentinel was not written."); + }); + it("normalizes Windows installed CLI paths to the cmd shim", () => { expect( normalizeWindowsInstalledCliPath( From bfdee5fa72ccf522fde3dde5e75b90179b95ed80 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:40:23 +0100 Subject: [PATCH 10/12] test(browser): close hanging attach-only sockets --- .../src/browser/client-fetch.attach-only.e2e.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts b/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts index 29ffd282b97..a82779d75c1 100644 --- a/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts +++ b/extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts @@ -22,7 +22,10 @@ describe("browser client fetch attachOnly diagnostics", () => { it("does not suggest gateway restart when an attachOnly CDP endpoint hangs", async () => { tempHome = await createTempHomeEnv("openclaw-browser-client-fetch-live-"); + const sockets = new Set(); const server = net.createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); socket.on("error", () => {}); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); @@ -63,6 +66,9 @@ describe("browser client fetch attachOnly diagnostics", () => { expect(message).not.toContain("Restart the OpenClaw gateway"); expect(message).not.toContain("Do NOT retry the browser tool"); } finally { + for (const socket of sockets) { + socket.destroy(); + } await new Promise((resolve) => server.close(() => resolve())); } }); From 84929bf85b0b29d72aea3e085f52a940c79fa8da Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 08:43:01 +0100 Subject: [PATCH 11/12] fix: clean runtime deps backup owner marker --- scripts/stage-bundled-plugin-runtime-deps.mjs | 1 + test/scripts/stage-bundled-plugin-runtime-deps.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index df5bdc8fa5a..2b9507253f5 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -106,6 +106,7 @@ function replaceDirAtomically(targetPath, sourcePath) { } catch (error) { if (movedExistingTarget && !fs.existsSync(targetPath) && fs.existsSync(backupPath)) { fs.renameSync(backupPath, targetPath); + removePathIfExists(path.join(targetPath, TEMP_OWNER_FILE)); } throw error; } diff --git a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts index fa543db54d1..db08f95df44 100644 --- a/test/scripts/stage-bundled-plugin-runtime-deps.test.ts +++ b/test/scripts/stage-bundled-plugin-runtime-deps.test.ts @@ -392,6 +392,7 @@ describe("stageBundledPluginRuntimeDeps", () => { ).toThrow("rename failed after backup"); expect(fs.readFileSync(path.join(targetPath, "marker.txt"), "utf8")).toBe("original\n"); + expect(fs.existsSync(path.join(targetPath, "owner.json"))).toBe(false); }); it("restages when installed root runtime dependency contents change", () => { From 0286bb98178b08427a38f594153fd350407d9e75 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 27 Apr 2026 00:43:07 -0700 Subject: [PATCH 12/12] docs: point maintainer triage at gitcrawl Update the OpenClaw PR maintainer skill to use gitcrawl for local triage commands. --- .../skills/openclaw-pr-maintainer/SKILL.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.agents/skills/openclaw-pr-maintainer/SKILL.md b/.agents/skills/openclaw-pr-maintainer/SKILL.md index 5bea778d261..4f775656201 100644 --- a/.agents/skills/openclaw-pr-maintainer/SKILL.md +++ b/.agents/skills/openclaw-pr-maintainer/SKILL.md @@ -7,20 +7,20 @@ description: Review, triage, close, label, comment on, or land OpenClaw PRs/issu Use this skill for maintainer-facing GitHub workflow, not for ordinary code changes. -## Start issue and PR triage with ghcrawl +## Start issue and PR triage with gitcrawl -- Anytime you inspect OpenClaw issues or PRs, check local `ghcrawl` data first for related threads, duplicate attempts, and already-landed fixes. -- Use `ghcrawl` for candidate discovery and clustering; use `gh`, `gh api`, and the current checkout to verify live state before commenting, labeling, closing, or landing. -- If `ghcrawl` is missing, stale, lacks the target thread, or has no embeddings for neighbor/search commands, fall back to the GitHub search workflow below. -- Do not run expensive/update commands such as `ghcrawl refresh`, `ghcrawl embed`, or `ghcrawl cluster` unless the user asked to update the local store or the stale data is blocking the decision. +- Anytime you inspect OpenClaw issues or PRs, check local `gitcrawl` data first for related threads, duplicate attempts, and already-landed fixes. +- Use `gitcrawl` for candidate discovery and clustering; use `gh`, `gh api`, and the current checkout to verify live state before commenting, labeling, closing, or landing. +- If `gitcrawl` is missing, stale, lacks the target thread, or has no embeddings for neighbor/search commands, fall back to the GitHub search workflow below. +- Do not run expensive/update commands such as `gitcrawl sync --include-comments`, future enrichment commands, or broad reclustering unless the user asked to update the local store or stale data is blocking the decision. Common read-only path: ```bash -ghcrawl threads openclaw/openclaw --numbers --include-closed --json -ghcrawl neighbors openclaw/openclaw --number --limit 12 --json -ghcrawl search openclaw/openclaw --query "" --mode hybrid --json -ghcrawl cluster-detail openclaw/openclaw --id --member-limit 20 --body-chars 280 --json +gitcrawl threads openclaw/openclaw --numbers --include-closed --json +gitcrawl neighbors openclaw/openclaw --number --limit 12 --json +gitcrawl search openclaw/openclaw --query "" --mode hybrid --json +gitcrawl cluster-detail openclaw/openclaw --id --member-limit 20 --body-chars 280 --json ``` ## Apply close and triage labels correctly @@ -75,7 +75,7 @@ ghcrawl cluster-detail openclaw/openclaw --id --member-limit 20 --b ## Search broadly before deciding -- Prefer `ghcrawl` first. Then use targeted GitHub keyword search to verify gaps, live status, comments, and candidates not present in the local store. +- Prefer `gitcrawl` first. Then use targeted GitHub keyword search to verify gaps, live status, comments, and candidates not present in the local store. - Use `--repo openclaw/openclaw` with `--match title,body` first when using `gh search`. - Add `--match comments` when triaging follow-up discussion or closed-as-duplicate chains. - Do not stop at the first 500 results when the task requires a full search.