From 1fc5b2b7032c12f64c2ce0011d52c89d4b72f456 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 27 Apr 2026 00:34:29 -0700 Subject: [PATCH] feat(migrations): add plugin-owned Hermes import * feat: add migration providers * feat: offer Hermes migration during onboarding * feat(hermes): map imported config surfaces * feat(onboard): require fresh migration imports * docs(cli): clarify Hermes import coverage * chore(migrations): rename Hermes importer package * chore(migrations): rewire Hermes importer id * fix(migrations): redact migration JSON details * fix(hermes): use provider runtime for config imports * test(hermes): cover missing source planning --------- Co-authored-by: Peter Steinberger --- .github/labeler.yml | 5 + CHANGELOG.md | 1 + .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/.i18n/glossary.zh-CN.json | 16 + docs/cli/migrate.md | 75 +++ docs/cli/onboard.md | 6 + docs/cli/setup.md | 7 +- docs/docs.json | 1 + docs/plugins/manifest.md | 2 + docs/plugins/sdk-subpaths.md | 14 +- extensions/migrate-hermes/apply.ts | 70 +++ extensions/migrate-hermes/config.test.ts | 224 +++++++++ extensions/migrate-hermes/config.ts | 434 ++++++++++++++++ .../migrate-hermes/files-and-skills.test.ts | 204 ++++++++ extensions/migrate-hermes/helpers.ts | 134 +++++ extensions/migrate-hermes/index.ts | 11 + extensions/migrate-hermes/items.ts | 113 +++++ extensions/migrate-hermes/model.apply.test.ts | 188 +++++++ extensions/migrate-hermes/model.plan.test.ts | 107 ++++ extensions/migrate-hermes/model.ts | 87 ++++ .../migrate-hermes/openclaw.plugin.json | 13 + extensions/migrate-hermes/package.json | 27 + extensions/migrate-hermes/plan.ts | 162 ++++++ .../provider.secret-failure.test.ts | 99 ++++ extensions/migrate-hermes/provider.test.ts | 129 +++++ extensions/migrate-hermes/provider.ts | 35 ++ extensions/migrate-hermes/secrets.test.ts | 159 ++++++ extensions/migrate-hermes/secrets.ts | 118 +++++ extensions/migrate-hermes/skills.ts | 70 +++ extensions/migrate-hermes/source.ts | 74 +++ extensions/migrate-hermes/targets.ts | 30 ++ .../migrate-hermes/test/provider-helpers.ts | 65 +++ .../src/bot-native-commands.registry.test.ts | 1 + package.json | 8 + pnpm-lock.yaml | 13 + scripts/lib/plugin-sdk-entrypoints.json | 2 + src/agents/agent-scope.test.ts | 54 ++ src/agents/agent-scope.ts | 38 ++ src/cli/command-catalog.ts | 1 + src/cli/program/command-registry-core.ts | 5 + src/cli/program/core-command-descriptors.ts | 5 + src/cli/program/register.migrate.ts | 117 +++++ src/cli/program/register.onboard.test.ts | 22 + src/cli/program/register.onboard.ts | 10 +- src/cli/program/register.setup.test.ts | 21 + src/cli/program/register.setup.ts | 9 + src/commands/migrate.test.ts | 471 ++++++++++++++++++ src/commands/migrate.ts | 162 ++++++ src/commands/migrate/apply.ts | 86 ++++ src/commands/migrate/context.ts | 51 ++ src/commands/migrate/output.ts | 103 ++++ src/commands/migrate/providers.ts | 38 ++ src/commands/migrate/types.ts | 21 + src/commands/onboard-types.ts | 5 +- src/flows/channel-setup.test.ts | 1 + src/gateway/server-plugins.test.ts | 1 + src/gateway/test-helpers.plugin-registry.ts | 1 + src/plugin-sdk/agent-runtime.ts | 1 + src/plugin-sdk/migration-runtime.test.ts | 123 +++++ src/plugin-sdk/migration-runtime.ts | 186 +++++++ src/plugin-sdk/migration.ts | 153 ++++++ src/plugin-sdk/plugin-entry.ts | 14 + src/plugin-sdk/provider-auth.ts | 1 + src/plugins/api-builder.ts | 3 + .../bundled-capability-metadata.test.ts | 8 + .../bundled-capability-runtime.test.ts | 23 +- src/plugins/bundled-capability-runtime.ts | 11 + src/plugins/captured-registration.ts | 7 + .../inventory/bundled-capability-metadata.ts | 5 + .../contracts/registry.contract.test.ts | 16 + src/plugins/contracts/registry.retry.test.ts | 8 + src/plugins/contracts/registry.ts | 6 + src/plugins/hooks.test-helpers.ts | 1 + .../installed-plugin-index-record-builder.ts | 1 + src/plugins/installed-plugin-index.test.ts | 37 ++ src/plugins/loader.ts | 4 + src/plugins/manifest-contract-runtime.ts | 53 ++ src/plugins/manifest-registry.ts | 3 +- src/plugins/manifest.ts | 3 + .../migration-provider-runtime.test.ts | 214 ++++++++ src/plugins/migration-provider-runtime.ts | 117 +++++ src/plugins/plugin-registry-snapshot.ts | 7 +- src/plugins/plugin-registry.test.ts | 36 ++ src/plugins/registry-empty.ts | 1 + src/plugins/registry-types.ts | 5 + src/plugins/registry.ts | 13 + src/plugins/status.test-helpers.ts | 2 + src/plugins/status.ts | 1 + src/plugins/types.ts | 96 ++++ src/test-utils/channel-plugins.ts | 1 + src/trajectory/metadata.test.ts | 1 + src/wizard/setup.migration-import.test.ts | 61 +++ src/wizard/setup.migration-import.ts | 304 +++++++++++ src/wizard/setup.ts | 43 +- test/helpers/plugins/plugin-api.ts | 1 + test/setup-openclaw-runtime.ts | 1 + 96 files changed, 5477 insertions(+), 24 deletions(-) create mode 100644 docs/cli/migrate.md create mode 100644 extensions/migrate-hermes/apply.ts create mode 100644 extensions/migrate-hermes/config.test.ts create mode 100644 extensions/migrate-hermes/config.ts create mode 100644 extensions/migrate-hermes/files-and-skills.test.ts create mode 100644 extensions/migrate-hermes/helpers.ts create mode 100644 extensions/migrate-hermes/index.ts create mode 100644 extensions/migrate-hermes/items.ts create mode 100644 extensions/migrate-hermes/model.apply.test.ts create mode 100644 extensions/migrate-hermes/model.plan.test.ts create mode 100644 extensions/migrate-hermes/model.ts create mode 100644 extensions/migrate-hermes/openclaw.plugin.json create mode 100644 extensions/migrate-hermes/package.json create mode 100644 extensions/migrate-hermes/plan.ts create mode 100644 extensions/migrate-hermes/provider.secret-failure.test.ts create mode 100644 extensions/migrate-hermes/provider.test.ts create mode 100644 extensions/migrate-hermes/provider.ts create mode 100644 extensions/migrate-hermes/secrets.test.ts create mode 100644 extensions/migrate-hermes/secrets.ts create mode 100644 extensions/migrate-hermes/skills.ts create mode 100644 extensions/migrate-hermes/source.ts create mode 100644 extensions/migrate-hermes/targets.ts create mode 100644 extensions/migrate-hermes/test/provider-helpers.ts create mode 100644 src/cli/program/register.migrate.ts create mode 100644 src/commands/migrate.test.ts create mode 100644 src/commands/migrate.ts create mode 100644 src/commands/migrate/apply.ts create mode 100644 src/commands/migrate/context.ts create mode 100644 src/commands/migrate/output.ts create mode 100644 src/commands/migrate/providers.ts create mode 100644 src/commands/migrate/types.ts create mode 100644 src/plugin-sdk/migration-runtime.test.ts create mode 100644 src/plugin-sdk/migration-runtime.ts create mode 100644 src/plugin-sdk/migration.ts create mode 100644 src/plugins/manifest-contract-runtime.ts create mode 100644 src/plugins/migration-provider-runtime.test.ts create mode 100644 src/plugins/migration-provider-runtime.ts create mode 100644 src/wizard/setup.migration-import.test.ts create mode 100644 src/wizard/setup.migration-import.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index f2391091284..84e4f084753 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -35,6 +35,11 @@ - any-glob-to-any-file: - "extensions/google-meet/**" - "docs/plugins/google-meet.md" +"plugin: migrate-hermes": + - changed-files: + - any-glob-to-any-file: + - "extensions/migrate-hermes/**" + - "docs/cli/migrate.md" "plugin: bonjour": - changed-files: - any-glob-to-any-file: diff --git a/CHANGELOG.md b/CHANGELOG.md index 69385c0f3c5..fc966249f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Docs: https://docs.openclaw.ai - Control UI: polish the quick settings dashboard grid so common cards align across desktop, tablet, and mobile layouts without wasting horizontal space. Thanks @BunsDev. - Matrix/E2EE: add `openclaw matrix encryption setup` to enable Matrix encryption, bootstrap recovery, and print verification status from one setup flow. Thanks @gumadeiras. - Agents/compaction: add an opt-in `agents.defaults.compaction.maxActiveTranscriptBytes` preflight trigger that runs normal local compaction when the active JSONL grows too large, requiring transcript rotation so successful compaction moves future turns onto a smaller successor file instead of raw byte-splitting history. Thanks @vincentkoc. +- CLI/migration: add `openclaw migrate` with plan, dry-run, JSON, pre-migration backup, onboarding detection, archive-only report copies, and a bundled Hermes importer for configuration, memory/plugin hints, model providers, MCP servers, skills, and supported credentials. Thanks @NousResearch. ### Fixes diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 2d76d9b4abf..e5b23fb4e6d 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -2a3fb85feb7420de8b166a695c3693dcc1eaa7a7f31de0dd139da856f10b2085 plugin-sdk-api-baseline.json -6bdb96f7f92c34d7ae698784c0073343c34fb4274ab7eeded49acebb81056074 plugin-sdk-api-baseline.jsonl +8371f19a19ceeae4eb20fbfe8e68e51f6f54f42c487d7d5c75f214ab1ba0922a plugin-sdk-api-baseline.json +a5f5e15e75f8cf27ebaa1302cfe0488974edd53121279c1e90705bc531a4761a plugin-sdk-api-baseline.jsonl diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 3a16f990b14..f48391bc82f 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -435,6 +435,22 @@ "source": "Setup", "target": "设置" }, + { + "source": "Migrate", + "target": "迁移" + }, + { + "source": "Migration", + "target": "迁移" + }, + { + "source": "Hermes", + "target": "Hermes" + }, + { + "source": "Archive-only", + "target": "仅归档" + }, { "source": "Channel Plugin SDK", "target": "渠道插件 SDK" diff --git a/docs/cli/migrate.md b/docs/cli/migrate.md new file mode 100644 index 00000000000..635dc6313cb --- /dev/null +++ b/docs/cli/migrate.md @@ -0,0 +1,75 @@ +--- +summary: "CLI reference for importing state from another agent system" +read_when: + - You want to migrate from Hermes or another agent system into OpenClaw + - You are adding a plugin-owned migration provider +title: "Migrate" +--- + +# `openclaw migrate` + +Import state from another agent system through a plugin-owned migration provider. + +```bash +openclaw migrate list +openclaw migrate hermes --dry-run +openclaw migrate hermes +openclaw migrate apply hermes --yes +openclaw migrate apply hermes --include-secrets --yes +openclaw onboard --flow import +openclaw onboard --import-from hermes --import-source ~/.hermes +``` + +## Safety model + +`openclaw migrate` is preview-first. The provider returns an itemized plan before anything changes, including conflicts, skipped items, and sensitive items. JSON plans, apply output, and migration reports redact nested secret-looking keys such as API keys, tokens, authorization headers, cookies, and passwords. + +`openclaw migrate apply ` previews the plan and prompts before changing state unless `--yes` is set. In non-interactive mode, apply requires `--yes`. With `--json` and no `--yes`, apply prints the JSON plan and does not mutate state. + +Apply creates and verifies an OpenClaw backup before applying the migration. If no local OpenClaw state exists yet, the backup step is skipped and the migration can continue. To skip a backup when state exists, pass both `--no-backup` and `--force`. + +Apply mode refuses to continue when the plan has conflicts. Review the plan, then rerun with `--overwrite` if replacing existing targets is intentional. Providers may still write item-level backups for overwritten files in the migration report directory. + +Secrets are never imported by default. Use `--include-secrets` to import supported credentials. + +## Hermes + +The bundled Hermes provider detects Hermes state at `~/.hermes` by default. Use `--from ` when Hermes lives elsewhere. + +The Hermes migration can import: + +- default model configuration from `config.yaml` +- configured model providers and custom OpenAI-compatible endpoints from `providers` and `custom_providers` +- MCP server definitions from `mcp_servers` or `mcp.servers` +- `SOUL.md` and `AGENTS.md` into the OpenClaw agent workspace +- `memories/MEMORY.md` and `memories/USER.md` by appending them to workspace memory files +- memory config defaults for OpenClaw file memory, plus archive/manual-review items for external memory providers such as Honcho +- skills with a `SKILL.md` file from `skills//` +- per-skill config values from `skills.config` +- supported API keys from `.env`, only with `--include-secrets` + +Archive-only Hermes state is copied into the migration report for manual review, but it is not loaded into live OpenClaw config or credentials. This preserves opaque or unsafe state such as `plugins/`, `sessions/`, `logs/`, `cron/`, `mcp-tokens/`, `auth.json`, and `state.db` without pretending OpenClaw can execute or trust it automatically. + +Supported Hermes `.env` keys include `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `GOOGLE_API_KEY`, `GEMINI_API_KEY`, `GROQ_API_KEY`, `XAI_API_KEY`, `MISTRAL_API_KEY`, and `DEEPSEEK_API_KEY`. + +After applying a migration, run: + +```bash +openclaw doctor +``` + +## Plugin contract + +Migration sources are plugins. A plugin declares its provider ids in `openclaw.plugin.json`: + +```json +{ + "contracts": { + "migrationProviders": ["hermes"] + } +} +``` + +At runtime the plugin calls `api.registerMigrationProvider(...)`. The provider implements `detect`, `plan`, and `apply`; core owns CLI orchestration, backup policy, prompts, JSON output, and conflict preflight. Core passes the reviewed plan into `apply(ctx, plan)`, and providers may rebuild the plan only when that argument is absent for compatibility. Provider plugins can use `openclaw/plugin-sdk/migration` for item construction and summary counts, plus `openclaw/plugin-sdk/migration-runtime` for conflict-aware file copies, archive-only report copies, and migration reports. + +Onboarding can also offer migration when a provider detects a known source. `openclaw onboard --flow import` and `openclaw setup --wizard --import-from hermes` use the same plugin migration provider and still show a preview before applying. Onboarding imports require a fresh OpenClaw setup; reset config, credentials, sessions, and the workspace first if you already have local state. Backup plus overwrite or merge imports are feature-gated for existing setups. diff --git a/docs/cli/onboard.md b/docs/cli/onboard.md index 8494367a293..e0c715034f2 100644 --- a/docs/cli/onboard.md +++ b/docs/cli/onboard.md @@ -36,10 +36,14 @@ openclaw onboard openclaw onboard --modern openclaw onboard --flow quickstart openclaw onboard --flow manual +openclaw onboard --flow import +openclaw onboard --import-from hermes --import-source ~/.hermes openclaw onboard --skip-bootstrap openclaw onboard --mode remote --remote-url wss://gateway-host:18789 ``` +`--flow import` uses plugin-owned migration providers such as Hermes. It only runs against a fresh OpenClaw setup; if existing config, credentials, sessions, or workspace memory/identity files are present, reset or choose a fresh setup before importing. + `--modern` starts the Crestodian conversational onboarding preview. Without `--modern`, `openclaw onboard` keeps the classic onboarding flow. @@ -176,6 +180,7 @@ openclaw onboard --non-interactive \ - `quickstart`: minimal prompts, auto-generates a gateway token. - `manual`: full prompts for port, bind, and auth (alias of `advanced`). + - `import`: runs a detected migration provider, previews the plan, then applies after confirmation. When an auth choice implies a preferred provider, onboarding prefilters the default-model and allowlist pickers to that provider. For Volcengine and BytePlus, this also matches the coding-plan variants (`volcengine-plan/*`, `byteplus-plan/*`). @@ -194,6 +199,7 @@ openclaw onboard --non-interactive \ - Local onboarding DM scope behavior: [CLI setup reference](/start/wizard-cli-reference#outputs-and-internals). - Fastest first chat: `openclaw dashboard` (Control UI, no channel setup). - Custom provider: connect any OpenAI or Anthropic compatible endpoint, including hosted providers not listed. Use Unknown to auto-detect. + - If Hermes state is detected, onboarding offers a migration flow. Use [Migrate](/cli/migrate) for dry-run plans, overwrite mode, reports, and exact mappings. diff --git a/docs/cli/setup.md b/docs/cli/setup.md index 2a0d6fc50d4..b1fbfcc2338 100644 --- a/docs/cli/setup.md +++ b/docs/cli/setup.md @@ -21,6 +21,7 @@ Related: openclaw setup openclaw setup --workspace ~/.openclaw/workspace openclaw setup --wizard +openclaw setup --wizard --import-from hermes --import-source ~/.hermes openclaw setup --non-interactive --mode remote --remote-url wss://gateway-host:18789 --remote-token ``` @@ -30,6 +31,9 @@ openclaw setup --non-interactive --mode remote --remote-url wss://gateway-host:1 - `--wizard`: run onboarding - `--non-interactive`: run onboarding without prompts - `--mode `: onboarding mode +- `--import-from `: migration provider to run during onboarding +- `--import-source `: source agent home for `--import-from` +- `--import-secrets`: import supported secrets during onboarding migration - `--remote-url `: remote Gateway WebSocket URL - `--remote-token `: remote Gateway token @@ -42,7 +46,8 @@ openclaw setup --wizard Notes: - Plain `openclaw setup` initializes config + workspace without the full onboarding flow. -- Onboarding auto-runs when any onboarding flags are present (`--wizard`, `--non-interactive`, `--mode`, `--remote-url`, `--remote-token`). +- Onboarding auto-runs when any onboarding flags are present (`--wizard`, `--non-interactive`, `--mode`, `--import-from`, `--import-source`, `--import-secrets`, `--remote-url`, `--remote-token`). +- If Hermes state is detected, interactive onboarding can offer migration automatically. Import onboarding requires a fresh setup; use [Migrate](/cli/migrate) for dry-run plans, backups, and overwrite mode outside onboarding. ## Related diff --git a/docs/docs.json b/docs/docs.json index 11e5a8d93e5..27e58e0c0c0 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1545,6 +1545,7 @@ "cli/gateway", "cli/health", "cli/logs", + "cli/migrate", "cli/onboard", "cli/reset", "cli/secrets", diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index 481c3896fe8..afac720bd5e 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -429,6 +429,7 @@ read without importing the plugin runtime. "videoGenerationProviders": ["qwen"], "webFetchProviders": ["firecrawl"], "webSearchProviders": ["gemini"], + "migrationProviders": ["hermes"], "tools": ["firecrawl_search", "firecrawl_scrape"] } } @@ -450,6 +451,7 @@ Each list is optional: | `videoGenerationProviders` | `string[]` | Video-generation provider ids this plugin owns. | | `webFetchProviders` | `string[]` | Web-fetch provider ids this plugin owns. | | `webSearchProviders` | `string[]` | Web-search provider ids this plugin owns. | +| `migrationProviders` | `string[]` | Import provider ids this plugin owns for `openclaw migrate`. | | `tools` | `string[]` | Agent tool names this plugin owns for bundled contract checks. | `contracts.embeddedExtensionFactories` is retained for bundled Codex diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 19c1256f6fe..e8b497f66b3 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -16,12 +16,14 @@ For the plugin authoring guide, see [Plugin SDK overview](/plugins/sdk-overview) ## Plugin entry -| Subpath | Key exports | -| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `plugin-sdk/plugin-entry` | `definePluginEntry` | -| `plugin-sdk/core` | `defineChannelPluginEntry`, `createChatChannelPlugin`, `createChannelPluginBase`, `defineSetupPluginEntry`, `buildChannelConfigSchema` | -| `plugin-sdk/config-schema` | `OpenClawSchema` | -| `plugin-sdk/provider-entry` | `defineSingleProviderPluginEntry` | +| Subpath | Key exports | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| `plugin-sdk/plugin-entry` | `definePluginEntry` | +| `plugin-sdk/core` | `defineChannelPluginEntry`, `createChatChannelPlugin`, `createChannelPluginBase`, `defineSetupPluginEntry`, `buildChannelConfigSchema` | +| `plugin-sdk/config-schema` | `OpenClawSchema` | +| `plugin-sdk/provider-entry` | `defineSingleProviderPluginEntry` | +| `plugin-sdk/migration` | Migration provider item helpers such as `createMigrationItem`, reason constants, item status markers, redaction helpers, and `summarizeMigrationItems` | +| `plugin-sdk/migration-runtime` | Runtime migration helpers such as `copyMigrationFileItem` and `writeMigrationReport` | diff --git a/extensions/migrate-hermes/apply.ts b/extensions/migrate-hermes/apply.ts new file mode 100644 index 00000000000..1af8852b4fd --- /dev/null +++ b/extensions/migrate-hermes/apply.ts @@ -0,0 +1,70 @@ +import path from "node:path"; +import { summarizeMigrationItems } from "openclaw/plugin-sdk/migration"; +import { + archiveMigrationItem, + copyMigrationFileItem, + writeMigrationReport, +} from "openclaw/plugin-sdk/migration-runtime"; +import type { + MigrationApplyResult, + MigrationItem, + MigrationPlan, + MigrationProviderContext, +} from "openclaw/plugin-sdk/plugin-entry"; +import { applyConfigItem, applyManualItem } from "./config.js"; +import { appendItem } from "./helpers.js"; +import { applyModelItem } from "./model.js"; +import { buildHermesPlan } from "./plan.js"; +import { applySecretItem } from "./secrets.js"; +import { resolveTargets } from "./targets.js"; + +export async function applyHermesPlan(params: { + ctx: MigrationProviderContext; + plan?: MigrationPlan; + runtime?: MigrationProviderContext["runtime"]; +}): Promise { + const plan = params.plan ?? (await buildHermesPlan(params.ctx)); + const reportDir = params.ctx.reportDir ?? path.join(params.ctx.stateDir, "migration", "hermes"); + const targets = resolveTargets(params.ctx); + const items: MigrationItem[] = []; + for (const item of plan.items) { + if (item.status !== "planned") { + items.push(item); + continue; + } + if (item.id === "config:default-model") { + items.push( + await applyModelItem( + { ...params.ctx, runtime: params.ctx.runtime ?? params.runtime }, + item, + ), + ); + } else if (item.kind === "config") { + items.push( + await applyConfigItem( + { ...params.ctx, runtime: params.ctx.runtime ?? params.runtime }, + item, + ), + ); + } else if (item.kind === "manual") { + items.push(applyManualItem(item)); + } else if (item.action === "archive") { + items.push(await archiveMigrationItem(item, reportDir)); + } else if (item.kind === "secret") { + items.push(await applySecretItem(params.ctx, item, targets)); + } else if (item.action === "append") { + items.push(await appendItem(item)); + } else { + items.push(await copyMigrationFileItem(item, reportDir, { overwrite: params.ctx.overwrite })); + } + } + const result: MigrationApplyResult = { + ...plan, + items, + summary: summarizeMigrationItems(items), + backupPath: params.ctx.backupPath, + reportDir, + }; + await writeMigrationReport(result, { title: "Hermes Migration Report" }); + return result; +} diff --git a/extensions/migrate-hermes/config.test.ts b/extensions/migrate-hermes/config.test.ts new file mode 100644 index 00000000000..4d8de242592 --- /dev/null +++ b/extensions/migrate-hermes/config.test.ts @@ -0,0 +1,224 @@ +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +function makeConfigRuntime(config: Record) { + return { + config: { + loadConfig: () => config, + writeConfigFile: async (next: Record) => { + Object.keys(config).forEach((key) => { + delete config[key]; + }); + Object.assign(config, next); + return next; + }, + }, + } as never; +} + +describe("Hermes migration config mapping", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("plans provider, MCP, skill, and memory plugin config as plugin-owned items", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + [ + "model:", + " provider: openai", + " model: gpt-5.4", + "providers:", + " openai:", + " base_url: https://api.openai.example/v1", + " api_key_env: OPENAI_API_KEY", + " models: [gpt-5.4]", + "custom_providers:", + " - name: local-llm", + " base_url: http://127.0.0.1:11434/v1", + " models: [local-model]", + "memory:", + " provider: honcho", + " honcho:", + " project: hermes", + "skills:", + " config:", + " ship-it:", + " mode: fast", + "mcp_servers:", + " time:", + " command: npx", + " args: ['-y', 'mcp-server-time']", + "", + ].join("\n"), + ); + await writeFile(path.join(source, "memories", "MEMORY.md"), "memory line\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:memory-plugin:honcho", + kind: "config", + action: "merge", + target: "plugins.entries.honcho", + }), + expect.objectContaining({ + id: "manual:memory-provider:honcho", + kind: "manual", + status: "skipped", + }), + expect.objectContaining({ + id: "config:model-providers", + details: expect.objectContaining({ + value: expect.objectContaining({ + openai: expect.objectContaining({ + baseUrl: "https://api.openai.example/v1", + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }), + "local-llm": expect.objectContaining({ + baseUrl: "http://127.0.0.1:11434/v1", + }), + }), + }), + }), + expect.objectContaining({ + id: "config:mcp-servers", + details: expect.objectContaining({ + value: { + time: { + command: "npx", + args: ["-y", "mcp-server-time"], + }, + }, + }), + }), + expect.objectContaining({ + id: "config:skill-entries", + details: expect.objectContaining({ + value: { + "ship-it": { + config: { + mode: "fast", + }, + }, + }, + }), + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([expect.stringContaining("manual review")]), + ); + }); + + it("applies mapped config items through the migration runtime config writer", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const config: Record = { + agents: { defaults: { workspace: workspaceDir } }, + }; + await writeFile( + path.join(source, "config.yaml"), + [ + "providers:", + " openai:", + " api_key_env: OPENAI_API_KEY", + " models: [gpt-5.4]", + "mcp_servers:", + " time:", + " command: npx", + "skills:", + " config:", + " ship-it:", + " mode: fast", + "", + ].join("\n"), + ); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + runtime: makeConfigRuntime(config), + }), + ); + + expect(result.summary.errors).toBe(0); + expect(config).toMatchObject({ + models: { + providers: { + openai: { + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }, + }, + }, + mcp: { + servers: { + time: { + command: "npx", + }, + }, + }, + skills: { + entries: { + "ship-it": { + config: { + mode: "fast", + }, + }, + }, + }, + }); + }); + + it("uses the provider runtime for CLI-applied config items", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const config: Record = { + agents: { defaults: { workspace: workspaceDir } }, + }; + await writeFile( + path.join(source, "config.yaml"), + [ + "mcp_servers:", + " time:", + " command: npx", + " env:", + " OPENAI_API_KEY: short-dev-key", + "", + ].join("\n"), + ); + + const provider = buildHermesMigrationProvider({ runtime: makeConfigRuntime(config) }); + const result = await provider.apply(makeContext({ source, stateDir, workspaceDir })); + + expect(result.summary.errors).toBe(0); + expect(config).toMatchObject({ + mcp: { + servers: { + time: { + command: "npx", + env: { + OPENAI_API_KEY: "short-dev-key", + }, + }, + }, + }, + }); + }); +}); diff --git a/extensions/migrate-hermes/config.ts b/extensions/migrate-hermes/config.ts new file mode 100644 index 00000000000..82170ee5718 --- /dev/null +++ b/extensions/migrate-hermes/config.ts @@ -0,0 +1,434 @@ +import { + createMigrationItem, + markMigrationItemConflict, + markMigrationItemError, + markMigrationItemSkipped, + MIGRATION_REASON_TARGET_EXISTS, +} from "openclaw/plugin-sdk/migration"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { childRecord, isRecord, readString, readStringArray } from "./helpers.js"; + +type HermesProviderConfig = { + id: string; + baseUrl?: string; + apiKeyEnv?: string; + models: string[]; +}; + +type ConfigPatchDetails = { + path: string[]; + value: unknown; +}; + +const CONFIG_RUNTIME_UNAVAILABLE = "config runtime unavailable"; +const MISSING_CONFIG_PATCH = "missing config patch"; + +function envKeyForProvider(providerId: string): string { + return `${providerId.toUpperCase().replaceAll(/[^A-Z0-9]/gu, "_")}_API_KEY`; +} + +function splitProviderModel(modelRef: string | undefined): { provider?: string; model?: string } { + if (!modelRef) { + return {}; + } + const slash = modelRef.indexOf("/"); + if (slash > 0 && slash < modelRef.length - 1) { + return { provider: modelRef.slice(0, slash), model: modelRef.slice(slash + 1) }; + } + return { model: modelRef }; +} + +function modelDefinition(modelId: string, baseUrl?: string): Record { + return { + id: modelId, + name: modelId, + api: baseUrl ? "openai-completions" : "openai-responses", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 8192, + ...(baseUrl ? { baseUrl } : {}), + metadataSource: "models-add", + }; +} + +function providerConfig(entry: HermesProviderConfig): Record { + const models = entry.models.length > 0 ? entry.models : [`${entry.id}/default`]; + return { + baseUrl: entry.baseUrl ?? "", + ...(entry.apiKeyEnv + ? { apiKey: { source: "env", provider: "default", id: entry.apiKeyEnv } } + : {}), + api: "openai-completions", + models: models.map((modelId) => modelDefinition(modelId, entry.baseUrl)), + }; +} + +export function collectHermesProviders( + config: Record, + modelRef?: string, +): HermesProviderConfig[] { + const collected: HermesProviderConfig[] = []; + for (const [id, raw] of Object.entries(childRecord(config, "providers"))) { + if (!isRecord(raw)) { + continue; + } + const baseUrl = + readString(raw.base_url) ?? + readString(raw.baseUrl) ?? + readString(raw.url) ?? + readString(raw.api); + const apiKeyEnv = + readString(raw.api_key_env) ?? + readString(raw.apiKeyEnv) ?? + readString(raw.env) ?? + envKeyForProvider(id); + const models = [ + ...readStringArray(raw.models), + ...Object.keys(childRecord(raw, "models")), + readString(raw.model), + ].filter((value): value is string => Boolean(value)); + collected.push({ id, baseUrl, apiKeyEnv, models: [...new Set(models)] }); + } + + const customProviders = config.custom_providers; + if (Array.isArray(customProviders)) { + for (const raw of customProviders) { + if (!isRecord(raw)) { + continue; + } + const id = readString(raw.name) ?? readString(raw.id); + if (!id) { + continue; + } + const baseUrl = readString(raw.base_url) ?? readString(raw.baseUrl) ?? readString(raw.url); + const apiKeyEnv = readString(raw.api_key_env) ?? readString(raw.apiKeyEnv); + const models = [ + ...readStringArray(raw.models), + ...Object.keys(childRecord(raw, "models")), + readString(raw.model), + ].filter((value): value is string => Boolean(value)); + collected.push({ id, baseUrl, apiKeyEnv, models: [...new Set(models)] }); + } + } + + const defaultRef = splitProviderModel(modelRef); + if (defaultRef.provider && !collected.some((entry) => entry.id === defaultRef.provider)) { + collected.push({ + id: defaultRef.provider, + apiKeyEnv: envKeyForProvider(defaultRef.provider), + models: defaultRef.model ? [defaultRef.model] : [], + }); + } + return collected; +} + +function mapMcpServers(raw: unknown): Record | undefined { + if (!isRecord(raw)) { + return undefined; + } + const mapped: Record = {}; + for (const [name, value] of Object.entries(raw)) { + if (!isRecord(value)) { + continue; + } + const next: Record = {}; + for (const key of [ + "command", + "args", + "env", + "cwd", + "workingDirectory", + "url", + "transport", + "headers", + "connectionTimeoutMs", + ]) { + if (value[key] !== undefined) { + next[key] = value[key]; + } + } + if (Object.keys(next).length > 0) { + mapped[name] = next; + } + } + return Object.keys(mapped).length > 0 ? mapped : undefined; +} + +function mapSkillEntries(config: Record): Record | undefined { + const entries: Record = {}; + for (const [skillKey, value] of Object.entries( + childRecord(childRecord(config, "skills"), "config"), + )) { + if (isRecord(value)) { + entries[skillKey] = { config: value }; + } + } + return Object.keys(entries).length > 0 ? entries : undefined; +} + +function readPath(root: Record, path: readonly string[]): unknown { + let current: unknown = root; + for (const segment of path) { + if (!isRecord(current)) { + return undefined; + } + current = current[segment]; + } + return current; +} + +function mergeValue(left: unknown, right: unknown): unknown { + if (!isRecord(left) || !isRecord(right)) { + return structuredClone(right); + } + const next: Record = { ...left }; + for (const [key, value] of Object.entries(right)) { + next[key] = mergeValue(next[key], value); + } + return next; +} + +function writePath(root: Record, path: readonly string[], value: unknown): void { + let current = root; + for (const segment of path.slice(0, -1)) { + const existing = current[segment]; + if (!isRecord(existing)) { + current[segment] = {}; + } + current = current[segment] as Record; + } + const leaf = path.at(-1); + if (!leaf) { + return; + } + current[leaf] = mergeValue(current[leaf], value); +} + +function hasPatchConflict( + config: MigrationProviderContext["config"], + path: readonly string[], + value: unknown, +): boolean { + if (!isRecord(value)) { + return readPath(config as Record, path) !== undefined; + } + const existing = readPath(config as Record, path); + if (!isRecord(existing)) { + return false; + } + return Object.keys(value).some((key) => existing[key] !== undefined); +} + +function createConfigPatchItem(params: { + id: string; + target: string; + path: string[]; + value: unknown; + message: string; + conflict?: boolean; +}): MigrationItem { + return createMigrationItem({ + id: params.id, + kind: "config", + action: "merge", + target: params.target, + status: params.conflict ? "conflict" : "planned", + reason: params.conflict ? MIGRATION_REASON_TARGET_EXISTS : undefined, + message: params.message, + details: { path: params.path, value: params.value }, + }); +} + +function createManualItem(params: { + id: string; + source: string; + message: string; + recommendation: string; +}): MigrationItem { + return createMigrationItem({ + id: params.id, + kind: "manual", + action: "manual", + source: params.source, + status: "skipped", + message: params.message, + reason: params.recommendation, + }); +} + +export function buildConfigItems(params: { + ctx: MigrationProviderContext; + config: Record; + modelRef?: string; + hasMemoryFiles?: boolean; +}): MigrationItem[] { + const items: MigrationItem[] = []; + const memory = childRecord(params.config, "memory"); + const memoryProvider = readString(memory.provider); + + if (params.hasMemoryFiles || memoryProvider) { + items.push( + createConfigPatchItem({ + id: "config:memory", + target: "memory", + path: ["memory"], + value: { backend: "builtin" }, + message: "Use OpenClaw built-in file memory for imported Hermes memory files.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["memory"], { backend: true }), + }), + ); + items.push( + createConfigPatchItem({ + id: "config:memory-plugin-slot", + target: "plugins.slots", + path: ["plugins", "slots"], + value: { memory: "memory-core" }, + message: "Select the default OpenClaw memory plugin for imported file memory.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["plugins", "slots"], { memory: true }), + }), + ); + } + + if (memoryProvider === "honcho") { + const value = { + honcho: { + enabled: true, + config: childRecord(memory, "honcho"), + }, + }; + items.push( + createConfigPatchItem({ + id: "config:memory-plugin:honcho", + target: "plugins.entries.honcho", + path: ["plugins", "entries"], + value, + message: "Preserve Hermes Honcho memory settings as a plugin entry for manual activation.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["plugins", "entries"], value), + }), + ); + items.push( + createManualItem({ + id: "manual:memory-provider:honcho", + source: "config.yaml:memory.provider", + message: + "Hermes used Honcho memory. OpenClaw keeps built-in memory selected until the matching plugin is installed and reviewed.", + recommendation: + "Install or review the Honcho memory plugin before selecting it for plugins.slots.memory.", + }), + ); + } else if (memoryProvider && !["builtin", "file", "files"].includes(memoryProvider)) { + items.push( + createManualItem({ + id: `manual:memory-provider:${memoryProvider}`, + source: "config.yaml:memory.provider", + message: `Hermes memory provider "${memoryProvider}" does not have a known OpenClaw mapping.`, + recommendation: "Install or configure an equivalent OpenClaw memory plugin manually.", + }), + ); + } + + const providers = collectHermesProviders(params.config, params.modelRef); + if (providers.length > 0) { + const value = Object.fromEntries(providers.map((entry) => [entry.id, providerConfig(entry)])); + items.push( + createConfigPatchItem({ + id: "config:model-providers", + target: "models.providers", + path: ["models", "providers"], + value, + message: "Import Hermes provider and custom endpoint config.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["models", "providers"], value), + }), + ); + } + + const mcpConfig = params.config.mcp; + const rawMcpServers = + params.config.mcp_servers ?? + (isRecord(mcpConfig) && isRecord(mcpConfig.servers) ? mcpConfig.servers : mcpConfig); + const mcpServers = mapMcpServers(rawMcpServers); + if (mcpServers) { + items.push( + createConfigPatchItem({ + id: "config:mcp-servers", + target: "mcp.servers", + path: ["mcp", "servers"], + value: mcpServers, + message: "Import Hermes MCP server definitions.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["mcp", "servers"], mcpServers), + }), + ); + } + + const skillEntries = mapSkillEntries(params.config); + if (skillEntries) { + items.push( + createConfigPatchItem({ + id: "config:skill-entries", + target: "skills.entries", + path: ["skills", "entries"], + value: skillEntries, + message: "Import Hermes skill config values.", + conflict: + !params.ctx.overwrite && + hasPatchConflict(params.ctx.config, ["skills", "entries"], skillEntries), + }), + ); + } + + return items; +} + +function readConfigPatchDetails(item: MigrationItem): ConfigPatchDetails | undefined { + const path = item.details?.path; + if ( + !Array.isArray(path) || + !path.every((segment): segment is string => typeof segment === "string") + ) { + return undefined; + } + return { path, value: item.details?.value }; +} + +export async function applyConfigItem( + ctx: MigrationProviderContext, + item: MigrationItem, +): Promise { + if (item.status !== "planned") { + return item; + } + const details = readConfigPatchDetails(item); + if (!details) { + return markMigrationItemError(item, MISSING_CONFIG_PATCH); + } + if (!ctx.runtime?.config.writeConfigFile) { + return markMigrationItemError(item, CONFIG_RUNTIME_UNAVAILABLE); + } + try { + const nextConfig = structuredClone(ctx.runtime.config.loadConfig?.() ?? ctx.config); + if (!ctx.overwrite && hasPatchConflict(nextConfig, details.path, details.value)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + writePath(nextConfig as Record, details.path, details.value); + await ctx.runtime.config.writeConfigFile(nextConfig); + return { ...item, status: "migrated" }; + } catch (err) { + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export function applyManualItem(item: MigrationItem): MigrationItem { + return markMigrationItemSkipped(item, item.reason ?? "manual follow-up required"); +} diff --git a/extensions/migrate-hermes/files-and-skills.test.ts b/extensions/migrate-hermes/files-and-skills.test.ts new file mode 100644 index 00000000000..8f8f0d4bb43 --- /dev/null +++ b/extensions/migrate-hermes/files-and-skills.test.ts @@ -0,0 +1,204 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { MIGRATION_REASON_TARGET_EXISTS } from "openclaw/plugin-sdk/migration"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration file and skill items", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + function configRuntime(config: Record) { + return { + config: { + loadConfig: () => config, + writeConfigFile: async (next: Record) => { + Object.keys(config).forEach((key) => { + delete config[key]; + }); + Object.assign(config, next); + return next; + }, + }, + } as never; + } + + it("reports normalized skill-name collisions instead of overwriting during apply", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(source, "skills", "ship-it", "SKILL.md"), "# ship-it\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + const skillItems = plan.items.filter((item) => item.kind === "skill"); + + expect(skillItems).toHaveLength(2); + expect(skillItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "skill:ship-it", + status: "conflict", + reason: 'multiple Hermes skill directories normalize to "ship-it"', + target: path.join(workspaceDir, "skills", "ship-it"), + }), + ]), + ); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + overwrite: true, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.summary.conflicts).toBe(2); + await expect(fs.access(path.join(workspaceDir, "skills", "ship-it"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("reports late-created copy targets as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, "AGENTS.md"), "# Hermes agents\n"); + + const provider = buildHermesMigrationProvider(); + const ctx = makeContext({ source, stateDir, workspaceDir, reportDir }); + const plan = await provider.plan(ctx); + await writeFile(path.join(workspaceDir, "AGENTS.md"), "# Late agents\n"); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "workspace:AGENTS.md", + status: "conflict", + reason: MIGRATION_REASON_TARGET_EXISTS, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + expect(await fs.readFile(path.join(workspaceDir, "AGENTS.md"), "utf8")).toBe("# Late agents\n"); + }); + + it("applies files, appended memories, item backups, reports, and opt-in API keys", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile(path.join(source, "AGENTS.md"), "# Hermes agents\n"); + await writeFile(path.join(source, "memories", "MEMORY.md"), "memory line\n"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(workspaceDir, "AGENTS.md"), "# Existing agents\n"); + + const provider = buildHermesMigrationProvider(); + const config: Record = {}; + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + includeSecrets: true, + overwrite: true, + reportDir, + runtime: configRuntime(config), + }), + ); + + expect(result.summary.errors).toBe(0); + expect(result.summary.conflicts).toBe(0); + expect(await fs.readFile(path.join(workspaceDir, "AGENTS.md"), "utf8")).toBe( + "# Hermes agents\n", + ); + expect( + await fs.readFile(path.join(workspaceDir, "skills", "ship-it", "SKILL.md"), "utf8"), + ).toBe("# Ship It\n"); + await expect(fs.access(path.join(reportDir, "summary.md"))).resolves.toBeUndefined(); + expect(await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf8")).toContain( + "Imported from Hermes", + ); + const copiedAgentsItem = result.items.find((item) => item.id === "workspace:AGENTS.md"); + expect(copiedAgentsItem?.details?.backupPath).toEqual(expect.stringContaining("AGENTS.md")); + const authStore = JSON.parse( + await fs.readFile( + path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), + "utf8", + ), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]).toMatchObject({ + provider: "openai", + key: "sk-hermes", + }); + }); + + it("archives unsupported Hermes state into the report without importing it", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile(path.join(source, "logs", "session.log"), "log line\n"); + await writeFile(path.join(source, "auth.json"), '{"token":"opaque"}\n'); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir, reportDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "archive:logs", + kind: "archive", + action: "archive", + status: "planned", + }), + expect.objectContaining({ + id: "archive:auth.json", + kind: "archive", + action: "archive", + status: "planned", + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([expect.stringContaining("archive-only")]), + ); + + const result = await provider.apply(makeContext({ source, stateDir, workspaceDir, reportDir })); + + expect(result.summary.errors).toBe(0); + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "archive:logs", + status: "migrated", + target: path.join(reportDir, "archive", "logs"), + }), + expect.objectContaining({ + id: "archive:auth.json", + status: "migrated", + target: path.join(reportDir, "archive", "auth.json"), + }), + ]), + ); + expect(await fs.readFile(path.join(reportDir, "archive", "logs", "session.log"), "utf8")).toBe( + "log line\n", + ); + await expect(fs.access(path.join(workspaceDir, "logs", "session.log"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); +}); diff --git a/extensions/migrate-hermes/helpers.ts b/extensions/migrate-hermes/helpers.ts new file mode 100644 index 00000000000..ad11ab6d7c0 --- /dev/null +++ b/extensions/migrate-hermes/helpers.ts @@ -0,0 +1,134 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + markMigrationItemError, + MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, +} from "openclaw/plugin-sdk/migration"; +import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { parse as parseYaml } from "yaml"; + +export function resolveHomePath(input: string): string { + if (input === "~") { + return os.homedir(); + } + if (input.startsWith("~/")) { + return path.join(os.homedir(), input.slice(2)); + } + return path.resolve(input); +} + +export async function exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +export async function isDirectory(dirPath: string): Promise { + try { + return (await fs.stat(dirPath)).isDirectory(); + } catch { + return false; + } +} + +export function sanitizeName(name: string): string { + return name + .trim() + .toLowerCase() + .replaceAll(/[^a-z0-9._-]+/g, "-") + .replaceAll(/^-+|-+$/g, ""); +} + +export async function readText(filePath: string | undefined): Promise { + if (!filePath) { + return undefined; + } + try { + return await fs.readFile(filePath, "utf8"); + } catch { + return undefined; + } +} + +export function parseEnv(content: string | undefined): Record { + const env: Record = {}; + if (!content) { + return env; + } + for (const line of content.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u.exec(trimmed); + if (!match) { + continue; + } + const key = match[1]; + let value = match[2] ?? ""; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[key] = value; + } + return env; +} + +export function parseHermesConfig(content: string | undefined): Record { + if (!content) { + return {}; + } + try { + const parsed = parseYaml(content); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +export function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +export function childRecord( + root: Record | undefined, + key: string, +): Record { + const value = root?.[key]; + return isRecord(value) ? value : {}; +} + +export function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +export function readStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.filter((entry): entry is string => typeof entry === "string" && entry.trim() !== ""); +} + +export async function appendItem(item: MigrationItem): Promise { + if (!item.source || !item.target) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const content = await fs.readFile(item.source, "utf8"); + const header = `\n\n\n\n`; + await fs.mkdir(path.dirname(item.target), { recursive: true }); + await fs.appendFile(item.target, `${header}${content.trimEnd()}\n`, "utf8"); + return { ...item, status: "migrated" }; + } catch (err) { + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} diff --git a/extensions/migrate-hermes/index.ts b/extensions/migrate-hermes/index.ts new file mode 100644 index 00000000000..ff87eba7bb5 --- /dev/null +++ b/extensions/migrate-hermes/index.ts @@ -0,0 +1,11 @@ +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { buildHermesMigrationProvider } from "./provider.js"; + +export default definePluginEntry({ + id: "migrate-hermes", + name: "Hermes Migration", + description: "Imports Hermes state into OpenClaw.", + register(api) { + api.registerMigrationProvider(buildHermesMigrationProvider({ runtime: api.runtime })); + }, +}); diff --git a/extensions/migrate-hermes/items.ts b/extensions/migrate-hermes/items.ts new file mode 100644 index 00000000000..598c7cb6de2 --- /dev/null +++ b/extensions/migrate-hermes/items.ts @@ -0,0 +1,113 @@ +import type { MigrationItem } from "openclaw/plugin-sdk/migration"; +import { + createMigrationItem, + markMigrationItemConflict, + markMigrationItemError, + markMigrationItemSkipped, +} from "openclaw/plugin-sdk/migration"; +import { readString } from "./helpers.js"; + +export type HermesModelDetails = { + model: string; +}; + +export type HermesSecretDetails = { + envVar: string; + provider: string; + profileId: string; +}; + +export type HermesModelItem = MigrationItem & { + id: "config:default-model"; + kind: "config"; + action: "skip" | "update"; + details: HermesModelDetails; +}; + +export type HermesSecretItem = MigrationItem & { + kind: "secret"; + action: "skip" | "create"; + details: HermesSecretDetails; +}; + +export const HERMES_REASON_ALREADY_CONFIGURED = "already configured"; +export const HERMES_REASON_DEFAULT_MODEL_CONFIGURED = "default model already configured"; +export const HERMES_REASON_INCLUDE_SECRETS = "use --include-secrets to import"; +export const HERMES_REASON_AUTH_PROFILE_EXISTS = "auth profile exists"; +export const HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE = "config runtime unavailable"; +export const HERMES_REASON_MISSING_SECRET_METADATA = "missing secret metadata"; +export const HERMES_REASON_SECRET_NO_LONGER_PRESENT = "secret no longer present"; +export const HERMES_REASON_AUTH_PROFILE_WRITE_FAILED = "failed to write auth profile"; + +export function createHermesModelItem(params: { + model: string; + currentModel?: string; + overwrite?: boolean; +}): HermesModelItem { + const alreadyConfigured = params.currentModel === params.model; + const conflict = Boolean(params.currentModel && !params.overwrite && !alreadyConfigured); + return createMigrationItem({ + id: "config:default-model", + kind: "config", + action: alreadyConfigured ? "skip" : "update", + target: "agents.defaults.model", + status: alreadyConfigured ? "skipped" : conflict ? "conflict" : "planned", + reason: alreadyConfigured + ? HERMES_REASON_ALREADY_CONFIGURED + : conflict + ? HERMES_REASON_DEFAULT_MODEL_CONFIGURED + : undefined, + details: { model: params.model }, + }) as HermesModelItem; +} + +export function readHermesModelDetails(item: MigrationItem): HermesModelDetails | undefined { + const model = readString(item.details?.model); + return model ? { model } : undefined; +} + +export function createHermesSecretItem(params: { + id: string; + source?: string; + target: string; + includeSecrets?: boolean; + existsAlready?: boolean; + details: HermesSecretDetails; +}): HermesSecretItem { + const skipped = !params.includeSecrets; + const conflict = Boolean(params.existsAlready && !skipped); + return createMigrationItem({ + id: params.id, + kind: "secret", + action: skipped ? "skip" : "create", + source: params.source, + target: params.target, + status: skipped ? "skipped" : conflict ? "conflict" : "planned", + sensitive: true, + reason: skipped + ? HERMES_REASON_INCLUDE_SECRETS + : conflict + ? HERMES_REASON_AUTH_PROFILE_EXISTS + : undefined, + details: params.details, + }) as HermesSecretItem; +} + +export function readHermesSecretDetails(item: MigrationItem): HermesSecretDetails | undefined { + const envVar = readString(item.details?.envVar); + const provider = readString(item.details?.provider); + const profileId = readString(item.details?.profileId); + return envVar && provider && profileId ? { envVar, provider, profileId } : undefined; +} + +export function hermesItemConflict(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemConflict(item, reason); +} + +export function hermesItemError(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemError(item, reason); +} + +export function hermesItemSkipped(item: MigrationItem, reason: string): MigrationItem { + return markMigrationItemSkipped(item, reason); +} diff --git a/extensions/migrate-hermes/model.apply.test.ts b/extensions/migrate-hermes/model.apply.test.ts new file mode 100644 index 00000000000..9e8edf20d9c --- /dev/null +++ b/extensions/migrate-hermes/model.apply.test.ts @@ -0,0 +1,188 @@ +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_DEFAULT_MODEL_CONFIGURED } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration model apply", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("updates only the primary model when applying over object-form model config", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const existingConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + timeoutMs: 120_000, + }, + }, + }, + } as OpenClawConfig; + let writtenConfig: OpenClawConfig | undefined; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => existingConfig, + writeConfigFile: async (next: OpenClawConfig) => { + writtenConfig = next; + }, + }, + } as never, + }); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + overwrite: true, + model: existingConfig.agents?.defaults?.model, + reportDir, + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "migrated", + }), + ]), + ); + expect(writtenConfig?.agents?.defaults?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + timeoutMs: 120_000, + }); + }); + + it("updates the default-agent model override when applying with overwrite", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const existingConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: { + primary: "google/gemini-3-pro", + fallbacks: ["openai/gpt-5.4"], + }, + }, + list: [ + { + id: "main", + default: true, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }, + }, + ], + }, + } as OpenClawConfig; + let writtenConfig: OpenClawConfig | undefined; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => existingConfig, + writeConfigFile: async (next: OpenClawConfig) => { + writtenConfig = next; + }, + }, + } as never, + }); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + config: existingConfig, + overwrite: true, + reportDir, + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "migrated", + }), + ]), + ); + expect(writtenConfig?.agents?.list?.[0]?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }); + expect(writtenConfig?.agents?.defaults?.model).toEqual(existingConfig.agents?.defaults?.model); + }); + + it("reports late-created default models as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const lateConfig = { + agents: { + defaults: { + workspace: workspaceDir, + model: "anthropic/claude-sonnet-4.6", + }, + }, + } as OpenClawConfig; + let writeCalled = false; + const provider = buildHermesMigrationProvider({ + runtime: { + config: { + loadConfig: () => lateConfig, + writeConfigFile: async () => { + writeCalled = true; + }, + }, + } as never, + }); + const ctx = makeContext({ source, stateDir, workspaceDir, reportDir }); + const plan = await provider.plan(ctx); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + expect(writeCalled).toBe(false); + }); +}); diff --git a/extensions/migrate-hermes/model.plan.test.ts b/extensions/migrate-hermes/model.plan.test.ts new file mode 100644 index 00000000000..8a68cabc6fd --- /dev/null +++ b/extensions/migrate-hermes/model.plan.test.ts @@ -0,0 +1,107 @@ +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_DEFAULT_MODEL_CONFIGURED } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration model planning", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("preserves the provider for top-level string model refs", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, "config.yaml"), "provider: openai\nmodel: gpt-5.4\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + details: { model: "openai/gpt-5.4" }, + status: "planned", + }), + ]), + ); + }); + + it("treats existing object-form default model primaries as conflicts", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + model: { + primary: "anthropic/claude-sonnet-4.6", + fallbacks: ["openai/gpt-5.4"], + timeoutMs: 120_000, + }, + }), + ); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + }); + + it("treats default-agent model overrides as conflicts", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + const config = { + agents: { + defaults: { + workspace: workspaceDir, + model: "openai/gpt-5.4", + }, + list: [ + { + id: "main", + default: true, + model: "anthropic/claude-sonnet-4.6", + }, + ], + }, + } as OpenClawConfig; + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan(makeContext({ source, stateDir, workspaceDir, config })); + + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "config:default-model", + status: "conflict", + reason: HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + }), + ]), + ); + }); +}); diff --git a/extensions/migrate-hermes/model.ts b/extensions/migrate-hermes/model.ts new file mode 100644 index 00000000000..7dc8356a9dd --- /dev/null +++ b/extensions/migrate-hermes/model.ts @@ -0,0 +1,87 @@ +import { + resolveAgentEffectiveModelPrimary, + resolveDefaultAgentId, + setAgentEffectiveModelPrimary, +} from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { readString } from "./helpers.js"; +import { + HERMES_REASON_ALREADY_CONFIGURED, + HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE, + HERMES_REASON_DEFAULT_MODEL_CONFIGURED, + hermesItemConflict, + hermesItemError, + hermesItemSkipped, + readHermesModelDetails, +} from "./items.js"; + +export function resolveHermesModelRef(config: Record): string | undefined { + const model = config.model; + if (typeof model === "string" && model.trim()) { + const rawModel = model.trim(); + const provider = readString(config.provider); + if (provider && !rawModel.includes("/")) { + return `${provider}/${rawModel}`; + } + return rawModel; + } + if (model && typeof model === "object" && !Array.isArray(model)) { + const modelRecord = model as Record; + const rawModel = readString(modelRecord.default) ?? readString(modelRecord.model); + const provider = readString(modelRecord.provider); + if (rawModel && provider && !rawModel.includes("/")) { + return `${provider}/${rawModel}`; + } + return rawModel; + } + const rootModel = readString(config.default_model) ?? readString(config.model_name); + const rootProvider = readString(config.provider); + if (rootModel && rootProvider && !rootModel.includes("/")) { + return `${rootProvider}/${rootModel}`; + } + return rootModel; +} + +function resolveDefaultAgentModelState(config: MigrationProviderContext["config"]): { + agentId: string; + effectivePrimary?: string; +} { + const agentId = resolveDefaultAgentId(config); + const effectivePrimary = resolveAgentEffectiveModelPrimary(config, agentId); + return { + agentId, + effectivePrimary, + }; +} + +export function resolveCurrentModelRef(ctx: MigrationProviderContext): string | undefined { + return resolveDefaultAgentModelState(ctx.config).effectivePrimary; +} + +export async function applyModelItem( + ctx: MigrationProviderContext, + item: MigrationItem, +): Promise { + const details = readHermesModelDetails(item); + if (!details || item.status !== "planned") { + return item; + } + try { + if (!ctx.runtime?.config.writeConfigFile) { + return hermesItemError(item, HERMES_REASON_CONFIG_RUNTIME_UNAVAILABLE); + } + const nextConfig = structuredClone(ctx.runtime?.config.loadConfig?.() ?? ctx.config); + const currentState = resolveDefaultAgentModelState(nextConfig); + if (currentState.effectivePrimary === details.model) { + return hermesItemSkipped(item, HERMES_REASON_ALREADY_CONFIGURED); + } + if (currentState.effectivePrimary && !ctx.overwrite) { + return hermesItemConflict(item, HERMES_REASON_DEFAULT_MODEL_CONFIGURED); + } + setAgentEffectiveModelPrimary(nextConfig, currentState.agentId, details.model); + await ctx.runtime.config.writeConfigFile(nextConfig); + return { ...item, status: "migrated" }; + } catch (err) { + return hermesItemError(item, err instanceof Error ? err.message : String(err)); + } +} diff --git a/extensions/migrate-hermes/openclaw.plugin.json b/extensions/migrate-hermes/openclaw.plugin.json new file mode 100644 index 00000000000..0b848a4f627 --- /dev/null +++ b/extensions/migrate-hermes/openclaw.plugin.json @@ -0,0 +1,13 @@ +{ + "id": "migrate-hermes", + "name": "Hermes Migration", + "description": "Imports Hermes configuration, memories, skills, and supported credentials into OpenClaw.", + "contracts": { + "migrationProviders": ["hermes"] + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/migrate-hermes/package.json b/extensions/migrate-hermes/package.json new file mode 100644 index 00000000000..cc7485db619 --- /dev/null +++ b/extensions/migrate-hermes/package.json @@ -0,0 +1,27 @@ +{ + "name": "@openclaw/migrate-hermes", + "version": "2026.4.25", + "private": true, + "description": "Hermes to OpenClaw migration provider", + "type": "module", + "dependencies": { + "yaml": "^2.8.3" + }, + "devDependencies": { + "@openclaw/plugin-sdk": "workspace:*", + "openclaw": "workspace:*" + }, + "peerDependencies": { + "openclaw": ">=2026.4.25" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/extensions/migrate-hermes/plan.ts b/extensions/migrate-hermes/plan.ts new file mode 100644 index 00000000000..9930cd8021c --- /dev/null +++ b/extensions/migrate-hermes/plan.ts @@ -0,0 +1,162 @@ +import path from "node:path"; +import { + createMigrationItem, + MIGRATION_REASON_TARGET_EXISTS, + summarizeMigrationItems, +} from "openclaw/plugin-sdk/migration"; +import type { + MigrationItem, + MigrationPlan, + MigrationProviderContext, +} from "openclaw/plugin-sdk/plugin-entry"; +import { buildConfigItems } from "./config.js"; +import { exists, parseHermesConfig, readText } from "./helpers.js"; +import { createHermesModelItem } from "./items.js"; +import { resolveCurrentModelRef, resolveHermesModelRef } from "./model.js"; +import { buildSecretItems } from "./secrets.js"; +import { buildSkillItems } from "./skills.js"; +import { discoverHermesSource, hasHermesSource } from "./source.js"; +import { resolveTargets } from "./targets.js"; + +async function addFileItem(params: { + items: MigrationItem[]; + id: string; + source?: string; + target: string; + kind?: MigrationItem["kind"]; + action?: MigrationItem["action"]; + overwrite?: boolean; +}): Promise { + if (!params.source) { + return; + } + const targetExists = await exists(params.target); + params.items.push( + createMigrationItem({ + id: params.id, + kind: params.kind ?? "file", + action: params.action ?? "copy", + source: params.source, + target: params.target, + status: targetExists && !params.overwrite ? "conflict" : "planned", + reason: targetExists && !params.overwrite ? MIGRATION_REASON_TARGET_EXISTS : undefined, + }), + ); +} + +export async function buildHermesPlan(ctx: MigrationProviderContext): Promise { + const source = await discoverHermesSource(ctx.source); + if (!hasHermesSource(source)) { + throw new Error( + `Hermes state was not found at ${source.root}. Pass --from if it lives elsewhere.`, + ); + } + const targets = resolveTargets(ctx); + const config = parseHermesConfig(await readText(source.configPath)); + const modelRef = resolveHermesModelRef(config); + const items: MigrationItem[] = []; + + if (modelRef) { + const currentModel = resolveCurrentModelRef(ctx); + items.push( + createHermesModelItem({ + model: modelRef, + currentModel, + overwrite: ctx.overwrite, + }), + ); + } + items.push( + ...buildConfigItems({ + ctx, + config, + modelRef, + hasMemoryFiles: Boolean(source.memoryPath || source.userPath), + }), + ); + + await addFileItem({ + items, + id: "workspace:SOUL.md", + kind: "workspace", + source: source.soulPath, + target: path.join(targets.workspaceDir, "SOUL.md"), + overwrite: ctx.overwrite, + }); + await addFileItem({ + items, + id: "workspace:AGENTS.md", + kind: "workspace", + source: source.agentsPath, + target: path.join(targets.workspaceDir, "AGENTS.md"), + overwrite: ctx.overwrite, + }); + if (source.memoryPath) { + items.push( + createMigrationItem({ + id: "memory:MEMORY.md", + kind: "memory", + action: "append", + source: source.memoryPath, + target: path.join(targets.workspaceDir, "MEMORY.md"), + }), + ); + } + if (source.userPath) { + items.push( + createMigrationItem({ + id: "memory:USER.md", + kind: "memory", + action: "append", + source: source.userPath, + target: path.join(targets.workspaceDir, "USER.md"), + }), + ); + } + items.push(...(await buildSkillItems({ source, targets, overwrite: ctx.overwrite }))); + items.push(...(await buildSecretItems({ ctx, source, targets }))); + for (const archivePath of source.archivePaths) { + items.push( + createMigrationItem({ + id: archivePath.id, + kind: "archive", + action: "archive", + source: archivePath.path, + message: + "Archived in the migration report for manual review; not imported into live config.", + details: { archiveRelativePath: archivePath.relativePath }, + }), + ); + } + + const warnings = [ + ...(!ctx.includeSecrets && items.some((item) => item.kind === "secret") + ? [ + "Secrets were detected but skipped. Re-run with --include-secrets to import supported API keys.", + ] + : []), + ...(items.some((item) => item.status === "conflict") + ? [ + "Conflicts were found. Re-run with --overwrite to replace conflicting targets after item-level backups.", + ] + : []), + ...(source.archivePaths.length > 0 + ? [ + "Some Hermes files are archive-only. They will be copied into the migration report for manual review, not loaded into OpenClaw.", + ] + : []), + ...(items.some((item) => item.kind === "manual") + ? ["Some Hermes settings require manual review before they can be activated safely."] + : []), + ]; + return { + providerId: "hermes", + source: source.root, + target: targets.workspaceDir, + summary: summarizeMigrationItems(items), + items, + warnings, + nextSteps: ["Run openclaw doctor after applying the migration."], + metadata: { agentDir: targets.agentDir }, + }; +} diff --git a/extensions/migrate-hermes/provider.secret-failure.test.ts b/extensions/migrate-hermes/provider.secret-failure.test.ts new file mode 100644 index 00000000000..b11640ca480 --- /dev/null +++ b/extensions/migrate-hermes/provider.secret-failure.test.ts @@ -0,0 +1,99 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HERMES_REASON_AUTH_PROFILE_WRITE_FAILED } from "./items.js"; + +const mocks = vi.hoisted(() => ({ + updateAuthProfileStoreWithLock: vi.fn(async () => null), +})); + +vi.mock("openclaw/plugin-sdk/provider-auth", () => ({ + updateAuthProfileStoreWithLock: mocks.updateAuthProfileStoreWithLock, +})); + +const { buildHermesMigrationProvider } = await import("./provider.js"); + +const tempRoots = new Set(); +const logger = { + info() {}, + warn() {}, + error() {}, + debug() {}, +}; + +async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-hermes-secret-failure-")); + tempRoots.add(root); + return root; +} + +async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +function makeContext(params: { + source: string; + stateDir: string; + workspaceDir: string; + reportDir: string; +}): MigrationProviderContext { + return { + config: { + agents: { + defaults: { + workspace: params.workspaceDir, + }, + }, + } as OpenClawConfig, + stateDir: params.stateDir, + source: params.source, + includeSecrets: true, + overwrite: true, + reportDir: params.reportDir, + logger, + }; +} + +describe("Hermes migration provider secret write failures", () => { + afterEach(async () => { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); + mocks.updateAuthProfileStoreWithLock.mockClear(); + }); + + it("reports an error when a secret auth-profile write fails", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + status: "error", + reason: HERMES_REASON_AUTH_PROFILE_WRITE_FAILED, + }), + ]), + ); + expect(result.summary.errors).toBe(1); + expect(result.summary.migrated).toBe(0); + }); +}); diff --git a/extensions/migrate-hermes/provider.test.ts b/extensions/migrate-hermes/provider.test.ts new file mode 100644 index 00000000000..2819a5ec136 --- /dev/null +++ b/extensions/migrate-hermes/provider.test.ts @@ -0,0 +1,129 @@ +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createCapturedPluginRegistration } from "../../src/plugins/captured-registration.js"; +import pluginEntry from "./index.js"; +import { HERMES_REASON_INCLUDE_SECRETS } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration provider", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("registers the Hermes migration provider through the plugin entry", () => { + const captured = createCapturedPluginRegistration(); + pluginEntry.register(captured.api); + expect(captured.migrationProviders.map((provider) => provider.id)).toEqual(["hermes"]); + }); + + it("detects Hermes sources supported by planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + await writeFile(path.join(source, "SOUL.md"), "# Hermes soul\n"); + + const provider = buildHermesMigrationProvider(); + const detected = await provider.detect?.( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ); + + expect(detected).toMatchObject({ + found: true, + source, + confidence: "high", + }); + }); + + it("detects archive-only Hermes sources", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + await writeFile(path.join(source, "logs", "run.log"), "log line\n"); + + const provider = buildHermesMigrationProvider(); + const detected = await provider.detect?.( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ); + + expect(detected).toMatchObject({ + found: true, + source, + confidence: "high", + }); + }); + + it("rejects missing Hermes sources before planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "missing-hermes"); + + const provider = buildHermesMigrationProvider(); + + await expect( + provider.plan( + makeContext({ + source, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }), + ), + ).rejects.toThrow(`Hermes state was not found at ${source}`); + }); + + it("plans model, workspace, memory, skill, and secret items without importing secrets by default", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + await writeFile( + path.join(source, "config.yaml"), + "model:\n provider: openai\n model: gpt-5.4\n", + ); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile(path.join(source, "SOUL.md"), "# Hermes soul\n"); + await writeFile(path.join(source, "memories", "MEMORY.md"), "remember this\n"); + await writeFile(path.join(source, "skills", "Ship It", "SKILL.md"), "# Ship It\n"); + await writeFile(path.join(workspaceDir, "SOUL.md"), "# Existing soul\n"); + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + model: "anthropic/claude-sonnet-4.6", + }), + ); + + expect(plan.summary).toMatchObject({ total: 8, conflicts: 2, sensitive: 1 }); + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "config:default-model", status: "conflict" }), + expect.objectContaining({ id: "config:memory", status: "planned" }), + expect.objectContaining({ id: "config:memory-plugin-slot", status: "planned" }), + expect.objectContaining({ id: "config:model-providers", status: "planned" }), + expect.objectContaining({ id: "workspace:SOUL.md", status: "conflict" }), + expect.objectContaining({ id: "memory:MEMORY.md", action: "append", status: "planned" }), + expect.objectContaining({ id: "skill:ship-it", status: "planned" }), + expect.objectContaining({ + id: "secret:openai", + sensitive: true, + status: "skipped", + reason: HERMES_REASON_INCLUDE_SECRETS, + }), + ]), + ); + expect(plan.warnings).toEqual( + expect.arrayContaining([ + expect.stringContaining("Secrets were detected but skipped"), + expect.stringContaining("Conflicts were found"), + ]), + ); + }); +}); diff --git a/extensions/migrate-hermes/provider.ts b/extensions/migrate-hermes/provider.ts new file mode 100644 index 00000000000..212cf237122 --- /dev/null +++ b/extensions/migrate-hermes/provider.ts @@ -0,0 +1,35 @@ +import type { + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, +} from "openclaw/plugin-sdk/plugin-entry"; +import { applyHermesPlan } from "./apply.js"; +import { buildHermesPlan } from "./plan.js"; +import { discoverHermesSource, hasHermesSource } from "./source.js"; + +export function buildHermesMigrationProvider( + params: { + runtime?: MigrationProviderContext["runtime"]; + } = {}, +): MigrationProviderPlugin { + return { + id: "hermes", + label: "Hermes", + description: "Import Hermes config, memories, skills, and supported credentials.", + async detect(ctx) { + const source = await discoverHermesSource(ctx.source); + const found = hasHermesSource(source); + return { + found, + source: source.root, + label: "Hermes", + confidence: found ? "high" : "low", + message: found ? "Hermes state found." : "Hermes state not found.", + }; + }, + plan: buildHermesPlan, + async apply(ctx, plan?: MigrationPlan) { + return await applyHermesPlan({ ctx, plan, runtime: params.runtime }); + }, + }; +} diff --git a/extensions/migrate-hermes/secrets.test.ts b/extensions/migrate-hermes/secrets.test.ts new file mode 100644 index 00000000000..5f4bca0fb43 --- /dev/null +++ b/extensions/migrate-hermes/secrets.test.ts @@ -0,0 +1,159 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; +import { afterEach, describe, expect, it } from "vitest"; +import { HERMES_REASON_AUTH_PROFILE_EXISTS } from "./items.js"; +import { buildHermesMigrationProvider } from "./provider.js"; +import { cleanupTempRoots, makeContext, makeTempRoot, writeFile } from "./test/provider-helpers.js"; + +describe("Hermes migration secret items", () => { + afterEach(async () => { + await cleanupTempRoots(); + }); + + it("uses configured agentDir for secret planning and imports without runtime helpers", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const customAgentDir = path.join(root, "custom-agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + const config = { + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [ + { + id: "custom", + default: true, + agentDir: customAgentDir, + }, + ], + }, + } as OpenClawConfig; + + const provider = buildHermesMigrationProvider(); + const plan = await provider.plan( + makeContext({ + source, + stateDir, + workspaceDir, + config, + includeSecrets: true, + }), + ); + + expect(plan.metadata?.agentDir).toBe(customAgentDir); + expect(plan.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + target: `${customAgentDir}/auth-profiles.json#openai:hermes-import`, + status: "planned", + }), + ]), + ); + + const result = await provider.apply( + makeContext({ + source, + stateDir, + workspaceDir, + config, + includeSecrets: true, + overwrite: true, + reportDir: path.join(root, "report"), + }), + ); + + expect(result.summary.errors).toBe(0); + const authStore = JSON.parse( + await fs.readFile(path.join(customAgentDir, "auth-profiles.json"), "utf8"), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]).toMatchObject({ + provider: "openai", + key: "sk-hermes", + }); + await expect( + fs.access(path.join(stateDir, "agents", "custom", "agent", "auth-profiles.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("keeps secret conflict checks read-only during planning", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + await writeFile( + path.join(agentDir, "auth.json"), + JSON.stringify({ + openai: { type: "api_key", provider: "openai", key: "legacy-main-key" }, + }), + ); + + const provider = buildHermesMigrationProvider(); + await provider.plan(makeContext({ source, stateDir, workspaceDir, includeSecrets: true })); + + await expect(fs.access(path.join(agentDir, "auth.json"))).resolves.toBeUndefined(); + await expect(fs.access(path.join(agentDir, "auth-profiles.json"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("reports late-created auth profiles as conflicts without overwriting", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await writeFile(path.join(source, ".env"), "OPENAI_API_KEY=sk-hermes\n"); + + const provider = buildHermesMigrationProvider(); + const ctx = makeContext({ + source, + stateDir, + workspaceDir, + includeSecrets: true, + reportDir, + }); + const plan = await provider.plan(ctx); + await writeFile( + path.join(agentDir, "auth-profiles.json"), + JSON.stringify( + { + version: 1, + profiles: { + "openai:hermes-import": { + type: "api_key", + provider: "openai", + key: "sk-late", + }, + }, + }, + null, + 2, + ), + ); + + const result = await provider.apply(ctx, plan); + + expect(result.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "secret:openai", + status: "conflict", + reason: HERMES_REASON_AUTH_PROFILE_EXISTS, + }), + ]), + ); + expect(result.summary.conflicts).toBe(1); + const authStore = JSON.parse( + await fs.readFile(path.join(agentDir, "auth-profiles.json"), "utf8"), + ) as { profiles?: Record }; + expect(authStore.profiles?.["openai:hermes-import"]?.key).toBe("sk-late"); + }); +}); diff --git a/extensions/migrate-hermes/secrets.ts b/extensions/migrate-hermes/secrets.ts new file mode 100644 index 00000000000..0ecf876b1b1 --- /dev/null +++ b/extensions/migrate-hermes/secrets.ts @@ -0,0 +1,118 @@ +import { loadAuthProfileStoreWithoutExternalProfiles } from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationItem, MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { updateAuthProfileStoreWithLock } from "openclaw/plugin-sdk/provider-auth"; +import { parseEnv, readText } from "./helpers.js"; +import { + createHermesSecretItem, + HERMES_REASON_AUTH_PROFILE_EXISTS, + HERMES_REASON_AUTH_PROFILE_WRITE_FAILED, + HERMES_REASON_MISSING_SECRET_METADATA, + HERMES_REASON_SECRET_NO_LONGER_PRESENT, + hermesItemConflict, + hermesItemError, + hermesItemSkipped, + readHermesSecretDetails, +} from "./items.js"; +import type { HermesSource } from "./source.js"; +import type { PlannedTargets } from "./targets.js"; + +type SecretMapping = { + envVar: string; + provider: string; + profileId: string; +}; + +const SECRET_MAPPINGS: readonly SecretMapping[] = [ + { envVar: "OPENAI_API_KEY", provider: "openai", profileId: "openai:hermes-import" }, + { envVar: "ANTHROPIC_API_KEY", provider: "anthropic", profileId: "anthropic:hermes-import" }, + { envVar: "OPENROUTER_API_KEY", provider: "openrouter", profileId: "openrouter:hermes-import" }, + { envVar: "GOOGLE_API_KEY", provider: "google", profileId: "google:hermes-import" }, + { envVar: "GEMINI_API_KEY", provider: "google", profileId: "google:hermes-import" }, + { envVar: "GROQ_API_KEY", provider: "groq", profileId: "groq:hermes-import" }, + { envVar: "XAI_API_KEY", provider: "xai", profileId: "xai:hermes-import" }, + { envVar: "MISTRAL_API_KEY", provider: "mistral", profileId: "mistral:hermes-import" }, + { envVar: "DEEPSEEK_API_KEY", provider: "deepseek", profileId: "deepseek:hermes-import" }, +] as const; + +export async function buildSecretItems(params: { + ctx: MigrationProviderContext; + source: HermesSource; + targets: PlannedTargets; +}): Promise { + const env = parseEnv(await readText(params.source.envPath)); + const store = loadAuthProfileStoreWithoutExternalProfiles(params.targets.agentDir); + const seenProfiles = new Set(); + const items: MigrationItem[] = []; + for (const mapping of SECRET_MAPPINGS) { + const value = env[mapping.envVar]?.trim(); + if (!value || seenProfiles.has(mapping.profileId)) { + continue; + } + seenProfiles.add(mapping.profileId); + const existsAlready = Boolean(store.profiles[mapping.profileId]); + items.push( + createHermesSecretItem({ + id: `secret:${mapping.provider}`, + source: params.source.envPath, + target: `${params.targets.agentDir}/auth-profiles.json#${mapping.profileId}`, + includeSecrets: params.ctx.includeSecrets, + existsAlready: existsAlready && !params.ctx.overwrite, + details: { + envVar: mapping.envVar, + provider: mapping.provider, + profileId: mapping.profileId, + }, + }), + ); + } + return items; +} + +export async function applySecretItem( + ctx: MigrationProviderContext, + item: MigrationItem, + targets: PlannedTargets, +): Promise { + if (item.status !== "planned") { + return item; + } + const details = readHermesSecretDetails(item); + const source = item.source; + if (!details || !source) { + return hermesItemError(item, HERMES_REASON_MISSING_SECRET_METADATA); + } + const env = parseEnv(await readText(source)); + const key = env[details.envVar]?.trim(); + if (!key) { + return hermesItemSkipped(item, HERMES_REASON_SECRET_NO_LONGER_PRESENT); + } + let conflicted = false; + let wrote = false; + const store = await updateAuthProfileStoreWithLock({ + agentDir: targets.agentDir, + updater: (freshStore) => { + if (!ctx.overwrite && freshStore.profiles[details.profileId]) { + conflicted = true; + return false; + } + freshStore.profiles[details.profileId] = { + type: "api_key", + provider: details.provider, + key, + displayName: "Hermes import", + }; + wrote = true; + return true; + }, + }); + if (conflicted) { + return hermesItemConflict(item, HERMES_REASON_AUTH_PROFILE_EXISTS); + } + if (!store?.profiles[details.profileId]) { + return hermesItemError(item, HERMES_REASON_AUTH_PROFILE_WRITE_FAILED); + } + if (!wrote && !ctx.overwrite) { + return hermesItemConflict(item, HERMES_REASON_AUTH_PROFILE_EXISTS); + } + return { ...item, status: "migrated" }; +} diff --git a/extensions/migrate-hermes/skills.ts b/extensions/migrate-hermes/skills.ts new file mode 100644 index 00000000000..16b0e30bf7f --- /dev/null +++ b/extensions/migrate-hermes/skills.ts @@ -0,0 +1,70 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { createMigrationItem, MIGRATION_REASON_TARGET_EXISTS } from "openclaw/plugin-sdk/migration"; +import type { MigrationItem } from "openclaw/plugin-sdk/plugin-entry"; +import { exists, sanitizeName } from "./helpers.js"; +import type { HermesSource } from "./source.js"; +import type { PlannedTargets } from "./targets.js"; + +type PlannedSkill = { + name: string; + source: string; + target: string; +}; + +export async function buildSkillItems(params: { + source: HermesSource; + targets: PlannedTargets; + overwrite?: boolean; +}): Promise { + if (!params.source.skillsDir) { + return []; + } + const entries = await fs + .readdir(params.source.skillsDir, { withFileTypes: true }) + .catch(() => []); + const plannedSkills: PlannedSkill[] = []; + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const name = sanitizeName(entry.name); + if (!name) { + continue; + } + const source = path.join(params.source.skillsDir, entry.name); + if (!(await exists(path.join(source, "SKILL.md")))) { + continue; + } + plannedSkills.push({ + name, + source, + target: path.join(params.targets.workspaceDir, "skills", name), + }); + } + const counts = new Map(); + for (const skill of plannedSkills) { + counts.set(skill.name, (counts.get(skill.name) ?? 0) + 1); + } + const items: MigrationItem[] = []; + for (const skill of plannedSkills) { + const collides = (counts.get(skill.name) ?? 0) > 1; + const targetExists = await exists(skill.target); + items.push( + createMigrationItem({ + id: `skill:${skill.name}`, + kind: "skill", + action: "copy", + source: skill.source, + target: skill.target, + status: collides ? "conflict" : targetExists && !params.overwrite ? "conflict" : "planned", + reason: collides + ? `multiple Hermes skill directories normalize to "${skill.name}"` + : targetExists && !params.overwrite + ? MIGRATION_REASON_TARGET_EXISTS + : undefined, + }), + ); + } + return items; +} diff --git a/extensions/migrate-hermes/source.ts b/extensions/migrate-hermes/source.ts new file mode 100644 index 00000000000..83d5de8a65d --- /dev/null +++ b/extensions/migrate-hermes/source.ts @@ -0,0 +1,74 @@ +import path from "node:path"; +import { exists, isDirectory, resolveHomePath } from "./helpers.js"; + +export type HermesSource = { + root: string; + configPath?: string; + envPath?: string; + soulPath?: string; + agentsPath?: string; + memoryPath?: string; + userPath?: string; + skillsDir?: string; + archivePaths: HermesArchivePath[]; +}; + +export type HermesArchivePath = { + id: string; + path: string; + relativePath: string; +}; + +const HERMES_ARCHIVE_DIRS = ["plugins", "sessions", "logs", "cron", "mcp-tokens"] as const; +const HERMES_ARCHIVE_FILES = ["auth.json", "state.db"] as const; + +export async function discoverHermesSource(input?: string): Promise { + const root = resolveHomePath(input?.trim() || "~/.hermes"); + const archivePaths: HermesArchivePath[] = []; + for (const dir of HERMES_ARCHIVE_DIRS) { + const candidate = path.join(root, dir); + if (await isDirectory(candidate)) { + archivePaths.push({ id: `archive:${dir}`, path: candidate, relativePath: dir }); + } + } + for (const file of HERMES_ARCHIVE_FILES) { + const candidate = path.join(root, file); + if (await exists(candidate)) { + archivePaths.push({ id: `archive:${file}`, path: candidate, relativePath: file }); + } + } + return { + root, + archivePaths, + ...((await exists(path.join(root, "config.yaml"))) + ? { configPath: path.join(root, "config.yaml") } + : {}), + ...((await exists(path.join(root, ".env"))) ? { envPath: path.join(root, ".env") } : {}), + ...((await exists(path.join(root, "SOUL.md"))) ? { soulPath: path.join(root, "SOUL.md") } : {}), + ...((await exists(path.join(root, "AGENTS.md"))) + ? { agentsPath: path.join(root, "AGENTS.md") } + : {}), + ...((await exists(path.join(root, "memories", "MEMORY.md"))) + ? { memoryPath: path.join(root, "memories", "MEMORY.md") } + : {}), + ...((await exists(path.join(root, "memories", "USER.md"))) + ? { userPath: path.join(root, "memories", "USER.md") } + : {}), + ...((await isDirectory(path.join(root, "skills"))) + ? { skillsDir: path.join(root, "skills") } + : {}), + }; +} + +export function hasHermesSource(source: HermesSource): boolean { + return Boolean( + source.configPath || + source.envPath || + source.soulPath || + source.agentsPath || + source.memoryPath || + source.userPath || + source.skillsDir || + source.archivePaths.length > 0, + ); +} diff --git a/extensions/migrate-hermes/targets.ts b/extensions/migrate-hermes/targets.ts new file mode 100644 index 00000000000..c0e1923f821 --- /dev/null +++ b/extensions/migrate-hermes/targets.ts @@ -0,0 +1,30 @@ +import path from "node:path"; +import { + resolveAgentConfig, + resolveAgentWorkspaceDir, + resolveDefaultAgentId, +} from "openclaw/plugin-sdk/agent-runtime"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import { resolveHomePath } from "./helpers.js"; + +export type PlannedTargets = { + workspaceDir: string; + stateDir: string; + agentDir: string; +}; + +export function resolveTargets(ctx: MigrationProviderContext): PlannedTargets { + const cfg = ctx.config; + const agentId = resolveDefaultAgentId(cfg); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const configuredAgentDir = resolveAgentConfig(cfg, agentId)?.agentDir?.trim(); + const agentDir = + ctx.runtime?.agent?.resolveAgentDir(cfg, agentId) ?? + (configuredAgentDir ? resolveHomePath(configuredAgentDir) : undefined) ?? + path.join(ctx.stateDir, "agents", agentId, "agent"); + return { + workspaceDir, + stateDir: ctx.stateDir, + agentDir, + }; +} diff --git a/extensions/migrate-hermes/test/provider-helpers.ts b/extensions/migrate-hermes/test/provider-helpers.ts new file mode 100644 index 00000000000..ef79994a517 --- /dev/null +++ b/extensions/migrate-hermes/test/provider-helpers.ts @@ -0,0 +1,65 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { MigrationProviderContext } from "openclaw/plugin-sdk/plugin-entry"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-auth"; + +const tempRoots = new Set(); + +export const logger = { + info() {}, + warn() {}, + error() {}, + debug() {}, +}; + +export async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migrate-hermes-")); + tempRoots.add(root); + return root; +} + +export async function cleanupTempRoots() { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); +} + +export async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +export function makeContext(params: { + source: string; + stateDir: string; + workspaceDir: string; + config?: OpenClawConfig; + includeSecrets?: boolean; + overwrite?: boolean; + model?: NonNullable["defaults"]>["model"]; + reportDir?: string; + runtime?: MigrationProviderContext["runtime"]; +}): MigrationProviderContext { + const config = + params.config ?? + ({ + agents: { + defaults: { + workspace: params.workspaceDir, + ...(params.model !== undefined ? { model: params.model } : {}), + }, + }, + } as OpenClawConfig); + return { + config, + stateDir: params.stateDir, + source: params.source, + includeSecrets: params.includeSecrets, + overwrite: params.overwrite, + reportDir: params.reportDir, + runtime: params.runtime, + logger, + }; +} diff --git a/extensions/telegram/src/bot-native-commands.registry.test.ts b/extensions/telegram/src/bot-native-commands.registry.test.ts index acf93f67d30..9cb7abfde05 100644 --- a/extensions/telegram/src/bot-native-commands.registry.test.ts +++ b/extensions/telegram/src/bot-native-commands.registry.test.ts @@ -60,6 +60,7 @@ function createTelegramPluginRegistry() { videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], gatewayHandlers: {}, httpRoutes: [], cliRegistrars: [], diff --git a/package.json b/package.json index e1f45690c76..fd73a7e1fb7 100644 --- a/package.json +++ b/package.json @@ -442,6 +442,14 @@ "types": "./dist/plugin-sdk/logging-core.d.ts", "default": "./dist/plugin-sdk/logging-core.js" }, + "./plugin-sdk/migration": { + "types": "./dist/plugin-sdk/migration.d.ts", + "default": "./dist/plugin-sdk/migration.js" + }, + "./plugin-sdk/migration-runtime": { + "types": "./dist/plugin-sdk/migration-runtime.d.ts", + "default": "./dist/plugin-sdk/migration-runtime.js" + }, "./plugin-sdk/markdown-table-runtime": { "types": "./dist/plugin-sdk/markdown-table-runtime.d.ts", "default": "./dist/plugin-sdk/markdown-table-runtime.js" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52af352ac8f..8fa376a3b50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -877,6 +877,19 @@ importers: specifier: workspace:* version: link:../.. + extensions/migrate-hermes: + dependencies: + yaml: + specifier: ^2.8.3 + version: 2.8.3 + devDependencies: + '@openclaw/plugin-sdk': + specifier: workspace:* + version: link:../../packages/plugin-sdk + openclaw: + specifier: workspace:* + version: link:../.. + extensions/microsoft: dependencies: node-edge-tts: diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index ee0371c4f72..1522d709929 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -94,6 +94,8 @@ "testing", "temp-path", "logging-core", + "migration", + "migration-runtime", "markdown-table-runtime", "account-helpers", "account-core", diff --git a/src/agents/agent-scope.test.ts b/src/agents/agent-scope.test.ts index ea2825a9c60..ad349027db8 100644 --- a/src/agents/agent-scope.test.ts +++ b/src/agents/agent-scope.test.ts @@ -18,6 +18,7 @@ import { resolveAgentWorkspaceDir, resolveAgentIdByWorkspacePath, resolveAgentIdsByWorkspacePath, + setAgentEffectiveModelPrimary, } from "./agent-scope.js"; afterEach(() => { @@ -267,6 +268,59 @@ describe("resolveAgentConfig", () => { ).toEqual([]); }); + it("updates the effective model primary at the winning config layer", () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }, + }, + list: [ + { + id: "linus", + default: true, + model: { + primary: "anthropic/claude-sonnet-4-6", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }, + }, + ], + }, + }; + + expect(setAgentEffectiveModelPrimary(cfg, "linus", "google/gemini-3-pro")).toBe("agent"); + expect(cfg.agents?.list?.[0]?.model).toEqual({ + primary: "google/gemini-3-pro", + fallbacks: ["openrouter/anthropic/claude-opus-4.6"], + }); + expect(cfg.agents?.defaults?.model).toEqual({ + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }); + + const inheritedCfg: OpenClawConfig = { + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }, + }, + list: [{ id: "main", default: true }], + }, + }; + + expect(setAgentEffectiveModelPrimary(inheritedCfg, "main", "google/gemini-3-pro")).toBe( + "defaults", + ); + expect(inheritedCfg.agents?.defaults?.model).toEqual({ + primary: "google/gemini-3-pro", + fallbacks: ["anthropic/claude-sonnet-4-6"], + }); + }); + it("resolves fallback agent id from explicit agent id first", () => { expect( resolveFallbackAgentId({ diff --git a/src/agents/agent-scope.ts b/src/agents/agent-scope.ts index 19bd7ca1240..ee1cc6a44d6 100644 --- a/src/agents/agent-scope.ts +++ b/src/agents/agent-scope.ts @@ -2,6 +2,8 @@ import fs from "node:fs"; import path from "node:path"; import { resolveAgentModelFallbackValues } from "../config/model-input.js"; import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js"; +import type { AgentModelConfig } from "../config/types.agents-shared.js"; +import type { AgentConfig } from "../config/types.agents.js"; import type { OpenClawConfig } from "../config/types.js"; import { normalizeAgentId, @@ -108,6 +110,42 @@ export function resolveAgentEffectiveModelPrimary( ); } +function findMutableAgentEntry(cfg: OpenClawConfig, agentId: string): AgentConfig | undefined { + const id = normalizeAgentId(agentId); + return cfg.agents?.list?.find((entry) => normalizeAgentId(entry?.id) === id); +} + +function updateAgentModelPrimary( + existing: AgentModelConfig | undefined, + primary: string, +): AgentModelConfig { + if (existing && typeof existing === "object" && !Array.isArray(existing)) { + return { ...existing, primary }; + } + return primary; +} + +export type AgentModelPrimaryWriteTarget = "agent" | "defaults"; + +export function setAgentEffectiveModelPrimary( + cfg: OpenClawConfig, + agentId: string, + primary: string, +): AgentModelPrimaryWriteTarget { + const id = normalizeAgentId(agentId); + if (resolveAgentExplicitModelPrimary(cfg, id)) { + const entry = findMutableAgentEntry(cfg, id); + if (entry) { + entry.model = updateAgentModelPrimary(entry.model, primary); + return "agent"; + } + } + cfg.agents ??= {}; + cfg.agents.defaults ??= {}; + cfg.agents.defaults.model = updateAgentModelPrimary(cfg.agents.defaults.model, primary); + return "defaults"; +} + // Backward-compatible alias. Prefer explicit/effective helpers at new call sites. export function resolveAgentModelPrimary(cfg: OpenClawConfig, agentId: string): string | undefined { return resolveAgentExplicitModelPrimary(cfg, agentId); diff --git a/src/cli/command-catalog.ts b/src/cli/command-catalog.ts index a970430dc6c..4b0a5c0e6cb 100644 --- a/src/cli/command-catalog.ts +++ b/src/cli/command-catalog.ts @@ -67,6 +67,7 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ policy: { loadPlugins: "never" }, }, { commandPath: ["configure"], policy: { bypassConfigGuard: true, loadPlugins: "never" } }, + { commandPath: ["migrate"], policy: { bypassConfigGuard: true, loadPlugins: "never" } }, { commandPath: ["status"], policy: { diff --git a/src/cli/program/command-registry-core.ts b/src/cli/program/command-registry-core.ts index 6604387b096..98591862f9b 100644 --- a/src/cli/program/command-registry-core.ts +++ b/src/cli/program/command-registry-core.ts @@ -81,6 +81,11 @@ const coreEntrySpecs: readonly CommandGroupDescriptorSpec< loadModule: () => import("./register.backup.js"), exportName: "registerBackupCommand", }, + { + commandNames: ["migrate"], + loadModule: () => import("./register.migrate.js"), + exportName: "registerMigrateCommand", + }, { commandNames: ["doctor", "dashboard", "reset", "uninstall"], loadModule: () => import("./register.maintenance.js"), diff --git a/src/cli/program/core-command-descriptors.ts b/src/cli/program/core-command-descriptors.ts index 42f916bbe5a..fd996b86bef 100644 --- a/src/cli/program/core-command-descriptors.ts +++ b/src/cli/program/core-command-descriptors.ts @@ -35,6 +35,11 @@ const coreCliCommandCatalog = defineCommandDescriptorCatalog([ description: "Create and verify local backup archives for OpenClaw state", hasSubcommands: true, }, + { + name: "migrate", + description: "Import state from another agent system", + hasSubcommands: true, + }, { name: "doctor", description: "Health checks + quick fixes for the gateway and channels", diff --git a/src/cli/program/register.migrate.ts b/src/cli/program/register.migrate.ts new file mode 100644 index 00000000000..18e9f709a27 --- /dev/null +++ b/src/cli/program/register.migrate.ts @@ -0,0 +1,117 @@ +import type { Command } from "commander"; +import { + migrateApplyCommand, + migrateDefaultCommand, + migrateListCommand, + migratePlanCommand, +} from "../../commands/migrate.js"; +import { defaultRuntime } from "../../runtime.js"; +import { theme } from "../../terminal/theme.js"; +import { runCommandWithRuntime } from "../cli-utils.js"; +import { formatHelpExamples } from "../help-format.js"; + +function addMigrationOptions(command: Command): Command { + return command + .option("--from ", "Source directory to migrate from") + .option("--include-secrets", "Import supported credentials and secrets", false) + .option("--overwrite", "Overwrite conflicting target files after item-level backups", false) + .option("--json", "Output JSON", false); +} + +export function registerMigrateCommand(program: Command) { + const migrate = program + .command("migrate") + .description("Import state from another agent system") + .argument("[provider]", "Migration provider id, for example hermes") + .option("--from ", "Source directory to migrate from") + .option("--include-secrets", "Import supported credentials and secrets", false) + .option("--overwrite", "Overwrite conflicting target files after item-level backups", false) + .option("--dry-run", "Preview only; do not apply changes", false) + .option("--yes", "Apply without prompting after preview", false) + .option("--backup-output ", "Pre-migration backup archive path or directory") + .option("--no-backup", "Skip the pre-migration OpenClaw backup") + .option("--force", "Allow dangerous options such as --no-backup", false) + .option("--json", "Output JSON", false) + .addHelpText( + "after", + () => + `\n${theme.heading("Examples:")}\n${formatHelpExamples([ + ["openclaw migrate list", "Show available migration providers."], + ["openclaw migrate hermes", "Preview Hermes migration, then prompt before applying."], + ["openclaw migrate hermes --dry-run", "Preview Hermes migration only."], + [ + "openclaw migrate apply hermes --yes", + "Apply Hermes migration non-interactively after writing a verified backup.", + ], + [ + "openclaw migrate apply hermes --include-secrets --yes", + "Include supported credentials in the migration.", + ], + ])}`, + ) + .action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateDefaultCommand(defaultRuntime, { + provider: provider as string | undefined, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + dryRun: Boolean(opts.dryRun), + yes: Boolean(opts.yes), + backupOutput: opts.backupOutput as string | undefined, + noBackup: opts.backup === false, + force: Boolean(opts.force), + json: Boolean(opts.json), + }); + }); + }); + + migrate + .command("list") + .description("List migration providers") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateListCommand(defaultRuntime, { json: Boolean(opts.json) }); + }); + }); + + addMigrationOptions( + migrate + .command("plan ") + .description("Preview a migration without changing OpenClaw state"), + ).action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migratePlanCommand(defaultRuntime, { + provider: provider as string, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + json: Boolean(opts.json), + }); + }); + }); + + addMigrationOptions( + migrate.command("apply ").description("Apply a migration after a verified backup"), + ) + .option("--yes", "Apply without prompting", false) + .option("--backup-output ", "Pre-migration backup archive path or directory") + .option("--no-backup", "Skip the pre-migration OpenClaw backup") + .option("--force", "Allow dangerous options such as --no-backup", false) + .action(async (provider, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await migrateApplyCommand(defaultRuntime, { + provider: provider as string, + source: opts.from as string | undefined, + includeSecrets: Boolean(opts.includeSecrets), + overwrite: Boolean(opts.overwrite), + yes: Boolean(opts.yes), + backupOutput: opts.backupOutput as string | undefined, + noBackup: opts.backup === false, + force: Boolean(opts.force), + json: Boolean(opts.json), + }); + }); + }); +} diff --git a/src/cli/program/register.onboard.test.ts b/src/cli/program/register.onboard.test.ts index cefc608339e..c2275717e4a 100644 --- a/src/cli/program/register.onboard.test.ts +++ b/src/cli/program/register.onboard.test.ts @@ -181,6 +181,28 @@ describe("registerOnboardCommand", () => { ); }); + it("forwards onboarding migration flags", async () => { + await runCli([ + "onboard", + "--flow", + "import", + "--import-from", + "hermes", + "--import-source", + "/tmp/hermes", + "--import-secrets", + ]); + expect(setupWizardCommandMock).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "import", + importFrom: "hermes", + importSource: "/tmp/hermes", + importSecrets: true, + }), + runtime, + ); + }); + it("reports errors via runtime on setup wizard command failures", async () => { setupWizardCommandMock.mockRejectedValueOnce(new Error("setup failed")); diff --git a/src/cli/program/register.onboard.ts b/src/cli/program/register.onboard.ts index 75cffcd1b43..9f032358e07 100644 --- a/src/cli/program/register.onboard.ts +++ b/src/cli/program/register.onboard.ts @@ -111,7 +111,7 @@ export function registerOnboardCommand(program: Command) { "Acknowledge that agents are powerful and full system access is risky (required for --non-interactive)", false, ) - .option("--flow ", "Onboard flow: quickstart|advanced|manual") + .option("--flow ", "Onboard flow: quickstart|advanced|manual|import") .option("--mode ", "Onboard mode: local|remote") .option("--auth-choice ", `Auth: ${AUTH_CHOICE_HELP}`) .option( @@ -168,6 +168,9 @@ export function registerOnboardCommand(program: Command) { .option("--skip-health", "Skip health check") .option("--skip-ui", "Skip Control UI/TUI prompts") .option("--node-manager ", "Node manager for skills: npm|pnpm|bun") + .option("--import-from ", "Migration provider to run during onboarding") + .option("--import-source ", "Source agent home for --import-from") + .option("--import-secrets", "Import supported secrets during onboarding migration", false) .option("--json", "Output JSON summary", false); command.action(async (opts, commandRuntime) => { @@ -195,7 +198,7 @@ export function registerOnboardCommand(program: Command) { workspace: opts.workspace as string | undefined, nonInteractive: Boolean(opts.nonInteractive), acceptRisk: Boolean(opts.acceptRisk), - flow: opts.flow as "quickstart" | "advanced" | "manual" | undefined, + flow: opts.flow as "quickstart" | "advanced" | "manual" | "import" | undefined, mode: opts.mode as "local" | "remote" | undefined, authChoice: opts.authChoice as AuthChoice | undefined, tokenProvider: opts.tokenProvider as string | undefined, @@ -235,6 +238,9 @@ export function registerOnboardCommand(program: Command) { skipHealth: Boolean(opts.skipHealth), skipUi: Boolean(opts.skipUi), nodeManager: opts.nodeManager as NodeManagerChoice | undefined, + importFrom: opts.importFrom as string | undefined, + importSource: opts.importSource as string | undefined, + importSecrets: Boolean(opts.importSecrets), json: Boolean(opts.json), }, defaultRuntime, diff --git a/src/cli/program/register.setup.test.ts b/src/cli/program/register.setup.test.ts index c8b7ceacacf..a293ea35f51 100644 --- a/src/cli/program/register.setup.test.ts +++ b/src/cli/program/register.setup.test.ts @@ -79,6 +79,27 @@ describe("registerSetupCommand", () => { expect(setupCommandMock).not.toHaveBeenCalled(); }); + it("runs setup wizard command for migration import flags", async () => { + await runCli([ + "setup", + "--import-from", + "hermes", + "--import-source", + "/tmp/hermes", + "--import-secrets", + ]); + + expect(setupWizardCommandMock).toHaveBeenCalledWith( + expect.objectContaining({ + importFrom: "hermes", + importSource: "/tmp/hermes", + importSecrets: true, + }), + runtime, + ); + expect(setupCommandMock).not.toHaveBeenCalled(); + }); + it("reports setup errors through runtime", async () => { setupCommandMock.mockRejectedValueOnce(new Error("setup failed")); diff --git a/src/cli/program/register.setup.ts b/src/cli/program/register.setup.ts index 84b698c863c..0dc980189a0 100644 --- a/src/cli/program/register.setup.ts +++ b/src/cli/program/register.setup.ts @@ -23,6 +23,9 @@ export function registerSetupCommand(program: Command) { .option("--wizard", "Run interactive onboarding", false) .option("--non-interactive", "Run onboarding without prompts", false) .option("--mode ", "Onboard mode: local|remote") + .option("--import-from ", "Migration provider to run during onboarding") + .option("--import-source ", "Source agent home for --import-from") + .option("--import-secrets", "Import supported secrets during onboarding migration", false) .option("--remote-url ", "Remote Gateway WebSocket URL") .option("--remote-token ", "Remote Gateway token (optional)") .action(async (opts, command) => { @@ -31,6 +34,9 @@ export function registerSetupCommand(program: Command) { "wizard", "nonInteractive", "mode", + "importFrom", + "importSource", + "importSecrets", "remoteUrl", "remoteToken", ]); @@ -40,6 +46,9 @@ export function registerSetupCommand(program: Command) { workspace: opts.workspace as string | undefined, nonInteractive: Boolean(opts.nonInteractive), mode: opts.mode as "local" | "remote" | undefined, + importFrom: opts.importFrom as string | undefined, + importSource: opts.importSource as string | undefined, + importSecrets: Boolean(opts.importSecrets), remoteUrl: opts.remoteUrl as string | undefined, remoteToken: opts.remoteToken as string | undefined, }, diff --git a/src/commands/migrate.test.ts b/src/commands/migrate.test.ts new file mode 100644 index 00000000000..078c22f84ac --- /dev/null +++ b/src/commands/migrate.test.ts @@ -0,0 +1,471 @@ +import fs from "node:fs/promises"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { MigrationApplyResult, MigrationPlan } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; + +const mocks = vi.hoisted(() => ({ + backupCreateCommand: vi.fn(), + promptYesNo: vi.fn(), + provider: { + id: "hermes", + label: "Hermes", + plan: vi.fn(), + apply: vi.fn(), + }, +})); + +vi.mock("../config/config.js", () => ({ + loadConfig: () => ({}), +})); + +vi.mock("../config/paths.js", () => ({ + resolveStateDir: () => "/tmp/openclaw-migrate-command-test", +})); + +vi.mock("../cli/prompt.js", () => ({ + promptYesNo: mocks.promptYesNo, +})); + +vi.mock("../plugins/migration-provider-runtime.js", () => ({ + resolvePluginMigrationProvider: () => mocks.provider, + resolvePluginMigrationProviders: () => [mocks.provider], +})); + +vi.mock("./backup.js", () => ({ + backupCreateCommand: mocks.backupCreateCommand, +})); + +const { migrateApplyCommand, migrateDefaultCommand } = await import("./migrate.js"); + +function plan(overrides: Partial = {}): MigrationPlan { + return { + providerId: "hermes", + source: "/tmp/hermes", + summary: { + total: 1, + planned: 1, + migrated: 0, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [{ id: "workspace:AGENTS.md", kind: "workspace", action: "copy", status: "planned" }], + ...overrides, + }; +} + +const runtime: RuntimeEnv = { + log: vi.fn(), + error: vi.fn(), + exit(code) { + throw new Error(`exit ${code}`); + }, +}; + +describe("migrateApplyCommand", () => { + const originalIsTty = process.stdin.isTTY; + + beforeEach(async () => { + await fs.rm("/tmp/openclaw-migrate-command-test", { force: true, recursive: true }); + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: false, + }); + mocks.provider.plan.mockReset(); + mocks.provider.apply.mockReset(); + mocks.promptYesNo.mockReset(); + mocks.backupCreateCommand.mockReset(); + mocks.backupCreateCommand.mockResolvedValue({ archivePath: "/tmp/openclaw-backup.tgz" }); + }); + + afterEach(async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: originalIsTty, + }); + await fs.rm("/tmp/openclaw-migrate-command-test", { force: true, recursive: true }); + vi.clearAllMocks(); + }); + + it("requires explicit force before skipping the pre-migration backup", async () => { + await expect( + migrateApplyCommand(runtime, { provider: "hermes", yes: true, noBackup: true }), + ).rejects.toThrow("--no-backup requires --force"); + expect(mocks.provider.plan).not.toHaveBeenCalled(); + }); + + it("requires --yes in non-interactive apply mode", async () => { + await expect(migrateApplyCommand(runtime, { provider: "hermes" })).rejects.toThrow( + "requires --yes", + ); + expect(mocks.provider.plan).not.toHaveBeenCalled(); + }); + + it("previews and prompts before interactive apply without --yes", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + mocks.promptYesNo.mockResolvedValue(true); + + await migrateApplyCommand(runtime, { provider: "hermes" }); + + expect(mocks.provider.plan).toHaveBeenCalledTimes(1); + expect(mocks.promptYesNo).toHaveBeenCalledWith("Apply this migration now?", false); + expect(mocks.backupCreateCommand).toHaveBeenCalled(); + expect(mocks.provider.apply).toHaveBeenCalledWith(expect.any(Object), planned); + }); + + it("does not apply when interactive apply confirmation is declined", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan(); + mocks.provider.plan.mockResolvedValue(planned); + mocks.promptYesNo.mockResolvedValue(false); + + const result = await migrateApplyCommand(runtime, { provider: "hermes", overwrite: true }); + + expect(result).toBe(planned); + expect(mocks.promptYesNo).toHaveBeenCalledWith("Apply this migration now?", false); + expect(runtime.log).toHaveBeenCalledWith("Migration cancelled."); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("prints a JSON plan without applying when interactive apply uses --json without --yes", async () => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: true, + }); + const planned = plan({ + items: [ + { + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "planned", + details: { + value: { + time: { + env: { OPENAI_API_KEY: "short-dev-key", SAFE_FLAG: "visible" }, + headers: { Authorization: "Bearer short-dev-key" }, + }, + }, + }, + }, + ], + }); + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + + const result = await migrateApplyCommand(jsonRuntime, { + provider: "hermes", + json: true, + }); + + expect(result).toBe(planned); + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { planned: 1 }, + items: [ + { + details: { + value: { + time: { + env: { OPENAI_API_KEY: "[redacted]", SAFE_FLAG: "visible" }, + headers: { Authorization: "[redacted]" }, + }, + }, + }, + }, + ], + }); + expect(logs[0]).not.toContain("short-dev-key"); + expect(mocks.promptYesNo).not.toHaveBeenCalled(); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("does not create a backup or apply when the preflight plan has conflicts", async () => { + mocks.provider.plan.mockResolvedValue( + plan({ + summary: { + total: 1, + planned: 0, + migrated: 0, + skipped: 0, + conflicts: 1, + errors: 0, + sensitive: 0, + }, + items: [ + { + id: "workspace:SOUL.md", + kind: "workspace", + action: "copy", + status: "conflict", + }, + ], + }), + ); + + await expect(migrateApplyCommand(runtime, { provider: "hermes", yes: true })).rejects.toThrow( + "Migration has 1 conflict", + ); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + }); + + it("creates a verified backup before applying a conflict-free migration", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + const result = await migrateApplyCommand(runtime, { provider: "hermes", yes: true }); + + expect(mocks.backupCreateCommand).toHaveBeenCalledWith( + expect.objectContaining({ log: expect.any(Function) }), + { output: undefined, verify: true }, + ); + expect(mocks.provider.apply).toHaveBeenCalledWith( + expect.objectContaining({ + backupPath: "/tmp/openclaw-backup.tgz", + reportDir: expect.stringContaining("/migration/hermes/"), + }), + planned, + ); + expect(result.backupPath).toBe("/tmp/openclaw-backup.tgz"); + }); + + it("prints only the final result for root apply in JSON mode", async () => { + const planned = plan({ + items: [ + { + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "planned", + details: { + value: { + time: { + env: { OPENAI_API_KEY: "short-dev-key" }, + headers: { "x-api-key": "another-short-dev-key" }, + }, + }, + }, + }, + ], + }); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await migrateDefaultCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + backupPath: "/tmp/openclaw-backup.tgz", + items: [ + { + details: { + value: { + time: { + env: { OPENAI_API_KEY: "[redacted]" }, + headers: { "x-api-key": "[redacted]" }, + }, + }, + }, + }, + ], + }); + expect(logs[0]).not.toContain("short-dev-key"); + expect(logs[0]).not.toContain("another-short-dev-key"); + expect(logs[0]).not.toContain("Migration plan"); + }); + + it("keeps provider info logs off stdout in JSON mode", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + const logs: string[] = []; + const errors: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + error(message) { + errors.push(String(message)); + }, + }; + mocks.provider.plan.mockImplementation(async (ctx) => { + ctx.logger.info("provider planning"); + return planned; + }); + mocks.provider.apply.mockImplementation(async (ctx) => { + ctx.logger.info("provider applying"); + return applied; + }); + + await migrateDefaultCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ providerId: "hermes" }); + expect(errors).toEqual(["provider planning", "provider applying"]); + }); + + it("applies the already-reviewed default plan instead of planning again", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { ...planned.summary, planned: 0, migrated: 1 }, + items: planned.items.map((item) => ({ ...item, status: "migrated" })), + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await migrateDefaultCommand(runtime, { provider: "hermes", yes: true }); + + expect(mocks.provider.plan).toHaveBeenCalledTimes(1); + expect(mocks.provider.apply).toHaveBeenCalledWith(expect.any(Object), planned); + }); + + it("fails after writing JSON output when apply reports item errors", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { + ...planned.summary, + planned: 0, + errors: 1, + }, + items: planned.items.map((item) => ({ + ...item, + status: "error", + reason: "copy failed", + })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await expect( + migrateApplyCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }), + ).rejects.toThrow("Migration finished with 1 error"); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { errors: 1 }, + reportDir: expect.stringContaining("/migration/hermes/"), + }); + }); + + it("fails after writing JSON output when apply reports late conflicts", async () => { + const planned = plan(); + const applied: MigrationApplyResult = { + ...planned, + summary: { + ...planned.summary, + planned: 0, + conflicts: 1, + }, + items: planned.items.map((item) => ({ + ...item, + status: "conflict", + reason: "target exists", + })), + }; + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + mocks.provider.apply.mockResolvedValue(applied); + + await expect( + migrateApplyCommand(jsonRuntime, { provider: "hermes", yes: true, json: true }), + ).rejects.toThrow("Migration finished with 1 conflict"); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { conflicts: 1 }, + reportDir: expect.stringContaining("/migration/hermes/"), + }); + }); + + it("prints the dry-run plan in JSON mode even when --yes is set", async () => { + const planned = plan(); + const logs: string[] = []; + const jsonRuntime: RuntimeEnv = { + ...runtime, + log(message) { + logs.push(String(message)); + }, + }; + mocks.provider.plan.mockResolvedValue(planned); + + await migrateDefaultCommand(jsonRuntime, { + provider: "hermes", + yes: true, + dryRun: true, + json: true, + }); + + expect(logs).toHaveLength(1); + expect(JSON.parse(logs[0] ?? "{}")).toMatchObject({ + providerId: "hermes", + summary: { planned: 1 }, + }); + expect(mocks.provider.apply).not.toHaveBeenCalled(); + expect(mocks.backupCreateCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/migrate.ts b/src/commands/migrate.ts new file mode 100644 index 00000000000..dc104de837f --- /dev/null +++ b/src/commands/migrate.ts @@ -0,0 +1,162 @@ +import { promptYesNo } from "../cli/prompt.js"; +import { loadConfig } from "../config/config.js"; +import { redactMigrationPlan } from "../plugin-sdk/migration.js"; +import { resolvePluginMigrationProviders } from "../plugins/migration-provider-runtime.js"; +import type { MigrationApplyResult, MigrationPlan } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { writeRuntimeJson } from "../runtime.js"; +import { runMigrationApply } from "./migrate/apply.js"; +import { formatMigrationPlan } from "./migrate/output.js"; +import { createMigrationPlan, resolveMigrationProvider } from "./migrate/providers.js"; +import type { + MigrateApplyOptions, + MigrateCommonOptions, + MigrateDefaultOptions, +} from "./migrate/types.js"; + +export type { MigrateApplyOptions, MigrateCommonOptions, MigrateDefaultOptions }; + +export async function migrateListCommand(runtime: RuntimeEnv, opts: { json?: boolean } = {}) { + const providers = resolvePluginMigrationProviders({ cfg: loadConfig() }).map((provider) => ({ + id: provider.id, + label: provider.label, + description: provider.description, + })); + if (opts.json) { + writeRuntimeJson(runtime, { providers }); + return; + } + if (providers.length === 0) { + runtime.log("No migration providers found."); + return; + } + runtime.log( + providers + .map((provider) => + provider.description + ? `${provider.id}\t${provider.label} - ${provider.description}` + : `${provider.id}\t${provider.label}`, + ) + .join("\n"), + ); +} + +export async function migratePlanCommand( + runtime: RuntimeEnv, + opts: MigrateCommonOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + throw new Error("Migration provider is required."); + } + const plan = await createMigrationPlan(runtime, { ...opts, provider: providerId }); + if (opts.json) { + writeRuntimeJson(runtime, redactMigrationPlan(plan)); + } else { + runtime.log(formatMigrationPlan(plan).join("\n")); + } + return plan; +} + +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions & { yes: true }, +): Promise; +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, +): Promise; +export async function migrateApplyCommand( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + throw new Error("Migration provider is required."); + } + if (opts.noBackup && !opts.force) { + throw new Error("--no-backup requires --force."); + } + if (!opts.yes && !process.stdin.isTTY) { + throw new Error("openclaw migrate apply requires --yes in non-interactive mode."); + } + const provider = resolveMigrationProvider(providerId); + if (!opts.yes) { + const plan = await migratePlanCommand(runtime, { + ...opts, + provider: providerId, + json: opts.json, + }); + if (opts.json) { + return plan; + } + const ok = await promptYesNo("Apply this migration now?", false); + if (!ok) { + runtime.log("Migration cancelled."); + return plan; + } + return await runMigrationApply({ + runtime, + opts: { ...opts, provider: providerId, yes: true, preflightPlan: plan }, + providerId, + provider, + }); + } + return await runMigrationApply({ runtime, opts, providerId, provider }); +} + +export async function migrateDefaultCommand( + runtime: RuntimeEnv, + opts: MigrateDefaultOptions, +): Promise { + const providerId = opts.provider?.trim(); + if (!providerId) { + await migrateListCommand(runtime, { json: opts.json }); + return { + providerId: "list", + source: "", + summary: { + total: 0, + planned: 0, + migrated: 0, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [], + }; + } + const plan = + opts.json && opts.yes && !opts.dryRun + ? await createMigrationPlan(runtime, { ...opts, provider: providerId }) + : await migratePlanCommand(runtime, { + ...opts, + provider: providerId, + json: opts.json && (opts.dryRun || !opts.yes), + }); + if (opts.dryRun) { + return plan; + } + if (opts.json && !opts.yes) { + return plan; + } + if (!opts.yes) { + if (!process.stdin.isTTY) { + runtime.log("Re-run with --yes to apply this migration non-interactively."); + return plan; + } + const ok = await promptYesNo("Apply this migration now?", false); + if (!ok) { + runtime.log("Migration cancelled."); + return plan; + } + } + return await migrateApplyCommand(runtime, { + ...opts, + provider: providerId, + yes: true, + json: opts.json, + preflightPlan: plan, + }); +} diff --git a/src/commands/migrate/apply.ts b/src/commands/migrate/apply.ts new file mode 100644 index 00000000000..bc88412fd6f --- /dev/null +++ b/src/commands/migrate/apply.ts @@ -0,0 +1,86 @@ +import fs from "node:fs/promises"; +import { resolveStateDir } from "../../config/paths.js"; +import type { MigrationApplyResult, MigrationProviderPlugin } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { backupCreateCommand } from "../backup.js"; +import { buildMigrationContext, buildMigrationReportDir } from "./context.js"; +import { assertApplySucceeded, assertConflictFreePlan, writeApplyResult } from "./output.js"; +import type { MigrateApplyOptions } from "./types.js"; + +function shouldTreatMissingBackupAsEmptyState(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.includes("No local OpenClaw state was found to back up") || + message.includes("No OpenClaw config file was found to back up") + ); +} + +export async function createPreMigrationBackup(opts: { + output?: string; +}): Promise { + try { + const result = await backupCreateCommand( + { + log() {}, + error() {}, + exit(code) { + throw new Error(`backup exited with ${code}`); + }, + }, + { + output: opts.output, + verify: true, + }, + ); + return result.archivePath; + } catch (err) { + if (shouldTreatMissingBackupAsEmptyState(err)) { + return undefined; + } + throw err; + } +} + +export async function runMigrationApply(params: { + runtime: RuntimeEnv; + opts: MigrateApplyOptions; + providerId: string; + provider: MigrationProviderPlugin; +}): Promise { + const preflightPlan = + params.opts.preflightPlan ?? + (await params.provider.plan( + buildMigrationContext({ + source: params.opts.source, + includeSecrets: params.opts.includeSecrets, + overwrite: params.opts.overwrite, + runtime: params.runtime, + json: params.opts.json, + }), + )); + assertConflictFreePlan(preflightPlan, params.providerId); + const stateDir = resolveStateDir(); + const reportDir = buildMigrationReportDir(params.providerId, stateDir); + const backupPath = params.opts.noBackup + ? undefined + : await createPreMigrationBackup({ output: params.opts.backupOutput }); + await fs.mkdir(reportDir, { recursive: true }); + const ctx = buildMigrationContext({ + source: params.opts.source, + includeSecrets: params.opts.includeSecrets, + overwrite: params.opts.overwrite, + runtime: params.runtime, + backupPath, + reportDir, + json: params.opts.json, + }); + const result = await params.provider.apply(ctx, preflightPlan); + const withBackup = { + ...result, + backupPath: result.backupPath ?? backupPath, + reportDir: result.reportDir ?? reportDir, + }; + writeApplyResult(params.runtime, params.opts, withBackup); + assertApplySucceeded(withBackup); + return withBackup; +} diff --git a/src/commands/migrate/context.ts b/src/commands/migrate/context.ts new file mode 100644 index 00000000000..b51dca7a3c6 --- /dev/null +++ b/src/commands/migrate/context.ts @@ -0,0 +1,51 @@ +import path from "node:path"; +import { loadConfig } from "../../config/config.js"; +import { resolveStateDir } from "../../config/paths.js"; +import type { MigrationProviderContext } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; + +export function createMigrationLogger(runtime: RuntimeEnv, opts: { json?: boolean } = {}) { + const info = opts.json ? runtime.error : runtime.log; + return { + debug: (message: string) => { + if (process.env.OPENCLAW_VERBOSE === "1") { + info(message); + } + }, + info: (message: string) => info(message), + warn: (message: string) => runtime.error(message), + error: (message: string) => runtime.error(message), + }; +} + +export function buildMigrationReportDir( + providerId: string, + stateDir: string, + nowMs = Date.now(), +): string { + const stamp = new Date(nowMs).toISOString().replaceAll(":", "-"); + return path.join(stateDir, "migration", providerId, stamp); +} + +export function buildMigrationContext(params: { + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + backupPath?: string; + runtime: RuntimeEnv; + reportDir?: string; + json?: boolean; +}): MigrationProviderContext { + const config = loadConfig(); + const stateDir = resolveStateDir(); + return { + config, + stateDir, + source: params.source, + includeSecrets: Boolean(params.includeSecrets), + overwrite: Boolean(params.overwrite), + backupPath: params.backupPath, + reportDir: params.reportDir, + logger: createMigrationLogger(params.runtime, { json: params.json }), + }; +} diff --git a/src/commands/migrate/output.ts b/src/commands/migrate/output.ts new file mode 100644 index 00000000000..03415e82bdc --- /dev/null +++ b/src/commands/migrate/output.ts @@ -0,0 +1,103 @@ +import type { MigrationApplyResult, MigrationItem, MigrationPlan } from "../../plugins/types.js"; +import { redactMigrationPlan } from "../../plugin-sdk/migration.js"; +import { writeRuntimeJson } from "../../runtime.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { theme } from "../../terminal/theme.js"; +import type { MigrateApplyOptions } from "./types.js"; + +export function formatCount(value: number, label: string): string { + return `${value} ${label}${value === 1 ? "" : "s"}`; +} + +export function formatMigrationPlan(plan: MigrationPlan): string[] { + const lines = [ + `${theme.heading("Migration plan:")} ${plan.providerId}`, + `Source: ${plan.source}`, + ]; + if (plan.target) { + lines.push(`Target: ${plan.target}`); + } + lines.push( + [ + formatCount(plan.summary.total, "item"), + formatCount(plan.summary.conflicts, "conflict"), + formatCount(plan.summary.sensitive, "sensitive item"), + ].join(", "), + ); + if (plan.warnings && plan.warnings.length > 0) { + lines.push(""); + lines.push(theme.warn("Warnings:")); + for (const warning of plan.warnings) { + lines.push(`- ${warning}`); + } + } + const visibleItems = plan.items.slice(0, 25); + if (visibleItems.length > 0) { + lines.push(""); + lines.push(theme.heading("Items:")); + for (const item of visibleItems) { + lines.push(formatMigrationItem(item)); + } + if (plan.items.length > visibleItems.length) { + lines.push(`- ... ${plan.items.length - visibleItems.length} more`); + } + } + if (plan.nextSteps && plan.nextSteps.length > 0) { + lines.push(""); + lines.push(theme.heading("Next:")); + for (const step of plan.nextSteps) { + lines.push(`- ${step}`); + } + } + return lines; +} + +export function formatMigrationItem(item: MigrationItem): string { + const target = item.target ? ` -> ${item.target}` : ""; + const message = item.message ? ` (${item.message})` : item.reason ? ` (${item.reason})` : ""; + const sensitive = item.sensitive ? " [sensitive]" : ""; + return `- ${item.status}: ${item.kind}/${item.action} ${item.id}${target}${sensitive}${message}`; +} + +export function assertConflictFreePlan(plan: MigrationPlan, providerId: string): void { + if (plan.summary.conflicts > 0) { + throw new Error( + `Migration has ${formatCount(plan.summary.conflicts, "conflict")}. Re-run with --overwrite after reviewing openclaw migrate plan ${providerId}.`, + ); + } +} + +export function writeApplyResult( + runtime: RuntimeEnv, + opts: MigrateApplyOptions, + result: MigrationApplyResult, +): void { + if (opts.json) { + writeRuntimeJson(runtime, redactMigrationPlan(result)); + return; + } + runtime.log(formatMigrationPlan(result).join("\n")); + if (result.backupPath) { + runtime.log(`Backup: ${result.backupPath}`); + } else if (!opts.noBackup) { + runtime.log("Backup: skipped (no existing OpenClaw state found)"); + } + if (result.reportDir) { + runtime.log(`Report: ${result.reportDir}`); + } +} + +export function assertApplySucceeded(result: MigrationApplyResult): void { + if (result.summary.errors === 0 && result.summary.conflicts === 0) { + return; + } + const reportHint = result.reportDir ? ` See report: ${result.reportDir}.` : ""; + if (result.summary.errors > 0) { + throw new Error( + `Migration finished with ${formatCount(result.summary.errors, "error")}.${reportHint}`, + ); + } + throw new Error( + `Migration finished with ${formatCount(result.summary.conflicts, "conflict")}.${reportHint}`, + ); +} diff --git a/src/commands/migrate/providers.ts b/src/commands/migrate/providers.ts new file mode 100644 index 00000000000..ed2012c62cc --- /dev/null +++ b/src/commands/migrate/providers.ts @@ -0,0 +1,38 @@ +import { loadConfig } from "../../config/config.js"; +import { + resolvePluginMigrationProvider, + resolvePluginMigrationProviders, +} from "../../plugins/migration-provider-runtime.js"; +import type { MigrationPlan, MigrationProviderPlugin } from "../../plugins/types.js"; +import type { RuntimeEnv } from "../../runtime.js"; +import { buildMigrationContext } from "./context.js"; +import type { MigrateCommonOptions } from "./types.js"; + +export function resolveMigrationProvider(providerId: string): MigrationProviderPlugin { + const config = loadConfig(); + const provider = resolvePluginMigrationProvider({ providerId, cfg: config }); + if (!provider) { + const available = resolvePluginMigrationProviders({ cfg: config }).map((entry) => entry.id); + const suffix = + available.length > 0 + ? ` Available providers: ${available.join(", ")}.` + : " No providers found."; + throw new Error(`Unknown migration provider "${providerId}".${suffix}`); + } + return provider; +} + +export async function createMigrationPlan( + runtime: RuntimeEnv, + opts: MigrateCommonOptions & { provider: string }, +): Promise { + const provider = resolveMigrationProvider(opts.provider); + const ctx = buildMigrationContext({ + source: opts.source, + includeSecrets: opts.includeSecrets, + overwrite: opts.overwrite, + runtime, + json: opts.json, + }); + return await provider.plan(ctx); +} diff --git a/src/commands/migrate/types.ts b/src/commands/migrate/types.ts new file mode 100644 index 00000000000..6b9bbfcf29c --- /dev/null +++ b/src/commands/migrate/types.ts @@ -0,0 +1,21 @@ +import type { MigrationPlan } from "../../plugins/types.js"; + +export type MigrateCommonOptions = { + provider?: string; + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + json?: boolean; +}; + +export type MigrateApplyOptions = MigrateCommonOptions & { + yes?: boolean; + noBackup?: boolean; + force?: boolean; + backupOutput?: string; + preflightPlan?: MigrationPlan; +}; + +export type MigrateDefaultOptions = MigrateApplyOptions & { + dryRun?: boolean; +}; diff --git a/src/commands/onboard-types.ts b/src/commands/onboard-types.ts index 91672cea0d8..c1ee73a8ea6 100644 --- a/src/commands/onboard-types.ts +++ b/src/commands/onboard-types.ts @@ -35,7 +35,7 @@ type OnboardDynamicProviderOptions = { export type OnboardOptions = OnboardDynamicProviderOptions & { mode?: OnboardMode; /** "manual" is an alias for "advanced". */ - flow?: "quickstart" | "advanced" | "manual"; + flow?: "quickstart" | "advanced" | "manual" | "import"; workspace?: string; nonInteractive?: boolean; /** Required for non-interactive setup; skips the interactive risk prompt when true. */ @@ -83,5 +83,8 @@ export type OnboardOptions = OnboardDynamicProviderOptions & { nodeManager?: NodeManagerChoice; remoteUrl?: string; remoteToken?: string; + importFrom?: string; + importSource?: string; + importSecrets?: boolean; json?: boolean; }; diff --git a/src/flows/channel-setup.test.ts b/src/flows/channel-setup.test.ts index 6f958836ad6..dcbc98c2663 100644 --- a/src/flows/channel-setup.test.ts +++ b/src/flows/channel-setup.test.ts @@ -53,6 +53,7 @@ function makePluginRegistry(overrides: Partial = {}): PluginRegi authRequirements: [], webSearchProviders: [], webFetchProviders: [], + migrationProviders: [], mediaUnderstandingProviders: [], imageGenerationProviders: [], videoGenerationProviders: [], diff --git a/src/gateway/server-plugins.test.ts b/src/gateway/server-plugins.test.ts index 5fdcb06d898..00e50107428 100644 --- a/src/gateway/server-plugins.test.ts +++ b/src/gateway/server-plugins.test.ts @@ -87,6 +87,7 @@ const createRegistry = (diagnostics: PluginDiagnostic[]): PluginRegistry => ({ videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], memoryEmbeddingProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], diff --git a/src/gateway/test-helpers.plugin-registry.ts b/src/gateway/test-helpers.plugin-registry.ts index 8aa03fd6f2d..d78e6cf91fd 100644 --- a/src/gateway/test-helpers.plugin-registry.ts +++ b/src/gateway/test-helpers.plugin-registry.ts @@ -22,6 +22,7 @@ function createStubPluginRegistry(): PluginRegistry { musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugin-sdk/agent-runtime.ts b/src/plugin-sdk/agent-runtime.ts index 13ed82bb22b..6781f9718be 100644 --- a/src/plugin-sdk/agent-runtime.ts +++ b/src/plugin-sdk/agent-runtime.ts @@ -38,6 +38,7 @@ export { suggestOAuthProfileIdForLegacyDefault, clearRuntimeAuthProfileStoreSnapshots, ensureAuthProfileStore, + loadAuthProfileStoreWithoutExternalProfiles, loadAuthProfileStoreForSecretsRuntime, loadAuthProfileStoreForRuntime, replaceRuntimeAuthProfileStoreSnapshots, diff --git a/src/plugin-sdk/migration-runtime.test.ts b/src/plugin-sdk/migration-runtime.test.ts new file mode 100644 index 00000000000..ed7d648b62e --- /dev/null +++ b/src/plugin-sdk/migration-runtime.test.ts @@ -0,0 +1,123 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { copyMigrationFileItem, writeMigrationReport } from "./migration-runtime.js"; +import { createMigrationItem } from "./migration.js"; + +async function writeFile(filePath: string, contents: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents, "utf8"); +} + +describe("copyMigrationFileItem", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("uses unique backup paths for same-basename targets in the same millisecond", async () => { + vi.spyOn(Date, "now").mockReturnValue(123); + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migration-runtime-")); + const reportDir = path.join(root, "report"); + const sourceOne = path.join(root, "source-one", "AGENTS.md"); + const sourceTwo = path.join(root, "source-two", "AGENTS.md"); + const targetOne = path.join(root, "target-one", "AGENTS.md"); + const targetTwo = path.join(root, "target-two", "AGENTS.md"); + + await writeFile(sourceOne, "new one"); + await writeFile(sourceTwo, "new two"); + await writeFile(targetOne, "old one"); + await writeFile(targetTwo, "old two"); + + const first = await copyMigrationFileItem( + createMigrationItem({ + id: "first", + kind: "file", + action: "copy", + source: sourceOne, + target: targetOne, + }), + reportDir, + { overwrite: true }, + ); + const second = await copyMigrationFileItem( + createMigrationItem({ + id: "second", + kind: "file", + action: "copy", + source: sourceTwo, + target: targetTwo, + }), + reportDir, + { overwrite: true }, + ); + + expect(first.status).toBe("migrated"); + expect(second.status).toBe("migrated"); + const firstBackup = first.details?.backupPath; + const secondBackup = second.details?.backupPath; + expect(firstBackup).toEqual(expect.stringContaining("AGENTS.md")); + expect(secondBackup).toEqual(expect.stringContaining("AGENTS.md")); + expect(firstBackup).not.toBe(secondBackup); + await expect(fs.readFile(firstBackup as string, "utf8")).resolves.toBe("old one"); + await expect(fs.readFile(secondBackup as string, "utf8")).resolves.toBe("old two"); + }); +}); + +describe("writeMigrationReport", () => { + it("redacts nested secret-looking config values in JSON reports", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-migration-report-")); + const reportDir = path.join(root, "report"); + + await writeMigrationReport({ + providerId: "hermes", + source: path.join(root, "hermes"), + summary: { + total: 1, + planned: 0, + migrated: 1, + skipped: 0, + conflicts: 0, + errors: 0, + sensitive: 0, + }, + items: [ + createMigrationItem({ + id: "config:mcp-servers", + kind: "config", + action: "merge", + status: "migrated", + details: { + value: { + mcp: { + env: { + OPENAI_API_KEY: "short-dev-key", + SAFE_FLAG: "visible", + }, + headers: { + Authorization: "Bearer short-dev-key", + "x-api-key": "another-short-dev-key", + }, + }, + }, + }, + }), + ], + reportDir, + }); + + const report = await fs.readFile(path.join(reportDir, "report.json"), "utf8"); + expect(report).not.toContain("short-dev-key"); + expect(report).not.toContain("another-short-dev-key"); + expect(JSON.parse(report).items[0].details.value.mcp).toEqual({ + env: { + OPENAI_API_KEY: "[redacted]", + SAFE_FLAG: "visible", + }, + headers: { + Authorization: "[redacted]", + "x-api-key": "[redacted]", + }, + }); + }); +}); diff --git a/src/plugin-sdk/migration-runtime.ts b/src/plugin-sdk/migration-runtime.ts new file mode 100644 index 00000000000..e8f97988c72 --- /dev/null +++ b/src/plugin-sdk/migration-runtime.ts @@ -0,0 +1,186 @@ +// Runtime helpers for migration providers that need filesystem side effects. + +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MigrationApplyResult, MigrationItem } from "../plugins/types.js"; +import { + MIGRATION_REASON_MISSING_SOURCE_OR_TARGET, + MIGRATION_REASON_TARGET_EXISTS, + markMigrationItemConflict, + markMigrationItemError, + redactMigrationPlan, +} from "./migration.js"; + +export type { MigrationApplyResult, MigrationItem } from "../plugins/types.js"; + +async function exists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function backupExistingMigrationTarget( + target: string, + reportDir: string, +): Promise { + if (!(await exists(target))) { + return undefined; + } + const backupRoot = path.join(reportDir, "item-backups"); + await fs.mkdir(backupRoot, { recursive: true }); + const targetHash = crypto + .createHash("sha256") + .update(path.resolve(target)) + .digest("hex") + .slice(0, 12); + const backupDir = await fs.mkdtemp( + path.join(backupRoot, `${Date.now()}-${targetHash}-${path.basename(target)}-`), + ); + const backupPath = path.join(backupDir, path.basename(target)); + await fs.cp(target, backupPath, { recursive: true, force: true }); + return backupPath; +} + +function isFileAlreadyExistsError(err: unknown): boolean { + return Boolean( + err && + typeof err === "object" && + "code" in err && + ((err as { code?: unknown }).code === "ERR_FS_CP_EEXIST" || + (err as { code?: unknown }).code === "EEXIST"), + ); +} + +function readArchiveRelativePath(item: MigrationItem): string { + const detailPath = item.details?.archiveRelativePath; + const raw = typeof detailPath === "string" && detailPath.trim() ? detailPath : undefined; + const fallback = item.source ? path.basename(item.source) : item.id; + const normalized = path + .normalize(raw ?? fallback) + .split(path.sep) + .filter((part) => part && part !== "." && part !== "..") + .join(path.sep); + return normalized || "item"; +} + +async function resolveUniqueArchivePath( + archiveRoot: string, + relativePath: string, +): Promise { + const parsed = path.parse(relativePath); + let candidate = path.join(archiveRoot, relativePath); + let index = 2; + while (await exists(candidate)) { + const filename = `${parsed.name}-${index}${parsed.ext}`; + candidate = path.join(archiveRoot, parsed.dir, filename); + index += 1; + } + return candidate; +} + +export async function archiveMigrationItem( + item: MigrationItem, + reportDir: string, +): Promise { + if (!item.source) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const sourceStat = await fs.lstat(item.source); + if (sourceStat.isSymbolicLink()) { + return markMigrationItemError(item, "archive source is a symlink"); + } + const archiveRoot = path.join(reportDir, "archive"); + const relativePath = readArchiveRelativePath(item); + const archivePath = await resolveUniqueArchivePath(archiveRoot, relativePath); + await fs.mkdir(path.dirname(archivePath), { recursive: true }); + await fs.cp(item.source, archivePath, { + recursive: true, + force: false, + errorOnExist: true, + verbatimSymlinks: true, + }); + return { + ...item, + status: "migrated", + target: archivePath, + details: { ...item.details, archivePath, archiveRelativePath: relativePath }, + }; + } catch (err) { + if (isFileAlreadyExistsError(err)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export async function copyMigrationFileItem( + item: MigrationItem, + reportDir: string, + opts: { overwrite?: boolean } = {}, +): Promise { + if (!item.source || !item.target) { + return markMigrationItemError(item, MIGRATION_REASON_MISSING_SOURCE_OR_TARGET); + } + try { + const targetExists = await exists(item.target); + if (targetExists && !opts.overwrite) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + const backupPath = opts.overwrite + ? await backupExistingMigrationTarget(item.target, reportDir) + : undefined; + await fs.mkdir(path.dirname(item.target), { recursive: true }); + await fs.cp(item.source, item.target, { + recursive: true, + force: Boolean(opts.overwrite), + errorOnExist: !opts.overwrite, + }); + return { + ...item, + status: "migrated", + details: { ...item.details, ...(backupPath ? { backupPath } : {}) }, + }; + } catch (err) { + if (isFileAlreadyExistsError(err)) { + return markMigrationItemConflict(item, MIGRATION_REASON_TARGET_EXISTS); + } + return markMigrationItemError(item, err instanceof Error ? err.message : String(err)); + } +} + +export async function writeMigrationReport( + result: MigrationApplyResult, + opts: { title?: string } = {}, +): Promise { + if (!result.reportDir) { + return; + } + await fs.mkdir(result.reportDir, { recursive: true }); + await fs.writeFile( + path.join(result.reportDir, "report.json"), + `${JSON.stringify(redactMigrationPlan(result), null, 2)}\n`, + "utf8", + ); + const lines = [ + `# ${opts.title ?? "Migration Report"}`, + "", + `Source: ${result.source}`, + result.target ? `Target: ${result.target}` : undefined, + result.backupPath ? `Backup: ${result.backupPath}` : undefined, + "", + `Migrated: ${result.summary.migrated}`, + `Skipped: ${result.summary.skipped}`, + `Conflicts: ${result.summary.conflicts}`, + `Errors: ${result.summary.errors}`, + "", + ...result.items.map( + (item) => `- ${item.status}: ${item.id}${item.reason ? ` (${item.reason})` : ""}`, + ), + ].filter((line): line is string => typeof line === "string"); + await fs.writeFile(path.join(result.reportDir, "summary.md"), `${lines.join("\n")}\n`, "utf8"); +} diff --git a/src/plugin-sdk/migration.ts b/src/plugin-sdk/migration.ts new file mode 100644 index 00000000000..2e68eb8bb7e --- /dev/null +++ b/src/plugin-sdk/migration.ts @@ -0,0 +1,153 @@ +// Shared migration-provider helpers for plan/apply item bookkeeping. + +import type { + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, +} from "../plugins/types.js"; + +export type { + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, +}; + +export const MIGRATION_REASON_MISSING_SOURCE_OR_TARGET = "missing source or target"; +export const MIGRATION_REASON_TARGET_EXISTS = "target exists"; + +export function createMigrationItem( + params: Omit & { status?: MigrationItem["status"] }, +): MigrationItem { + return { + ...params, + status: params.status ?? "planned", + }; +} + +export function markMigrationItemConflict(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "conflict", reason }; +} + +export function markMigrationItemError(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "error", reason }; +} + +export function markMigrationItemSkipped(item: MigrationItem, reason: string): MigrationItem { + return { ...item, status: "skipped", reason }; +} + +export function summarizeMigrationItems(items: readonly MigrationItem[]): MigrationSummary { + return { + total: items.length, + planned: items.filter((item) => item.status === "planned").length, + migrated: items.filter((item) => item.status === "migrated").length, + skipped: items.filter((item) => item.status === "skipped").length, + conflicts: items.filter((item) => item.status === "conflict").length, + errors: items.filter((item) => item.status === "error").length, + sensitive: items.filter((item) => item.sensitive).length, + }; +} + +const REDACTED_MIGRATION_VALUE = "[redacted]"; +const SECRET_KEY_MARKERS = [ + "accesstoken", + "apikey", + "authorization", + "bearertoken", + "clientsecret", + "cookie", + "credential", + "password", + "privatekey", + "refreshtoken", + "secret", +] as const; + +const SECRET_VALUE_PATTERNS = [ + /\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, + /\bsk-[A-Za-z0-9_-]{8,}\b/gu, + /\bgh[pousr]_[A-Za-z0-9_]{16,}\b/gu, + /\bxox[abprs]-[A-Za-z0-9-]{8,}\b/gu, + /\bAIza[0-9A-Za-z_-]{12,}\b/gu, +] as const; + +function normalizeSecretKey(key: string): string { + return key.toLowerCase().replaceAll(/[^a-z0-9]/gu, ""); +} + +function isSecretKey(key: string): boolean { + const normalized = normalizeSecretKey(key); + if (normalized === "token" || normalized.endsWith("token")) { + return true; + } + if (normalized === "auth" || normalized === "authorization") { + return true; + } + return SECRET_KEY_MARKERS.some((marker) => normalized.includes(marker)); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function isSecretReferenceLike(value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + return ( + value.source === "env" && + typeof value.id === "string" && + (value.provider === undefined || typeof value.provider === "string") + ); +} + +function redactString(value: string): string { + let next = value; + for (const pattern of SECRET_VALUE_PATTERNS) { + next = next.replace(pattern, REDACTED_MIGRATION_VALUE); + } + return next; +} + +function redactMigrationValueInternal(value: unknown, seen: WeakSet): unknown { + if (typeof value === "string") { + return redactString(value); + } + if (Array.isArray(value)) { + return value.map((entry) => redactMigrationValueInternal(entry, seen)); + } + if (!value || typeof value !== "object") { + return value; + } + if (seen.has(value)) { + return REDACTED_MIGRATION_VALUE; + } + seen.add(value); + const next: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (isSecretKey(key) && !isSecretReferenceLike(entry)) { + next[key] = REDACTED_MIGRATION_VALUE; + continue; + } + next[key] = redactMigrationValueInternal(entry, seen); + } + return next; +} + +export function redactMigrationValue(value: unknown): unknown { + return redactMigrationValueInternal(value, new WeakSet()); +} + +export function redactMigrationItem(item: MigrationItem): MigrationItem { + return redactMigrationValue(item) as MigrationItem; +} + +export function redactMigrationPlan(plan: T): T { + return redactMigrationValue(plan) as T; +} diff --git a/src/plugin-sdk/plugin-entry.ts b/src/plugin-sdk/plugin-entry.ts index 8242c8f33bf..53b1a6d6943 100644 --- a/src/plugin-sdk/plugin-entry.ts +++ b/src/plugin-sdk/plugin-entry.ts @@ -5,6 +5,13 @@ import type { AnyAgentTool, AgentHarness, MediaUnderstandingProviderPlugin, + MigrationApplyResult, + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, OpenClawPluginApi, OpenClawPluginCommandDefinition, OpenClawPluginConfigSchema, @@ -80,6 +87,13 @@ export type { AnyAgentTool, AgentHarness, MediaUnderstandingProviderPlugin, + MigrationApplyResult, + MigrationDetection, + MigrationItem, + MigrationPlan, + MigrationProviderContext, + MigrationProviderPlugin, + MigrationSummary, OpenClawPluginApi, OpenClawPluginNodeHostCommand, OpenClawPluginReloadRegistration, diff --git a/src/plugin-sdk/provider-auth.ts b/src/plugin-sdk/provider-auth.ts index 8ef12652f83..e647a457c8d 100644 --- a/src/plugin-sdk/provider-auth.ts +++ b/src/plugin-sdk/provider-auth.ts @@ -19,6 +19,7 @@ export { CLAUDE_CLI_PROFILE_ID, CODEX_CLI_PROFILE_ID } from "../agents/auth-prof export { ensureAuthProfileStore, ensureAuthProfileStoreForLocalUpdate, + updateAuthProfileStoreWithLock, } from "../agents/auth-profiles/store.js"; export { listProfilesForProvider, diff --git a/src/plugins/api-builder.ts b/src/plugins/api-builder.ts index 84d6d827ad5..d9ce38a5e0c 100644 --- a/src/plugins/api-builder.ts +++ b/src/plugins/api-builder.ts @@ -32,6 +32,7 @@ export type BuildPluginApiParams = { | "registerCliBackend" | "registerTextTransforms" | "registerConfigMigration" + | "registerMigrationProvider" | "registerAutoEnableProbe" | "registerProvider" | "registerSpeechProvider" @@ -80,6 +81,7 @@ const noopRegisterGatewayDiscoveryService: OpenClawPluginApi["registerGatewayDis const noopRegisterCliBackend: OpenClawPluginApi["registerCliBackend"] = () => {}; const noopRegisterTextTransforms: OpenClawPluginApi["registerTextTransforms"] = () => {}; const noopRegisterConfigMigration: OpenClawPluginApi["registerConfigMigration"] = () => {}; +const noopRegisterMigrationProvider: OpenClawPluginApi["registerMigrationProvider"] = () => {}; const noopRegisterAutoEnableProbe: OpenClawPluginApi["registerAutoEnableProbe"] = () => {}; const noopRegisterProvider: OpenClawPluginApi["registerProvider"] = () => {}; const noopRegisterSpeechProvider: OpenClawPluginApi["registerSpeechProvider"] = () => {}; @@ -151,6 +153,7 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi registerCliBackend: handlers.registerCliBackend ?? noopRegisterCliBackend, registerTextTransforms: handlers.registerTextTransforms ?? noopRegisterTextTransforms, registerConfigMigration: handlers.registerConfigMigration ?? noopRegisterConfigMigration, + registerMigrationProvider: handlers.registerMigrationProvider ?? noopRegisterMigrationProvider, registerAutoEnableProbe: handlers.registerAutoEnableProbe ?? noopRegisterAutoEnableProbe, registerProvider: handlers.registerProvider ?? noopRegisterProvider, registerSpeechProvider: handlers.registerSpeechProvider ?? noopRegisterSpeechProvider, diff --git a/src/plugins/bundled-capability-metadata.test.ts b/src/plugins/bundled-capability-metadata.test.ts index 1a5cff4a428..4f09e565a15 100644 --- a/src/plugins/bundled-capability-metadata.test.ts +++ b/src/plugins/bundled-capability-metadata.test.ts @@ -45,6 +45,14 @@ describe("bundled capability metadata", () => { .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)); expect(BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS).toEqual(expected); + expect(BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + pluginId: "migrate-hermes", + migrationProviderIds: ["hermes"], + }), + ]), + ); }); it("keeps lightweight alias maps aligned with bundled plugin manifests", () => { diff --git a/src/plugins/bundled-capability-runtime.test.ts b/src/plugins/bundled-capability-runtime.test.ts index 67ee7b334b5..42adae5a6fe 100644 --- a/src/plugins/bundled-capability-runtime.test.ts +++ b/src/plugins/bundled-capability-runtime.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { buildVitestCapabilityShimAliasMap } from "./bundled-capability-runtime.js"; +import { + buildVitestCapabilityShimAliasMap, + loadBundledCapabilityRuntimeRegistry, +} from "./bundled-capability-runtime.js"; describe("buildVitestCapabilityShimAliasMap", () => { it("keeps scoped and unscoped capability shim aliases aligned", () => { @@ -22,3 +25,21 @@ describe("buildVitestCapabilityShimAliasMap", () => { ); }); }); + +describe("loadBundledCapabilityRuntimeRegistry", () => { + it("captures bundled migration providers", () => { + const registry = loadBundledCapabilityRuntimeRegistry({ + pluginIds: ["migrate-hermes"], + pluginSdkResolution: "dist", + }); + + const record = registry.plugins.find((entry) => entry.id === "migrate-hermes"); + expect(record?.migrationProviderIds).toEqual(["hermes"]); + expect( + registry.migrationProviders.map((entry) => ({ + pluginId: entry.pluginId, + providerId: entry.provider.id, + })), + ).toEqual([{ pluginId: "migrate-hermes", providerId: "hermes" }]); + }); +}); diff --git a/src/plugins/bundled-capability-runtime.ts b/src/plugins/bundled-capability-runtime.ts index d8f9e5c8c63..90c918f23ed 100644 --- a/src/plugins/bundled-capability-runtime.ts +++ b/src/plugins/bundled-capability-runtime.ts @@ -158,6 +158,7 @@ function createCapabilityPluginRecord(params: { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], gatewayMethods: [], @@ -335,6 +336,7 @@ export function loadBundledCapabilityRuntimeRegistry(params: { ); record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id)); record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id)); + record.migrationProviderIds.push(...captured.migrationProviders.map((entry) => entry.id)); record.memoryEmbeddingProviderIds.push( ...captured.memoryEmbeddingProviders.map((entry) => entry.id), ); @@ -449,6 +451,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: { rootDir: record.rootDir, })), ); + registry.migrationProviders.push( + ...captured.migrationProviders.map((provider) => ({ + pluginId: record.id, + pluginName: record.name, + provider, + source: record.source, + rootDir: record.rootDir, + })), + ); registry.memoryEmbeddingProviders.push( ...captured.memoryEmbeddingProviders.map((provider) => ({ pluginId: record.id, diff --git a/src/plugins/captured-registration.ts b/src/plugins/captured-registration.ts index 7bbba7dbbca..4fb6a13d40c 100644 --- a/src/plugins/captured-registration.ts +++ b/src/plugins/captured-registration.ts @@ -16,6 +16,7 @@ import type { OpenClawPluginApi, ImageGenerationProviderPlugin, MediaUnderstandingProviderPlugin, + MigrationProviderPlugin, MusicGenerationProviderPlugin, OpenClawPluginCliCommandDescriptor, OpenClawPluginCliRegistrar, @@ -53,6 +54,7 @@ export type CapturedPluginRegistration = { musicGenerationProviders: MusicGenerationProviderPlugin[]; webFetchProviders: WebFetchProviderPlugin[]; webSearchProviders: WebSearchProviderPlugin[]; + migrationProviders: MigrationProviderPlugin[]; memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[]; tools: AnyAgentTool[]; }; @@ -77,6 +79,7 @@ export function createCapturedPluginRegistration(params?: { const musicGenerationProviders: MusicGenerationProviderPlugin[] = []; const webFetchProviders: WebFetchProviderPlugin[] = []; const webSearchProviders: WebSearchProviderPlugin[] = []; + const migrationProviders: MigrationProviderPlugin[] = []; const memoryEmbeddingProviders: MemoryEmbeddingProviderAdapter[] = []; const tools: AnyAgentTool[] = []; const noopLogger = { @@ -103,6 +106,7 @@ export function createCapturedPluginRegistration(params?: { musicGenerationProviders, webFetchProviders, webSearchProviders, + migrationProviders, memoryEmbeddingProviders, tools, api: buildPluginApi({ @@ -194,6 +198,9 @@ export function createCapturedPluginRegistration(params?: { registerWebSearchProvider(provider: WebSearchProviderPlugin) { webSearchProviders.push(provider); }, + registerMigrationProvider(provider: MigrationProviderPlugin) { + migrationProviders.push(provider); + }, registerMemoryEmbeddingProvider(adapter: MemoryEmbeddingProviderAdapter) { memoryEmbeddingProviders.push(adapter); }, diff --git a/src/plugins/contracts/inventory/bundled-capability-metadata.ts b/src/plugins/contracts/inventory/bundled-capability-metadata.ts index 403908f5a9b..39d22fb8800 100644 --- a/src/plugins/contracts/inventory/bundled-capability-metadata.ts +++ b/src/plugins/contracts/inventory/bundled-capability-metadata.ts @@ -28,6 +28,7 @@ export type BundledPluginContractSnapshot = { webContentExtractorIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; toolNames: string[]; }; @@ -164,6 +165,9 @@ export function buildBundledPluginContractSnapshot( webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders, (value) => value.trim(), ), + migrationProviderIds: uniqueStrings(manifest.contracts?.migrationProviders, (value) => + value.trim(), + ), toolNames: uniqueStrings(manifest.contracts?.tools, (value) => value.trim()), }; } @@ -185,6 +189,7 @@ export function hasBundledPluginContractSnapshotCapabilities( entry.webContentExtractorIds.length > 0 || entry.webFetchProviderIds.length > 0 || entry.webSearchProviderIds.length > 0 || + entry.migrationProviderIds.length > 0 || entry.toolNames.length > 0 ); } diff --git a/src/plugins/contracts/registry.contract.test.ts b/src/plugins/contracts/registry.contract.test.ts index 6a3573748ff..b3decb14734 100644 --- a/src/plugins/contracts/registry.contract.test.ts +++ b/src/plugins/contracts/registry.contract.test.ts @@ -22,6 +22,7 @@ describe("plugin contract registry", () => { speechProviders?: unknown[]; realtimeTranscriptionProviders?: unknown[]; realtimeVoiceProviders?: unknown[]; + migrationProviders?: unknown[]; }; }) => boolean; }) { @@ -38,6 +39,7 @@ describe("plugin contract registry", () => { speechProviders?: unknown[]; realtimeTranscriptionProviders?: unknown[]; realtimeVoiceProviders?: unknown[]; + migrationProviders?: unknown[]; }; }) => boolean, ) { @@ -65,6 +67,10 @@ describe("plugin contract registry", () => { name: "does not duplicate bundled web search provider ids", ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.webSearchProviderIds), }, + { + name: "does not duplicate bundled migration provider ids", + ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.migrationProviderIds), + }, { name: "does not duplicate bundled media provider ids", ids: () => @@ -200,4 +206,14 @@ describe("plugin contract registry", () => { ), ).toEqual(bundledWebSearchPluginIds); }); + + it("covers every bundled migration provider plugin discovered from manifests", () => { + expectRegistryPluginIds({ + actualPluginIds: pluginRegistrationContractRegistry + .filter((entry) => entry.migrationProviderIds.length > 0) + .map((entry) => entry.pluginId), + predicate: (plugin) => + plugin.origin === "bundled" && (plugin.contracts?.migrationProviders?.length ?? 0) > 0, + }); + }); }); diff --git a/src/plugins/contracts/registry.retry.test.ts b/src/plugins/contracts/registry.retry.test.ts index 4a3244ad635..6c1c7b1be38 100644 --- a/src/plugins/contracts/registry.retry.test.ts +++ b/src/plugins/contracts/registry.retry.test.ts @@ -8,6 +8,7 @@ type MockPluginRecord = { providerIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; }; type MockRuntimeRegistry = { @@ -52,6 +53,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [{ pluginId: "arcee", message: "transient arcee load failure" }], }), @@ -64,6 +66,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: ["arcee"], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, providers: [ { @@ -106,6 +109,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [{ pluginId: "searxng", message: "transient searxng load failure" }], }), @@ -118,6 +122,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: ["searxng"], + migrationProviderIds: [], }, webSearchProviders: [ { @@ -170,6 +175,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: ["byteplus"], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, providers: [ { @@ -311,6 +317,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], }, diagnostics: [ { pluginId: "firecrawl", message: "transient firecrawl fetch load failure" }, @@ -325,6 +332,7 @@ describe("plugin contract registry scoped retries", () => { providerIds: [], webFetchProviderIds: ["firecrawl"], webSearchProviderIds: ["firecrawl"], + migrationProviderIds: [], }, webFetchProviders: [ { diff --git a/src/plugins/contracts/registry.ts b/src/plugins/contracts/registry.ts index 0875e6cdd15..a9f094eb26b 100644 --- a/src/plugins/contracts/registry.ts +++ b/src/plugins/contracts/registry.ts @@ -69,6 +69,7 @@ type ManifestContractKey = | "webContentExtractors" | "webFetchProviders" | "webSearchProviders" + | "migrationProviders" | "tools"; type ManifestRegistryContractKey = "webFetchProviders" | "webSearchProviders"; @@ -102,6 +103,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { webContentExtractorIds: [...entry.webContentExtractorIds], webFetchProviderIds: [...entry.webFetchProviderIds], webSearchProviderIds: [...entry.webSearchProviderIds], + migrationProviderIds: [...entry.migrationProviderIds], toolNames: [...entry.toolNames], })); } @@ -122,6 +124,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { (plugin.contracts?.webContentExtractors?.length ?? 0) > 0 || (plugin.contracts?.webFetchProviders?.length ?? 0) > 0 || (plugin.contracts?.webSearchProviders?.length ?? 0) > 0 || + (plugin.contracts?.migrationProviders?.length ?? 0) > 0 || (plugin.contracts?.tools?.length ?? 0) > 0), ) .map((plugin) => ({ @@ -144,6 +147,7 @@ function resolveBundledManifestContracts(): PluginRegistrationContractEntry[] { webContentExtractorIds: uniqueStrings(plugin.contracts?.webContentExtractors ?? []), webFetchProviderIds: uniqueStrings(plugin.contracts?.webFetchProviders ?? []), webSearchProviderIds: uniqueStrings(plugin.contracts?.webSearchProviders ?? []), + migrationProviderIds: uniqueStrings(plugin.contracts?.migrationProviders ?? []), toolNames: uniqueStrings(plugin.contracts?.tools ?? []), })); } @@ -204,6 +208,8 @@ function resolveBundledManifestPluginIdsForContract(contract: ManifestContractKe return entry.webFetchProviderIds.length > 0; case "webSearchProviders": return entry.webSearchProviderIds.length > 0; + case "migrationProviders": + return entry.migrationProviderIds.length > 0; case "tools": return entry.toolNames.length > 0; } diff --git a/src/plugins/hooks.test-helpers.ts b/src/plugins/hooks.test-helpers.ts index 12ea7b8bcb3..3ea07cab615 100644 --- a/src/plugins/hooks.test-helpers.ts +++ b/src/plugins/hooks.test-helpers.ts @@ -42,6 +42,7 @@ export function createMockPluginRegistry( musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/installed-plugin-index-record-builder.ts b/src/plugins/installed-plugin-index-record-builder.ts index d63c83d0a06..c91cb11713a 100644 --- a/src/plugins/installed-plugin-index-record-builder.ts +++ b/src/plugins/installed-plugin-index-record-builder.ts @@ -43,6 +43,7 @@ function hasRuntimeContractSurface(record: PluginManifestRecord): boolean { record.contracts?.webContentExtractors?.length || record.contracts?.webFetchProviders?.length || record.contracts?.webSearchProviders?.length || + record.contracts?.migrationProviders?.length || record.contracts?.memoryEmbeddingProviders?.length || hasKind(record.kind, "memory"), ); diff --git a/src/plugins/installed-plugin-index.test.ts b/src/plugins/installed-plugin-index.test.ts index 9d2dff291e2..decff32735a 100644 --- a/src/plugins/installed-plugin-index.test.ts +++ b/src/plugins/installed-plugin-index.test.ts @@ -226,6 +226,43 @@ describe("installed plugin index", () => { expect(index.plugins[0]?.installRecordHash).toBeUndefined(); }); + it("does not classify migration-provider-only plugins as gateway startup sidecars", () => { + const rootDir = makeTempDir(); + writeRuntimeEntry(rootDir); + writePackageJson(rootDir, { + name: "@vendor/migration-plugin", + version: "1.0.0", + }); + writePluginManifest(rootDir, { + id: "migration-plugin", + name: "Migration Plugin", + enabledByDefault: true, + configSchema: { type: "object" }, + contracts: { + migrationProviders: ["legacy-import"], + }, + }); + + const index = loadInstalledPluginIndex({ + candidates: [ + createPluginCandidate({ + rootDir, + packageName: "@vendor/migration-plugin", + packageVersion: "1.0.0", + }), + ], + env: hermeticEnv(), + }); + + expect(index.plugins[0]).toMatchObject({ + pluginId: "migration-plugin", + enabledByDefault: true, + startup: { + sidecar: false, + }, + }); + }); + it("keeps bundle format metadata needed for manifest reconstruction", () => { const rootDir = makeTempDir(); fs.mkdirSync(path.join(rootDir, ".claude-plugin"), { recursive: true }); diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 5f9788fc910..d78f25948b9 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -291,6 +291,7 @@ type PluginRegistrySnapshot = { musicGenerationProviders: PluginRegistry["musicGenerationProviders"]; webFetchProviders: PluginRegistry["webFetchProviders"]; webSearchProviders: PluginRegistry["webSearchProviders"]; + migrationProviders: PluginRegistry["migrationProviders"]; codexAppServerExtensionFactories: PluginRegistry["codexAppServerExtensionFactories"]; agentToolResultMiddlewares: PluginRegistry["agentToolResultMiddlewares"]; memoryEmbeddingProviders: PluginRegistry["memoryEmbeddingProviders"]; @@ -329,6 +330,7 @@ function snapshotPluginRegistry(registry: PluginRegistry): PluginRegistrySnapsho musicGenerationProviders: [...registry.musicGenerationProviders], webFetchProviders: [...registry.webFetchProviders], webSearchProviders: [...registry.webSearchProviders], + migrationProviders: [...registry.migrationProviders], codexAppServerExtensionFactories: [...registry.codexAppServerExtensionFactories], agentToolResultMiddlewares: [...registry.agentToolResultMiddlewares], memoryEmbeddingProviders: [...registry.memoryEmbeddingProviders], @@ -366,6 +368,7 @@ function restorePluginRegistry(registry: PluginRegistry, snapshot: PluginRegistr registry.musicGenerationProviders = snapshot.arrays.musicGenerationProviders; registry.webFetchProviders = snapshot.arrays.webFetchProviders; registry.webSearchProviders = snapshot.arrays.webSearchProviders; + registry.migrationProviders = snapshot.arrays.migrationProviders; registry.codexAppServerExtensionFactories = snapshot.arrays.codexAppServerExtensionFactories; registry.agentToolResultMiddlewares = snapshot.arrays.agentToolResultMiddlewares; registry.memoryEmbeddingProviders = snapshot.arrays.memoryEmbeddingProviders; @@ -1830,6 +1833,7 @@ function createPluginRecord(params: { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], contextEngineIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], diff --git a/src/plugins/manifest-contract-runtime.ts b/src/plugins/manifest-contract-runtime.ts new file mode 100644 index 00000000000..ed21db7edf3 --- /dev/null +++ b/src/plugins/manifest-contract-runtime.ts @@ -0,0 +1,53 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginManifestContractListKey } from "./manifest-registry.js"; +import { loadPluginManifestRegistryForPluginRegistry } from "./plugin-registry.js"; + +export type ManifestContractRuntimePluginResolution = { + pluginIds: string[]; + bundledCompatPluginIds: string[]; +}; + +const DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS = { + preferPersisted: false, +} as const; + +function hasManifestContractValue( + plugin: ReturnType["plugins"][number], + contract: PluginManifestContractListKey, + value?: string, +): boolean { + const values = plugin.contracts?.[contract] ?? []; + return values.length > 0 && (!value || values.includes(value)); +} + +export function resolveManifestContractRuntimePluginResolution(params: { + cfg?: OpenClawConfig; + contract: PluginManifestContractListKey; + value?: string; +}): ManifestContractRuntimePluginResolution { + const allContractPlugins = loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env: process.env, + includeDisabled: true, + ...DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS, + }).plugins.filter((plugin) => hasManifestContractValue(plugin, params.contract, params.value)); + const bundledCompatPluginIds = allContractPlugins + .filter((plugin) => plugin.origin === "bundled") + .map((plugin) => plugin.id); + const enabledPluginIds = new Set( + loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env: process.env, + ...DEMAND_ONLY_CONTRACT_LOOKUP_OPTIONS, + }).plugins.map((plugin) => plugin.id), + ); + const pluginIds = allContractPlugins + .filter((plugin) => plugin.origin === "bundled" || enabledPluginIds.has(plugin.id)) + .map((plugin) => plugin.id); + return { + pluginIds: [...new Set(pluginIds)].toSorted((left, right) => left.localeCompare(right)), + bundledCompatPluginIds: [...new Set(bundledCompatPluginIds)].toSorted((left, right) => + left.localeCompare(right), + ), + }; +} diff --git a/src/plugins/manifest-registry.ts b/src/plugins/manifest-registry.ts index 51ecf2d88d7..633267b44be 100644 --- a/src/plugins/manifest-registry.ts +++ b/src/plugins/manifest-registry.ts @@ -79,7 +79,8 @@ export type PluginManifestContractListKey = | "memoryEmbeddingProviders" | "webContentExtractors" | "webFetchProviders" - | "webSearchProviders"; + | "webSearchProviders" + | "migrationProviders"; type SeenIdEntry = { candidate: PluginCandidate; diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 52be7ccb505..bfb13022957 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -294,6 +294,7 @@ export type PluginManifestContracts = { webContentExtractors?: string[]; webFetchProviders?: string[]; webSearchProviders?: string[]; + migrationProviders?: string[]; tools?: string[]; }; @@ -488,6 +489,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u const webContentExtractors = normalizeTrimmedStringList(value.webContentExtractors); const webFetchProviders = normalizeTrimmedStringList(value.webFetchProviders); const webSearchProviders = normalizeTrimmedStringList(value.webSearchProviders); + const migrationProviders = normalizeTrimmedStringList(value.migrationProviders); const tools = normalizeTrimmedStringList(value.tools); const contracts = { ...(embeddedExtensionFactories.length > 0 ? { embeddedExtensionFactories } : {}), @@ -505,6 +507,7 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u ...(webContentExtractors.length > 0 ? { webContentExtractors } : {}), ...(webFetchProviders.length > 0 ? { webFetchProviders } : {}), ...(webSearchProviders.length > 0 ? { webSearchProviders } : {}), + ...(migrationProviders.length > 0 ? { migrationProviders } : {}), ...(tools.length > 0 ? { tools } : {}), } satisfies PluginManifestContracts; diff --git a/src/plugins/migration-provider-runtime.test.ts b/src/plugins/migration-provider-runtime.test.ts new file mode 100644 index 00000000000..4d51c1788f4 --- /dev/null +++ b/src/plugins/migration-provider-runtime.test.ts @@ -0,0 +1,214 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginRegistry } from "./registry-types.js"; +import { createEmptyPluginRegistry } from "./registry.js"; + +type MockManifestRegistry = { + plugins: Array>; + diagnostics: unknown[]; +}; + +function createEmptyMockManifestRegistry(): MockManifestRegistry { + return { plugins: [], diagnostics: [] }; +} + +const mocks = vi.hoisted(() => ({ + resolveRuntimePluginRegistry: vi.fn<(params?: unknown) => PluginRegistry | undefined>( + () => undefined, + ), + loadPluginManifestRegistry: vi.fn<(params?: Record) => MockManifestRegistry>( + () => createEmptyMockManifestRegistry(), + ), + withBundledPluginAllowlistCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), + withBundledPluginEnablementCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), + withBundledPluginVitestCompat: vi.fn( + ({ config }: { config?: OpenClawConfig; pluginIds: string[] }) => config, + ), +})); + +vi.mock("./loader.js", () => ({ + resolveRuntimePluginRegistry: mocks.resolveRuntimePluginRegistry, +})); + +vi.mock("./plugin-registry.js", () => ({ + loadPluginManifestRegistryForPluginRegistry: mocks.loadPluginManifestRegistry, +})); + +vi.mock("./bundled-compat.js", () => ({ + withBundledPluginAllowlistCompat: mocks.withBundledPluginAllowlistCompat, + withBundledPluginEnablementCompat: mocks.withBundledPluginEnablementCompat, + withBundledPluginVitestCompat: mocks.withBundledPluginVitestCompat, +})); + +let resolvePluginMigrationProvider: typeof import("./migration-provider-runtime.js").resolvePluginMigrationProvider; +let resolvePluginMigrationProviders: typeof import("./migration-provider-runtime.js").resolvePluginMigrationProviders; + +function createMigrationProvider(id: string) { + return { + id, + label: id, + plan: vi.fn(), + apply: vi.fn(), + }; +} + +describe("migration provider runtime", () => { + beforeEach(async () => { + vi.clearAllMocks(); + mocks.resolveRuntimePluginRegistry.mockReturnValue(createEmptyPluginRegistry()); + mocks.loadPluginManifestRegistry.mockReturnValue(createEmptyMockManifestRegistry()); + const runtime = await import("./migration-provider-runtime.js"); + resolvePluginMigrationProvider = runtime.resolvePluginMigrationProvider; + resolvePluginMigrationProviders = runtime.resolvePluginMigrationProviders; + }); + + it("loads configured external migration-provider plugins from manifest contracts", () => { + const cfg = { + plugins: { entries: { "external-migration": { enabled: true } } }, + } as OpenClawConfig; + const provider = createMigrationProvider("external-import"); + const active = createEmptyPluginRegistry(); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "external-migration", + pluginName: "External Migration", + source: "test", + provider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => ({ + diagnostics: [], + plugins: params?.includeDisabled + ? [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + { + id: "disabled-external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ] + : [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ], + })); + + const resolved = resolvePluginMigrationProvider({ providerId: "external-import", cfg }); + + expect(resolved).toBe(provider); + expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({ + config: cfg, + env: process.env, + includeDisabled: true, + preferPersisted: false, + }); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith(); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ + config: cfg, + onlyPluginIds: ["external-migration"], + activate: false, + }); + }); + + it("derives a fresh manifest registry so newly bundled migration providers are discoverable", () => { + const provider = createMigrationProvider("hermes"); + const active = createEmptyPluginRegistry(); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "migrate-hermes", + pluginName: "Hermes Migration", + source: "test", + provider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => { + if (params?.preferPersisted !== false) { + return createEmptyMockManifestRegistry(); + } + return { + diagnostics: [], + plugins: [ + { + id: "migrate-hermes", + origin: "bundled", + contracts: { migrationProviders: ["hermes"] }, + }, + ], + }; + }); + + const resolved = resolvePluginMigrationProvider({ providerId: "hermes" }); + + expect(resolved).toBe(provider); + expect(mocks.loadPluginManifestRegistry).toHaveBeenCalledWith({ + config: undefined, + env: process.env, + includeDisabled: true, + preferPersisted: false, + }); + expect(mocks.resolveRuntimePluginRegistry).toHaveBeenCalledWith({ + onlyPluginIds: ["migrate-hermes"], + activate: false, + }); + }); + + it("lists configured external migration providers alongside active providers", () => { + const activeProvider = createMigrationProvider("active-import"); + const externalProvider = createMigrationProvider("external-import"); + const active = createEmptyPluginRegistry(); + active.migrationProviders.push({ + pluginId: "active-migration", + pluginName: "Active Migration", + source: "test", + provider: activeProvider, + } as never); + const loaded = createEmptyPluginRegistry(); + loaded.migrationProviders.push({ + pluginId: "external-migration", + pluginName: "External Migration", + source: "test", + provider: externalProvider, + } as never); + mocks.resolveRuntimePluginRegistry.mockImplementation((params?: unknown) => + params === undefined ? active : loaded, + ); + mocks.loadPluginManifestRegistry.mockImplementation((params?: Record) => ({ + diagnostics: [], + plugins: params?.includeDisabled + ? [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ] + : [ + { + id: "external-migration", + origin: "installed", + contracts: { migrationProviders: ["external-import"] }, + }, + ], + })); + + expect(resolvePluginMigrationProviders().map((provider) => provider.id)).toEqual([ + "active-import", + "external-import", + ]); + }); +}); diff --git a/src/plugins/migration-provider-runtime.ts b/src/plugins/migration-provider-runtime.ts new file mode 100644 index 00000000000..2601f975248 --- /dev/null +++ b/src/plugins/migration-provider-runtime.ts @@ -0,0 +1,117 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { + withBundledPluginAllowlistCompat, + withBundledPluginEnablementCompat, + withBundledPluginVitestCompat, +} from "./bundled-compat.js"; +import { resolveRuntimePluginRegistry } from "./loader.js"; +import { resolveManifestContractRuntimePluginResolution } from "./manifest-contract-runtime.js"; +import type { MigrationProviderPlugin } from "./types.js"; + +function resolveMigrationProviderConfig(params: { + cfg?: OpenClawConfig; + bundledCompatPluginIds: string[]; +}): OpenClawConfig | undefined { + const allowlistCompat = withBundledPluginAllowlistCompat({ + config: params.cfg, + pluginIds: params.bundledCompatPluginIds, + }); + const enablementCompat = withBundledPluginEnablementCompat({ + config: allowlistCompat, + pluginIds: params.bundledCompatPluginIds, + }); + return withBundledPluginVitestCompat({ + config: enablementCompat, + pluginIds: params.bundledCompatPluginIds, + env: process.env, + }); +} + +function findMigrationProviderById( + entries: ReadonlyArray<{ provider: MigrationProviderPlugin }>, + providerId: string, +): MigrationProviderPlugin | undefined { + return entries.find((entry) => entry.provider.id === providerId)?.provider; +} + +function resolveMigrationProviderRegistry(params: { + cfg?: OpenClawConfig; + pluginIds: string[]; + bundledCompatPluginIds: string[]; +}) { + const compatConfig = resolveMigrationProviderConfig({ + cfg: params.cfg, + bundledCompatPluginIds: params.bundledCompatPluginIds, + }); + return resolveRuntimePluginRegistry({ + ...(compatConfig === undefined ? {} : { config: compatConfig }), + onlyPluginIds: params.pluginIds, + activate: false, + }); +} + +function mergeMigrationProviders( + left: ReadonlyArray<{ provider: MigrationProviderPlugin }>, + right: ReadonlyArray<{ provider: MigrationProviderPlugin }>, +): MigrationProviderPlugin[] { + const merged = new Map(); + for (const entry of [...left, ...right]) { + if (!merged.has(entry.provider.id)) { + merged.set(entry.provider.id, entry.provider); + } + } + return [...merged.values()].toSorted((a, b) => a.id.localeCompare(b.id)); +} + +export function resolvePluginMigrationProvider(params: { + providerId: string; + cfg?: OpenClawConfig; +}): MigrationProviderPlugin | undefined { + const activeRegistry = resolveRuntimePluginRegistry(); + const activeProvider = findMigrationProviderById( + activeRegistry?.migrationProviders ?? [], + params.providerId, + ); + if (activeProvider) { + return activeProvider; + } + + const resolution = resolveManifestContractRuntimePluginResolution({ + cfg: params.cfg, + contract: "migrationProviders", + value: params.providerId, + }); + const pluginIds = resolution.pluginIds; + if (pluginIds.length === 0) { + return undefined; + } + const registry = resolveMigrationProviderRegistry({ + cfg: params.cfg, + pluginIds, + bundledCompatPluginIds: resolution.bundledCompatPluginIds, + }); + return findMigrationProviderById(registry?.migrationProviders ?? [], params.providerId); +} + +export function resolvePluginMigrationProviders( + params: { + cfg?: OpenClawConfig; + } = {}, +): MigrationProviderPlugin[] { + const activeRegistry = resolveRuntimePluginRegistry(); + const activeProviders = activeRegistry?.migrationProviders ?? []; + const resolution = resolveManifestContractRuntimePluginResolution({ + cfg: params.cfg, + contract: "migrationProviders", + }); + const pluginIds = resolution.pluginIds; + if (pluginIds.length === 0) { + return mergeMigrationProviders(activeProviders, []); + } + const registry = resolveMigrationProviderRegistry({ + cfg: params.cfg, + pluginIds, + bundledCompatPluginIds: resolution.bundledCompatPluginIds, + }); + return mergeMigrationProviders(activeProviders, registry?.migrationProviders ?? []); +} diff --git a/src/plugins/plugin-registry-snapshot.ts b/src/plugins/plugin-registry-snapshot.ts index 778cc1747ef..af24b9f5dc3 100644 --- a/src/plugins/plugin-registry-snapshot.ts +++ b/src/plugins/plugin-registry-snapshot.ts @@ -76,10 +76,11 @@ export function loadPluginRegistrySnapshotWithMetadata( const disabledByCaller = params.preferPersisted === false; const disabledByEnv = hasEnvFlag(env, DISABLE_PERSISTED_PLUGIN_REGISTRY_ENV); const persistedReadsEnabled = !disabledByCaller && !disabledByEnv; + const persistedInstallRecordReadsEnabled = !disabledByEnv; let persistedIndex: InstalledPluginIndex | null = null; - if (persistedReadsEnabled) { + if (persistedInstallRecordReadsEnabled) { persistedIndex = readPersistedInstalledPluginIndexSync(params); - if (persistedIndex) { + if (persistedReadsEnabled && persistedIndex) { if ( params.config && persistedIndex.policyHash !== resolveInstalledPluginIndexPolicyHash(params.config) @@ -97,7 +98,7 @@ export function loadPluginRegistrySnapshotWithMetadata( diagnostics, }; } - } else { + } else if (persistedReadsEnabled) { diagnostics.push({ level: "info", code: "persisted-registry-missing", diff --git a/src/plugins/plugin-registry.test.ts b/src/plugins/plugin-registry.test.ts index 6dda1b905a4..0078effb693 100644 --- a/src/plugins/plugin-registry.test.ts +++ b/src/plugins/plugin-registry.test.ts @@ -388,6 +388,42 @@ describe("plugin registry facade", () => { ]); }); + it("derives a fresh registry without dropping persisted install records", async () => { + const stateDir = makeTempDir(); + const rootDir = makeTempDir(); + const candidate = createCandidate(rootDir); + await writePersistedInstalledPluginIndex( + createIndex("persisted", { + installRecords: { + persisted: { + source: "npm", + spec: "persisted-plugin@1.0.0", + installPath: path.join(stateDir, "plugins", "persisted"), + }, + }, + }), + { stateDir }, + ); + + const result = loadPluginRegistrySnapshotWithMetadata({ + stateDir, + candidates: [candidate], + env: hermeticEnv(), + preferPersisted: false, + }); + + expect(result.source).toBe("derived"); + expect(listPluginRecords({ index: result.snapshot }).map((plugin) => plugin.pluginId)).toEqual([ + "demo", + ]); + expect(result.snapshot.installRecords).toMatchObject({ + persisted: { + source: "npm", + spec: "persisted-plugin@1.0.0", + }, + }); + }); + it("exposes explicit persisted registry inspect and refresh operations", async () => { const stateDir = makeTempDir(); const pluginDir = path.join(stateDir, "plugins", "demo"); diff --git a/src/plugins/registry-empty.ts b/src/plugins/registry-empty.ts index 96d77148d28..ace8481395e 100644 --- a/src/plugins/registry-empty.ts +++ b/src/plugins/registry-empty.ts @@ -20,6 +20,7 @@ export function createEmptyPluginRegistry(): PluginRegistry { musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/registry-types.ts b/src/plugins/registry-types.ts index f77e805a37f..f1421ff9ef3 100644 --- a/src/plugins/registry-types.ts +++ b/src/plugins/registry-types.ts @@ -43,6 +43,7 @@ import type { PluginLogger, PluginOrigin, PluginTextTransformRegistration, + MigrationProviderPlugin, ProviderPlugin, RealtimeTranscriptionProviderPlugin, RealtimeVoiceProviderPlugin, @@ -149,6 +150,8 @@ export type PluginWebFetchProviderRegistration = PluginOwnedProviderRegistration; export type PluginWebSearchProviderRegistration = PluginOwnedProviderRegistration; +export type PluginMigrationProviderRegistration = + PluginOwnedProviderRegistration; export type PluginMemoryEmbeddingProviderRegistration = PluginOwnedProviderRegistration; export type PluginCodexAppServerExtensionFactoryRegistration = { @@ -279,6 +282,7 @@ export type PluginRecord = { musicGenerationProviderIds: string[]; webFetchProviderIds: string[]; webSearchProviderIds: string[]; + migrationProviderIds: string[]; contextEngineIds?: string[]; memoryEmbeddingProviderIds: string[]; agentHarnessIds: string[]; @@ -315,6 +319,7 @@ export type PluginRegistry = { musicGenerationProviders: PluginMusicGenerationProviderRegistration[]; webFetchProviders: PluginWebFetchProviderRegistration[]; webSearchProviders: PluginWebSearchProviderRegistration[]; + migrationProviders: PluginMigrationProviderRegistration[]; codexAppServerExtensionFactories: PluginCodexAppServerExtensionFactoryRegistration[]; agentToolResultMiddlewares: PluginAgentToolResultMiddlewareRegistration[]; memoryEmbeddingProviders: PluginMemoryEmbeddingProviderRegistration[]; diff --git a/src/plugins/registry.ts b/src/plugins/registry.ts index 2cc1fea101c..ac528582f9a 100644 --- a/src/plugins/registry.ts +++ b/src/plugins/registry.ts @@ -113,6 +113,7 @@ import type { OpenClawPluginReloadRegistration, OpenClawPluginSecurityAuditCollector, MediaUnderstandingProviderPlugin, + MigrationProviderPlugin, OpenClawPluginService, OpenClawPluginToolContext, OpenClawPluginToolFactory, @@ -1016,6 +1017,16 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { }); }; + const registerMigrationProvider = (record: PluginRecord, provider: MigrationProviderPlugin) => { + registerUniqueProviderLike({ + record, + provider, + kindLabel: "migration provider", + registrations: registry.migrationProviders, + ownedIds: record.migrationProviderIds, + }); + }; + const registerCli = ( record: PluginRecord, registrar: OpenClawPluginCliRegistrar, @@ -1487,6 +1498,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerMusicGenerationProvider(record, provider), registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider), registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider), + registerMigrationProvider: (provider) => registerMigrationProvider(record, provider), registerGatewayMethod: (method, handler, opts) => registerGatewayMethod(record, method, handler, opts), registerService: (service) => registerService(record, service), @@ -1764,6 +1776,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) { registerVideoGenerationProvider, registerMusicGenerationProvider, registerWebSearchProvider, + registerMigrationProvider, registerGatewayMethod, registerCli, registerReload, diff --git a/src/plugins/status.test-helpers.ts b/src/plugins/status.test-helpers.ts index d063049f53e..acf64cb91ee 100644 --- a/src/plugins/status.test-helpers.ts +++ b/src/plugins/status.test-helpers.ts @@ -61,6 +61,7 @@ export function createPluginRecord( musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], contextEngineIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], @@ -131,6 +132,7 @@ export function createPluginLoadResult( musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/plugins/status.ts b/src/plugins/status.ts index 848622b90f7..74214837364 100644 --- a/src/plugins/status.ts +++ b/src/plugins/status.ts @@ -192,6 +192,7 @@ function buildPluginRecordFromInstalledIndex( musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: [], gatewayMethods: [], diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 22b242b8876..dbcefa6142f 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -2049,6 +2049,100 @@ export type PluginConfigMigration = (config: OpenClawConfig) => | null | undefined; +export type MigrationItemStatus = "planned" | "migrated" | "skipped" | "conflict" | "error"; +export type MigrationItemKind = + | "config" + | "secret" + | "memory" + | "skill" + | "workspace" + | "session" + | "file" + | "archive" + | "manual"; +export type MigrationItemAction = + | "copy" + | "create" + | "update" + | "merge" + | "append" + | "archive" + | "skip" + | "manual"; + +export type MigrationItem = { + id: string; + kind: MigrationItemKind | (string & {}); + action: MigrationItemAction | (string & {}); + status: MigrationItemStatus; + source?: string; + target?: string; + message?: string; + reason?: string; + sensitive?: boolean; + details?: Record; +}; + +export type MigrationSummary = { + total: number; + planned: number; + migrated: number; + skipped: number; + conflicts: number; + errors: number; + sensitive: number; +}; + +export type MigrationDetection = { + found: boolean; + source?: string; + label?: string; + confidence?: "low" | "medium" | "high"; + message?: string; +}; + +export type MigrationPlan = { + providerId: string; + source: string; + target?: string; + summary: MigrationSummary; + items: MigrationItem[]; + warnings?: string[]; + nextSteps?: string[]; + metadata?: Record; +}; + +export type MigrationApplyResult = MigrationPlan & { + backupPath?: string; + reportDir?: string; +}; + +export type MigrationProviderContext = { + config: OpenClawConfig; + runtime?: PluginRuntime; + logger: PluginLogger; + stateDir: string; + source?: string; + includeSecrets?: boolean; + overwrite?: boolean; + backupPath?: string; + reportDir?: string; + signal?: AbortSignal; +}; + +/** Migration source implemented by a plugin and orchestrated by `openclaw migrate`. */ +export type MigrationProviderPlugin = { + id: string; + label: string; + description?: string; + detect?: (ctx: MigrationProviderContext) => MigrationDetection | Promise; + plan: (ctx: MigrationProviderContext) => MigrationPlan | Promise; + apply: ( + ctx: MigrationProviderContext, + plan?: MigrationPlan, + ) => MigrationApplyResult | Promise; +}; + export type PluginSetupAutoEnableContext = { config: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -2128,6 +2222,8 @@ export type OpenClawPluginApi = { registerTextTransforms: (transforms: PluginTextTransformRegistration) => void; /** Register a lightweight config migration that can run before plugin runtime loads. */ registerConfigMigration: (migrate: PluginConfigMigration) => void; + /** Register an importer for `openclaw migrate` (migration capability). */ + registerMigrationProvider: (provider: MigrationProviderPlugin) => void; /** Register a lightweight config probe that can auto-enable this plugin generically. */ registerAutoEnableProbe: (probe: PluginSetupAutoEnableProbe) => void; /** Register a native model/provider plugin (text inference capability). */ diff --git a/src/test-utils/channel-plugins.ts b/src/test-utils/channel-plugins.ts index 53062f220f9..b68c42d3864 100644 --- a/src/test-utils/channel-plugins.ts +++ b/src/test-utils/channel-plugins.ts @@ -35,6 +35,7 @@ export const createTestRegistry = (channels: TestChannelRegistration[] = []): Pl musicGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], codexAppServerExtensionFactories: [], agentToolResultMiddlewares: [], memoryEmbeddingProviders: [], diff --git a/src/trajectory/metadata.test.ts b/src/trajectory/metadata.test.ts index ee1d7854516..a1894fe784b 100644 --- a/src/trajectory/metadata.test.ts +++ b/src/trajectory/metadata.test.ts @@ -96,6 +96,7 @@ describe("trajectory metadata", () => { musicGenerationProviderIds: [], webFetchProviderIds: [], webSearchProviderIds: [], + migrationProviderIds: [], memoryEmbeddingProviderIds: [], agentHarnessIds: ["pi"], gatewayMethods: [], diff --git a/src/wizard/setup.migration-import.test.ts b/src/wizard/setup.migration-import.test.ts new file mode 100644 index 00000000000..83d541d7d40 --- /dev/null +++ b/src/wizard/setup.migration-import.test.ts @@ -0,0 +1,61 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectSetupMigrationFreshness } from "./setup.migration-import.js"; + +const tempRoots = new Set(); + +async function makeTempRoot() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-setup-migration-")); + tempRoots.add(root); + return root; +} + +async function writeFile(filePath: string, content: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +describe("setup migration import freshness", () => { + afterEach(async () => { + for (const root of tempRoots) { + await fs.rm(root, { force: true, recursive: true }); + } + tempRoots.clear(); + }); + + it("allows empty config and empty target directories", async () => { + const root = await makeTempRoot(); + const result = await inspectSetupMigrationFreshness({ + baseConfig: {}, + stateDir: path.join(root, "state"), + workspaceDir: path.join(root, "workspace"), + }); + + expect(result).toEqual({ fresh: true, reasons: [] }); + }); + + it("rejects existing config, workspace files, and state", async () => { + const root = await makeTempRoot(); + const stateDir = path.join(root, "state"); + const workspaceDir = path.join(root, "workspace"); + await writeFile(path.join(workspaceDir, "MEMORY.md"), "existing memory\n"); + await writeFile(path.join(stateDir, "agents", "main", "agent", "auth-profiles.json"), "{}\n"); + + const result = await inspectSetupMigrationFreshness({ + baseConfig: { gateway: { port: 3131 } }, + stateDir, + workspaceDir, + }); + + expect(result.fresh).toBe(false); + expect(result.reasons).toEqual( + expect.arrayContaining([ + "existing config values are loaded", + "workspace MEMORY.md exists", + "state agents/ exists", + ]), + ); + }); +}); diff --git a/src/wizard/setup.migration-import.ts b/src/wizard/setup.migration-import.ts new file mode 100644 index 00000000000..150cce46da3 --- /dev/null +++ b/src/wizard/setup.migration-import.ts @@ -0,0 +1,304 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OnboardOptions } from "../commands/onboard-types.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import type { MigrationProviderPlugin } from "../plugins/types.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { resolveUserPath } from "../utils.js"; +import { WizardCancelledError, type WizardPrompter } from "./prompts.js"; + +export type SetupMigrationDetection = { + providerId: string; + label: string; + source?: string; + message?: string; +}; + +const MEANINGFUL_CONFIG_IGNORED_KEYS = new Set(["$schema", "meta"]); +const MEANINGFUL_WORKSPACE_ENTRIES = [ + "AGENTS.md", + "SOUL.md", + "USER.md", + "IDENTITY.md", + "MEMORY.md", + "skills", +] as const; +const MEANINGFUL_STATE_ENTRIES = ["credentials", "sessions", "agents"] as const; + +async function exists(candidate: string): Promise { + try { + await fs.access(candidate); + return true; + } catch { + return false; + } +} + +async function hasDirectoryEntries(candidate: string): Promise { + try { + return (await fs.readdir(candidate)).length > 0; + } catch { + return false; + } +} + +function hasMeaningfulConfig(config: OpenClawConfig): boolean { + return Object.keys(config as Record).some( + (key) => !MEANINGFUL_CONFIG_IGNORED_KEYS.has(key), + ); +} + +export async function inspectSetupMigrationFreshness(params: { + baseConfig: OpenClawConfig; + stateDir: string; + workspaceDir: string; +}): Promise<{ fresh: boolean; reasons: string[] }> { + const reasons: string[] = []; + if (hasMeaningfulConfig(params.baseConfig)) { + reasons.push("existing config values are loaded"); + } + for (const entry of MEANINGFUL_WORKSPACE_ENTRIES) { + if (await exists(path.join(params.workspaceDir, entry))) { + reasons.push(`workspace ${entry} exists`); + } + } + for (const entry of MEANINGFUL_STATE_ENTRIES) { + if (await hasDirectoryEntries(path.join(params.stateDir, entry))) { + reasons.push(`state ${entry}/ exists`); + } + } + return { fresh: reasons.length === 0, reasons }; +} + +function assertFreshSetupMigrationTarget(freshness: { + fresh: boolean; + reasons: readonly string[]; +}): void { + if (freshness.fresh || process.env.OPENCLAW_MIGRATION_EXISTING_IMPORT === "1") { + return; + } + throw new Error( + [ + "Migration import during onboarding requires a fresh OpenClaw setup.", + "Create a fresh setup or reset config, credentials, sessions, and workspace before importing.", + "Backup plus overwrite/merge imports are feature-gated for now.", + "Existing setup:", + ...freshness.reasons.map((reason) => `- ${reason}`), + ].join("\n"), + ); +} + +export async function detectSetupMigrationSources(params: { + config: OpenClawConfig; + runtime: RuntimeEnv; +}): Promise { + const [{ resolvePluginMigrationProviders }, { createMigrationLogger }, { resolveStateDir }] = + await Promise.all([ + import("../plugins/migration-provider-runtime.js"), + import("../commands/migrate/context.js"), + import("../config/paths.js"), + ]); + const stateDir = resolveStateDir(); + const logger = createMigrationLogger(params.runtime); + const detections: SetupMigrationDetection[] = []; + for (const provider of resolvePluginMigrationProviders({ cfg: params.config })) { + if (!provider.detect) { + continue; + } + try { + const detection = await provider.detect({ + config: params.config, + stateDir, + logger, + }); + if (detection.found) { + detections.push({ + providerId: provider.id, + label: detection.label ?? provider.label, + ...(detection.source ? { source: detection.source } : {}), + ...(detection.message ? { message: detection.message } : {}), + }); + } + } catch (error) { + logger.debug?.( + `Migration provider ${provider.id} detection failed: ${formatErrorMessage(error)}`, + ); + } + } + return detections; +} + +function resolveImportSourceDefault(params: { + providerId: string; + detections: readonly SetupMigrationDetection[]; +}): string { + const detected = params.detections.find( + (detection) => detection.providerId === params.providerId, + ); + if (detected?.source) { + return detected.source; + } + return params.providerId === "hermes" ? "~/.hermes" : ""; +} + +async function selectSetupMigrationProvider(params: { + opts: OnboardOptions; + baseConfig: OpenClawConfig; + detections: readonly SetupMigrationDetection[]; + prompter: WizardPrompter; +}): Promise<{ + provider: MigrationProviderPlugin; + providerId: string; +}> { + const { resolvePluginMigrationProvider, resolvePluginMigrationProviders } = + await import("../plugins/migration-provider-runtime.js"); + const providers = resolvePluginMigrationProviders({ cfg: params.baseConfig }); + if (providers.length === 0) { + throw new Error("No migration providers found."); + } + const providerById = new Map(providers.map((provider) => [provider.id, provider])); + const providerId = + params.opts.importFrom?.trim() || + (await params.prompter.select({ + message: "Migration source", + options: [ + ...params.detections.map((detection) => ({ + value: detection.providerId, + label: detection.label, + ...(detection.source || detection.message + ? { hint: detection.source ?? detection.message } + : {}), + })), + ...providers + .filter( + (provider) => + !params.detections.some((detection) => detection.providerId === provider.id), + ) + .map((provider) => ({ + value: provider.id, + label: provider.label, + hint: provider.description ?? "Enter a source path next", + })), + ], + initialValue: params.detections[0]?.providerId ?? providers[0]?.id, + })); + const provider = + providerById.get(providerId) ?? + resolvePluginMigrationProvider({ providerId, cfg: params.baseConfig }); + if (!provider) { + throw new Error(`Unknown migration provider "${providerId}".`); + } + return { provider, providerId }; +} + +export async function runSetupMigrationImport(params: { + opts: OnboardOptions; + baseConfig: OpenClawConfig; + detections: readonly SetupMigrationDetection[]; + prompter: WizardPrompter; + runtime: RuntimeEnv; + writeConfigFile: (config: OpenClawConfig) => Promise; +}): Promise { + const [ + { applyLocalSetupWorkspaceConfig, applySkipBootstrapConfig }, + { createMigrationLogger, buildMigrationReportDir }, + { createPreMigrationBackup }, + { assertApplySucceeded, assertConflictFreePlan, formatMigrationPlan }, + { resolveStateDir }, + onboardHelpers, + ] = await Promise.all([ + import("../commands/onboard-config.js"), + import("../commands/migrate/context.js"), + import("../commands/migrate/apply.js"), + import("../commands/migrate/output.js"), + import("../config/paths.js"), + import("../commands/onboard-helpers.js"), + ]); + const { provider, providerId } = await selectSetupMigrationProvider({ + opts: params.opts, + baseConfig: params.baseConfig, + detections: params.detections, + prompter: params.prompter, + }); + const sourceDefault = resolveImportSourceDefault({ providerId, detections: params.detections }); + const sourceDir = + params.opts.importSource?.trim() || + sourceDefault || + (params.opts.nonInteractive + ? (() => { + throw new Error("--import-source is required for non-interactive migration import."); + })() + : await params.prompter.text({ + message: "Source agent home", + initialValue: providerId === "hermes" ? "~/.hermes" : undefined, + })); + const workspaceInput = + params.opts.workspace ?? + (params.opts.nonInteractive + ? (params.baseConfig.agents?.defaults?.workspace ?? onboardHelpers.DEFAULT_WORKSPACE) + : await params.prompter.text({ + message: "Target workspace directory", + initialValue: + params.baseConfig.agents?.defaults?.workspace ?? onboardHelpers.DEFAULT_WORKSPACE, + })); + const workspaceDir = resolveUserPath(workspaceInput.trim() || onboardHelpers.DEFAULT_WORKSPACE); + let targetConfig = applyLocalSetupWorkspaceConfig(params.baseConfig, workspaceDir); + if (params.opts.skipBootstrap) { + targetConfig = applySkipBootstrapConfig(targetConfig); + } + + const stateDir = resolveStateDir(); + assertFreshSetupMigrationTarget( + await inspectSetupMigrationFreshness({ + baseConfig: params.baseConfig, + stateDir, + workspaceDir, + }), + ); + const ctx = { + config: targetConfig, + stateDir, + source: sourceDir, + includeSecrets: Boolean(params.opts.importSecrets), + overwrite: false, + logger: createMigrationLogger(params.runtime), + }; + const plan = await provider.plan(ctx); + await params.prompter.note(formatMigrationPlan(plan).join("\n"), "Migration preview"); + assertConflictFreePlan(plan, providerId); + + const confirmed = + params.opts.nonInteractive === true + ? true + : await params.prompter.confirm({ + message: "Apply this migration now?", + initialValue: false, + }); + if (!confirmed) { + throw new WizardCancelledError("migration cancelled"); + } + + const reportDir = buildMigrationReportDir(providerId, stateDir); + const backupPath = await createPreMigrationBackup({}); + targetConfig = onboardHelpers.applyWizardMetadata(targetConfig, { + command: "onboard", + mode: "local", + }); + targetConfig = await params.writeConfigFile(targetConfig); + const applyCtx = { + ...ctx, + config: targetConfig, + ...(backupPath ? { backupPath } : {}), + reportDir, + }; + const result = await provider.apply(applyCtx, plan); + const withReport = { + ...result, + ...((result.backupPath ?? backupPath) ? { backupPath: result.backupPath ?? backupPath } : {}), + reportDir: result.reportDir ?? reportDir, + }; + assertApplySucceeded(withReport); + await params.prompter.note(formatMigrationPlan(withReport).join("\n"), "Migration applied"); + await params.prompter.outro("Migration complete. Run `openclaw doctor` next."); +} diff --git a/src/wizard/setup.ts b/src/wizard/setup.ts index 48761819632..eb8288e500d 100644 --- a/src/wizard/setup.ts +++ b/src/wizard/setup.ts @@ -20,6 +20,7 @@ import type { RuntimeEnv } from "../runtime.js"; import { defaultRuntime } from "../runtime.js"; import { resolveUserPath } from "../utils.js"; import { WizardCancelledError, type WizardPrompter } from "./prompts.js"; +import { detectSetupMigrationSources, runSetupMigrationImport } from "./setup.migration-import.js"; import { resolveSetupSecretInputString } from "./setup.secret-input.js"; import { SECURITY_CONFIRM_MESSAGE, @@ -28,6 +29,8 @@ import { } from "./setup.security-note.js"; import type { QuickstartGatewayDefaults, WizardFlow } from "./setup.types.js"; +type SetupFlowChoice = WizardFlow | "import"; + type AuthChoiceModule = typeof import("../commands/auth-choice.js"); type ConfigLoggingModule = typeof import("../config/logging.js"); type ModelPickerModule = typeof import("../commands/model-picker.js"); @@ -229,28 +232,41 @@ export async function runSetupWizard( const quickstartHint = `Configure details later via ${formatCliCommand("openclaw configure")}.`; const manualHint = "Configure port, network, Tailscale, and auth options."; + const migrationDetections = await detectSetupMigrationSources({ config: baseConfig, runtime }); + const firstMigrationDetection = migrationDetections[0]; + const importOption = firstMigrationDetection + ? { + value: "import" as const, + label: `Import from ${firstMigrationDetection.label}`, + ...(firstMigrationDetection.source ? { hint: firstMigrationDetection.source } : {}), + } + : undefined; const explicitFlowRaw = opts.flow?.trim(); const normalizedExplicitFlow = explicitFlowRaw === "manual" ? "advanced" : explicitFlowRaw; if ( normalizedExplicitFlow && normalizedExplicitFlow !== "quickstart" && - normalizedExplicitFlow !== "advanced" + normalizedExplicitFlow !== "advanced" && + normalizedExplicitFlow !== "import" ) { - runtime.error("Invalid --flow (use quickstart, manual, or advanced)."); + runtime.error("Invalid --flow (use quickstart, manual, advanced, or import)."); runtime.exit(1); return; } - const explicitFlow: WizardFlow | undefined = - normalizedExplicitFlow === "quickstart" || normalizedExplicitFlow === "advanced" + const explicitFlow: SetupFlowChoice | undefined = + normalizedExplicitFlow === "quickstart" || + normalizedExplicitFlow === "advanced" || + normalizedExplicitFlow === "import" ? normalizedExplicitFlow : undefined; - let flow: WizardFlow = + let flow: SetupFlowChoice = explicitFlow ?? (await prompter.select({ message: "Setup mode", options: [ { value: "quickstart", label: "QuickStart", hint: quickstartHint }, { value: "advanced", label: "Manual", hint: manualHint }, + ...(importOption ? [importOption] : []), ], initialValue: "quickstart", })); @@ -300,6 +316,19 @@ export async function runSetupWizard( } } + if (opts.importFrom || flow === "import") { + await runSetupMigrationImport({ + opts, + baseConfig, + detections: migrationDetections, + prompter, + runtime, + writeConfigFile: writeWizardConfigFile, + }); + return; + } + const wizardFlow: WizardFlow = flow; + const quickstartGateway: QuickstartGatewayDefaults = (() => { const hasExisting = typeof baseConfig.gateway?.port === "number" || @@ -669,7 +698,7 @@ export async function runSetupWizard( const { configureGatewayForSetup } = await import("./setup.gateway-config.js"); const gateway = await configureGatewayForSetup({ - flow, + flow: wizardFlow, baseConfig, nextConfig, localPort, @@ -746,7 +775,7 @@ export async function runSetupWizard( const { finalizeSetupWizard } = await import("./setup.finalize.js"); const { launchedTui } = await finalizeSetupWizard({ - flow, + flow: wizardFlow, opts, baseConfig, nextConfig, diff --git a/test/helpers/plugins/plugin-api.ts b/test/helpers/plugins/plugin-api.ts index e8fed03cf37..68db5993506 100644 --- a/test/helpers/plugins/plugin-api.ts +++ b/test/helpers/plugins/plugin-api.ts @@ -25,6 +25,7 @@ export function createTestPluginApi(api: TestPluginApiInput = {}): OpenClawPlugi registerNodeHostCommand() {}, registerSecurityAuditCollector() {}, registerConfigMigration() {}, + registerMigrationProvider() {}, registerAutoEnableProbe() {}, registerProvider() {}, registerSpeechProvider() {}, diff --git a/test/setup-openclaw-runtime.ts b/test/setup-openclaw-runtime.ts index a1783104daf..e7c7e5e248a 100644 --- a/test/setup-openclaw-runtime.ts +++ b/test/setup-openclaw-runtime.ts @@ -153,6 +153,7 @@ function createTestRegistryForSetup( videoGenerationProviders: [], webFetchProviders: [], webSearchProviders: [], + migrationProviders: [], memoryEmbeddingProviders: [], gatewayHandlers: {}, gatewayMethodScopes: {},