diff --git a/CHANGELOG.md b/CHANGELOG.md index bc37bc4e3cc..0121f790ab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -203,6 +203,7 @@ Docs: https://docs.openclaw.ai OpenClaw-owned package manifest so Linux updates cannot accidentally write to a parent `$HOME/node_modules` tree. Fixes #71730. - Plugins/install: pass onboarding plugin config into plugin index writes so local plugin installs outside default discovery roots keep their install records. Thanks @shakkernerd. +- Plugins/install: migrate shipped `plugins.installs` config records into the plugin index while stripping them from runtime config and future writes. Thanks @shakkernerd. - Plugins/security: keep plugin audit JSON check ids stable while reporting plugin index install-record findings with updated wording. Thanks @shakkernerd. - CLI/config: reject direct `plugins.installs` edits with guidance to use `openclaw plugins install`, `openclaw plugins update`, or `openclaw plugins uninstall` instead. Thanks @shakkernerd. - Live tests/voice: accept common STT variants for OpenClaw and ElevenLabs diff --git a/src/commands/doctor/shared/plugin-registry-migration.test.ts b/src/commands/doctor/shared/plugin-registry-migration.test.ts index 5925620e951..309416c6486 100644 --- a/src/commands/doctor/shared/plugin-registry-migration.test.ts +++ b/src/commands/doctor/shared/plugin-registry-migration.test.ts @@ -238,6 +238,63 @@ describe("plugin registry install migration", () => { }); }); + it("seeds first-run install records from shipped plugins.installs config", async () => { + const stateDir = makeTempDir(); + const pluginDir = path.join(stateDir, "plugins", "demo"); + fs.mkdirSync(pluginDir, { recursive: true }); + + await expect( + migratePluginRegistryForInstall({ + stateDir, + candidates: [createCandidate(pluginDir)], + readConfig: async () => ({ + plugins: { + entries: { + demo: { + enabled: true, + }, + }, + installs: { + demo: { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }, + }, + }, + }), + env: hermeticEnv(), + }), + ).resolves.toMatchObject({ + status: "migrated", + current: { + plugins: [ + expect.objectContaining({ + pluginId: "demo", + installRecord: { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }, + }), + ], + }, + }); + + await expect(readPersistedInstalledPluginIndex({ stateDir })).resolves.toMatchObject({ + plugins: [ + expect.objectContaining({ + pluginId: "demo", + installRecord: { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }, + }), + ], + }); + }); + it("marks force migration env as deprecated break-glass", () => { expect( preflightPluginRegistryInstallMigration({ diff --git a/src/commands/doctor/shared/plugin-registry-migration.ts b/src/commands/doctor/shared/plugin-registry-migration.ts index 0dd76e05c46..2f689354a38 100644 --- a/src/commands/doctor/shared/plugin-registry-migration.ts +++ b/src/commands/doctor/shared/plugin-registry-migration.ts @@ -1,5 +1,9 @@ import fs from "node:fs"; import { normalizeProviderId } from "../../../agents/provider-id.js"; +import { + extractShippedPluginInstallConfigRecords, + stripShippedPluginInstallConfigRecords, +} from "../../../config/plugin-install-config-migration.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import { loadInstalledPluginIndexInstallRecords } from "../../../plugins/installed-plugin-index-records.js"; import { @@ -252,8 +256,12 @@ export async function migratePluginRegistryForInstall( return { status: "dry-run", migrated: false, preflight }; } - const config = await readMigrationConfig(params); - const installRecords = await loadInstalledPluginIndexInstallRecords(params); + const rawConfig = await readMigrationConfig(params); + const config = stripShippedPluginInstallConfigRecords(rawConfig) as OpenClawConfig; + const installRecords = { + ...extractShippedPluginInstallConfigRecords(rawConfig), + ...(await loadInstalledPluginIndexInstallRecords(params)), + }; const migrationParams = { ...params, config, diff --git a/src/config/io.ts b/src/config/io.ts index 0bb6bf2bf4f..1876d74dbb3 100644 --- a/src/config/io.ts +++ b/src/config/io.ts @@ -18,6 +18,10 @@ import { collectRelevantDoctorPluginIds, listPluginDoctorLegacyConfigRules, } from "../plugins/doctor-contract-registry.js"; +import { + loadInstalledPluginIndexInstallRecordsSync, + writePersistedInstalledPluginIndexInstallRecordsSync, +} from "../plugins/installed-plugin-index-records.js"; import { sanitizeTerminalText } from "../terminal/safe-text.js"; import { isRecord } from "../utils.js"; import { VERSION } from "../version.js"; @@ -60,6 +64,7 @@ import { projectSourceOntoRuntimeShape, restoreEnvRefsFromMap, resolvePersistCandidateForWrite, + resolveManagedUnsetPathsForWrite, resolveWriteEnvSnapshotForPath, } from "./io.write-prepare.js"; import { findLegacyConfigIssues } from "./legacy.js"; @@ -70,6 +75,10 @@ import { } from "./materialize.js"; import { applyMergePatch } from "./merge-patch.js"; import { resolveConfigPath, resolveStateDir } from "./paths.js"; +import { + extractShippedPluginInstallConfigRecords, + stripShippedPluginInstallConfigRecords, +} from "./plugin-install-config-migration.js"; import { applyConfigOverrides } from "./runtime-overrides.js"; import { clearRuntimeConfigSnapshot as clearRuntimeConfigSnapshotState, @@ -1009,9 +1018,12 @@ async function recoverConfigFromJsonRootSuffixWithDeps(params: { readResolution.resolvedConfigRaw, suffixRecovery.parsed, ); - const validated = validateConfigObjectWithPlugins(legacyResolution.effectiveConfigRaw, { - env: params.deps.env, - }); + const validated = validateConfigObjectWithPlugins( + stripShippedPluginInstallConfigRecords(legacyResolution.effectiveConfigRaw), + { + env: params.deps.env, + }, + ); if (!validated.ok) { return false; } @@ -1198,6 +1210,41 @@ export function createConfigIO( return applyConfigOverrides(cfgWithOwnerDisplaySecret); } + function migrateAndStripShippedPluginInstallConfigRecords(configRaw: unknown): unknown { + const installRecords = extractShippedPluginInstallConfigRecords(configRaw); + const stripped = stripShippedPluginInstallConfigRecords(configRaw); + if (Object.keys(installRecords).length === 0) { + return stripped; + } + + try { + const stateDir = resolveStateDir(deps.env, deps.homedir); + const existingRecords = loadInstalledPluginIndexInstallRecordsSync({ + env: deps.env, + stateDir, + }); + const nextRecords = { + ...installRecords, + ...existingRecords, + }; + if (Object.keys(installRecords).some((pluginId) => !(pluginId in existingRecords))) { + writePersistedInstalledPluginIndexInstallRecordsSync(nextRecords, { + config: coerceConfig(stripped), + env: deps.env, + stateDir, + }); + } + } catch (err) { + deps.logger.warn( + `Config (${configPath}): could not migrate shipped plugins.installs records into the plugin index: ${formatErrorMessage( + err, + )}`, + ); + } + + return stripped; + } + function loadConfig(): OpenClawConfig { try { maybeLoadDotEnvForConfig(deps.env); @@ -1230,7 +1277,9 @@ export function createConfigIO( ); const resolvedConfig = readResolution.resolvedConfigRaw; const legacyResolution = resolveLegacyConfigForRead(resolvedConfig, effectiveParsed); - const effectiveConfigRaw = legacyResolution.effectiveConfigRaw; + const effectiveConfigRaw = migrateAndStripShippedPluginInstallConfigRecords( + legacyResolution.effectiveConfigRaw, + ); for (const w of readResolution.envWarnings) { deps.logger.warn( `Config (${configPath}): missing env var "${w.varName}" at ${w.configPath} - feature using this value will be unavailable`, @@ -1439,7 +1488,9 @@ export function createConfigIO( const resolvedConfigRaw = readResolution.resolvedConfigRaw; const legacyResolution = resolveLegacyConfigForRead(resolvedConfigRaw, effectiveParsed); - const effectiveConfigRaw = legacyResolution.effectiveConfigRaw; + const effectiveConfigRaw = migrateAndStripShippedPluginInstallConfigRecords( + legacyResolution.effectiveConfigRaw, + ); fallbackSourceConfig = coerceConfig(effectiveConfigRaw); const validated = validateConfigObjectWithPlugins(effectiveConfigRaw, { env: deps.env, @@ -1562,6 +1613,7 @@ export function createConfigIO( writeOptions: { envSnapshotForRestore: result.envSnapshotForRestore, expectedConfigPath: configPath, + unsetPaths: resolveManagedUnsetPathsForWrite(undefined), }, }; } @@ -1609,7 +1661,9 @@ export function createConfigIO( readResolution.resolvedConfigRaw, recovered.parsed, ); - return coerceConfig(legacyResolution.effectiveConfigRaw); + return coerceConfig( + stripShippedPluginInstallConfigRecords(legacyResolution.effectiveConfigRaw), + ); } catch { return {}; } @@ -1620,6 +1674,7 @@ export function createConfigIO( options: ConfigWriteOptions = {}, ): Promise<{ persistedHash: string; persistedConfig: OpenClawConfig }> { clearConfigCache(); + const unsetPaths = resolveManagedUnsetPathsForWrite(options.unsetPaths); let persistCandidate: unknown = cfg; const snapshot = options.baseSnapshot ?? (await readConfigFileSnapshotInternal()).snapshot; let envRefMap: Map | null = null; @@ -1655,10 +1710,7 @@ export function createConfigIO( } } - persistCandidate = applyUnsetPathsForWrite( - persistCandidate as OpenClawConfig, - options.unsetPaths, - ); + persistCandidate = applyUnsetPathsForWrite(persistCandidate as OpenClawConfig, unsetPaths); const validated = validateConfigObjectRawWithPlugins(persistCandidate, { env: deps.env }); if (!validated.ok) { @@ -1720,7 +1772,7 @@ export function createConfigIO( envRefMap && changedPaths ? (restoreEnvRefsFromMap(cfgToWrite, "", envRefMap, changedPaths) as OpenClawConfig) : cfgToWrite; - const outputConfig = applyUnsetPathsForWrite(outputConfigBase, options.unsetPaths); + const outputConfig = applyUnsetPathsForWrite(outputConfigBase, unsetPaths); // Do NOT apply runtime defaults when writing - user config should only contain // explicitly set values. Runtime defaults are applied when loading (issue #6070). const stampedOutputConfig = stampConfigVersion(outputConfig); @@ -2054,7 +2106,7 @@ export async function writeConfigFile( expectedConfigPath: options.expectedConfigPath, envSnapshotForRestore: options.envSnapshotForRestore, }), - unsetPaths: options.unsetPaths, + unsetPaths: resolveManagedUnsetPathsForWrite(options.unsetPaths), allowDestructiveWrite: options.allowDestructiveWrite, skipRuntimeSnapshotRefresh: options.skipRuntimeSnapshotRefresh, skipOutputLogs: options.skipOutputLogs, diff --git a/src/config/io.write-config.test.ts b/src/config/io.write-config.test.ts index 56735106617..35aaee5a74c 100644 --- a/src/config/io.write-config.test.ts +++ b/src/config/io.write-config.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { readPersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { @@ -99,6 +100,91 @@ describe("config io write", () => { logger: silentLogger, }); + it("migrates shipped plugin install config records into the plugin index", async () => { + await withSuiteHome(async (home) => { + const configPath = path.join(home, ".openclaw", "openclaw.json"); + const pluginDir = path.join(home, ".openclaw", "plugins", "demo"); + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + const source = path.join(pluginDir, "index.ts"); + await fs.mkdir(pluginDir, { recursive: true }); + await fs.writeFile(source, "export function register() {}\n", "utf-8"); + await fs.writeFile( + manifestPath, + `${JSON.stringify({ id: "demo", configSchema: { type: "object" } }, null, 2)}\n`, + "utf-8", + ); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify( + { + plugins: { + entries: { demo: { enabled: true } }, + installs: { + demo: { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }, + }, + }, + }, + null, + 2, + )}\n`, + "utf-8", + ); + mockLoadPluginManifestRegistry.mockReturnValue({ + diagnostics: [], + plugins: [ + { + id: "demo", + origin: "global", + channels: [], + providers: [], + cliBackends: [], + skills: [], + hooks: [], + rootDir: pluginDir, + source, + manifestPath, + configSchema: { + type: "object", + }, + }, + ], + } satisfies PluginManifestRegistry); + + const io = createFastConfigIO(home); + try { + const cfg = io.loadConfig(); + + expect(cfg.plugins?.installs).toBeUndefined(); + await expect( + readPersistedInstalledPluginIndex({ + stateDir: path.join(home, ".openclaw"), + }), + ).resolves.toMatchObject({ + plugins: [ + expect.objectContaining({ + pluginId: "demo", + installRecord: { + source: "npm", + spec: "demo@1.0.0", + installPath: pluginDir, + }, + }), + ], + }); + } finally { + mockLoadPluginManifestRegistry.mockReturnValue({ + diagnostics: [], + plugins: [], + } satisfies PluginManifestRegistry); + } + }); + }); + const writeGatewayPortAndReadConfig = async (home: string, configPath: string) => { const io = createFastConfigIO(home); diff --git a/src/config/io.write-prepare.ts b/src/config/io.write-prepare.ts index 0869855c9dd..161f3c0cd60 100644 --- a/src/config/io.write-prepare.ts +++ b/src/config/io.write-prepare.ts @@ -7,6 +7,8 @@ import type { OpenClawConfig } from "./types.js"; const OPEN_DM_POLICY_ALLOW_FROM_RE = /^(?[a-z0-9_.-]+)\s*=\s*"open"\s+requires\s+(?[a-z0-9_.-]+)(?:\s+\(or\s+[a-z0-9_.-]+\))?\s+to include "\*"$/i; +const MANAGED_CONFIG_UNSET_PATHS = [["plugins", "installs"]] as const; + function cloneUnknown(value: T): T { return structuredClone(value); } @@ -337,6 +339,25 @@ export function applyUnsetPathsForWrite( return next; } +export function resolveManagedUnsetPathsForWrite( + unsetPaths: readonly string[][] | undefined, +): string[][] { + const next: string[][] = []; + for (const managedPath of MANAGED_CONFIG_UNSET_PATHS) { + next.push(Array.from(managedPath)); + } + for (const unsetPath of unsetPaths ?? []) { + if (!Array.isArray(unsetPath) || unsetPath.length === 0) { + continue; + } + if (next.some((existing) => isDeepStrictEqual(existing, unsetPath))) { + continue; + } + next.push([...unsetPath]); + } + return next; +} + export function collectChangedPaths( base: unknown, target: unknown, diff --git a/src/config/mutate.ts b/src/config/mutate.ts index 3c9aa1fab1e..e3f3a25c85e 100644 --- a/src/config/mutate.ts +++ b/src/config/mutate.ts @@ -13,7 +13,7 @@ import { writeConfigFile, type ConfigWriteOptions, } from "./io.js"; -import { applyUnsetPathsForWrite } from "./io.write-prepare.js"; +import { applyUnsetPathsForWrite, resolveManagedUnsetPathsForWrite } from "./io.write-prepare.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "./types.js"; import { validateConfigObjectWithPlugins } from "./validation.js"; @@ -114,7 +114,10 @@ async function tryWriteSingleTopLevelIncludeMutation(params: { nextConfig: OpenClawConfig; writeOptions?: ConfigWriteOptions; }): Promise { - const nextConfig = applyUnsetPathsForWrite(params.nextConfig, params.writeOptions?.unsetPaths); + const nextConfig = applyUnsetPathsForWrite( + params.nextConfig, + resolveManagedUnsetPathsForWrite(params.writeOptions?.unsetPaths), + ); const changedKeys = getChangedTopLevelKeys(params.snapshot.sourceConfig, nextConfig); if (changedKeys.length !== 1 || changedKeys[0] === "") { return false; diff --git a/src/config/plugin-install-config-migration.ts b/src/config/plugin-install-config-migration.ts new file mode 100644 index 00000000000..fa46b9409f9 --- /dev/null +++ b/src/config/plugin-install-config-migration.ts @@ -0,0 +1,48 @@ +import { z } from "zod"; +import type { PluginInstallRecord } from "./types.plugins.js"; +import { PluginInstallRecordShape } from "./zod-schema.installs.js"; + +const PluginInstallRecordsSchema = z.record( + z.string(), + z.object(PluginInstallRecordShape).passthrough(), +); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function pruneEmptyPluginsObject(plugins: Record): unknown { + const { installs: _installs, ...rest } = plugins; + return Object.keys(rest).length === 0 ? undefined : rest; +} + +export function extractShippedPluginInstallConfigRecords( + config: unknown, +): Record { + if (!isRecord(config) || !isRecord(config.plugins)) { + return {}; + } + const parsed = PluginInstallRecordsSchema.safeParse(config.plugins.installs); + return parsed.success + ? (structuredClone(parsed.data) as Record) + : {}; +} + +export function stripShippedPluginInstallConfigRecords(config: unknown): unknown { + if (!isRecord(config) || !isRecord(config.plugins) || !("installs" in config.plugins)) { + return config; + } + const plugins = pruneEmptyPluginsObject(config.plugins); + const { plugins: _plugins, ...rest } = config; + return plugins === undefined ? rest : { ...rest, plugins }; +} + +export function prepareShippedPluginInstallConfigMigration(config: unknown): { + config: unknown; + installRecords: Record; +} { + return { + config: stripShippedPluginInstallConfigRecords(config), + installRecords: extractShippedPluginInstallConfigRecords(config), + }; +} diff --git a/src/plugins/installed-plugin-index-records.ts b/src/plugins/installed-plugin-index-records.ts index 2452c81195a..9140a5f8107 100644 --- a/src/plugins/installed-plugin-index-records.ts +++ b/src/plugins/installed-plugin-index-records.ts @@ -7,7 +7,10 @@ import { readPersistedInstalledPluginIndexInstallRecordsSync, } from "./installed-plugin-index-record-reader.js"; import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; -import { refreshPersistedInstalledPluginIndex } from "./installed-plugin-index-store.js"; +import { + refreshPersistedInstalledPluginIndex, + refreshPersistedInstalledPluginIndexSync, +} from "./installed-plugin-index-store.js"; import { type RefreshInstalledPluginIndexParams } from "./installed-plugin-index.js"; import { recordPluginInstall, type PluginInstallUpdate } from "./installs.js"; @@ -49,6 +52,18 @@ export async function writePersistedInstalledPluginIndexInstallRecords( return resolveInstalledPluginIndexRecordsStorePath(options); } +export function writePersistedInstalledPluginIndexInstallRecordsSync( + records: Record, + options: InstalledPluginIndexRecordRefreshOptions = {}, +): string { + refreshPersistedInstalledPluginIndexSync({ + ...options, + reason: "source-changed", + installRecords: records, + }); + return resolveInstalledPluginIndexRecordsStorePath(options); +} + export function withPluginInstallRecords( config: OpenClawConfig, records: Record, diff --git a/src/plugins/installed-plugin-index-store.ts b/src/plugins/installed-plugin-index-store.ts index bce1eb9723e..4fb075767d2 100644 --- a/src/plugins/installed-plugin-index-store.ts +++ b/src/plugins/installed-plugin-index-store.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { saveJsonFile } from "../infra/json-file.js"; import { readJsonFile, readJsonFileSync, writeJsonAtomic } from "../infra/json-files.js"; import { safeParseWithSchema } from "../utils/zod-parse.js"; import { @@ -142,6 +143,15 @@ export async function writePersistedInstalledPluginIndex( return filePath; } +export function writePersistedInstalledPluginIndexSync( + index: InstalledPluginIndex, + options: InstalledPluginIndexStoreOptions = {}, +): string { + const filePath = resolveInstalledPluginIndexStorePath(options); + saveJsonFile(filePath, { ...index, warning: INSTALLED_PLUGIN_INDEX_WARNING }); + return filePath; +} + export async function inspectPersistedInstalledPluginIndex( params: LoadInstalledPluginIndexParams & InstalledPluginIndexStoreOptions = {}, ): Promise { @@ -176,3 +186,11 @@ export async function refreshPersistedInstalledPluginIndex( await writePersistedInstalledPluginIndex(index, params); return index; } + +export function refreshPersistedInstalledPluginIndexSync( + params: RefreshInstalledPluginIndexParams & InstalledPluginIndexStoreOptions, +): InstalledPluginIndex { + const index = refreshInstalledPluginIndex(params); + writePersistedInstalledPluginIndexSync(index, params); + return index; +}