diff --git a/CHANGELOG.md b/CHANGELOG.md index 12091267c35..0d44face5c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Docs: https://docs.openclaw.ai - CLI/onboarding: infer image input for common custom-provider vision model IDs, ask only for unknown models, and keep `--custom-image-input`/`--custom-text-input` overrides so vision-capable proxies do not get saved as text-only configs. Fixes #51869. Thanks @Antsoldier1974. - Models/OpenAI Codex: stop listing or resolving unsupported `openai-codex/gpt-5.4-mini` rows through Codex OAuth, keep stale discovery rows suppressed with a clear API-key-route hint, and leave direct `openai/gpt-5.4-mini` available. Fixes #73242. Thanks @0xCyda. - Plugin SDK: restore the root-alias bridge for `registerContextEngine` and expose missing legacy compat helpers `normalizeAccountId` and `resolvePreferredOpenClawTmpDir` so older external plugins such as `openclaw-weixin` can keep loading while migrating to focused SDK subpaths. Fixes #53497. Thanks @alanxchen85. +- Auth profiles: make `openclaw doctor --fix` migrate legacy flat `auth-profiles.json` files such as `{ "ollama-windows": { "apiKey": "ollama-local" } }` to canonical provider default API-key profiles with a backup, so custom Ollama/OpenAI-compatible providers recover cleanly after upgrading. Fixes #59629; supersedes #59642. Thanks @Xsanders555 and @Linux2010. - Memory/Dreaming: retry Dream Diary once with the session default when a configured dreaming model is unavailable, while leaving subagent trust and allowlist errors visible instead of silently masking configuration problems. Refs #67409 and #69209. Thanks @Ghiggins18 and @everySympathy. - Feishu/inbound files: recover CJK filenames from plain `Content-Disposition: filename=` download headers when Feishu exposes UTF-8 bytes through Latin-1 header decoding, while leaving valid Latin-1 and JSON-derived names unchanged. (#48578, #50435, #59431) Thanks @alex-xuweilong, @lishuaigit, and @DoChaoing. - Channels/Telegram: normalize accidental full `/bot` Telegram `apiRoot` values at runtime and teach `openclaw doctor --fix` to remove the suffix, so startup control calls no longer 404 when direct Bot API curl commands work. Fixes #55387. Thanks @brendanmatthewjones-cmyk, @techfindubai-ux, and @Sivlerback-Chris. diff --git a/docs/gateway/authentication.md b/docs/gateway/authentication.md index dac0dd0dd4b..25c725b1bb0 100644 --- a/docs/gateway/authentication.md +++ b/docs/gateway/authentication.md @@ -93,6 +93,23 @@ Manual token entry (any provider; writes `auth-profiles.json` + updates config): openclaw models auth paste-token --provider openrouter ``` +`auth-profiles.json` stores credentials only. The canonical shape is: + +```json +{ + "version": 1, + "profiles": { + "openrouter:default": { + "type": "api_key", + "provider": "openrouter", + "key": "OPENROUTER_API_KEY" + } + } +} +``` + +OpenClaw expects the canonical `version` + `profiles` shape at runtime. If an older install still has a flat file such as `{ "openrouter": { "apiKey": "..." } }`, run `openclaw doctor --fix` to rewrite it as an `openrouter:default` API-key profile; doctor keeps a `.legacy-flat.*.bak` copy beside the original. Endpoint details such as `baseUrl`, `api`, model ids, headers, and timeouts belong under `models.providers.` in `openclaw.json` or `models.json`, not in `auth-profiles.json`. + Auth profile refs are also supported for static credentials: - `api_key` credentials can use `keyRef: { source, provider, id }` diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index ded910bb30b..e0a0269ea05 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -800,6 +800,7 @@ Notes: - Per-agent profiles are stored at `/auth-profiles.json`. - `auth-profiles.json` supports value-level refs (`keyRef` for `api_key`, `tokenRef` for `token`) for static credential modes. +- Legacy flat `auth-profiles.json` maps such as `{ "provider": { "apiKey": "..." } }` are not a runtime format; `openclaw doctor --fix` rewrites them to canonical `provider:default` API-key profiles with a `.legacy-flat.*.bak` backup. - OAuth-mode profiles (`auth.profiles..mode = "oauth"`) do not support SecretRef-backed auth-profile credentials. - Static runtime credentials come from in-memory resolved snapshots; legacy static `auth.json` entries are scrubbed when discovered. - Legacy OAuth imports from `~/.openclaw/credentials/oauth.json`. diff --git a/docs/providers/ollama.md b/docs/providers/ollama.md index 3ee76cadd23..c144eada57a 100644 --- a/docs/providers/ollama.md +++ b/docs/providers/ollama.md @@ -27,6 +27,9 @@ Ollama provider config uses `baseUrl` as the canonical key. OpenClaw also accept Custom provider ids that set `api: "ollama"` follow the same rules. For example, an `ollama-remote` provider that points at a private LAN Ollama host can use `apiKey: "ollama-local"` and sub-agents will resolve that marker through the Ollama provider hook instead of treating it as a missing credential. Memory search can also set `agents.defaults.memorySearch.provider` to that custom provider id so embeddings use the matching Ollama endpoint. + + `auth-profiles.json` stores the credential for a provider id. Put endpoint settings (`baseUrl`, `api`, model ids, headers, timeouts) in `models.providers.`. Older flat auth-profile files such as `{ "ollama-windows": { "apiKey": "ollama-local" } }` are not a runtime format; run `openclaw doctor --fix` to rewrite them to the canonical `ollama-windows:default` API-key profile with a backup. `baseUrl` in that file is compatibility noise and should be moved to provider config. + When Ollama is used for memory embeddings, bearer auth is scoped to the host where it was declared: diff --git a/src/agents/auth-profiles.ensureauthprofilestore.test.ts b/src/agents/auth-profiles.ensureauthprofilestore.test.ts index be7841d2b7d..95d65a3e3cc 100644 --- a/src/agents/auth-profiles.ensureauthprofilestore.test.ts +++ b/src/agents/auth-profiles.ensureauthprofilestore.test.ts @@ -673,6 +673,27 @@ describe("ensureAuthProfileStore", () => { }); }); + it("does not load legacy flat auth-profiles.json entries at runtime", () => { + const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-auth-flat-profiles-")); + try { + const authPath = path.join(agentDir, "auth-profiles.json"); + const legacyFlatStore = { + "ollama-windows": { + apiKey: "ollama-local", + baseUrl: "http://10.0.2.2:11434/v1", + }, + }; + fs.writeFileSync(authPath, `${JSON.stringify(legacyFlatStore)}\n`, "utf8"); + + const store = ensureAuthProfileStore(agentDir); + + expect(store.profiles["ollama-windows:default"]).toBeUndefined(); + expect(JSON.parse(fs.readFileSync(authPath, "utf8"))).toEqual(legacyFlatStore); + } finally { + fs.rmSync(agentDir, { recursive: true, force: true }); + } + }); + it("merges legacy oauth.json into auth-profiles.json", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-oauth-migrate-")); const previousStateDir = process.env.OPENCLAW_STATE_DIR; diff --git a/src/commands/doctor-auth-flat-profiles.test.ts b/src/commands/doctor-auth-flat-profiles.test.ts new file mode 100644 index 00000000000..bd28a12dbc0 --- /dev/null +++ b/src/commands/doctor-auth-flat-profiles.test.ts @@ -0,0 +1,128 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { clearRuntimeAuthProfileStoreSnapshots } from "../agents/auth-profiles/store.js"; +import { maybeRepairLegacyFlatAuthProfileStores } from "./doctor-auth-flat-profiles.js"; +import type { DoctorPrompter } from "./doctor-prompter.js"; + +const roots: string[] = []; + +function makeTempRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-flat-auth-")); + roots.push(root); + return root; +} + +function makePrompter(shouldRepair: boolean): DoctorPrompter { + return { + confirm: vi.fn(async () => shouldRepair), + confirmAutoFix: vi.fn(async () => shouldRepair), + confirmAggressiveAutoFix: vi.fn(async () => shouldRepair), + confirmRuntimeRepair: vi.fn(async () => shouldRepair), + select: vi.fn(async (_params, fallback) => fallback), + shouldRepair, + shouldForce: false, + repairMode: { + shouldRepair, + shouldForce: false, + nonInteractive: false, + canPrompt: true, + updateInProgress: false, + }, + }; +} + +function withStateDir(root: string, run: () => T): T { + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + const previousAgentDir = process.env.OPENCLAW_AGENT_DIR; + process.env.OPENCLAW_STATE_DIR = root; + delete process.env.OPENCLAW_AGENT_DIR; + try { + return run(); + } finally { + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + if (previousAgentDir === undefined) { + delete process.env.OPENCLAW_AGENT_DIR; + } else { + process.env.OPENCLAW_AGENT_DIR = previousAgentDir; + } + } +} + +afterEach(() => { + clearRuntimeAuthProfileStoreSnapshots(); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +describe("maybeRepairLegacyFlatAuthProfileStores", () => { + it("rewrites legacy flat auth-profiles.json stores with a backup", async () => { + const root = makeTempRoot(); + await withStateDir(root, async () => { + const agentDir = path.join(root, "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + const authPath = path.join(agentDir, "auth-profiles.json"); + const legacy = { + "ollama-windows": { + apiKey: "ollama-local", + baseUrl: "http://10.0.2.2:11434/v1", + }, + }; + fs.writeFileSync(authPath, `${JSON.stringify(legacy)}\n`, "utf8"); + + const result = await maybeRepairLegacyFlatAuthProfileStores({ + cfg: {}, + prompter: makePrompter(true), + now: () => 123, + }); + + expect(result.detected).toEqual([authPath]); + expect(result.changes).toHaveLength(1); + expect(result.warnings).toEqual([]); + expect(JSON.parse(fs.readFileSync(authPath, "utf8"))).toEqual({ + version: 1, + profiles: { + "ollama-windows:default": { + type: "api_key", + provider: "ollama-windows", + key: "ollama-local", + }, + }, + }); + expect(JSON.parse(fs.readFileSync(`${authPath}.legacy-flat.123.bak`, "utf8"))).toEqual( + legacy, + ); + }); + }); + + it("reports legacy flat stores without rewriting when repair is declined", async () => { + const root = makeTempRoot(); + await withStateDir(root, async () => { + const agentDir = path.join(root, "agents", "main", "agent"); + fs.mkdirSync(agentDir, { recursive: true }); + const authPath = path.join(agentDir, "auth-profiles.json"); + const legacy = { + openai: { + apiKey: "sk-openai", + }, + }; + fs.writeFileSync(authPath, `${JSON.stringify(legacy)}\n`, "utf8"); + + const result = await maybeRepairLegacyFlatAuthProfileStores({ + cfg: {}, + prompter: makePrompter(false), + }); + + expect(result.detected).toEqual([authPath]); + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([]); + expect(JSON.parse(fs.readFileSync(authPath, "utf8"))).toEqual(legacy); + }); + }); +}); diff --git a/src/commands/doctor-auth-flat-profiles.ts b/src/commands/doctor-auth-flat-profiles.ts new file mode 100644 index 00000000000..06620850b0f --- /dev/null +++ b/src/commands/doctor-auth-flat-profiles.ts @@ -0,0 +1,271 @@ +import fs from "node:fs"; +import path from "node:path"; +import { resolveOpenClawAgentDir } from "../agents/agent-paths.js"; +import { resolveAgentDir, listAgentIds } from "../agents/agent-scope.js"; +import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js"; +import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; +import { + clearRuntimeAuthProfileStoreSnapshots, + saveAuthProfileStore, +} from "../agents/auth-profiles/store.js"; +import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { formatCliCommand } from "../cli/command-format.js"; +import { resolveStateDir } from "../config/paths.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { loadJsonFile } from "../infra/json-file.js"; +import { note } from "../terminal/note.js"; +import { shortenHomePath } from "../utils.js"; +import type { DoctorPrompter } from "./doctor-prompter.js"; + +type AuthProfileRepairCandidate = { + agentDir?: string; + authPath: string; +}; + +type LegacyFlatAuthProfileStore = { + agentDir?: string; + authPath: string; + store: AuthProfileStore; +}; + +export type LegacyFlatAuthProfileRepairResult = { + detected: string[]; + changes: string[]; + warnings: string[]; +}; + +const UNSAFE_LEGACY_AUTH_PROFILE_KEYS = new Set(["__proto__", "constructor", "prototype"]); + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} + +function readNonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +function isSafeLegacyProviderKey(key: string): boolean { + return key.trim().length > 0 && !UNSAFE_LEGACY_AUTH_PROFILE_KEYS.has(key); +} + +function inferLegacyCredentialType( + record: Record, +): AuthProfileCredential["type"] | undefined { + const explicit = readNonEmptyString(record.type) ?? readNonEmptyString(record.mode); + if (explicit === "api_key" || explicit === "token" || explicit === "oauth") { + return explicit; + } + if (readNonEmptyString(record.key) ?? readNonEmptyString(record.apiKey)) { + return "api_key"; + } + if (readNonEmptyString(record.token)) { + return "token"; + } + if ( + readNonEmptyString(record.access) && + readNonEmptyString(record.refresh) && + typeof record.expires === "number" + ) { + return "oauth"; + } + return undefined; +} + +function coerceLegacyFlatCredential( + providerId: string, + raw: unknown, +): AuthProfileCredential | null { + if (!isRecord(raw)) { + return null; + } + const provider = readNonEmptyString(raw.provider) ?? providerId; + const type = inferLegacyCredentialType(raw); + const email = readNonEmptyString(raw.email); + if (type === "api_key") { + const key = readNonEmptyString(raw.key) ?? readNonEmptyString(raw.apiKey); + return key ? { type, provider, key, ...(email ? { email } : {}) } : null; + } + if (type === "token") { + const token = readNonEmptyString(raw.token); + return token + ? { + type, + provider, + token, + ...(typeof raw.expires === "number" ? { expires: raw.expires } : {}), + ...(email ? { email } : {}), + } + : null; + } + if (type === "oauth") { + const access = readNonEmptyString(raw.access); + const refresh = readNonEmptyString(raw.refresh); + if (!access || !refresh || typeof raw.expires !== "number") { + return null; + } + return { + type, + provider, + access, + refresh, + expires: raw.expires, + ...(readNonEmptyString(raw.enterpriseUrl) + ? { enterpriseUrl: readNonEmptyString(raw.enterpriseUrl) } + : {}), + ...(readNonEmptyString(raw.projectId) + ? { projectId: readNonEmptyString(raw.projectId) } + : {}), + ...(readNonEmptyString(raw.accountId) + ? { accountId: readNonEmptyString(raw.accountId) } + : {}), + ...(email ? { email } : {}), + }; + } + return null; +} + +function coerceLegacyFlatAuthProfileStore(raw: unknown): AuthProfileStore | null { + if (!isRecord(raw) || "profiles" in raw) { + return null; + } + const store: AuthProfileStore = { + version: AUTH_STORE_VERSION, + profiles: {}, + }; + for (const [key, value] of Object.entries(raw)) { + const providerId = key.trim(); + if (!isSafeLegacyProviderKey(providerId)) { + continue; + } + const credential = coerceLegacyFlatCredential(providerId, value); + if (!credential) { + continue; + } + store.profiles[`${providerId}:default`] = credential; + } + return Object.keys(store.profiles).length > 0 ? store : null; +} + +function addCandidate( + candidates: Map, + agentDir: string | undefined, +): void { + const authPath = resolveAuthStorePath(agentDir); + candidates.set(path.resolve(authPath), { agentDir, authPath }); +} + +function listExistingAgentDirsFromState(): string[] { + const root = path.join(resolveStateDir(), "agents"); + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(root, { withFileTypes: true }); + } catch { + return []; + } + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(root, entry.name, "agent")) + .filter((agentDir) => { + try { + return fs.statSync(agentDir).isDirectory(); + } catch { + return false; + } + }); +} + +function listAuthProfileRepairCandidates(cfg: OpenClawConfig): AuthProfileRepairCandidate[] { + const candidates = new Map(); + addCandidate(candidates, resolveOpenClawAgentDir()); + for (const agentId of listAgentIds(cfg)) { + addCandidate(candidates, resolveAgentDir(cfg, agentId)); + } + for (const agentDir of listExistingAgentDirsFromState()) { + addCandidate(candidates, agentDir); + } + return [...candidates.values()]; +} + +function resolveLegacyFlatStore( + candidate: AuthProfileRepairCandidate, +): LegacyFlatAuthProfileStore | null { + if (!fs.existsSync(candidate.authPath)) { + return null; + } + const raw = loadJsonFile(candidate.authPath); + if (!raw || typeof raw !== "object" || "profiles" in raw) { + return null; + } + const store = coerceLegacyFlatAuthProfileStore(raw); + if (!store || Object.keys(store.profiles).length === 0) { + return null; + } + return { + ...candidate, + store, + }; +} + +function backupAuthProfileStore(authPath: string, now: () => number): string { + const backupPath = `${authPath}.legacy-flat.${now()}.bak`; + fs.copyFileSync(authPath, backupPath); + return backupPath; +} + +export async function maybeRepairLegacyFlatAuthProfileStores(params: { + cfg: OpenClawConfig; + prompter: DoctorPrompter; + now?: () => number; +}): Promise { + const now = params.now ?? Date.now; + const legacyStores = listAuthProfileRepairCandidates(params.cfg) + .map(resolveLegacyFlatStore) + .filter((entry): entry is LegacyFlatAuthProfileStore => entry !== null); + + const result: LegacyFlatAuthProfileRepairResult = { + detected: legacyStores.map((entry) => entry.authPath), + changes: [], + warnings: [], + }; + if (legacyStores.length === 0) { + return result; + } + + note( + [ + ...legacyStores.map( + (entry) => `- ${shortenHomePath(entry.authPath)} uses the legacy flat auth profile format.`, + ), + `- The gateway expects the canonical version/profiles store; ${formatCliCommand("openclaw doctor --fix")} rewrites this legacy shape with a backup.`, + ].join("\n"), + "Auth profiles", + ); + + const shouldRepair = await params.prompter.confirmAutoFix({ + message: "Rewrite legacy flat auth-profiles.json files now?", + initialValue: true, + }); + if (!shouldRepair) { + return result; + } + + for (const entry of legacyStores) { + try { + const backupPath = backupAuthProfileStore(entry.authPath, now); + saveAuthProfileStore(entry.store, entry.agentDir, { syncExternalCli: false }); + result.changes.push( + `Rewrote ${shortenHomePath(entry.authPath)} to the canonical auth profile format (backup: ${shortenHomePath(backupPath)}).`, + ); + } catch (err) { + result.warnings.push(`Failed to rewrite ${shortenHomePath(entry.authPath)}: ${String(err)}`); + } + } + clearRuntimeAuthProfileStoreSnapshots(); + if (result.changes.length > 0) { + note(result.changes.map((change) => `- ${change}`).join("\n"), "Doctor changes"); + } + if (result.warnings.length > 0) { + note(result.warnings.map((warning) => `- ${warning}`).join("\n"), "Doctor warnings"); + } + return result; +} diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index 543c87bdea6..433da1a322a 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -88,12 +88,18 @@ async function runGatewayConfigHealth(ctx: DoctorHealthFlowContext): Promise { + const { maybeRepairLegacyFlatAuthProfileStores } = + await import("../commands/doctor-auth-flat-profiles.js"); const { maybeRepairLegacyOAuthProfileIds } = await import("../commands/doctor-auth-legacy-oauth.js"); const { noteAuthProfileHealth, noteLegacyCodexProviderOverride } = await import("../commands/doctor-auth.js"); const { buildGatewayConnectionDetails } = await import("../gateway/call.js"); const { note } = await import("../terminal/note.js"); + await maybeRepairLegacyFlatAuthProfileStores({ + cfg: ctx.cfg, + prompter: ctx.prompter, + }); ctx.cfg = await maybeRepairLegacyOAuthProfileIds(ctx.cfg, ctx.prompter); await noteAuthProfileHealth({ cfg: ctx.cfg,