mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-12 07:20:45 +00:00
feat(secrets): finalize external secrets runtime and migration hardening
This commit is contained in:
committed by
Peter Steinberger
parent
c5b89fbaea
commit
0e69660c41
@@ -52,6 +52,23 @@ Skip `.env` scrubbing:
|
||||
openclaw secrets migrate --write --no-scrub-env
|
||||
```
|
||||
|
||||
`.env` scrub details (default behavior):
|
||||
|
||||
- Scrub target is `<config-dir>/.env`.
|
||||
- Only known secret env keys are considered.
|
||||
- Entries are removed only when the value exactly matches a migrated plaintext secret.
|
||||
- If `<config-dir>/.sops.yaml` or `<config-dir>/.sops.yml` exists, migrate passes it explicitly to `sops` so command behavior is cwd-independent.
|
||||
|
||||
Common migrate write failure:
|
||||
|
||||
- `config file not found, or has no creation rules, and no keys provided through command line options`
|
||||
|
||||
If you hit this:
|
||||
|
||||
- Add or fix `<config-dir>/.sops.yaml` / `.sops.yml` with valid `creation_rules`.
|
||||
- Ensure key access is available in the command environment (for example `SOPS_AGE_KEY_FILE`).
|
||||
- Re-run `openclaw secrets migrate --write`.
|
||||
|
||||
Rollback a previous migration:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -93,6 +93,7 @@ Contract:
|
||||
|
||||
- OpenClaw shells out to `sops` for decrypt/encrypt.
|
||||
- Minimum supported version: `sops >= 3.9.0`.
|
||||
- For migration, OpenClaw explicitly passes `--config <config-dir>/.sops.yaml` (or `.sops.yml`) when present, so behavior is not dependent on current working directory.
|
||||
- Decrypted payload must be a JSON object.
|
||||
- `id` is resolved as JSON pointer into decrypted payload.
|
||||
- Default timeout is `5000ms`.
|
||||
@@ -101,6 +102,13 @@ Common errors:
|
||||
|
||||
- Missing binary: `sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.`
|
||||
- Timeout: `sops decrypt timed out after <n>ms for <path>.`
|
||||
- Missing creation rules/key access (common during migrate write): `config file not found, or has no creation rules, and no keys provided through command line options`
|
||||
|
||||
Fix for creation-rules/key-access errors:
|
||||
|
||||
- Ensure `<config-dir>/.sops.yaml` or `<config-dir>/.sops.yml` contains a valid `creation_rules` entry for your secrets file.
|
||||
- Ensure the runtime environment for `openclaw secrets migrate --write` can access decryption/encryption keys (for example `SOPS_AGE_KEY_FILE` for age keys).
|
||||
- Re-run migration after confirming both config and key access.
|
||||
|
||||
## In-scope fields (v1)
|
||||
|
||||
@@ -158,6 +166,8 @@ Behavior:
|
||||
|
||||
- Degraded: runtime keeps last-known-good snapshot.
|
||||
- Recovered: emitted once after successful activation.
|
||||
- Repeated failures while already degraded only log warnings (no repeated system events).
|
||||
- Startup fail-fast does not emit degraded events because no runtime snapshot is active yet.
|
||||
|
||||
## Migration command
|
||||
|
||||
@@ -185,7 +195,15 @@ What migration covers:
|
||||
|
||||
- `openclaw.json` fields listed above
|
||||
- `auth-profiles.json` API key and token plaintext fields
|
||||
- optional scrub of matching secret values in `~/.openclaw/.env` (default on)
|
||||
- optional scrub of matching plaintext values from config-dir `.env` (default on)
|
||||
- if `<config-dir>/.sops.yaml` or `<config-dir>/.sops.yml` exists, migration uses it explicitly for sops decrypt/encrypt
|
||||
|
||||
`.env` scrub semantics:
|
||||
|
||||
- Scrub target path is `<config-dir>/.env` (`resolveConfigDir(...)`), not `OPENCLAW_HOME/.env`.
|
||||
- Only known secret env keys are eligible (for example `OPENAI_API_KEY`).
|
||||
- A line is removed only when its parsed value exactly matches a migrated plaintext value.
|
||||
- Non-secret keys, comments, and unmatched values are preserved.
|
||||
|
||||
Backups:
|
||||
|
||||
|
||||
@@ -47,4 +47,30 @@ describe("resolveModelAuthLabel", () => {
|
||||
|
||||
expect(label).toContain("token ref(env:GITHUB_TOKEN)");
|
||||
});
|
||||
|
||||
it("masks short api-key profile values", () => {
|
||||
const shortSecret = "abc123";
|
||||
ensureAuthProfileStoreMock.mockReturnValue({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: shortSecret,
|
||||
},
|
||||
},
|
||||
} as never);
|
||||
resolveAuthProfileOrderMock.mockReturnValue(["openai:default"]);
|
||||
resolveAuthProfileDisplayLabelMock.mockReturnValue("openai:default");
|
||||
|
||||
const label = resolveModelAuthLabel({
|
||||
provider: "openai",
|
||||
cfg: {},
|
||||
sessionEntry: { authProfileOverride: "openai:default" } as never,
|
||||
});
|
||||
|
||||
expect(label).toContain("api-key");
|
||||
expect(label).toContain("...");
|
||||
expect(label).not.toContain(shortSecret);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import { maskApiKey } from "../utils/mask-api-key.js";
|
||||
import {
|
||||
ensureAuthProfileStore,
|
||||
resolveAuthProfileDisplayLabel,
|
||||
@@ -13,10 +14,7 @@ function formatApiKeySnippet(apiKey: string): string {
|
||||
if (!compact) {
|
||||
return "unknown";
|
||||
}
|
||||
const edge = compact.length >= 12 ? 6 : 4;
|
||||
const head = compact.slice(0, edge);
|
||||
const tail = compact.slice(-edge);
|
||||
return `${head}…${tail}`;
|
||||
return maskApiKey(compact);
|
||||
}
|
||||
|
||||
function formatCredentialSnippet(params: {
|
||||
|
||||
@@ -77,6 +77,7 @@ describe("registerPreActionHooks", () => {
|
||||
program.command("status").action(async () => {});
|
||||
program.command("doctor").action(async () => {});
|
||||
program.command("completion").action(async () => {});
|
||||
program.command("secrets").action(async () => {});
|
||||
program.command("update").action(async () => {});
|
||||
program.command("channels").action(async () => {});
|
||||
program.command("directory").action(async () => {});
|
||||
@@ -155,7 +156,7 @@ describe("registerPreActionHooks", () => {
|
||||
expect(ensurePluginRegistryLoadedMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips config guard for doctor and completion commands", async () => {
|
||||
it("skips config guard for doctor, completion, and secrets commands", async () => {
|
||||
await runCommand({
|
||||
parseArgv: ["doctor"],
|
||||
processArgv: ["node", "openclaw", "doctor"],
|
||||
@@ -164,6 +165,10 @@ describe("registerPreActionHooks", () => {
|
||||
parseArgv: ["completion"],
|
||||
processArgv: ["node", "openclaw", "completion"],
|
||||
});
|
||||
await runCommand({
|
||||
parseArgv: ["secrets"],
|
||||
processArgv: ["node", "openclaw", "secrets"],
|
||||
});
|
||||
|
||||
expect(ensureConfigReadyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -29,6 +29,7 @@ const PLUGIN_REQUIRED_COMMANDS = new Set([
|
||||
"configure",
|
||||
"onboard",
|
||||
]);
|
||||
const CONFIG_GUARD_BYPASS_COMMANDS = new Set(["doctor", "completion", "secrets"]);
|
||||
|
||||
function getRootCommand(command: Command): Command {
|
||||
let current = command;
|
||||
@@ -75,7 +76,7 @@ export function registerPreActionHooks(program: Command, programVersion: string)
|
||||
if (!verbose) {
|
||||
process.env.NODE_NO_WARNINGS ??= "1";
|
||||
}
|
||||
if (commandPath[0] === "doctor" || commandPath[0] === "completion") {
|
||||
if (CONFIG_GUARD_BYPASS_COMMANDS.has(commandPath[0])) {
|
||||
return;
|
||||
}
|
||||
const { ensureConfigReady } = await import("./config-guard.js");
|
||||
|
||||
@@ -150,6 +150,61 @@ describe("ensureApiKeyFromEnvOrPrompt", () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses explicit inline env ref when secret-input-mode=ref selects existing env key", async () => {
|
||||
process.env.MINIMAX_API_KEY = "env-key";
|
||||
delete process.env.MINIMAX_OAUTH_TOKEN;
|
||||
|
||||
const { confirm, text } = createPromptSpies({
|
||||
confirmResult: true,
|
||||
textResult: "prompt-key",
|
||||
});
|
||||
const setCredential = vi.fn(async () => undefined);
|
||||
|
||||
const result = await ensureApiKeyFromEnvOrPrompt({
|
||||
provider: "minimax",
|
||||
envLabel: "MINIMAX_API_KEY",
|
||||
promptMessage: "Enter key",
|
||||
normalize: (value) => value.trim(),
|
||||
validate: () => undefined,
|
||||
prompter: createPrompter({ confirm, text }),
|
||||
secretInputMode: "ref",
|
||||
setCredential,
|
||||
});
|
||||
|
||||
expect(result).toBe("env-key");
|
||||
expect(setCredential).toHaveBeenCalledWith("${MINIMAX_API_KEY}", "ref");
|
||||
expect(text).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows a ref-mode note when plaintext input is provided in ref mode", async () => {
|
||||
delete process.env.MINIMAX_API_KEY;
|
||||
delete process.env.MINIMAX_OAUTH_TOKEN;
|
||||
|
||||
const { confirm, note, text } = createPromptSpies({
|
||||
confirmResult: false,
|
||||
textResult: " prompted-key ",
|
||||
});
|
||||
const setCredential = vi.fn(async () => undefined);
|
||||
|
||||
const result = await ensureApiKeyFromEnvOrPrompt({
|
||||
provider: "minimax",
|
||||
envLabel: "MINIMAX_API_KEY",
|
||||
promptMessage: "Enter key",
|
||||
normalize: (value) => value.trim(),
|
||||
validate: () => undefined,
|
||||
prompter: createPrompter({ confirm, note, text }),
|
||||
secretInputMode: "ref",
|
||||
setCredential,
|
||||
});
|
||||
|
||||
expect(result).toBe("prompted-key");
|
||||
expect(setCredential).toHaveBeenCalledWith("prompted-key", "ref");
|
||||
expect(note).toHaveBeenCalledWith(
|
||||
expect.stringContaining("secret-input-mode=ref stores an env reference"),
|
||||
"Ref mode note",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureApiKeyFromOptionEnvOrPrompt", () => {
|
||||
|
||||
@@ -5,6 +5,14 @@ import type { ApplyAuthChoiceParams } from "./auth-choice.apply.js";
|
||||
import { applyDefaultModelChoice } from "./auth-choice.default-model.js";
|
||||
import type { SecretInputMode } from "./onboard-types.js";
|
||||
|
||||
const INLINE_ENV_REF_RE = /^\$\{([A-Z][A-Z0-9_]*)\}$/;
|
||||
const ENV_SOURCE_LABEL_RE = /(?:^|:\s)([A-Z][A-Z0-9_]*)$/;
|
||||
|
||||
function extractEnvVarFromSourceLabel(source: string): string | undefined {
|
||||
const match = ENV_SOURCE_LABEL_RE.exec(source.trim());
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
export function createAuthChoiceAgentModelNoter(
|
||||
params: ApplyAuthChoiceParams,
|
||||
): (model: string) => Promise<void> {
|
||||
@@ -205,7 +213,14 @@ export async function ensureApiKeyFromEnvOrPrompt(params: {
|
||||
prompter: params.prompter,
|
||||
explicitMode: params.secretInputMode,
|
||||
});
|
||||
await params.setCredential(envKey.apiKey, mode);
|
||||
const explicitEnvRef =
|
||||
mode === "ref"
|
||||
? (() => {
|
||||
const envVar = extractEnvVarFromSourceLabel(envKey.source);
|
||||
return envVar ? `\${${envVar}}` : envKey.apiKey;
|
||||
})()
|
||||
: envKey.apiKey;
|
||||
await params.setCredential(explicitEnvRef, mode);
|
||||
return envKey.apiKey;
|
||||
}
|
||||
}
|
||||
@@ -215,6 +230,12 @@ export async function ensureApiKeyFromEnvOrPrompt(params: {
|
||||
validate: params.validate,
|
||||
});
|
||||
const apiKey = params.normalize(String(key ?? ""));
|
||||
if (params.secretInputMode === "ref" && !INLINE_ENV_REF_RE.test(apiKey)) {
|
||||
await params.prompter.note(
|
||||
"secret-input-mode=ref stores an env reference, not plaintext key input. Enter ${ENV_VAR} to target a specific variable, or keep current input to use the provider default env var.",
|
||||
"Ref mode note",
|
||||
);
|
||||
}
|
||||
await params.setCredential(apiKey, params.secretInputMode);
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
@@ -229,6 +229,35 @@ describe("modelsStatusCommand auth overview", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not emit raw short api-key values in JSON labels", async () => {
|
||||
const localRuntime = createRuntime();
|
||||
const shortSecret = "abc123";
|
||||
const originalProfiles = { ...mocks.store.profiles };
|
||||
mocks.store.profiles = {
|
||||
...mocks.store.profiles,
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: shortSecret,
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await modelsStatusCommand({ json: true }, localRuntime as never);
|
||||
const payload = JSON.parse(String((localRuntime.log as Mock).mock.calls[0]?.[0]));
|
||||
const providers = payload.auth.providers as Array<{
|
||||
provider: string;
|
||||
profiles: { labels: string[] };
|
||||
}>;
|
||||
const openai = providers.find((p) => p.provider === "openai");
|
||||
const labels = openai?.profiles.labels ?? [];
|
||||
expect(labels.join(" ")).toContain("...");
|
||||
expect(labels.join(" ")).not.toContain(shortSecret);
|
||||
} finally {
|
||||
mocks.store.profiles = originalProfiles;
|
||||
}
|
||||
});
|
||||
|
||||
it("uses agent overrides and reports sources", async () => {
|
||||
const localRuntime = createRuntime();
|
||||
await withAgentScopeOverrides(
|
||||
|
||||
@@ -279,4 +279,54 @@ describe("startGatewayConfigReloader", () => {
|
||||
|
||||
await reloader.stop();
|
||||
});
|
||||
|
||||
it("contains restart callback failures and retries on subsequent changes", async () => {
|
||||
const readSnapshot = vi
|
||||
.fn<() => Promise<ConfigFileSnapshot>>()
|
||||
.mockResolvedValueOnce(
|
||||
makeSnapshot({
|
||||
config: {
|
||||
gateway: { reload: { debounceMs: 0 }, port: 18790 },
|
||||
},
|
||||
hash: "restart-1",
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeSnapshot({
|
||||
config: {
|
||||
gateway: { reload: { debounceMs: 0 }, port: 18791 },
|
||||
},
|
||||
hash: "restart-2",
|
||||
}),
|
||||
);
|
||||
const { watcher, onHotReload, onRestart, log, reloader } = createReloaderHarness(readSnapshot);
|
||||
onRestart.mockRejectedValueOnce(new Error("restart-check failed"));
|
||||
onRestart.mockResolvedValueOnce(undefined);
|
||||
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => {
|
||||
unhandled.push(reason);
|
||||
};
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
try {
|
||||
watcher.emit("change");
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(onHotReload).not.toHaveBeenCalled();
|
||||
expect(onRestart).toHaveBeenCalledTimes(1);
|
||||
expect(log.error).toHaveBeenCalledWith("config restart failed: Error: restart-check failed");
|
||||
expect(unhandled).toEqual([]);
|
||||
|
||||
watcher.emit("change");
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(onRestart).toHaveBeenCalledTimes(2);
|
||||
expect(unhandled).toEqual([]);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
await reloader.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -291,7 +291,16 @@ export function startGatewayConfigReloader(opts: {
|
||||
return;
|
||||
}
|
||||
restartQueued = true;
|
||||
void opts.onRestart(plan, nextConfig);
|
||||
void (async () => {
|
||||
try {
|
||||
await opts.onRestart(plan, nextConfig);
|
||||
} catch (err) {
|
||||
// Restart checks can fail (for example unresolved SecretRefs). Keep the
|
||||
// reloader alive and allow a future change to retry restart scheduling.
|
||||
restartQueued = false;
|
||||
opts.log.error(`config restart failed: ${String(err)}`);
|
||||
}
|
||||
})();
|
||||
};
|
||||
|
||||
const handleMissingSnapshot = (snapshot: ConfigFileSnapshot): boolean => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveMainSessionKeyFromConfig } from "../config/sessions.js";
|
||||
import { drainSystemEvents } from "../infra/system-events.js";
|
||||
@@ -234,6 +235,45 @@ describe("gateway hot reload", () => {
|
||||
);
|
||||
}
|
||||
|
||||
async function writeAuthProfileEnvRefStore() {
|
||||
const stateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
if (!stateDir) {
|
||||
throw new Error("OPENCLAW_STATE_DIR is not set");
|
||||
}
|
||||
const authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
|
||||
await fs.mkdir(path.dirname(authStorePath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
authStorePath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
missing: {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
keyRef: { source: "env", id: "MISSING_OPENCLAW_AUTH_REF" },
|
||||
},
|
||||
},
|
||||
selectedProfileId: "missing",
|
||||
lastUsedProfileByModel: {},
|
||||
usageStats: {},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
|
||||
async function removeMainAuthProfileStore() {
|
||||
const stateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
if (!stateDir) {
|
||||
return;
|
||||
}
|
||||
const authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
|
||||
await fs.rm(authStorePath, { force: true });
|
||||
}
|
||||
|
||||
it("applies hot reload actions and emits restart signal", async () => {
|
||||
await withGatewayServer(async () => {
|
||||
const onHotReload = hoisted.getOnHotReload();
|
||||
@@ -347,6 +387,18 @@ describe("gateway hot reload", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("fails startup when auth-profile secret refs are unresolved", async () => {
|
||||
await writeAuthProfileEnvRefStore();
|
||||
delete process.env.MISSING_OPENCLAW_AUTH_REF;
|
||||
try {
|
||||
await expect(withGatewayServer(async () => {})).rejects.toThrow(
|
||||
'Environment variable "MISSING_OPENCLAW_AUTH_REF" is missing or empty.',
|
||||
);
|
||||
} finally {
|
||||
await removeMainAuthProfileStore();
|
||||
}
|
||||
});
|
||||
|
||||
it("emits one-shot degraded and recovered system events during secret reload transitions", async () => {
|
||||
await writeEnvRefConfig();
|
||||
process.env.OPENAI_API_KEY = "sk-startup";
|
||||
@@ -402,6 +454,24 @@ describe("gateway hot reload", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("serves secrets.reload immediately after startup without race failures", async () => {
|
||||
await writeEnvRefConfig();
|
||||
process.env.OPENAI_API_KEY = "sk-startup";
|
||||
const { server, ws } = await startServerWithClient();
|
||||
try {
|
||||
await connectOk(ws);
|
||||
const [first, second] = await Promise.all([
|
||||
rpcReq<{ warningCount: number }>(ws, "secrets.reload", {}),
|
||||
rpcReq<{ warningCount: number }>(ws, "secrets.reload", {}),
|
||||
]);
|
||||
expect(first.ok).toBe(true);
|
||||
expect(second.ok).toBe(true);
|
||||
} finally {
|
||||
ws.close();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("gateway agents", () => {
|
||||
|
||||
@@ -94,7 +94,7 @@ describe("secrets migrate", () => {
|
||||
|
||||
runExecMock.mockReset();
|
||||
runExecMock.mockImplementation(async (_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--encrypt") {
|
||||
if (args.includes("--encrypt")) {
|
||||
const outputPath = args[args.indexOf("--output") + 1];
|
||||
const inputPath = args.at(-1);
|
||||
if (!outputPath || !inputPath) {
|
||||
@@ -103,7 +103,7 @@ describe("secrets migrate", () => {
|
||||
await fs.copyFile(inputPath, outputPath);
|
||||
return { stdout: "", stderr: "" };
|
||||
}
|
||||
if (args[0] === "--decrypt") {
|
||||
if (args.includes("--decrypt")) {
|
||||
const sourcePath = args.at(-1);
|
||||
if (!sourcePath) {
|
||||
throw new Error("missing sops decrypt source");
|
||||
@@ -213,4 +213,20 @@ describe("secrets migrate", () => {
|
||||
expect(second.backupId).toBeTruthy();
|
||||
expect(second.backupId).not.toBe(first.backupId);
|
||||
});
|
||||
|
||||
it("passes --config for sops when .sops.yaml exists in config dir", async () => {
|
||||
const sopsConfigPath = path.join(stateDir, ".sops.yaml");
|
||||
await fs.writeFile(sopsConfigPath, "creation_rules:\n - path_regex: .*\n", "utf8");
|
||||
|
||||
await runSecretsMigration({ env, write: true });
|
||||
|
||||
const encryptCall = runExecMock.mock.calls.find((call) =>
|
||||
(call[1] as string[]).includes("--encrypt"),
|
||||
);
|
||||
expect(encryptCall).toBeTruthy();
|
||||
const args = encryptCall?.[1] as string[];
|
||||
const configIndex = args.indexOf("--config");
|
||||
expect(configIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(args[configIndex + 1]).toBe(sopsConfigPath);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import fs from "node:fs";
|
||||
import { createConfigIO } from "../../config/config.js";
|
||||
import { ensureDirForFile, writeJsonFileSecure } from "../shared.js";
|
||||
import { encryptSopsJsonFile } from "../sops.js";
|
||||
import {
|
||||
@@ -8,17 +7,20 @@ import {
|
||||
resolveUniqueBackupId,
|
||||
restoreFromManifest,
|
||||
} from "./backup.js";
|
||||
import { createSecretsMigrationConfigIO } from "./config-io.js";
|
||||
import type { MigrationPlan, SecretsMigrationRunResult } from "./types.js";
|
||||
|
||||
async function encryptSopsJson(params: {
|
||||
pathname: string;
|
||||
timeoutMs: number;
|
||||
payload: Record<string, unknown>;
|
||||
sopsConfigPath?: string;
|
||||
}): Promise<void> {
|
||||
await encryptSopsJsonFile({
|
||||
path: params.pathname,
|
||||
payload: params.payload,
|
||||
timeoutMs: params.timeoutMs,
|
||||
configPath: params.sopsConfigPath,
|
||||
missingBinaryMessage:
|
||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||
});
|
||||
@@ -54,11 +56,12 @@ export async function applyMigrationPlan(params: {
|
||||
pathname: plan.secretsFilePath,
|
||||
timeoutMs: plan.secretsFileTimeoutMs,
|
||||
payload: plan.nextPayload,
|
||||
sopsConfigPath: plan.sopsConfigPath,
|
||||
});
|
||||
}
|
||||
|
||||
if (plan.configChanged) {
|
||||
const io = createConfigIO({ env: params.env });
|
||||
const io = createSecretsMigrationConfigIO({ env: params.env });
|
||||
await io.writeConfigFile(plan.nextConfig, plan.configWriteOptions);
|
||||
}
|
||||
|
||||
|
||||
14
src/secrets/migrate/config-io.ts
Normal file
14
src/secrets/migrate/config-io.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { createConfigIO } from "../../config/config.js";
|
||||
|
||||
const silentConfigIoLogger = {
|
||||
error: () => {},
|
||||
warn: () => {},
|
||||
} as const;
|
||||
|
||||
export function createSecretsMigrationConfigIO(params: { env: NodeJS.ProcessEnv }) {
|
||||
// Migration output is owned by the CLI command so --json remains machine-parseable.
|
||||
return createConfigIO({
|
||||
env: params.env,
|
||||
logger: silentConfigIoLogger,
|
||||
});
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { listAgentIds, resolveAgentDir } from "../../agents/agent-scope.js";
|
||||
import { resolveAuthStorePath } from "../../agents/auth-profiles/paths.js";
|
||||
import { createConfigIO, resolveStateDir, type OpenClawConfig } from "../../config/config.js";
|
||||
import { resolveStateDir, type OpenClawConfig } from "../../config/config.js";
|
||||
import { isSecretRef } from "../../config/types.secrets.js";
|
||||
import { resolveConfigDir, resolveUserPath } from "../../utils.js";
|
||||
import {
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import { listKnownSecretEnvVarNames } from "../provider-env-vars.js";
|
||||
import { isNonEmptyString, isRecord, normalizePositiveInt } from "../shared.js";
|
||||
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "../sops.js";
|
||||
import { createSecretsMigrationConfigIO } from "./config-io.js";
|
||||
import type { AuthStoreChange, EnvChange, MigrationCounters, MigrationPlan } from "./types.js";
|
||||
|
||||
const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json";
|
||||
@@ -112,6 +113,7 @@ function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string {
|
||||
async function decryptSopsJson(
|
||||
pathname: string,
|
||||
timeoutMs: number,
|
||||
sopsConfigPath?: string,
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (!fs.existsSync(pathname)) {
|
||||
return {};
|
||||
@@ -119,6 +121,7 @@ async function decryptSopsJson(
|
||||
const parsed = await decryptSopsJsonFile({
|
||||
path: pathname,
|
||||
timeoutMs,
|
||||
configPath: sopsConfigPath,
|
||||
missingBinaryMessage:
|
||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||
});
|
||||
@@ -128,6 +131,17 @@ async function decryptSopsJson(
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function resolveExistingSopsConfigPath(env: NodeJS.ProcessEnv): string | undefined {
|
||||
const configDir = resolveConfigDir(env, os.homedir);
|
||||
const candidates = [".sops.yaml", ".sops.yml"].map((name) => path.join(configDir, name));
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function migrateModelProviderSecrets(params: {
|
||||
config: OpenClawConfig;
|
||||
payload: Record<string, unknown>;
|
||||
@@ -385,7 +399,7 @@ export async function buildMigrationPlan(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
scrubEnv: boolean;
|
||||
}): Promise<MigrationPlan> {
|
||||
const io = createConfigIO({ env: params.env });
|
||||
const io = createSecretsMigrationConfigIO({ env: params.env });
|
||||
const { snapshot, writeOptions } = await io.readConfigFileSnapshotForWrite();
|
||||
if (!snapshot.valid) {
|
||||
const issues =
|
||||
@@ -398,7 +412,12 @@ export async function buildMigrationPlan(params: {
|
||||
const stateDir = resolveStateDir(params.env, os.homedir);
|
||||
const nextConfig = structuredClone(snapshot.config);
|
||||
const fileSource = resolveFileSource(nextConfig, params.env);
|
||||
const previousPayload = await decryptSopsJson(fileSource.path, fileSource.timeoutMs);
|
||||
const sopsConfigPath = resolveExistingSopsConfigPath(params.env);
|
||||
const previousPayload = await decryptSopsJson(
|
||||
fileSource.path,
|
||||
fileSource.timeoutMs,
|
||||
sopsConfigPath,
|
||||
);
|
||||
const nextPayload = structuredClone(previousPayload);
|
||||
|
||||
const counters: MigrationCounters = {
|
||||
@@ -518,6 +537,7 @@ export async function buildMigrationPlan(params: {
|
||||
nextPayload,
|
||||
secretsFilePath: fileSource.path,
|
||||
secretsFileTimeoutMs: fileSource.timeoutMs,
|
||||
sopsConfigPath,
|
||||
envChange,
|
||||
backupTargets: [...backupTargets],
|
||||
};
|
||||
|
||||
@@ -46,6 +46,7 @@ export type MigrationPlan = {
|
||||
nextPayload: Record<string, unknown>;
|
||||
secretsFilePath: string;
|
||||
secretsFileTimeoutMs: number;
|
||||
sopsConfigPath?: string;
|
||||
envChange: EnvChange | null;
|
||||
backupTargets: string[];
|
||||
};
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { SecretRef } from "../config/types.secrets.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import { readJsonPointer } from "./json-pointer.js";
|
||||
import { isNonEmptyString, normalizePositiveInt } from "./shared.js";
|
||||
import { isNonEmptyString, isRecord, normalizePositiveInt } from "./shared.js";
|
||||
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
|
||||
|
||||
export type SecretRefResolveCache = {
|
||||
@@ -39,6 +39,11 @@ async function resolveFileSecretPayload(options: ResolveSecretRefOptions): Promi
|
||||
path: resolveUserPath(fileSource.path),
|
||||
timeoutMs: normalizePositiveInt(fileSource.timeoutMs, DEFAULT_SOPS_TIMEOUT_MS),
|
||||
missingBinaryMessage: options.missingBinaryMessage ?? DEFAULT_SOPS_MISSING_BINARY_MESSAGE,
|
||||
}).then((payload) => {
|
||||
if (!isRecord(payload)) {
|
||||
throw new Error("sops decrypt failed: decrypted payload is not a JSON object");
|
||||
}
|
||||
return payload;
|
||||
});
|
||||
if (cache) {
|
||||
cache.fileSecretsPromise = promise;
|
||||
|
||||
@@ -133,6 +133,39 @@ describe("secrets runtime snapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("fails when sops decrypt payload is not a JSON object", async () => {
|
||||
runExecMock.mockResolvedValueOnce({
|
||||
stdout: JSON.stringify(["not-an-object"]),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
await expect(
|
||||
prepareSecretsRuntimeSnapshot({
|
||||
config: {
|
||||
secrets: {
|
||||
sources: {
|
||||
file: {
|
||||
type: "sops",
|
||||
path: "~/.openclaw/secrets.enc.json",
|
||||
},
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "file", id: "/providers/openai/apiKey" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({ version: 1, profiles: {} }),
|
||||
}),
|
||||
).rejects.toThrow("sops decrypt failed: decrypted payload is not a JSON object");
|
||||
});
|
||||
|
||||
it("activates runtime snapshots for loadConfig and ensureAuthProfileStore", async () => {
|
||||
const prepared = await prepareSecretsRuntimeSnapshot({
|
||||
config: {
|
||||
|
||||
@@ -34,10 +34,16 @@ export async function decryptSopsJsonFile(params: {
|
||||
path: string;
|
||||
timeoutMs?: number;
|
||||
missingBinaryMessage: string;
|
||||
configPath?: string;
|
||||
}): Promise<unknown> {
|
||||
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
|
||||
try {
|
||||
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], {
|
||||
const args: string[] = [];
|
||||
if (typeof params.configPath === "string" && params.configPath.trim().length > 0) {
|
||||
args.push("--config", params.configPath);
|
||||
}
|
||||
args.push("--decrypt", "--output-type", "json", params.path);
|
||||
const { stdout } = await runExec("sops", args, {
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
});
|
||||
@@ -61,6 +67,7 @@ export async function encryptSopsJsonFile(params: {
|
||||
payload: Record<string, unknown>;
|
||||
timeoutMs?: number;
|
||||
missingBinaryMessage: string;
|
||||
configPath?: string;
|
||||
}): Promise<void> {
|
||||
ensureDirForFile(params.path);
|
||||
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
|
||||
@@ -77,23 +84,24 @@ export async function encryptSopsJsonFile(params: {
|
||||
fs.chmodSync(tmpPlain, 0o600);
|
||||
|
||||
try {
|
||||
await runExec(
|
||||
"sops",
|
||||
[
|
||||
"--encrypt",
|
||||
"--input-type",
|
||||
"json",
|
||||
"--output-type",
|
||||
"json",
|
||||
"--output",
|
||||
tmpEncrypted,
|
||||
tmpPlain,
|
||||
],
|
||||
{
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
},
|
||||
const args: string[] = [];
|
||||
if (typeof params.configPath === "string" && params.configPath.trim().length > 0) {
|
||||
args.push("--config", params.configPath);
|
||||
}
|
||||
args.push(
|
||||
"--encrypt",
|
||||
"--input-type",
|
||||
"json",
|
||||
"--output-type",
|
||||
"json",
|
||||
"--output",
|
||||
tmpEncrypted,
|
||||
tmpPlain,
|
||||
);
|
||||
await runExec("sops", args, {
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
});
|
||||
fs.renameSync(tmpEncrypted, params.path);
|
||||
fs.chmodSync(params.path, 0o600);
|
||||
} catch (err) {
|
||||
|
||||
@@ -7,9 +7,11 @@ describe("maskApiKey", () => {
|
||||
expect(maskApiKey(" ")).toBe("missing");
|
||||
});
|
||||
|
||||
it("returns trimmed value when length is 16 chars or less", () => {
|
||||
expect(maskApiKey(" abcdefghijklmnop ")).toBe("abcdefghijklmnop");
|
||||
expect(maskApiKey(" short ")).toBe("short");
|
||||
it("masks short and medium values without returning raw secrets", () => {
|
||||
expect(maskApiKey(" abcdefghijklmnop ")).toBe("ab...op");
|
||||
expect(maskApiKey(" short ")).toBe("s...t");
|
||||
expect(maskApiKey(" a ")).toBe("a...a");
|
||||
expect(maskApiKey(" ab ")).toBe("a...b");
|
||||
});
|
||||
|
||||
it("masks long values with first and last 8 chars", () => {
|
||||
|
||||
@@ -3,8 +3,11 @@ export const maskApiKey = (value: string): string => {
|
||||
if (!trimmed) {
|
||||
return "missing";
|
||||
}
|
||||
if (trimmed.length <= 6) {
|
||||
return `${trimmed.slice(0, 1)}...${trimmed.slice(-1)}`;
|
||||
}
|
||||
if (trimmed.length <= 16) {
|
||||
return trimmed;
|
||||
return `${trimmed.slice(0, 2)}...${trimmed.slice(-2)}`;
|
||||
}
|
||||
return `${trimmed.slice(0, 8)}...${trimmed.slice(-8)}`;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user