refactor: unify sensitive URL config hints

This commit is contained in:
Peter Steinberger
2026-03-29 20:41:34 +01:00
parent 1318479a2c
commit e45cc3890b
11 changed files with 253 additions and 20 deletions

View File

@@ -1,6 +1,10 @@
import JSON5 from "json5";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { redactSensitiveUrlLikeString } from "../shared/net/redact-sensitive-url.js";
import {
hasSensitiveUrlHintTag,
isSensitiveUrlConfigPath,
redactSensitiveUrlLikeString,
} from "../shared/net/redact-sensitive-url.js";
import {
replaceSensitiveValuesInRaw,
shouldFallbackToStructuredRawRedaction,
@@ -30,10 +34,14 @@ function isWholeObjectSensitivePath(path: string): boolean {
}
function isSensitiveUrlPath(path: string): boolean {
if (path.endsWith(".baseUrl") || path.endsWith(".httpUrl")) {
return true;
return isSensitiveUrlConfigPath(path);
}
function hasSensitiveUrlHintPath(hints: ConfigUiHints | undefined, paths: string[]): boolean {
if (!hints) {
return false;
}
return /^mcp\.servers\.[^.]+\.url$/.test(path);
return paths.some((path) => hasSensitiveUrlHintTag(hints[path]));
}
function collectSensitiveStrings(value: unknown, values: string[]): void {
@@ -220,7 +228,11 @@ function redactObjectWithLookup(
) {
// Keep primitives at explicitly-sensitive paths fully redacted.
result[key] = REDACTED_SENTINEL;
} else if (typeof value === "string" && isSensitiveUrlPath(path)) {
} else if (
typeof value === "string" &&
(hasSensitiveUrlHintPath(hints, [candidate, path, wildcardPath]) ||
isSensitiveUrlPath(path))
) {
const scrubbed = redactSensitiveUrlLikeString(value);
if (scrubbed !== value) {
values.push(value);
@@ -245,7 +257,10 @@ function redactObjectWithLookup(
) {
result[key] = REDACTED_SENTINEL;
values.push(value);
} else if (typeof value === "string" && isSensitiveUrlPath(path)) {
} else if (
typeof value === "string" &&
(hasSensitiveUrlHintPath(hints, [path, wildcardPath]) || isSensitiveUrlPath(path))
) {
const scrubbed = redactSensitiveUrlLikeString(value);
if (scrubbed !== value) {
values.push(value);
@@ -317,7 +332,10 @@ function redactObjectGuessing(
) {
collectSensitiveStrings(value, values);
result[key] = REDACTED_SENTINEL;
} else if (typeof value === "string" && isSensitiveUrlPath(dotPath)) {
} else if (
typeof value === "string" &&
(hasSensitiveUrlHintPath(hints, [dotPath, wildcardPath]) || isSensitiveUrlPath(dotPath))
) {
const scrubbed = redactSensitiveUrlLikeString(value);
if (scrubbed !== value) {
values.push(value);
@@ -658,7 +676,9 @@ function restoreRedactedValuesWithLookup(
matched = true;
if (
value === REDACTED_SENTINEL &&
(hints[candidate]?.sensitive === true || isSensitiveUrlPath(path))
(hints[candidate]?.sensitive === true ||
hasSensitiveUrlHintPath(hints, [candidate, path, wildcardPath]) ||
isSensitiveUrlPath(path))
) {
result[key] = restoreOriginalValueOrThrow({ key, path: candidate, original: orig });
} else if (typeof value === "object" && value !== null) {
@@ -672,7 +692,9 @@ function restoreRedactedValuesWithLookup(
if (
!markedNonSensitive &&
value === REDACTED_SENTINEL &&
(isSensitivePath(path) || isSensitiveUrlPath(path))
(isSensitivePath(path) ||
hasSensitiveUrlHintPath(hints, [path, wildcardPath]) ||
isSensitiveUrlPath(path))
) {
result[key] = restoreOriginalValueOrThrow({ key, path, original: orig });
} else if (typeof value === "object" && value !== null) {
@@ -714,7 +736,9 @@ function restoreRedactedValuesGuessing(
if (
!isExplicitlyNonSensitivePath(hints, [path, wildcardPath]) &&
value === REDACTED_SENTINEL &&
(isSensitivePath(path) || isSensitiveUrlPath(path))
(isSensitivePath(path) ||
hasSensitiveUrlHintPath(hints, [path, wildcardPath]) ||
isSensitiveUrlPath(path))
) {
result[key] = restoreOriginalValueOrThrow({ key, path, original: orig });
} else if (typeof value === "object" && value !== null) {

View File

@@ -1,6 +1,12 @@
import { isSensitiveUrlConfigPath } from "../shared/net/redact-sensitive-url.js";
import { VERSION } from "../version.js";
import type { ConfigUiHints } from "./schema.hints.js";
import { buildBaseHints, mapSensitivePaths } from "./schema.hints.js";
import {
applySensitiveUrlHints,
buildBaseHints,
collectMatchingSchemaPaths,
mapSensitivePaths,
} from "./schema.hints.js";
import { asSchemaObject, cloneSchema } from "./schema.shared.js";
import { applyDerivedTags } from "./schema.tags.js";
import { OpenClawSchema } from "./zod-schema.js";
@@ -61,9 +67,15 @@ function computeBaseConfigSchemaStablePayload(): BaseConfigSchemaStablePayload {
unrepresentable: "any",
});
schema.title = "OpenClawConfig";
const baseHints = mapSensitivePaths(OpenClawSchema, "", buildBaseHints());
const sensitiveUrlPaths = collectMatchingSchemaPaths(
OpenClawSchema,
"",
isSensitiveUrlConfigPath,
);
const stablePayload = {
schema: stripChannelSchema(schema),
uiHints: applyDerivedTags(mapSensitivePaths(OpenClawSchema, "", buildBaseHints())),
uiHints: applyDerivedTags(applySensitiveUrlHints(baseHints, sensitiveUrlPaths)),
version: VERSION,
} satisfies BaseConfigSchemaStablePayload;
baseConfigSchemaStablePayload = stablePayload;

View File

@@ -1,4 +1,5 @@
import { describe, expect, it } from "vitest";
import { SENSITIVE_URL_HINT_TAG } from "../shared/net/redact-sensitive-url.js";
import { computeBaseConfigSchemaResponse } from "./schema-base.js";
import { GENERATED_BASE_CONFIG_SCHEMA } from "./schema.base.generated.js";
@@ -10,4 +11,13 @@ describe("generated base config schema", () => {
}),
).toEqual(GENERATED_BASE_CONFIG_SCHEMA);
});
it("includes explicit URL-secret tags for sensitive URL fields", () => {
expect(GENERATED_BASE_CONFIG_SCHEMA.uiHints["mcp.servers.*.url"]?.tags).toContain(
SENSITIVE_URL_HINT_TAG,
);
expect(GENERATED_BASE_CONFIG_SCHEMA.uiHints["models.providers.*.baseUrl"]?.tags).toContain(
SENSITIVE_URL_HINT_TAG,
);
});
});

View File

@@ -12941,7 +12941,7 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
"tools.web.fetch.firecrawl.baseUrl": {
label: "Firecrawl Base URL",
help: "Firecrawl base URL (e.g. https://api.firecrawl.dev or custom endpoint).",
tags: ["tools"],
tags: ["tools", "url-secret"],
},
"tools.web.fetch.firecrawl.onlyMainContent": {
label: "Firecrawl Main Content Only",
@@ -13046,7 +13046,7 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
label: "Gateway APNs Relay Base URL",
help: "Base HTTPS URL for the external APNs relay service used by official/TestFlight iOS builds. Keep this aligned with the relay URL baked into the iOS build so registration and send traffic hit the same deployment.",
placeholder: "https://relay.example.com",
tags: ["network", "advanced"],
tags: ["network", "advanced", "url-secret"],
},
"gateway.push.apns.relay.timeoutMs": {
label: "Gateway APNs Relay Timeout (ms)",
@@ -13386,7 +13386,7 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
"agents.defaults.memorySearch.remote.baseUrl": {
label: "Remote Embedding Base URL",
help: "Overrides the embedding API endpoint, such as an OpenAI-compatible proxy or custom Gemini base URL. Use this only when routing through your own gateway or vendor endpoint; keep provider defaults otherwise.",
tags: ["advanced"],
tags: ["advanced", "url-secret"],
},
"agents.defaults.memorySearch.remote.apiKey": {
label: "Remote Embedding API Key",
@@ -13837,7 +13837,7 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
"models.providers.*.baseUrl": {
label: "Model Provider Base URL",
help: "Base URL for the provider endpoint used to serve model requests for that provider entry. Use HTTPS endpoints and keep URLs environment-specific through config templating where needed.",
tags: ["models"],
tags: ["models", "url-secret"],
},
"models.providers.*.apiKey": {
label: "Model Provider API Key",
@@ -15532,6 +15532,51 @@ export const GENERATED_BASE_CONFIG_SCHEMA = {
sensitive: true,
tags: ["security", "auth"],
},
"agents.list[].memorySearch.remote.baseUrl": {
tags: ["advanced", "url-secret"],
},
"tools.web.search.brave.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.web.search.firecrawl.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.web.search.gemini.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.web.search.grok.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.web.search.kimi.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.web.search.perplexity.baseUrl": {
tags: ["tools", "url-secret"],
},
"tools.media.models[].baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.image.baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.image.models[].baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.audio.baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.audio.models[].baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.video.baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"tools.media.video.models[].baseUrl": {
tags: ["media", "tools", "url-secret"],
},
"mcp.servers.*.url": {
tags: ["advanced", "url-secret"],
},
},
version: "2026.3.29",
generatedAt: "2026-03-22T21:17:33.302Z",

View File

@@ -1,13 +1,14 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { buildSecretInputSchema } from "../plugin-sdk/secret-input-schema.js";
import { isSensitiveUrlConfigPath } from "../shared/net/redact-sensitive-url.js";
import { FIELD_HELP } from "./schema.help.js";
import { __test__, isPluginOwnedChannelHintPath, isSensitiveConfigPath } from "./schema.hints.js";
import { FIELD_LABELS } from "./schema.labels.js";
import { OpenClawSchema } from "./zod-schema.js";
import { sensitive } from "./zod-schema.sensitive.js";
const { mapSensitivePaths } = __test__;
const { collectMatchingSchemaPaths, mapSensitivePaths } = __test__;
const BUNDLED_CHANNEL_HINT_PREFIXES = [
"channels.bluebubbles",
"channels.discord",
@@ -183,3 +184,12 @@ describe("mapSensitivePaths", () => {
expect(hints["nested.verificationToken"]?.sensitive).toBe(true);
});
});
describe("collectMatchingSchemaPaths", () => {
it("finds base-config URL fields that may embed secrets", () => {
const paths = collectMatchingSchemaPaths(OpenClawSchema, "", isSensitiveUrlConfigPath);
expect(paths.has("mcp.servers.*.url")).toBe(true);
expect(paths.has("models.providers.*.baseUrl")).toBe(true);
});
});

View File

@@ -1,6 +1,10 @@
import { z } from "zod";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { ConfigUiHints } from "../shared/config-ui-hints-types.js";
import {
isSensitiveUrlConfigPath,
SENSITIVE_URL_HINT_TAG,
} from "../shared/net/redact-sensitive-url.js";
import { FIELD_HELP } from "./schema.help.js";
import { FIELD_LABELS } from "./schema.labels.js";
import { applyDerivedTags } from "./schema.tags.js";
@@ -200,6 +204,80 @@ export function applySensitiveHints(
return next;
}
export function applySensitiveUrlHints(
hints: ConfigUiHints,
allowedKeys?: ReadonlySet<string>,
): ConfigUiHints {
const next = { ...hints };
const keys = allowedKeys ? [...allowedKeys] : Object.keys(next);
for (const key of keys) {
if (!isSensitiveUrlConfigPath(key)) {
continue;
}
const current = next[key];
const tags = new Set(current?.tags ?? []);
tags.add(SENSITIVE_URL_HINT_TAG);
next[key] = {
...current,
tags: [...tags],
};
}
return next;
}
export function collectMatchingSchemaPaths(
schema: z.ZodType,
path: string,
matchesPath: (path: string) => boolean,
paths: Set<string> = new Set(),
): Set<string> {
let currentSchema = schema;
while (isUnwrappable(currentSchema)) {
currentSchema = currentSchema.unwrap();
}
if (path && matchesPath(path)) {
paths.add(path);
}
if (currentSchema instanceof z.ZodObject) {
const shape = currentSchema.shape;
for (const key in shape) {
const nextPath = path ? `${path}.${key}` : key;
collectMatchingSchemaPaths(shape[key], nextPath, matchesPath, paths);
}
const catchallSchema = currentSchema._def.catchall as z.ZodType | undefined;
if (catchallSchema && !(catchallSchema instanceof z.ZodNever)) {
const nextPath = path ? `${path}.*` : "*";
collectMatchingSchemaPaths(catchallSchema, nextPath, matchesPath, paths);
}
} else if (currentSchema instanceof z.ZodArray) {
const nextPath = path ? `${path}[]` : "[]";
collectMatchingSchemaPaths(currentSchema.element as z.ZodType, nextPath, matchesPath, paths);
} else if (currentSchema instanceof z.ZodRecord) {
const nextPath = path ? `${path}.*` : "*";
collectMatchingSchemaPaths(
currentSchema._def.valueType as z.ZodType,
nextPath,
matchesPath,
paths,
);
} else if (
currentSchema instanceof z.ZodUnion ||
currentSchema instanceof z.ZodDiscriminatedUnion
) {
for (const option of currentSchema.options) {
collectMatchingSchemaPaths(option as z.ZodType, path, matchesPath, paths);
}
} else if (currentSchema instanceof z.ZodIntersection) {
collectMatchingSchemaPaths(currentSchema._def.left as z.ZodType, path, matchesPath, paths);
collectMatchingSchemaPaths(currentSchema._def.right as z.ZodType, path, matchesPath, paths);
}
return paths;
}
// Seems to be the only way tsgo accepts us to check if we have a ZodClass
// with an unwrap() method. And it's overly complex because oxlint and
// tsgo are each forbidding what the other allows.
@@ -270,5 +348,6 @@ export function mapSensitivePaths(
/** @internal */
export const __test__ = {
collectMatchingSchemaPaths,
mapSensitivePaths,
};

View File

@@ -101,6 +101,18 @@ function normalizeTags(tags: ReadonlyArray<string>): ConfigTag[] {
return [...out].toSorted((a, b) => TAG_PRIORITY[a] - TAG_PRIORITY[b]);
}
function collectUnknownTags(tags: ReadonlyArray<string>): string[] {
const out = new Set<string>();
for (const tag of tags) {
const normalized = tag.trim().toLowerCase();
if (!normalized || normalizeTag(normalized)) {
continue;
}
out.add(normalized);
}
return [...out];
}
function patternToRegExp(pattern: string): RegExp {
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[^.]+");
return new RegExp(`^${escaped}$`, "i");
@@ -180,7 +192,10 @@ export function applyDerivedTags(hints: ConfigUiHints): ConfigUiHints {
for (const [path, hint] of Object.entries(hints)) {
const existingTags = Array.isArray(hint?.tags) ? hint.tags : [];
const derivedTags = deriveTagsForPath(path, hint);
const tags = normalizeTags([...derivedTags, ...existingTags]);
const tags = [
...normalizeTags([...derivedTags, ...existingTags]),
...collectUnknownTags(existingTags),
];
next[path] = { ...hint, tags };
}
return next;

View File

@@ -1,4 +1,5 @@
import { beforeAll, describe, expect, it } from "vitest";
import { SENSITIVE_URL_HINT_TAG } from "../shared/net/redact-sensitive-url.js";
import { buildConfigSchema, lookupConfigSchema } from "./schema.js";
import { applyDerivedTags, CONFIG_TAGS, deriveTagsForPath } from "./schema.tags.js";
import { ToolsSchema } from "./zod-schema.agent-runtime.js";
@@ -101,6 +102,8 @@ describe("config schema", () => {
expect(res.uiHints["gateway.auth.token"]?.sensitive).toBe(true);
expect(res.uiHints["channels.defaults.groupPolicy"]?.label).toBeTruthy();
expect(res.uiHints["mcp.servers.*.headers.*"]?.sensitive).toBe(true);
expect(res.uiHints["mcp.servers.*.url"]?.tags).toContain(SENSITIVE_URL_HINT_TAG);
expect(res.uiHints["models.providers.*.baseUrl"]?.tags).toContain(SENSITIVE_URL_HINT_TAG);
expect(res.uiHints["channels.discord.threadBindings.spawnAcpSessions"]?.label).toBeTruthy();
expect(res.version).toBeTruthy();
expect(res.generatedAt).toBeTruthy();

View File

@@ -3,7 +3,7 @@ import { CHANNEL_IDS } from "../channels/registry.js";
import { GENERATED_BUNDLED_CHANNEL_CONFIG_METADATA } from "./bundled-channel-config-metadata.generated.js";
import { GENERATED_BASE_CONFIG_SCHEMA } from "./schema.base.generated.js";
import type { ConfigUiHint, ConfigUiHints } from "./schema.hints.js";
import { applySensitiveHints } from "./schema.hints.js";
import { applySensitiveHints, applySensitiveUrlHints } from "./schema.hints.js";
import {
asSchemaObject,
cloneSchema,
@@ -509,7 +509,10 @@ export function buildConfigSchema(params?: {
channels,
);
const mergedHints = applyDerivedTags(
applySensitiveHints(mergedWithoutSensitiveHints, extensionHintKeys),
applySensitiveUrlHints(
applySensitiveHints(mergedWithoutSensitiveHints, extensionHintKeys),
extensionHintKeys,
),
);
const mergedSchema = applyChannelSchemas(applyPluginSchemas(base.schema, plugins), channels);
const merged = {

View File

@@ -1,6 +1,9 @@
import { describe, expect, it } from "vitest";
import {
isSensitiveUrlQueryParamName,
isSensitiveUrlConfigPath,
SENSITIVE_URL_HINT_TAG,
hasSensitiveUrlHintTag,
redactSensitiveUrl,
redactSensitiveUrlLikeString,
} from "./redact-sensitive-url.js";
@@ -40,3 +43,17 @@ describe("isSensitiveUrlQueryParamName", () => {
expect(isSensitiveUrlQueryParamName("safe")).toBe(false);
});
});
describe("sensitive URL config metadata", () => {
it("recognizes config paths that may embed URL secrets", () => {
expect(isSensitiveUrlConfigPath("models.providers.*.baseUrl")).toBe(true);
expect(isSensitiveUrlConfigPath("mcp.servers.remote.url")).toBe(true);
expect(isSensitiveUrlConfigPath("gateway.remote.url")).toBe(false);
});
it("uses an explicit url-secret hint tag", () => {
expect(SENSITIVE_URL_HINT_TAG).toBe("url-secret");
expect(hasSensitiveUrlHintTag({ tags: [SENSITIVE_URL_HINT_TAG] })).toBe(true);
expect(hasSensitiveUrlHintTag({ tags: ["security"] })).toBe(false);
});
});

View File

@@ -1,3 +1,7 @@
import type { ConfigUiHint } from "../config-ui-hints-types.js";
export const SENSITIVE_URL_HINT_TAG = "url-secret";
const SENSITIVE_URL_QUERY_PARAM_NAMES = new Set([
"token",
"key",
@@ -16,6 +20,17 @@ export function isSensitiveUrlQueryParamName(name: string): boolean {
return SENSITIVE_URL_QUERY_PARAM_NAMES.has(name.toLowerCase());
}
export function isSensitiveUrlConfigPath(path: string): boolean {
if (path.endsWith(".baseUrl") || path.endsWith(".httpUrl")) {
return true;
}
return /^mcp\.servers\.(?:\*|[^.]+)\.url$/.test(path);
}
export function hasSensitiveUrlHintTag(hint: Pick<ConfigUiHint, "tags"> | undefined): boolean {
return hint?.tags?.includes(SENSITIVE_URL_HINT_TAG) === true;
}
export function redactSensitiveUrl(value: string): string {
try {
const parsed = new URL(value);