diff --git a/src/agents/session-tool-result-guard-wrapper.ts b/src/agents/session-tool-result-guard-wrapper.ts index 768cab827fb..618f2f5b102 100644 --- a/src/agents/session-tool-result-guard-wrapper.ts +++ b/src/agents/session-tool-result-guard-wrapper.ts @@ -166,6 +166,7 @@ export function guardSessionManager( missingToolResultText: opts?.missingToolResultText, allowedToolNames: opts?.allowedToolNames, beforeMessageWriteHook: beforeMessageWrite, + redactLoggingConfig: opts?.config?.logging, maxToolResultChars: typeof opts?.contextWindowTokens === "number" ? resolveLiveToolResultMaxChars({ @@ -180,4 +181,4 @@ export function guardSessionManager( (sessionManager as GuardedSessionManager).flushPendingToolResults = guard.flushPendingToolResults; (sessionManager as GuardedSessionManager).clearPendingToolResults = guard.clearPendingToolResults; return sessionManager as GuardedSessionManager; -} +} \ No newline at end of file diff --git a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts index 3f3a532926a..42689852b37 100644 --- a/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts +++ b/src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts @@ -13,6 +13,8 @@ import { guardSessionManager } from "./session-tool-result-guard-wrapper.js"; const EMPTY_PLUGIN_SCHEMA = { type: "object", additionalProperties: false, properties: {} }; const originalBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; +const originalConfigPath = process.env.OPENCLAW_CONFIG_PATH; +let tempDirs: string[] = []; function writeTempPlugin(params: { dir: string; id: string; body: string }): string { const pluginDir = path.join(params.dir, params.id); @@ -111,6 +113,15 @@ afterEach(() => { } else { process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = originalBundledPluginsDir; } + if (originalConfigPath === undefined) { + delete process.env.OPENCLAW_CONFIG_PATH; + } else { + process.env.OPENCLAW_CONFIG_PATH = originalConfigPath; + } + for (const dir of tempDirs) { + fs.rmSync(dir, { force: true, recursive: true }); + } + tempDirs = []; }); describe("tool_result_persist hook", () => { @@ -128,6 +139,156 @@ describe("tool_result_persist hook", () => { expect(toolResult.details.originalDetailsBytesAtLeast).toBeGreaterThan(8_192); }); + it("redacts small toolResult details before persistence", () => { + const tokenValue = "abcdefghijklmnopqrstuvwx1234567890"; + const bearerValue = "bearerdiagnosticvalue1234567890"; + const adjacentLongGithubToken = `ghp_${"a".repeat(5_000)}`; + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + status: "completed", + token: tokenValue, + GITHUB_TOKEN: tokenValue, + github_token: tokenValue, + openai_api_key: tokenValue, + card_number: 4242424242424242, + cvc: 123, + authToken: [tokenValue], + aggregated: `GITHUB_TOKEN=${tokenValue}`, + adjacentLongGithubToken: `${"x".repeat(1_000)}${adjacentLongGithubToken} z`, + nested: { + apiKey: { value: bearerValue }, + stdout: `Authorization: Bearer ${bearerValue}`, + items: [`curl --token ${tokenValue} https://example.test`], + }, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(toolResult.content[0]?.text).toBe("visible output stays small"); + expect(serialized).toContain("GITHUB_TOKEN="); + expect(serialized).toContain("Bearer"); + expect(serialized).toContain("…"); + expect(serialized).not.toContain(tokenValue); + expect(serialized).not.toContain(bearerValue); + expect(serialized).not.toContain(adjacentLongGithubToken); + expect(serialized).not.toContain("a".repeat(100)); + expect(serialized).not.toContain("4242424242424242"); + }); + + it("applies in-memory redaction config to persisted details", () => { + const customSecret = "customsecret=abcdef1234567890ghij"; + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + config: { + logging: { + redactPatterns: [String.raw`customsecret=([^\s]+)`], + }, + }, + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: customSecret }], + details: { + diagnostic: customSecret, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult); + expect(serialized).toContain("customsecret=abcdef…ghij"); + expect(serialized).not.toContain(customSecret); + }); + + it("keeps sensitive parent keys when custom value patterns match the key probe", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-redact-config-")); + tempDirs.push(tempDir); + process.env.OPENCLAW_CONFIG_PATH = path.join(tempDir, "openclaw.json"); + fs.writeFileSync( + process.env.OPENCLAW_CONFIG_PATH, + JSON.stringify({ logging: { redactPatterns: ["/[a-z0-9]{30,}/g"] } }), + "utf-8", + ); + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + token: { value: "shortsecret" }, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(serialized).toContain("***"); + expect(serialized).not.toContain("shortsecret"); + }); + + it("redacts secret-bearing keys and too-deep detail branches before persistence", () => { + const tokenValue = "abcdefghijklmnopqrstuvwx1234567890"; + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + let deepDetails: Record = { token: tokenValue }; + for (let index = 0; index < 10; index += 1) { + deepDetails = { child: deepDetails }; + } + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + [`https://example.test/callback?token=${tokenValue}`]: "ok", + deepDetails, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(serialized).toContain("token="); + expect(serialized).toContain("***"); + expect(serialized).toContain("max depth exceeded"); + expect(serialized).not.toContain(tokenValue); + }); + it("caps oversized toolResult details before persistence", () => { const sm = guardSessionManager(SessionManager.inMemory(), { agentId: "main", @@ -165,6 +326,165 @@ describe("tool_result_persist hook", () => { expectPersistedToolResultDetailsCapped(sm); }); + it("redacts summarized oversized toolResult details before persistence", () => { + const tokenValue = "abcdefghijklmnopqrstuvwx1234567890"; + const boundaryGhToken = "ghp_1234567890abcdefghij1234567890abcdef"; + const leadingTailToken = "a".repeat(5_000); + const omittedTailToken = "b".repeat(5_000); + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + status: { state: "completed", token: tokenValue }, + sessionId: "exec-1", + [`https://example.test/callback?token=${tokenValue}`]: "ok", + aggregated: "x".repeat(120_000), + tail: `GITHUB_TOKEN=${tokenValue} ${"x".repeat( + 1_940, + )} ${boundaryGhToken} GITHUB_TOKEN=${leadingTailToken} {"token":"${omittedTailToken}"}`, + sessions: [ + { + sessionId: "proc-1", + status: { state: "completed", token: tokenValue }, + command: `${"x".repeat(490)} --token ${tokenValue} ${"y".repeat(6_000)}`, + aggregated: "a".repeat(80_000), + tail: "z".repeat(8_000), + }, + ], + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(toolResult.content[0]?.text).toBe("visible output stays small"); + expect(toolResult.details.persistedDetailsTruncated).toBe(true); + expect(serialized).toContain("token=***"); + expect(serialized).toContain("partial secret span omitted"); + expect(serialized).toContain("boundary overlap omitted"); + expect(serialized).not.toContain(tokenValue); + expect(serialized).not.toContain(boundaryGhToken.slice(0, 12)); + expect(serialized).not.toContain("a".repeat(100)); + expect(serialized).not.toContain("b".repeat(100)); + }); + + it("redacts retained structured fields in fallback oversized details summaries", () => { + const tokenValue = "fallback-token-abcdefghijklmnopqrstuv"; + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + status: { state: "completed", token: tokenValue }, + sessionId: "exec-1", + cwd: "/tmp/".concat("workspace/".repeat(400)), + name: "oversized fallback command ".repeat(200), + fullOutputPath: "/tmp/".concat("output/".repeat(400)), + aggregated: "x".repeat(120_000), + tail: "tail ".repeat(800), + sessions: Array.from({ length: 10 }, (_, i) => ({ + sessionId: `proc-${i}`, + status: "completed", + command: `node script-${i}.js ${"x".repeat(6_000)}`, + })), + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const details = toolResult.details; + const serialized = JSON.stringify(details); + expect(details.persistedDetailsTruncated).toBe(true); + expect(details.finalDetailsTruncated).toBe(true); + expect(details.status?.token).toBe("***"); + expect(serialized).not.toContain(tokenValue); + }); + + it("does not persist lookahead text after redaction shrinks an oversized detail prefix", () => { + const tokenValue = "abcdefghijklmnopqrstuvwx1234567890"; + const postBoundarySecret = "UNREDACTED_AFTER_LIMIT_SECRET"; + const shrinkPrefix = `${Array.from({ length: 20 }, () => `GITHUB_TOKEN=${tokenValue}`).join( + " ", + )} `; + const tail = `${shrinkPrefix}${"x".repeat( + 2_300 - shrinkPrefix.length, + )}${postBoundarySecret}${"z".repeat(5_000)}`; + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + status: "completed", + aggregated: "x".repeat(120_000), + tail, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(toolResult.details.persistedDetailsTruncated).toBe(true); + expect(serialized).toContain("partial secret span omitted"); + expect(serialized).not.toContain(tokenValue); + expect(serialized).not.toContain(postBoundarySecret); + }); + + it("fails closed for partially scanned oversized structured secret values", () => { + const longSecret = "r".repeat(10_000); + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + const appendMessage = sm.appendMessage.bind(sm) as unknown as (message: AgentMessage) => void; + appendMessage({ + role: "assistant", + content: [{ type: "toolCall", id: "call_1", name: "exec", arguments: {} }], + } as AgentMessage); + appendMessage({ + role: "toolResult", + toolCallId: "call_1", + isError: false, + content: [{ type: "text", text: "visible output stays small" }], + details: { + status: "completed", + tail: `${"x".repeat(1_000)}{"token":"${longSecret}${"z".repeat(1_000)}`, + }, + } as any); + + const toolResult = requirePersistedToolResult(sm); + const serialized = JSON.stringify(toolResult.details); + expect(serialized).toContain("partial secret span omitted"); + expect(serialized).not.toContain("r".repeat(100)); + }); + it("caps oversized toolResult details without serializing the original payload", () => { const sm = guardSessionManager(SessionManager.inMemory(), { agentId: "main", @@ -419,6 +739,28 @@ describe("tool_result_persist hook", () => { appendToolCallAndResult(sm); expectPersistedToolResultDetailsCapped(sm); }); + + it("reapplies the details cap after redaction expands hook details", () => { + const deepItems = Array.from({ length: 2_000 }, () => ({})); + const hookDetails = { a: { b: { c: { d: { e: { f: { g: deepItems } } } } } } }; + initializeTempPlugin({ + tmpPrefix: "openclaw-toolpersist-details-redaction-expand-", + id: "persist-details-redaction-expand", + body: `export default { id: "persist-details-redaction-expand", register(api) { + api.on("tool_result_persist", (event) => { + return { message: { ...event.message, details: ${JSON.stringify(hookDetails)} } }; + }, { priority: 10 }); +} };`, + }); + + const sm = guardSessionManager(SessionManager.inMemory(), { + agentId: "main", + sessionKey: "main", + }); + + appendToolCallAndResult(sm); + expectPersistedToolResultDetailsCapped(sm); + }); }); describe("before_message_write hook", () => { @@ -479,4 +821,4 @@ describe("before_message_write hook", () => { appendToolCallAndResult(sm); expectPersistedToolResultTextCapped(sm); }); -}); +}); \ No newline at end of file diff --git a/src/agents/session-tool-result-guard.ts b/src/agents/session-tool-result-guard.ts index 529a81105b1..695a6d5dd4b 100644 --- a/src/agents/session-tool-result-guard.ts +++ b/src/agents/session-tool-result-guard.ts @@ -6,6 +6,11 @@ import { jsonUtf8BytesOrInfinity, type BoundedJsonUtf8Bytes, } from "../infra/json-utf8-bytes.js"; +import { + isSensitiveFieldKey, + redactSensitiveFieldValueWithConfig, + redactToolPayloadTextWithConfig, +} from "../logging/redact.js"; import type { PluginHookBeforeMessageWriteEvent, PluginHookBeforeMessageWriteResult, @@ -58,26 +63,148 @@ const MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES = 8_192; const MAX_PERSISTED_DETAIL_STRING_CHARS = 2_000; const MAX_PERSISTED_DETAIL_SESSION_COUNT = 10; const MAX_PERSISTED_DETAIL_FALLBACK_STRING_CHARS = 200; +const MAX_PERSISTED_DETAIL_REDACTION_LOOKAHEAD_CHARS = 1_024; +const MAX_PERSISTED_DETAIL_BOUNDARY_OVERLAP_CHARS = 512; +const PERSISTED_DETAIL_REDACTION_BOUNDARY = "\u0000OPENCLAW_PERSISTED_DETAIL_BOUNDARY\u0000"; +const PARTIAL_STRUCTURED_SECRET_VALUE_RE = + /(?:["']?(?:api[-_]?key|apikey|token|secret|password|passwd|access[-_]?token|accesstoken|refresh[-_]?token|refreshtoken|auth[-_]?token|authtoken|client[-_]?secret|clientsecret|app[-_]?secret|appsecret|card[-_]?number|cardnumber|cvc|cvv)["']?\s*[:=]\s*["']?)(?!\*{3})(?=[^\s"',}\]]{8,})/i; +const PARTIAL_PRIVATE_KEY_BLOCK_RE = + /-----BEGIN [A-Z0-9 ]*(?:PRIVATE KEY|OPENSSH PRIVATE KEY|RSA PRIVATE KEY|EC PRIVATE KEY|DSA PRIVATE KEY)-----/i; +type ToolResultDetailRedactionConfig = Parameters[1]; function originalDetailsSizeFields(size: BoundedJsonUtf8Bytes): Record { return size.complete ? { originalDetailsBytes: size.bytes } : { originalDetailsBytesAtLeast: size.bytes }; } -function truncatePersistedDetailString( +function redactPersistedDetailString( value: string, maxChars = MAX_PERSISTED_DETAIL_STRING_CHARS, + redactionConfig?: ToolResultDetailRedactionConfig, ): string { if (value.length <= maxChars) { - return value; + return redactToolPayloadTextWithConfig(value, redactionConfig); } - return `${value.slice(0, maxChars)}\n\n[OpenClaw persisted detail truncated: ${ - value.length - maxChars - } chars omitted]`; + + const scan = `${value.slice(0, maxChars)}${PERSISTED_DETAIL_REDACTION_BOUNDARY}${value.slice( + maxChars, + maxChars + MAX_PERSISTED_DETAIL_REDACTION_LOOKAHEAD_CHARS, + )}`; + const redactedScan = redactToolPayloadTextWithConfig(scan, redactionConfig); + const boundaryIndex = redactedScan.indexOf(PERSISTED_DETAIL_REDACTION_BOUNDARY); + const redactedPrefix = + boundaryIndex >= 0 + ? redactedScan.slice(0, boundaryIndex) + : "[OpenClaw persisted detail redacted: boundary marker removed]"; + const safePrefixChars = Math.max( + 0, + maxChars - Math.min(maxChars, MAX_PERSISTED_DETAIL_BOUNDARY_OVERLAP_CHARS), + ); + const initialPersistedPrefix = redactedPrefix.slice(0, safePrefixChars); + const persistedPrefix = + PARTIAL_STRUCTURED_SECRET_VALUE_RE.test(initialPersistedPrefix) || + PARTIAL_PRIVATE_KEY_BLOCK_RE.test(initialPersistedPrefix) + ? "[OpenClaw persisted detail redacted: partial secret span omitted]" + : initialPersistedPrefix; + const boundaryNotice = "[OpenClaw persisted detail redacted: boundary overlap omitted]"; + return `${persistedPrefix}${persistedPrefix ? "\n" : ""}${boundaryNotice}\n\n[OpenClaw persisted detail truncated: ${Math.max( + 0, + value.length - maxChars, + )} original chars omitted]`; } -function sanitizePersistedSessionDetail(value: unknown): unknown { +function isSensitivePersistedDetailKey(key: string | undefined): boolean { + return Boolean(key && isSensitiveFieldKey(key)); +} + +function selectPersistedDetailRedactionKey( + key: string, + inheritedKey: string | undefined, +): string | undefined { + return isSensitivePersistedDetailKey(key) ? key : inheritedKey; +} + +function redactedOriginalDetailKeys( + src: Record, + redactionConfig?: ToolResultDetailRedactionConfig, +): string[] { + return firstEnumerableOwnKeys(src, 40).map((key) => + redactToolPayloadTextWithConfig(key, redactionConfig), + ); +} + +function redactPersistedDetailValue( + value: unknown, + depth = 0, + redactionKey?: string, + redactionConfig?: ToolResultDetailRedactionConfig, +): unknown { + if (typeof value === "string") { + return redactionKey + ? redactSensitiveFieldValueWithConfig(redactionKey, value, redactionConfig) + : redactToolPayloadTextWithConfig(value, redactionConfig); + } + if ( + redactionKey && + (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") + ) { + return redactSensitiveFieldValueWithConfig(redactionKey, String(value), redactionConfig); + } + if (value === null || value === undefined || typeof value !== "object") { + return value; + } + if (depth >= 8) { + return "[OpenClaw persisted detail redacted: max depth exceeded]"; + } + if (Array.isArray(value)) { + let changed = false; + const next = value.map((item) => { + const redacted = redactPersistedDetailValue(item, depth + 1, redactionKey, redactionConfig); + changed ||= redacted !== item; + return redacted; + }); + return changed ? next : value; + } + + const source = value as Record; + let changed = false; + const next: Record = {}; + for (const [key, field] of Object.entries(source)) { + const redactedKey = redactToolPayloadTextWithConfig(key, redactionConfig); + const redacted = redactPersistedDetailValue( + field, + depth + 1, + selectPersistedDetailRedactionKey(key, redactionKey), + redactionConfig, + ); + changed ||= redactedKey !== key || redacted !== field; + next[redactedKey] = redacted; + } + return changed ? next : value; +} + +function redactPersistedSummaryField( + key: string, + value: unknown, + maxStringChars: number, + redactionConfig?: ToolResultDetailRedactionConfig, +): unknown { + if (typeof value === "string") { + return redactPersistedDetailString(value, maxStringChars, redactionConfig); + } + return redactPersistedDetailValue( + value, + 0, + selectPersistedDetailRedactionKey(key, undefined), + redactionConfig, + ); +} + +function sanitizePersistedSessionDetail( + value: unknown, + redactionConfig?: ToolResultDetailRedactionConfig, +): unknown { if (!value || typeof value !== "object") { return value; } @@ -98,11 +225,11 @@ function sanitizePersistedSessionDetail(value: unknown): unknown { ]) { const field = src[key]; if (field !== undefined) { - out[key] = typeof field === "string" ? truncatePersistedDetailString(field, 500) : field; + out[key] = redactPersistedSummaryField(key, field, 500, redactionConfig); } } if (typeof src.command === "string") { - out.command = truncatePersistedDetailString(src.command, 500); + out.command = redactPersistedDetailString(src.command, 500, redactionConfig); } return out; } @@ -111,6 +238,7 @@ function buildPersistedDetailsFallback( src: Record | undefined, originalSize: BoundedJsonUtf8Bytes, sanitizedBytes?: number, + redactionConfig?: ToolResultDetailRedactionConfig, ): Record { // If even the structured summary is too large, keep only shape and stable // status fields. This preserves "what happened?" without persisting the raw @@ -124,14 +252,16 @@ function buildPersistedDetailsFallback( fallback.sanitizedDetailsBytes = sanitizedBytes; } if (src) { - fallback.originalDetailKeys = firstEnumerableOwnKeys(src, 40); + fallback.originalDetailKeys = redactedOriginalDetailKeys(src, redactionConfig); for (const key of ["status", "sessionId", "pid", "exitCode", "exitSignal", "truncated"]) { const field = src[key]; if (field !== undefined) { - fallback[key] = - typeof field === "string" - ? truncatePersistedDetailString(field, MAX_PERSISTED_DETAIL_FALLBACK_STRING_CHARS) - : field; + fallback[key] = redactPersistedSummaryField( + key, + field, + MAX_PERSISTED_DETAIL_FALLBACK_STRING_CHARS, + redactionConfig, + ); } } } @@ -142,12 +272,18 @@ function enforcePersistedDetailsByteCap( value: Record, src: Record | undefined, originalSize: BoundedJsonUtf8Bytes, + redactionConfig?: ToolResultDetailRedactionConfig, ): Record { const sanitizedBytes = jsonUtf8BytesOrInfinity(value); if (sanitizedBytes <= MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES) { return value; } - const fallback = buildPersistedDetailsFallback(src, originalSize, sanitizedBytes); + const fallback = buildPersistedDetailsFallback( + src, + originalSize, + sanitizedBytes, + redactionConfig, + ); if (jsonUtf8BytesOrInfinity(fallback) <= MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES) { return fallback; } @@ -159,7 +295,36 @@ function enforcePersistedDetailsByteCap( }; } -function sanitizeToolResultDetailsForPersistence(details: unknown): unknown { +function enforceRedactedPersistedDetailsByteCap( + redacted: unknown, + originalDetails: unknown, + originalSize: BoundedJsonUtf8Bytes, + redactionConfig?: ToolResultDetailRedactionConfig, +): unknown { + const redactedBytes = jsonUtf8BytesOrInfinity(redacted); + if (redactedBytes <= MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES) { + return redacted; + } + if (originalDetails && typeof originalDetails === "object" && !Array.isArray(originalDetails)) { + return buildPersistedDetailsFallback( + originalDetails as Record, + originalSize, + redactedBytes, + redactionConfig, + ); + } + return { + persistedDetailsTruncated: true, + finalDetailsTruncated: true, + ...originalDetailsSizeFields(originalSize), + sanitizedDetailsBytes: redactedBytes, + }; +} + +function sanitizeToolResultDetailsForPersistence( + details: unknown, + redactionConfig?: ToolResultDetailRedactionConfig, +): unknown { if (details === undefined || details === null) { return details; } @@ -167,7 +332,12 @@ function sanitizeToolResultDetailsForPersistence(details: unknown): unknown { // need to be fully stringified just to learn they exceed the persistence cap. const originalSize = boundedJsonUtf8Bytes(details, MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES); if (originalSize.complete && originalSize.bytes <= MAX_PERSISTED_TOOL_RESULT_DETAILS_BYTES) { - return details; + return enforceRedactedPersistedDetailsByteCap( + redactPersistedDetailValue(details, 0, undefined, redactionConfig), + details, + originalSize, + redactionConfig, + ); } if (typeof details !== "object") { return enforcePersistedDetailsByteCap( @@ -178,13 +348,14 @@ function sanitizeToolResultDetailsForPersistence(details: unknown): unknown { }, undefined, originalSize, + redactionConfig, ); } const src = details as Record; const out: Record = { persistedDetailsTruncated: true, ...originalDetailsSizeFields(originalSize), - originalDetailKeys: firstEnumerableOwnKeys(src, 40), + originalDetailKeys: redactedOriginalDetailKeys(src, redactionConfig), }; for (const key of [ "status", @@ -206,29 +377,41 @@ function sanitizeToolResultDetailsForPersistence(details: unknown): unknown { ]) { const field = src[key]; if (field !== undefined) { - out[key] = typeof field === "string" ? truncatePersistedDetailString(field) : field; + out[key] = redactPersistedSummaryField( + key, + field, + MAX_PERSISTED_DETAIL_STRING_CHARS, + redactionConfig, + ); } } if (typeof src.tail === "string") { - out.tail = truncatePersistedDetailString(src.tail); + out.tail = redactPersistedDetailString( + src.tail, + MAX_PERSISTED_DETAIL_STRING_CHARS, + redactionConfig, + ); } if (Array.isArray(src.sessions)) { out.sessions = src.sessions .slice(0, MAX_PERSISTED_DETAIL_SESSION_COUNT) - .map(sanitizePersistedSessionDetail); + .map((session) => sanitizePersistedSessionDetail(session, redactionConfig)); if (src.sessions.length > MAX_PERSISTED_DETAIL_SESSION_COUNT) { out.sessionsTruncated = src.sessions.length - MAX_PERSISTED_DETAIL_SESSION_COUNT; } } - return enforcePersistedDetailsByteCap(out, src, originalSize); + return enforcePersistedDetailsByteCap(out, src, originalSize, redactionConfig); } -function capToolResultDetails(msg: AgentMessage): AgentMessage { +function capToolResultDetails( + msg: AgentMessage, + redactionConfig?: ToolResultDetailRedactionConfig, +): AgentMessage { if ((msg as { role?: string }).role !== "toolResult") { return msg; } const details = (msg as { details?: unknown }).details; - const sanitizedDetails = sanitizeToolResultDetailsForPersistence(details); + const sanitizedDetails = sanitizeToolResultDetailsForPersistence(details, redactionConfig); if (sanitizedDetails === details) { return msg; } @@ -237,8 +420,12 @@ function capToolResultDetails(msg: AgentMessage): AgentMessage { return next; } -function capToolResultForPersistence(msg: AgentMessage, maxChars: number): AgentMessage { - return capToolResultDetails(capToolResultSize(msg, maxChars)); +function capToolResultForPersistence( + msg: AgentMessage, + maxChars: number, + redactionConfig?: ToolResultDetailRedactionConfig, +): AgentMessage { + return capToolResultDetails(capToolResultSize(msg, maxChars), redactionConfig); } function normalizePersistedToolResultName( @@ -316,6 +503,7 @@ export function installSessionToolResultGuard( beforeMessageWriteHook?: ( event: PluginHookBeforeMessageWriteEvent, ) => PluginHookBeforeMessageWriteResult | undefined; + redactLoggingConfig?: ToolResultDetailRedactionConfig; maxToolResultChars?: number; suppressNextUserMessagePersistence?: boolean; onUserMessagePersisted?: ( @@ -346,6 +534,7 @@ export function installSessionToolResultGuard( const allowSyntheticToolResults = opts?.allowSyntheticToolResults ?? true; const missingToolResultText = opts?.missingToolResultText; const beforeWrite = opts?.beforeMessageWriteHook; + const redactionConfig = opts?.redactLoggingConfig; const maxToolResultChars = resolveMaxToolResultChars(opts); let suppressNextUserMessagePersistence = opts?.suppressNextUserMessagePersistence === true; @@ -386,7 +575,9 @@ export function installSessionToolResultGuard( }), ); if (flushed) { - originalAppend(capToolResultForPersistence(flushed, maxToolResultChars) as never); + originalAppend( + capToolResultForPersistence(flushed, maxToolResultChars, redactionConfig) as never, + ); } } } @@ -426,6 +617,7 @@ export function installSessionToolResultGuard( const capped = capToolResultForPersistence( persistMessage(normalizedToolResult), maxToolResultChars, + redactionConfig, ); const persisted = applyBeforeWriteHook( persistToolResult(capped, { @@ -437,7 +629,9 @@ export function installSessionToolResultGuard( if (!persisted) { return undefined; } - return originalAppend(capToolResultForPersistence(persisted, maxToolResultChars) as never); + return originalAppend( + capToolResultForPersistence(persisted, maxToolResultChars, redactionConfig) as never, + ); } // Skip tool call extraction for aborted/errored assistant messages. @@ -513,4 +707,4 @@ export function installSessionToolResultGuard( clearPendingToolResults, getPendingIds: pendingState.getPendingIds, }; -} +} \ No newline at end of file diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index ac102702c10..7872b21377a 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -186,6 +186,19 @@ describe("redactSensitiveText", () => { expect(redactSensitiveFieldValue("amount", "4200")).toBe("4200"); }); + it("masks structured uppercase env-style field values by key", () => { + expect(redactSensitiveFieldValue("GITHUB_TOKEN", "abcdefghijklmnopqrstuvwx1234567890")).toBe( + "abcdef…7890", + ); + expect(redactSensitiveFieldValue("github_token", "abcdefghijklmnopqrstuvwx1234567890")).toBe( + "abcdef…7890", + ); + expect(redactSensitiveFieldValue("openai_api_key", "abcdefghijklmnopqrstuvwx1234567890")).toBe( + "abcdef…7890", + ); + expect(redactSensitiveFieldValue("MONKEY", "banana")).toBe("banana"); + }); + it("masks bearer tokens", () => { const input = "Authorization: Bearer abcdef1234567890ghij"; const output = redactSensitiveText(input, { @@ -195,6 +208,17 @@ describe("redactSensitiveText", () => { expect(output).toBe("Authorization: Bearer abcdef…ghij"); }); + it("masks token prefixes embedded after adjacent text", () => { + const token = `ghp_${"a".repeat(5_000)}`; + const output = redactSensitiveText(`prefix-${token} suffix`, { + mode: "tools", + patterns: defaults, + }); + expect(output).toBe("prefix-ghp_aa…aaaa suffix"); + expect(output).not.toContain(token); + expect(output).not.toContain("a".repeat(100)); + }); + it("masks URL query tokens", () => { const input = "GET /_matrix/client/v3/sync?access_token=abcdef1234567890ghij"; const output = redactSensitiveText(input, { @@ -543,4 +567,4 @@ describe("redactSensitiveLines", () => { expect(joined).toContain("…redacted…"); expect(joined).not.toContain("ABCDEF1234567890"); }); -}); +}); \ No newline at end of file diff --git a/src/logging/redact.ts b/src/logging/redact.ts index a6149479ecb..b52b4db5841 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -1,9 +1,11 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { compileConfigRegex } from "../security/config-regex.js"; import { readLoggingConfig } from "./config.js"; import { replacePatternBounded } from "./redact-bounded.js"; export type RedactSensitiveMode = "off" | "tools"; type RedactPattern = string | RegExp; +type LoggingConfig = OpenClawConfig["logging"]; const DEFAULT_REDACT_MODE: RedactSensitiveMode = "tools"; const DEFAULT_REDACT_MIN_LENGTH = 18; @@ -14,7 +16,7 @@ const PAYMENT_CREDENTIAL_ENV_KEYS = String.raw`CARD[_-]?NUMBER|CARD[_-]?CVC|CARD const PAYMENT_CREDENTIAL_QUERY_KEYS = String.raw`card[-_]?number|card[-_]?cvc|card[-_]?cvv|cvc|cvv|security[-_]?code|payment[-_]?credential|shared[-_]?payment[-_]?token`; const PAYMENT_CREDENTIAL_JSON_KEYS = String.raw`cardNumber|card_number|cardCvc|card_cvc|cardCvv|card_cvv|cvc|cvv|securityCode|security_code|paymentCredential|payment_credential|sharedPaymentToken|shared_payment_token`; const STRUCTURED_SECRET_FIELD_RE = new RegExp( - String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|client[-_]?secret|clientSecret|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`, + String.raw`^(?:api[-_]?key|apiKey|token|secret|password|passwd|access[-_]?token|accessToken|refresh[-_]?token|refreshToken|id[-_]?token|idToken|auth[-_]?token|authToken|client[-_]?secret|clientSecret|app[-_]?secret|appSecret|${PAYMENT_CREDENTIAL_QUERY_KEYS}|${PAYMENT_CREDENTIAL_JSON_KEYS})$`, "i", ); const STRUCTURED_APP_PASSWORD_FIELD_RE = @@ -32,6 +34,10 @@ const BENIGN_APP_PASSWORD_WORDS = new Set([ "slug", "test", ]); +const STRUCTURED_SECRET_ENV_FIELD_RE = new RegExp( + String.raw`^(?:(?:[A-Z0-9]+[_-])+(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD)|API[_-]?KEY|TOKEN|SECRET|PASSWORD|PASSWD|${PAYMENT_CREDENTIAL_ENV_KEYS})$`, + "i", +); const DEFAULT_REDACT_PATTERNS: string[] = [ // ENV-style assignments. Keep this case-sensitive so diagnostics like @@ -58,22 +64,22 @@ const DEFAULT_REDACT_PATTERNS: string[] = [ String.raw`-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----`, // Common token prefixes. String.raw`\b(sk-[A-Za-z0-9_-]{8,})\b`, - String.raw`\b(ghp_[A-Za-z0-9]{20,})\b`, - String.raw`\b(github_pat_[A-Za-z0-9_]{20,})\b`, - String.raw`\b(xox[baprs]-[A-Za-z0-9-]{10,})\b`, - String.raw`\b(xapp-[A-Za-z0-9-]{10,})\b`, - String.raw`\b(gsk_[A-Za-z0-9_-]{10,})\b`, - String.raw`\b(AIza[0-9A-Za-z\-_]{20,})\b`, - String.raw`\b(ya29\.[0-9A-Za-z_\-./+=]{10,})\b`, - String.raw`\b(1//0[0-9A-Za-z_\-./+=]{10,})\b`, - String.raw`\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b`, - String.raw`\b(pplx-[A-Za-z0-9_-]{10,})\b`, - String.raw`\b(npm_[A-Za-z0-9]{10,})\b`, + String.raw`(ghp_[A-Za-z0-9]{20,})`, + String.raw`(github_pat_[A-Za-z0-9_]{20,})`, + String.raw`(xox[baprs]-[A-Za-z0-9-]{10,})`, + String.raw`(xapp-[A-Za-z0-9-]{10,})`, + String.raw`(gsk_[A-Za-z0-9_-]{10,})`, + String.raw`(AIza[0-9A-Za-z\-_]{20,})`, + String.raw`(ya29\.[0-9A-Za-z_\-./+=]{10,})`, + String.raw`(1//0[0-9A-Za-z_\-./+=]{10,})`, + String.raw`(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})`, + String.raw`(pplx-[A-Za-z0-9_-]{10,})`, + String.raw`(npm_[A-Za-z0-9]{10,})`, // Additional access-key and token-style prefixes. - String.raw`\b(AKID[A-Za-z0-9]{10,})\b`, - String.raw`\b(LTAI[A-Za-z0-9]{10,})\b`, - String.raw`\b(hf_[A-Za-z0-9]{10,})\b`, - String.raw`\b(r8_[A-Za-z0-9]{10,})\b`, + String.raw`(AKID[A-Za-z0-9]{10,})`, + String.raw`(LTAI[A-Za-z0-9]{10,})`, + String.raw`(hf_[A-Za-z0-9]{10,})`, + String.raw`(r8_[A-Za-z0-9]{10,})`, // Telegram Bot API URLs embed the token as `/bot/...` (no word-boundary before digits). String.raw`\bbot(\d{6,}:[A-Za-z0-9_-]{20,})\b`, String.raw`\b(\d{6,}:[A-Za-z0-9_-]{20,})\b`, @@ -210,9 +216,10 @@ export function redactToolDetail(detail: string): string { return redactSensitiveText(detail, resolved); } -function resolveToolPayloadRedaction(): RedactOptions { - const cfg = readLoggingConfig(); - const userPatterns = cfg?.redactPatterns; +function resolveToolPayloadRedaction( + loggingConfig: LoggingConfig | undefined = readLoggingConfig(), +): RedactOptions { + const userPatterns = loggingConfig?.redactPatterns; const patterns = userPatterns && userPatterns.length > 0 ? [...userPatterns, ...DEFAULT_REDACT_PATTERNS] @@ -224,10 +231,21 @@ function resolveToolPayloadRedaction(): RedactOptions { // output, not UI surfaces), and merges user `logging.redactPatterns` with the // built-in defaults so both apply. export function redactToolPayloadText(text: string): string { + return redactToolPayloadTextWithConfig(text, readLoggingConfig()); +} + +export function redactToolPayloadTextWithConfig( + text: string, + loggingConfig?: LoggingConfig, +): string { if (!text) { return text; } - return redactSensitiveText(text, resolveToolPayloadRedaction()); + return redactSensitiveText(text, resolveToolPayloadRedaction(loggingConfig)); +} + +export function isSensitiveFieldKey(key: string): boolean { + return STRUCTURED_SECRET_FIELD_RE.test(key) || STRUCTURED_SECRET_ENV_FIELD_RE.test(key); } function redactSensitiveFieldValueWithOptions( @@ -242,17 +260,30 @@ function redactSensitiveFieldValueWithOptions( if (appRedacted !== value) { return appRedacted; } - } else if (redacted !== value) { + } + if (redacted !== value) { return redacted; } - if (STRUCTURED_SECRET_FIELD_RE.test(key)) { + if (isSensitiveFieldKey(key)) { return maskToken(value); } return value; } export function redactSensitiveFieldValue(key: string, value: string): string { - return redactSensitiveFieldValueWithOptions(key, value, resolveToolPayloadRedaction()); + return redactSensitiveFieldValueWithConfig(key, value, readLoggingConfig()); +} + +export function redactSensitiveFieldValueWithConfig( + key: string, + value: string, + loggingConfig?: LoggingConfig, +): string { + return redactSensitiveFieldValueWithOptions( + key, + value, + resolveToolPayloadRedaction(loggingConfig), + ); } function isPlainRedactableObject(value: object): value is Record { @@ -329,4 +360,4 @@ export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOp return lines; } return redactText(lines.join("\n"), resolved.patterns).split("\n"); -} +} \ No newline at end of file