fix(secrets): resolve web tool SecretRefs atomically at runtime

This commit is contained in:
Josh Avant
2026-03-09 22:57:03 -05:00
committed by GitHub
parent 93c44e3dad
commit f0eb67923c
28 changed files with 2059 additions and 112 deletions

View File

@@ -1,5 +1,6 @@
import type { OpenClawConfig } from "../config/config.js";
import { resolvePluginTools } from "../plugins/tools.js";
import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime.js";
import type { GatewayMessageChannel } from "../utils/message-channel.js";
import { resolveSessionAgentId } from "./agent-scope.js";
import type { SandboxFsBridge } from "./sandbox/fs-bridge.js";
@@ -72,6 +73,7 @@ export function createOpenClawTools(
} & SpawnedToolContext,
): AnyAgentTool[] {
const workspaceDir = resolveWorkspaceRoot(options?.workspaceDir);
const runtimeWebTools = getActiveRuntimeWebToolsMetadata();
const imageTool = options?.agentDir?.trim()
? createImageTool({
config: options?.config,
@@ -100,10 +102,12 @@ export function createOpenClawTools(
const webSearchTool = createWebSearchTool({
config: options?.config,
sandboxed: options?.sandboxed,
runtimeWebSearch: runtimeWebTools?.search,
});
const webFetchTool = createWebFetchTool({
config: options?.config,
sandboxed: options?.sandboxed,
runtimeFirecrawl: runtimeWebTools?.fetch.firecrawl,
});
const messageTool = options?.disableMessageTool
? null

View File

@@ -0,0 +1,135 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} from "../secrets/runtime.js";
import { withFetchPreconnect } from "../test-utils/fetch-mock.js";
import { createOpenClawTools } from "./openclaw-tools.js";
vi.mock("../plugins/tools.js", () => ({
resolvePluginTools: () => [],
}));
function asConfig(value: unknown): OpenClawConfig {
return value as OpenClawConfig;
}
function findTool(name: string, config: OpenClawConfig) {
const allTools = createOpenClawTools({ config, sandboxed: true });
const tool = allTools.find((candidate) => candidate.name === name);
expect(tool).toBeDefined();
if (!tool) {
throw new Error(`missing ${name} tool`);
}
return tool;
}
function makeHeaders(map: Record<string, string>): { get: (key: string) => string | null } {
return {
get: (key) => map[key.toLowerCase()] ?? null,
};
}
async function prepareAndActivate(params: { config: OpenClawConfig; env?: NodeJS.ProcessEnv }) {
const snapshot = await prepareSecretsRuntimeSnapshot({
config: params.config,
env: params.env,
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({ version: 1, profiles: {} }),
});
activateSecretsRuntimeSnapshot(snapshot);
return snapshot;
}
describe("openclaw tools runtime web metadata wiring", () => {
const priorFetch = global.fetch;
afterEach(() => {
global.fetch = priorFetch;
clearSecretsRuntimeSnapshot();
});
it("uses runtime-selected provider when higher-precedence provider ref is unresolved", async () => {
const snapshot = await prepareAndActivate({
config: asConfig({
tools: {
web: {
search: {
apiKey: { source: "env", provider: "default", id: "MISSING_BRAVE_KEY_REF" },
gemini: {
apiKey: { source: "env", provider: "default", id: "GEMINI_WEB_KEY_REF" },
},
},
},
},
}),
env: {
GEMINI_WEB_KEY_REF: "gemini-runtime-key",
},
});
expect(snapshot.webTools.search.selectedProvider).toBe("gemini");
const mockFetch = vi.fn((_input?: unknown, _init?: unknown) =>
Promise.resolve({
ok: true,
json: () =>
Promise.resolve({
candidates: [
{
content: { parts: [{ text: "runtime gemini ok" }] },
groundingMetadata: { groundingChunks: [] },
},
],
}),
} as Response),
);
global.fetch = withFetchPreconnect(mockFetch);
const webSearch = findTool("web_search", snapshot.config);
const result = await webSearch.execute("call-runtime-search", { query: "runtime search" });
expect(mockFetch).toHaveBeenCalled();
expect(String(mockFetch.mock.calls[0]?.[0])).toContain("generativelanguage.googleapis.com");
expect((result.details as { provider?: string }).provider).toBe("gemini");
});
it("skips Firecrawl key resolution when runtime marks Firecrawl inactive", async () => {
const snapshot = await prepareAndActivate({
config: asConfig({
tools: {
web: {
fetch: {
firecrawl: {
enabled: false,
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_KEY_REF" },
},
},
},
},
}),
});
const mockFetch = vi.fn((_input?: unknown, _init?: unknown) =>
Promise.resolve({
ok: true,
status: 200,
headers: makeHeaders({ "content-type": "text/html; charset=utf-8" }),
text: () =>
Promise.resolve(
"<html><body><article><h1>Runtime Off</h1><p>Use direct fetch.</p></article></body></html>",
),
} as Response),
);
global.fetch = withFetchPreconnect(mockFetch);
const webFetch = findTool("web_fetch", snapshot.config);
await webFetch.execute("call-runtime-fetch", { url: "https://example.com/runtime-off" });
expect(mockFetch).toHaveBeenCalled();
expect(mockFetch.mock.calls[0]?.[0]).toBe("https://example.com/runtime-off");
expect(String(mockFetch.mock.calls[0]?.[0])).not.toContain("api.firecrawl.dev");
});
});

View File

@@ -84,6 +84,47 @@ describe("web_fetch Cloudflare Markdown for Agents", () => {
expect(details?.contentType).toBe("text/html");
});
it("bypasses Firecrawl when runtime metadata marks Firecrawl inactive", async () => {
const fetchSpy = vi
.fn()
.mockResolvedValue(
htmlResponse(
"<html><body><article><h1>Runtime Off</h1><p>Use direct fetch.</p></article></body></html>",
),
);
global.fetch = withFetchPreconnect(fetchSpy);
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: {
firecrawl: {
enabled: true,
apiKey: {
source: "env",
provider: "default",
id: "MISSING_FIRECRAWL_KEY_REF",
},
},
},
},
},
},
sandboxed: false,
runtimeFirecrawl: {
active: false,
apiKeySource: "secretRef",
diagnostics: [],
},
});
await tool?.execute?.("call", { url: "https://example.com/runtime-firecrawl-off" });
expect(fetchSpy).toHaveBeenCalled();
expect(fetchSpy.mock.calls[0]?.[0]).toBe("https://example.com/runtime-firecrawl-off");
});
it("logs x-markdown-tokens when header is present", async () => {
const logSpy = vi.spyOn(logger, "logDebug").mockImplementation(() => {});
const fetchSpy = vi

View File

@@ -1,7 +1,9 @@
import { Type } from "@sinclair/typebox";
import type { OpenClawConfig } from "../../config/config.js";
import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js";
import { SsrFBlockedError } from "../../infra/net/ssrf.js";
import { logDebug } from "../../logger.js";
import type { RuntimeWebFetchFirecrawlMetadata } from "../../secrets/runtime-web-tools.js";
import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
import { stringEnum } from "../schema/typebox.js";
@@ -71,7 +73,7 @@ type WebFetchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer
type FirecrawlFetchConfig =
| {
enabled?: boolean;
apiKey?: string;
apiKey?: unknown;
baseUrl?: string;
onlyMainContent?: boolean;
maxAgeMs?: number;
@@ -136,10 +138,14 @@ function resolveFirecrawlConfig(fetch?: WebFetchConfig): FirecrawlFetchConfig {
}
function resolveFirecrawlApiKey(firecrawl?: FirecrawlFetchConfig): string | undefined {
const fromConfig =
firecrawl && "apiKey" in firecrawl && typeof firecrawl.apiKey === "string"
? normalizeSecretInput(firecrawl.apiKey)
: "";
const fromConfigRaw =
firecrawl && "apiKey" in firecrawl
? normalizeResolvedSecretInputString({
value: firecrawl.apiKey,
path: "tools.web.fetch.firecrawl.apiKey",
})
: undefined;
const fromConfig = normalizeSecretInput(fromConfigRaw);
const fromEnv = normalizeSecretInput(process.env.FIRECRAWL_API_KEY);
return fromConfig || fromEnv || undefined;
}
@@ -712,6 +718,7 @@ function resolveFirecrawlEndpoint(baseUrl: string): string {
export function createWebFetchTool(options?: {
config?: OpenClawConfig;
sandboxed?: boolean;
runtimeFirecrawl?: RuntimeWebFetchFirecrawlMetadata;
}): AnyAgentTool | null {
const fetch = resolveFetchConfig(options?.config);
if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed })) {
@@ -719,8 +726,14 @@ export function createWebFetchTool(options?: {
}
const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
const firecrawl = resolveFirecrawlConfig(fetch);
const firecrawlApiKey = resolveFirecrawlApiKey(firecrawl);
const firecrawlEnabled = resolveFirecrawlEnabled({ firecrawl, apiKey: firecrawlApiKey });
const runtimeFirecrawlActive = options?.runtimeFirecrawl?.active;
const shouldResolveFirecrawlApiKey =
runtimeFirecrawlActive === undefined ? firecrawl?.enabled !== false : runtimeFirecrawlActive;
const firecrawlApiKey = shouldResolveFirecrawlApiKey
? resolveFirecrawlApiKey(firecrawl)
: undefined;
const firecrawlEnabled =
runtimeFirecrawlActive ?? resolveFirecrawlEnabled({ firecrawl, apiKey: firecrawlApiKey });
const firecrawlBaseUrl = resolveFirecrawlBaseUrl(firecrawl);
const firecrawlOnlyMainContent = resolveFirecrawlOnlyMainContent(firecrawl);
const firecrawlMaxAgeMs = resolveFirecrawlMaxAgeMsOrDefault(firecrawl);

View File

@@ -3,6 +3,7 @@ import { formatCliCommand } from "../../cli/command-format.js";
import type { OpenClawConfig } from "../../config/config.js";
import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js";
import { logVerbose } from "../../globals.js";
import type { RuntimeWebSearchMetadata } from "../../secrets/runtime-web-tools.js";
import { wrapWebContent } from "../../security/external-content.js";
import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
import type { AnyAgentTool } from "./common.js";
@@ -193,6 +194,33 @@ function createWebSearchSchema(params: {
),
} as const;
const perplexityStructuredFilterSchema = {
country: Type.Optional(
Type.String({
description:
"Native Perplexity Search API only. 2-letter country code for region-specific results (e.g., 'DE', 'US', 'ALL'). Default: 'US'.",
}),
),
language: Type.Optional(
Type.String({
description:
"Native Perplexity Search API only. ISO 639-1 language code for results (e.g., 'en', 'de', 'fr').",
}),
),
date_after: Type.Optional(
Type.String({
description:
"Native Perplexity Search API only. Only results published after this date (YYYY-MM-DD).",
}),
),
date_before: Type.Optional(
Type.String({
description:
"Native Perplexity Search API only. Only results published before this date (YYYY-MM-DD).",
}),
),
} as const;
if (params.provider === "brave") {
return Type.Object({
...querySchema,
@@ -221,7 +249,8 @@ function createWebSearchSchema(params: {
}
return Type.Object({
...querySchema,
...filterSchema,
freshness: filterSchema.freshness,
...perplexityStructuredFilterSchema,
domain_filter: Type.Optional(
Type.Array(Type.String(), {
description:
@@ -742,6 +771,16 @@ function resolvePerplexityTransport(perplexity?: PerplexityConfig): {
};
}
function resolvePerplexitySchemaTransportHint(
perplexity?: PerplexityConfig,
): PerplexityTransport | undefined {
const hasLegacyOverride = Boolean(
(perplexity?.baseUrl && perplexity.baseUrl.trim()) ||
(perplexity?.model && perplexity.model.trim()),
);
return hasLegacyOverride ? "chat_completions" : undefined;
}
function resolveGrokConfig(search?: WebSearchConfig): GrokConfig {
if (!search || typeof search !== "object") {
return {};
@@ -1809,15 +1848,21 @@ async function runWebSearch(params: {
export function createWebSearchTool(options?: {
config?: OpenClawConfig;
sandboxed?: boolean;
runtimeWebSearch?: RuntimeWebSearchMetadata;
}): AnyAgentTool | null {
const search = resolveSearchConfig(options?.config);
if (!resolveSearchEnabled({ search, sandboxed: options?.sandboxed })) {
return null;
}
const provider = resolveSearchProvider(search);
const provider =
options?.runtimeWebSearch?.selectedProvider ??
options?.runtimeWebSearch?.providerConfigured ??
resolveSearchProvider(search);
const perplexityConfig = resolvePerplexityConfig(search);
const perplexityTransport = resolvePerplexityTransport(perplexityConfig);
const perplexitySchemaTransportHint =
options?.runtimeWebSearch?.perplexityTransport ??
resolvePerplexitySchemaTransportHint(perplexityConfig);
const grokConfig = resolveGrokConfig(search);
const geminiConfig = resolveGeminiConfig(search);
const kimiConfig = resolveKimiConfig(search);
@@ -1826,9 +1871,9 @@ export function createWebSearchTool(options?: {
const description =
provider === "perplexity"
? perplexityTransport.transport === "chat_completions"
? perplexitySchemaTransportHint === "chat_completions"
? "Search the web using Perplexity Sonar via Perplexity/OpenRouter chat completions. Returns AI-synthesized answers with citations from web-grounded search."
: "Search the web using the Perplexity Search API. Returns structured results (title, URL, snippet) for fast research. Supports domain, region, language, and freshness filtering."
: "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path."
: provider === "grok"
? "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search."
: provider === "kimi"
@@ -1845,10 +1890,13 @@ export function createWebSearchTool(options?: {
description,
parameters: createWebSearchSchema({
provider,
perplexityTransport: provider === "perplexity" ? perplexityTransport.transport : undefined,
perplexityTransport: provider === "perplexity" ? perplexitySchemaTransportHint : undefined,
}),
execute: async (_toolCallId, args) => {
const perplexityRuntime = provider === "perplexity" ? perplexityTransport : undefined;
// Resolve Perplexity auth/transport lazily at execution time so unrelated providers
// do not touch Perplexity-only credential surfaces during tool construction.
const perplexityRuntime =
provider === "perplexity" ? resolvePerplexityTransport(perplexityConfig) : undefined;
const apiKey =
provider === "perplexity"
? perplexityRuntime?.apiKey

View File

@@ -166,6 +166,39 @@ describe("web tools defaults", () => {
const tool = createWebSearchTool({ config: {}, sandboxed: false });
expect(tool?.name).toBe("web_search");
});
it("prefers runtime-selected web_search provider over local provider config", async () => {
const mockFetch = installMockFetch(createProviderSuccessPayload("gemini"));
const tool = createWebSearchTool({
config: {
tools: {
web: {
search: {
provider: "brave",
apiKey: "brave-config-test", // pragma: allowlist secret
gemini: {
apiKey: "gemini-config-test", // pragma: allowlist secret
},
},
},
},
},
sandboxed: true,
runtimeWebSearch: {
providerConfigured: "brave",
providerSource: "auto-detect",
selectedProvider: "gemini",
selectedProviderKeySource: "secretRef",
diagnostics: [],
},
});
const result = await tool?.execute?.("call-runtime-provider", { query: "runtime override" });
expect(mockFetch).toHaveBeenCalled();
expect(String(mockFetch.mock.calls[0]?.[0])).toContain("generativelanguage.googleapis.com");
expect((result?.details as { provider?: string } | undefined)?.provider).toBe("gemini");
});
});
describe("web_search country and language parameters", () => {
@@ -489,20 +522,56 @@ describe("web_search perplexity OpenRouter compatibility", () => {
expect(result?.details).toMatchObject({ error: "unsupported_domain_filter" });
});
it("hides Search API-only schema params on the compatibility path", () => {
it("keeps Search API schema params visible before runtime auth routing", () => {
vi.stubEnv("OPENROUTER_API_KEY", "sk-or-v1-test"); // pragma: allowlist secret
const tool = createPerplexitySearchTool();
const properties = (tool?.parameters as { properties?: Record<string, unknown> } | undefined)
?.properties;
expect(properties?.freshness).toBeDefined();
expect(properties?.country).toBeUndefined();
expect(properties?.language).toBeUndefined();
expect(properties?.date_after).toBeUndefined();
expect(properties?.date_before).toBeUndefined();
expect(properties?.domain_filter).toBeUndefined();
expect(properties?.max_tokens).toBeUndefined();
expect(properties?.max_tokens_per_page).toBeUndefined();
expect(properties?.country).toBeDefined();
expect(properties?.language).toBeDefined();
expect(properties?.date_after).toBeDefined();
expect(properties?.date_before).toBeDefined();
expect(properties?.domain_filter).toBeDefined();
expect(properties?.max_tokens).toBeDefined();
expect(properties?.max_tokens_per_page).toBeDefined();
expect(
(
properties?.country as
| {
description?: string;
}
| undefined
)?.description,
).toContain("Native Perplexity Search API only.");
expect(
(
properties?.language as
| {
description?: string;
}
| undefined
)?.description,
).toContain("Native Perplexity Search API only.");
expect(
(
properties?.date_after as
| {
description?: string;
}
| undefined
)?.description,
).toContain("Native Perplexity Search API only.");
expect(
(
properties?.date_before as
| {
description?: string;
}
| undefined
)?.description,
).toContain("Native Perplexity Search API only.");
});
it("keeps structured schema params on the native Search API path", () => {
@@ -522,6 +591,61 @@ describe("web_search perplexity OpenRouter compatibility", () => {
});
});
describe("web_search Perplexity lazy resolution", () => {
const priorFetch = global.fetch;
afterEach(() => {
vi.unstubAllEnvs();
global.fetch = priorFetch;
});
it("does not read Perplexity credentials while creating non-Perplexity tools", () => {
const perplexityConfig: Record<string, unknown> = {};
Object.defineProperty(perplexityConfig, "apiKey", {
enumerable: true,
get() {
throw new Error("perplexity-apiKey-getter-called");
},
});
const tool = createWebSearchTool({
config: {
tools: {
web: {
search: {
provider: "gemini",
gemini: { apiKey: "gemini-config-test" },
perplexity: perplexityConfig as { apiKey?: string; baseUrl?: string; model?: string },
},
},
},
},
sandboxed: true,
});
expect(tool?.name).toBe("web_search");
});
it("defers Perplexity credential reads until execute", async () => {
const perplexityConfig: Record<string, unknown> = {};
Object.defineProperty(perplexityConfig, "apiKey", {
enumerable: true,
get() {
throw new Error("perplexity-apiKey-getter-called");
},
});
const tool = createPerplexitySearchTool(
perplexityConfig as { apiKey?: string; baseUrl?: string; model?: string },
);
expect(tool?.name).toBe("web_search");
await expect(tool?.execute?.("call-1", { query: "test" })).rejects.toThrow(
/perplexity-apiKey-getter-called/,
);
});
});
describe("web_search kimi provider", () => {
const priorFetch = global.fetch;