mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 07:40:44 +00:00
fix(web-fetch): late-bind runtime config
This commit is contained in:
@@ -56,6 +56,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- Web fetch: late-bind `web_fetch` config and provider fallback metadata from the active runtime snapshot, matching `web_search` so long-lived tools do not use stale fetch provider settings. Thanks @vincentkoc.
|
||||
- Discord: clear stale startup probe bot/application status when the async bot probe throws, not just when it returns a degraded probe result. Thanks @vincentkoc.
|
||||
- Web search: scope explicit bundled `web_search` provider runtime loading through manifest ownership, so selecting DuckDuckGo/Gemini/etc. does not import unrelated bundled providers or log their optional dependency failures. Thanks @vincentkoc.
|
||||
- Plugins/discovery: demote the source-only TypeScript runtime check on already-installed `origin: "global"` plugin packages from a config-blocking error to a warning and let the runtime fall through to the TypeScript source via jiti, so a single broken installed package no longer blocks `plugins install` for unrelated plugins; install-time rejection of newly-installed source-only packages is unchanged. Thanks @romneyda.
|
||||
|
||||
@@ -455,6 +455,7 @@ export function createOpenClawTools(
|
||||
config: options?.config,
|
||||
sandboxed: options?.sandboxed,
|
||||
runtimeWebFetch: runtimeWebTools?.fetch,
|
||||
lateBindRuntimeConfig: true,
|
||||
});
|
||||
options?.recordToolPrepStage?.("openclaw-tools:web-fetch-tool");
|
||||
const messageTool = options?.disableMessageTool
|
||||
|
||||
@@ -6,21 +6,35 @@ import { createWebFetchTool } from "./web-fetch.js";
|
||||
const { resolveWebFetchDefinitionMock } = vi.hoisted(() => ({
|
||||
resolveWebFetchDefinitionMock: vi.fn(),
|
||||
}));
|
||||
const runtimeState = vi.hoisted(() => ({
|
||||
activeSecretsRuntimeSnapshot: null as null | { config: unknown },
|
||||
activeRuntimeWebToolsMetadata: null as null | Record<string, unknown>,
|
||||
}));
|
||||
|
||||
vi.mock("../../web-fetch/runtime.js", () => ({
|
||||
resolveWebFetchDefinition: resolveWebFetchDefinitionMock,
|
||||
}));
|
||||
vi.mock("../../secrets/runtime.js", () => ({
|
||||
getActiveSecretsRuntimeSnapshot: () => runtimeState.activeSecretsRuntimeSnapshot,
|
||||
}));
|
||||
vi.mock("../../secrets/runtime-web-tools-state.js", () => ({
|
||||
getActiveRuntimeWebToolsMetadata: () => runtimeState.activeRuntimeWebToolsMetadata,
|
||||
}));
|
||||
|
||||
describe("web_fetch provider fallback normalization", () => {
|
||||
const priorFetch = global.fetch;
|
||||
|
||||
beforeEach(() => {
|
||||
resolveWebFetchDefinitionMock.mockReset();
|
||||
runtimeState.activeSecretsRuntimeSnapshot = null;
|
||||
runtimeState.activeRuntimeWebToolsMetadata = null;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = priorFetch;
|
||||
vi.restoreAllMocks();
|
||||
runtimeState.activeSecretsRuntimeSnapshot = null;
|
||||
runtimeState.activeRuntimeWebToolsMetadata = null;
|
||||
});
|
||||
|
||||
it("re-wraps and truncates provider fallback payloads before caching or returning", async () => {
|
||||
@@ -124,4 +138,87 @@ describe("web_fetch provider fallback normalization", () => {
|
||||
expect(details.url).toBe("https://example.com/fallback");
|
||||
expect(details.finalUrl).toBe("https://example.com/fallback");
|
||||
});
|
||||
|
||||
it("late-binds provider fallback config and runtime metadata from the active runtime snapshot", async () => {
|
||||
global.fetch = withFetchPreconnect(
|
||||
vi.fn(async () => {
|
||||
throw new Error("network failed");
|
||||
}),
|
||||
);
|
||||
const runtimeConfig = {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: "firecrawl",
|
||||
maxChars: 640,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
runtimeState.activeSecretsRuntimeSnapshot = { config: runtimeConfig };
|
||||
runtimeState.activeRuntimeWebToolsMetadata = {
|
||||
fetch: {
|
||||
providerConfigured: "firecrawl",
|
||||
providerSource: "configured",
|
||||
selectedProvider: "firecrawl",
|
||||
selectedProviderKeySource: "config",
|
||||
diagnostics: [],
|
||||
},
|
||||
diagnostics: [],
|
||||
};
|
||||
resolveWebFetchDefinitionMock.mockReturnValue({
|
||||
provider: { id: "firecrawl" },
|
||||
definition: {
|
||||
description: "firecrawl",
|
||||
parameters: {},
|
||||
execute: async () => ({
|
||||
text: "runtime fallback body ".repeat(200),
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
const tool = createWebFetchTool({
|
||||
config: {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: "stale",
|
||||
maxChars: 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
sandboxed: false,
|
||||
runtimeWebFetch: {
|
||||
providerConfigured: "stale",
|
||||
providerSource: "configured",
|
||||
selectedProvider: "stale",
|
||||
selectedProviderKeySource: "config",
|
||||
diagnostics: [],
|
||||
},
|
||||
lateBindRuntimeConfig: true,
|
||||
});
|
||||
|
||||
const result = await tool?.execute?.("call-provider-fallback", {
|
||||
url: "https://example.com/fallback",
|
||||
});
|
||||
const details = result?.details as {
|
||||
wrappedLength?: number;
|
||||
externalContent?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
expect(details.wrappedLength).toBeGreaterThan(200);
|
||||
expect(details.wrappedLength).toBeLessThanOrEqual(640);
|
||||
expect(details.externalContent).toMatchObject({
|
||||
provider: "firecrawl",
|
||||
});
|
||||
expect(resolveWebFetchDefinitionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
config: runtimeConfig,
|
||||
runtimeWebFetch: expect.objectContaining({
|
||||
selectedProvider: "firecrawl",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -616,38 +616,13 @@ export function createWebFetchTool(options?: {
|
||||
config?: OpenClawConfig;
|
||||
sandboxed?: boolean;
|
||||
runtimeWebFetch?: RuntimeWebFetchMetadata;
|
||||
lateBindRuntimeConfig?: boolean;
|
||||
lookupFn?: LookupFn;
|
||||
}): AnyAgentTool | null {
|
||||
const fetch = resolveFetchConfig(options?.config);
|
||||
if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed })) {
|
||||
return null;
|
||||
}
|
||||
const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
|
||||
const userAgent =
|
||||
(fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) ||
|
||||
DEFAULT_FETCH_USER_AGENT;
|
||||
const maxResponseBytes = resolveFetchMaxResponseBytes(fetch);
|
||||
let providerFallbackResolved = false;
|
||||
let providerFallbackCache: WebFetchProviderFallback;
|
||||
const resolveProviderFallback = async () => {
|
||||
if (!providerFallbackResolved) {
|
||||
const { resolveWebFetchDefinition } = await loadWebFetchRuntime();
|
||||
const { config, preferRuntimeProviders, runtimeWebFetch } = resolveWebFetchToolRuntimeContext(
|
||||
{
|
||||
config: options?.config,
|
||||
runtimeWebFetch: options?.runtimeWebFetch,
|
||||
},
|
||||
);
|
||||
providerFallbackCache = resolveWebFetchDefinition({
|
||||
config,
|
||||
sandboxed: options?.sandboxed,
|
||||
runtimeWebFetch,
|
||||
preferRuntimeProviders,
|
||||
});
|
||||
providerFallbackResolved = true;
|
||||
}
|
||||
return providerFallbackCache;
|
||||
};
|
||||
return {
|
||||
label: "Web Fetch",
|
||||
name: "web_fetch",
|
||||
@@ -655,28 +630,68 @@ export function createWebFetchTool(options?: {
|
||||
"Fetch and extract readable content from a URL (HTML → markdown/text). Use for lightweight page access without browser automation.",
|
||||
parameters: WebFetchSchema,
|
||||
execute: async (_toolCallId, args) => {
|
||||
const { config, preferRuntimeProviders, runtimeWebFetch } = resolveWebFetchToolRuntimeContext(
|
||||
{
|
||||
config: options?.config,
|
||||
lateBindRuntimeConfig: options?.lateBindRuntimeConfig,
|
||||
runtimeWebFetch: options?.runtimeWebFetch,
|
||||
},
|
||||
);
|
||||
const executionFetch = resolveFetchConfig(config);
|
||||
if (!resolveFetchEnabled({ fetch: executionFetch, sandboxed: options?.sandboxed })) {
|
||||
throw new Error("web_fetch is disabled.");
|
||||
}
|
||||
const readabilityEnabled = resolveFetchReadabilityEnabled(executionFetch);
|
||||
const userAgent =
|
||||
(executionFetch &&
|
||||
"userAgent" in executionFetch &&
|
||||
typeof executionFetch.userAgent === "string" &&
|
||||
executionFetch.userAgent) ||
|
||||
DEFAULT_FETCH_USER_AGENT;
|
||||
const maxResponseBytes = resolveFetchMaxResponseBytes(executionFetch);
|
||||
let providerFallbackResolved = false;
|
||||
let providerFallbackCache: WebFetchProviderFallback;
|
||||
const resolveProviderFallback = async () => {
|
||||
if (!providerFallbackResolved) {
|
||||
const { resolveWebFetchDefinition } = await loadWebFetchRuntime();
|
||||
providerFallbackCache = resolveWebFetchDefinition({
|
||||
config,
|
||||
sandboxed: options?.sandboxed,
|
||||
runtimeWebFetch,
|
||||
preferRuntimeProviders,
|
||||
});
|
||||
providerFallbackResolved = true;
|
||||
}
|
||||
return providerFallbackCache;
|
||||
};
|
||||
const params = args as Record<string, unknown>;
|
||||
const url = readStringParam(params, "url", { required: true });
|
||||
const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown";
|
||||
const maxChars = readNumberParam(params, "maxChars", { integer: true });
|
||||
const maxCharsCap = resolveFetchMaxCharsCap(fetch);
|
||||
const maxCharsCap = resolveFetchMaxCharsCap(executionFetch);
|
||||
const result = await runWebFetch({
|
||||
url,
|
||||
extractMode,
|
||||
maxChars: resolveMaxChars(
|
||||
maxChars ?? fetch?.maxChars,
|
||||
maxChars ?? executionFetch?.maxChars,
|
||||
DEFAULT_FETCH_MAX_CHARS,
|
||||
maxCharsCap,
|
||||
),
|
||||
maxResponseBytes,
|
||||
maxRedirects: resolveMaxRedirects(fetch?.maxRedirects, DEFAULT_FETCH_MAX_REDIRECTS),
|
||||
timeoutSeconds: resolveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS),
|
||||
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
||||
maxRedirects: resolveMaxRedirects(
|
||||
executionFetch?.maxRedirects,
|
||||
DEFAULT_FETCH_MAX_REDIRECTS,
|
||||
),
|
||||
timeoutSeconds: resolveTimeoutSeconds(
|
||||
executionFetch?.timeoutSeconds,
|
||||
DEFAULT_TIMEOUT_SECONDS,
|
||||
),
|
||||
cacheTtlMs: resolveCacheTtlMs(executionFetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
||||
userAgent,
|
||||
readabilityEnabled,
|
||||
config: options?.config,
|
||||
useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(fetch),
|
||||
ssrfPolicy: fetch?.ssrfPolicy,
|
||||
config,
|
||||
useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(executionFetch),
|
||||
ssrfPolicy: executionFetch?.ssrfPolicy,
|
||||
lookupFn: options?.lookupFn,
|
||||
resolveProviderFallback,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user