diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index 0533f6a3a80..512232b8e39 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -234,10 +234,6 @@ vi.mock("../infra/outbound/session-context.js", () => ({ buildOutboundSessionContext: () => ({}), })); -vi.mock("../infra/skills-remote.js", () => ({ - getRemoteSkillEligibility: () => ({ eligible: false }), -})); - vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => { const logger = { @@ -596,18 +592,52 @@ vi.mock("./provider-auth-aliases.js", () => ({ provider.trim().toLowerCase() === "codex-cli" ? "openai-codex" : provider.trim().toLowerCase(), })); -vi.mock("../skills/index.js", () => ({ - buildWorkspaceSkillSnapshot: (workspaceDir: string, opts: unknown) => - state.buildWorkspaceSkillSnapshotMock(workspaceDir, opts), +vi.mock("../skills/agent-filter.js", () => ({ + resolveEffectiveAgentSkillFilter: (_cfg: unknown, agentId: string) => + state.resolveAgentSkillsFilterMock(_cfg, agentId), })); -vi.mock("../skills/filter.js", () => ({ - matchesSkillFilter: () => true, +vi.mock("../skills/remote.js", () => ({ + getRemoteSkillEligibility: () => ({ eligible: false }), })); -vi.mock("../skills/refresh-state.js", () => ({ - getSkillsSnapshotVersion: () => 0, - shouldRefreshSnapshotForVersion: () => false, +vi.mock("../skills/session-snapshot.js", () => ({ + resolveReusableWorkspaceSkillSnapshot: (params: { + workspaceDir: string; + existingSnapshot?: { resolvedSkills?: unknown }; + skillFilter?: string[]; + }) => { + if (params.skillFilter !== undefined && params.skillFilter.length === 0) { + return { + snapshot: { + prompt: "", + skills: [], + resolvedSkills: [], + skillFilter: params.skillFilter, + version: 0, + }, + shouldRefresh: !params.existingSnapshot, + snapshotVersion: 0, + }; + } + if (params.existingSnapshot?.resolvedSkills !== undefined) { + return { + snapshot: params.existingSnapshot, + shouldRefresh: false, + snapshotVersion: 0, + }; + } + const rebuilt = state.buildWorkspaceSkillSnapshotMock(params.workspaceDir, params) as { + resolvedSkills?: unknown; + }; + return { + snapshot: params.existingSnapshot + ? { ...params.existingSnapshot, resolvedSkills: rebuilt?.resolvedSkills } + : rebuilt, + shouldRefresh: !params.existingSnapshot, + snapshotVersion: 0, + }; + }, })); vi.mock("./spawned-context.js", () => ({ diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index 32db4862fa9..754e5a7ed43 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -39,7 +39,9 @@ import { import { resolveSendPolicy } from "../sessions/send-policy.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; -import { hydrateResolvedSkillsAsync } from "../skills/snapshot-hydration.js"; +import { resolveEffectiveAgentSkillFilter } from "../skills/agent-filter.js"; +import type { getRemoteSkillEligibility } from "../skills/remote.js"; +import type { resolveReusableWorkspaceSkillSnapshot } from "../skills/session-snapshot.js"; import type { SkillSnapshot } from "../skills/types.js"; import { sanitizeForLog } from "../terminal/ansi.js"; import { createTrajectoryRuntimeRecorder } from "../trajectory/runtime.js"; @@ -58,7 +60,6 @@ import { resolveDefaultAgentId, resolveEffectiveModelFallbacks, resolveSessionAgentId, - resolveAgentSkillsFilter, resolveAgentWorkspaceDir, } from "./agent-scope.js"; import { isStoredCredentialCompatibleWithAuthProvider } from "./auth-profiles/order.js"; @@ -117,10 +118,10 @@ type CliCompactionRuntime = typeof import("./command/cli-compaction.js"); type TranscriptResolveRuntime = typeof import("../config/sessions/transcript-resolve.runtime.js"); type CliDepsRuntime = typeof import("../cli/deps.js"); type ExecDefaultsRuntime = typeof import("./exec-defaults.js"); -type SkillsRuntime = typeof import("../skills/index.js"); -type SkillsFilterRuntime = typeof import("../skills/filter.js"); -type SkillsRefreshStateRuntime = typeof import("../skills/refresh-state.js"); -type SkillsRemoteRuntime = typeof import("../infra/skills-remote.js"); +type SkillsRuntime = { + getRemoteSkillEligibility: typeof getRemoteSkillEligibility; + resolveReusableWorkspaceSkillSnapshot: typeof resolveReusableWorkspaceSkillSnapshot; +}; const attemptExecutionRuntimeLoader = createLazyImportLoader( () => import("./command/attempt-execution.runtime.js"), @@ -153,31 +154,16 @@ const cliDepsRuntimeLoader = createLazyImportLoader(() => import const execDefaultsRuntimeLoader = createLazyImportLoader( () => import("./exec-defaults.js"), ); -const skillsRuntimeLoader = createLazyImportLoader( - () => import("../skills/index.js"), -); -const skillsFilterRuntimeLoader = createLazyImportLoader( - () => import("../skills/filter.js"), -); -const skillsRefreshStateRuntimeLoader = createLazyImportLoader( - () => import("../skills/refresh-state.js"), -); -const skillsRemoteRuntimeLoader = createLazyImportLoader( - () => import("../infra/skills-remote.js"), -); - -function createEmptySkillsSnapshot(params: { - skillFilter: string[]; - version?: number; -}): SkillSnapshot { +const skillsRuntimeLoader = createLazyImportLoader(async () => { + const [remote, sessionSnapshot] = await Promise.all([ + import("../skills/remote.js"), + import("../skills/session-snapshot.js"), + ]); return { - prompt: "", - skills: [], - resolvedSkills: [], - skillFilter: params.skillFilter, - version: params.version, + getRemoteSkillEligibility: remote.getRemoteSkillEligibility, + resolveReusableWorkspaceSkillSnapshot: sessionSnapshot.resolveReusableWorkspaceSkillSnapshot, }; -} +}); function loadAttemptExecutionRuntime(): Promise { return attemptExecutionRuntimeLoader.load(); @@ -227,18 +213,6 @@ function loadSkillsRuntime(): Promise { return skillsRuntimeLoader.load(); } -function loadSkillsFilterRuntime(): Promise { - return skillsFilterRuntimeLoader.load(); -} - -function loadSkillsRefreshStateRuntime(): Promise { - return skillsRefreshStateRuntimeLoader.load(); -} - -function loadSkillsRemoteRuntime(): Promise { - return skillsRemoteRuntimeLoader.load(); -} - async function resolveAgentCommandDeps(deps: CliDeps | undefined): Promise { if (deps) { return deps; @@ -819,55 +793,33 @@ async function agentCommandInternal( }); } - const [{ getSkillsSnapshotVersion, shouldRefreshSnapshotForVersion }, { matchesSkillFilter }] = - await Promise.all([loadSkillsRefreshStateRuntime(), loadSkillsFilterRuntime()]); - const skillsSnapshotVersion = getSkillsSnapshotVersion(workspaceDir); - const skillFilter = resolveAgentSkillsFilter(cfg, sessionAgentId); + const skillFilter = resolveEffectiveAgentSkillFilter(cfg, sessionAgentId); const currentSkillsSnapshot = sessionEntry?.skillsSnapshot; - const shouldRefreshSkillsSnapshot = - !currentSkillsSnapshot || - shouldRefreshSnapshotForVersion(currentSkillsSnapshot.version, skillsSnapshotVersion) || - !matchesSkillFilter(currentSkillsSnapshot.skillFilter, skillFilter); - const needsSkillsSnapshot = isNewSession || shouldRefreshSkillsSnapshot; - const emptySkillsFilter = skillFilter !== undefined && skillFilter.length === 0; - const buildSkillsSnapshot = async () => { - if (emptySkillsFilter) { - return createEmptySkillsSnapshot({ - skillFilter, - version: skillsSnapshotVersion, - }); - } - const [ - { buildWorkspaceSkillSnapshot }, - { getRemoteSkillEligibility }, - { canExecRequestNode }, - ] = await Promise.all([ - loadSkillsRuntime(), - loadSkillsRemoteRuntime(), - loadExecDefaultsRuntime(), - ]); - return buildWorkspaceSkillSnapshot(workspaceDir, { - config: cfg, - eligibility: { - remote: getRemoteSkillEligibility({ - advertiseExecNode: canExecRequestNode({ - cfg, - sessionEntry, - sessionKey, - agentId: sessionAgentId, - }), + const [ + { getRemoteSkillEligibility, resolveReusableWorkspaceSkillSnapshot }, + { canExecRequestNode }, + ] = await Promise.all([loadSkillsRuntime(), loadExecDefaultsRuntime()]); + const skillSnapshotState = resolveReusableWorkspaceSkillSnapshot({ + workspaceDir, + config: cfg, + agentId: sessionAgentId, + existingSnapshot: isNewSession ? undefined : currentSkillsSnapshot, + skillFilter, + eligibility: { + remote: getRemoteSkillEligibility({ + advertiseExecNode: canExecRequestNode({ + cfg, + sessionEntry, + sessionKey, + agentId: sessionAgentId, }), - }, - snapshotVersion: skillsSnapshotVersion, - skillFilter, - agentId: sessionAgentId, - }); - }; - const skillsSnapshot = needsSkillsSnapshot - ? await buildSkillsSnapshot() - : !currentSkillsSnapshot - ? undefined - : await hydrateResolvedSkillsAsync(currentSkillsSnapshot, buildSkillsSnapshot); + }), + }, + watch: false, + }); + const needsSkillsSnapshot = + isNewSession || !currentSkillsSnapshot || skillSnapshotState.shouldRefresh; + const skillsSnapshot = skillSnapshotState.snapshot; if ( skillsSnapshot && diff --git a/src/agents/sandbox.resolveSandboxContext.test.ts b/src/agents/sandbox.resolveSandboxContext.test.ts index 10e638eae5a..fac0eb15c7c 100644 --- a/src/agents/sandbox.resolveSandboxContext.test.ts +++ b/src/agents/sandbox.resolveSandboxContext.test.ts @@ -33,15 +33,15 @@ vi.mock("../plugin-sdk/browser-control-auth.js", () => browserControlAuthMock); vi.mock("../plugin-sdk/browser-profiles.js", () => browserProfilesMock); -vi.mock("../infra/skills-remote.js", () => ({ - getRemoteSkillEligibility: vi.fn(() => ({ note: "test-remote" })), -})); - vi.mock("./exec-defaults.js", () => ({ canExecRequestNode: vi.fn(() => false), })); -vi.mock("../skills/index.js", () => ({ +vi.mock("../skills/remote.js", () => ({ + getRemoteSkillEligibility: vi.fn(() => ({ note: "test-remote" })), +})); + +vi.mock("../skills/workspace.js", () => ({ syncSkillsToWorkspace: syncSkillsToWorkspaceMock, })); diff --git a/src/agents/sandbox/context.ts b/src/agents/sandbox/context.ts index 1fdfc9d17a8..dfc1b75cc04 100644 --- a/src/agents/sandbox/context.ts +++ b/src/agents/sandbox/context.ts @@ -53,11 +53,11 @@ async function ensureSandboxWorkspaceLayout(params: { ); if (cfg.workspaceAccess !== "rw") { try { - const [{ getRemoteSkillEligibility }, { canExecRequestNode }, { syncSkillsToWorkspace }] = + const [{ syncSkillsToWorkspace }, { getRemoteSkillEligibility }, { canExecRequestNode }] = await Promise.all([ - import("../../infra/skills-remote.js"), + import("../../skills/workspace.js"), + import("../../skills/remote.js"), import("../exec-defaults.js"), - import("../../skills/index.js"), ]); await syncSkillsToWorkspace({ sourceWorkspaceDir: agentWorkspaceDir, diff --git a/src/auto-reply/reply/commands-system-prompt.test.ts b/src/auto-reply/reply/commands-system-prompt.test.ts index a1a92cb99a4..db9621e33b0 100644 --- a/src/auto-reply/reply/commands-system-prompt.test.ts +++ b/src/auto-reply/reply/commands-system-prompt.test.ts @@ -22,12 +22,16 @@ vi.mock("../../agents/sandbox.js", () => ({ resolveSandboxRuntimeStatus: vi.fn(() => ({ sandboxed: false, mode: "off" })), })); -vi.mock("../../skills/index.js", () => ({ - buildWorkspaceSkillSnapshot: vi.fn(() => ({ prompt: "", skills: [], resolvedSkills: [] })), +vi.mock("../../skills/remote.js", () => ({ + getRemoteSkillEligibility: vi.fn(() => false), })); -vi.mock("../../skills/refresh.js", () => ({ - getSkillsSnapshotVersion: vi.fn(() => "test-snapshot"), +vi.mock("../../skills/session-snapshot.js", () => ({ + resolveReusableWorkspaceSkillSnapshot: vi.fn(() => ({ + snapshot: { prompt: "", skills: [], resolvedSkills: [] }, + shouldRefresh: false, + snapshotVersion: "test-snapshot", + })), })); vi.mock("../../agents/agent-scope.js", () => ({ @@ -60,10 +64,6 @@ vi.mock("../../tts/tts.js", () => ({ buildTtsSystemPromptHint: vi.fn(() => undefined), })); -vi.mock("../../infra/skills-remote.js", () => ({ - getRemoteSkillEligibility: vi.fn(() => false), -})); - function makeParams(): HandleCommandsParams { return { ctx: { diff --git a/src/auto-reply/reply/commands-system-prompt.ts b/src/auto-reply/reply/commands-system-prompt.ts index c2a23bd0aa1..2f21a0576ae 100644 --- a/src/auto-reply/reply/commands-system-prompt.ts +++ b/src/auto-reply/reply/commands-system-prompt.ts @@ -12,10 +12,9 @@ import { resolveSandboxRuntimeStatus } from "../../agents/sandbox.js"; import { buildConfiguredAgentSystemPrompt } from "../../agents/system-prompt-config.js"; import { buildSystemPromptParams } from "../../agents/system-prompt-params.js"; import type { WorkspaceBootstrapFile } from "../../agents/workspace.js"; -import { getRemoteSkillEligibility } from "../../infra/skills-remote.js"; import { listRegisteredPluginAgentPromptGuidance } from "../../plugins/command-registry-state.js"; -import { buildWorkspaceSkillSnapshot } from "../../skills/index.js"; -import { getSkillsSnapshotVersion } from "../../skills/refresh-state.js"; +import { getRemoteSkillEligibility } from "../../skills/remote.js"; +import { resolveReusableWorkspaceSkillSnapshot } from "../../skills/session-snapshot.js"; import type { HandleCommandsParams } from "./commands-types.js"; import { resolveRuntimePolicySessionKey } from "./runtime-policy-session-key.js"; @@ -60,7 +59,8 @@ export async function resolveCommandsSystemPromptBundle( }); const skillsSnapshot = (() => { try { - return buildWorkspaceSkillSnapshot(workspaceDir, { + return resolveReusableWorkspaceSkillSnapshot({ + workspaceDir, config: params.cfg, agentId: sessionAgentId, eligibility: { @@ -73,13 +73,13 @@ export async function resolveCommandsSystemPromptBundle( }), }), }, - snapshotVersion: getSkillsSnapshotVersion(workspaceDir), + watch: false, }); } catch { - return { prompt: "", skills: [], resolvedSkills: [] }; + return { snapshot: { prompt: "", skills: [], resolvedSkills: [] } }; } })(); - const skillsPrompt = skillsSnapshot.prompt ?? ""; + const skillsPrompt = skillsSnapshot.snapshot.prompt ?? ""; const tools = (() => { try { return createOpenClawCodingTools({ diff --git a/src/auto-reply/reply/session-updates.test.ts b/src/auto-reply/reply/session-updates.test.ts index d856bb0222c..82a826262bb 100644 --- a/src/auto-reply/reply/session-updates.test.ts +++ b/src/auto-reply/reply/session-updates.test.ts @@ -53,7 +53,11 @@ vi.mock("../../agents/agent-scope.js", () => ({ resolveSessionAgentId: resolveSessionAgentIdMock, })); -vi.mock("../../skills/index.js", () => ({ +vi.mock("../../skills/remote.js", () => ({ + getRemoteSkillEligibility: getRemoteSkillEligibilityMock, +})); + +vi.mock("../../skills/service.js", () => ({ buildWorkspaceSkillSnapshot: buildWorkspaceSkillSnapshotMock, })); @@ -72,10 +76,6 @@ vi.mock("../../config/sessions.js", () => ({ resolveSessionFilePathOptions: vi.fn(), })); -vi.mock("../../infra/skills-remote.js", () => ({ - getRemoteSkillEligibility: getRemoteSkillEligibilityMock, -})); - vi.mock("../../routing/session-key.js", () => ({ normalizeAgentId: (id: string) => id, normalizeMainKey: (key?: string) => key ?? "main", diff --git a/src/auto-reply/reply/session-updates.ts b/src/auto-reply/reply/session-updates.ts index 9f8eccc8bdc..44ed62f789c 100644 --- a/src/auto-reply/reply/session-updates.ts +++ b/src/auto-reply/reply/session-updates.ts @@ -2,7 +2,6 @@ import crypto from "node:crypto"; import path from "node:path"; import { resolveSessionAgentId } from "../../agents/agent-scope.js"; import { canExecRequestNode } from "../../agents/exec-defaults.js"; -import { stableStringify } from "../../agents/stable-stringify.js"; import { canonicalizeAbsoluteSessionFilePath, resolveSessionFilePath, @@ -18,112 +17,15 @@ import { } from "../../gateway/active-sessions-shutdown-tracker.js"; import { resolveStableSessionEndTranscript } from "../../gateway/session-transcript-files.fs.js"; import { logVerbose } from "../../globals.js"; -import { getRemoteSkillEligibility } from "../../infra/skills-remote.js"; import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; -import { matchesSkillFilter } from "../../skills/filter.js"; -import { buildWorkspaceSkillSnapshot, type SkillSnapshot } from "../../skills/index.js"; -import { - getSkillsSnapshotVersion, - shouldRefreshSnapshotForVersion, -} from "../../skills/refresh-state.js"; -import { ensureSkillsWatcher } from "../../skills/refresh.js"; -import { hydrateResolvedSkills } from "../../skills/snapshot-hydration.js"; +import { getRemoteSkillEligibility } from "../../skills/remote.js"; +import { resolveReusableWorkspaceSkillSnapshot } from "../../skills/session-snapshot.js"; import { buildSessionEndHookPayload, buildSessionStartHookPayload } from "./session-hooks.js"; export { drainFormattedSystemEvents } from "./session-system-events.js"; +export { resetResolvedSkillsCacheForTests } from "../../skills/session-snapshot.js"; -// Warm-start resolvedSkills cache: avoids redundant buildSnapshot calls when -// stripPersistedSkillsCache has removed resolvedSkills between turns. -// Bounded to 10 entries to prevent unbounded growth in long-lived gateways. -const resolvedSkillsCache = new Map(); -const RESOLVED_SKILLS_CACHE_MAX = 10; - -export function resetResolvedSkillsCacheForTests(): void { - resolvedSkillsCache.clear(); -} - -function isSensitiveConfigKey(key: string): boolean { - const normalized = key.toLowerCase().replaceAll(/[^a-z0-9]/g, ""); - return ( - normalized.endsWith("apikey") || - normalized.endsWith("token") || - normalized.endsWith("secret") || - normalized.endsWith("password") || - normalized.endsWith("privatekey") || - normalized.endsWith("clientsecret") - ); -} - -function redactSensitiveConfigValue(value: unknown): unknown { - if (value === undefined || value === null || value === false || value === "") { - return value; - } - if (typeof value === "string") { - return value.trim() ? "[redacted:string]" : ""; - } - if (typeof value === "number") { - return Number.isFinite(value) && value !== 0 ? "[redacted:number]" : value; - } - if (typeof value === "boolean") { - return value; - } - if (Array.isArray(value)) { - return value.length === 0 ? [] : "[redacted:array]"; - } - return "[redacted:object]"; -} - -function redactConfigForSkillSnapshotCache(value: unknown, stack = new WeakSet()): unknown { - if (!value || typeof value !== "object") { - return value; - } - if (stack.has(value)) { - return "[Circular]"; - } - stack.add(value); - try { - if (Array.isArray(value)) { - return value.map((entry) => redactConfigForSkillSnapshotCache(entry, stack)); - } - const redacted: Record = {}; - for (const key of Object.keys(value as Record).toSorted()) { - const field = (value as Record)[key]; - redacted[key] = isSensitiveConfigKey(key) - ? redactSensitiveConfigValue(field) - : redactConfigForSkillSnapshotCache(field, stack); - } - return redacted; - } finally { - stack.delete(value); - } -} - -// Skill frontmatter `requires.config` reads the full OpenClaw config, so cache -// reuse must follow the same boundary without putting raw secrets in Map keys. -function fingerprintSkillSnapshotConfig(config: OpenClawConfig): string { - return crypto - .createHash("sha256") - .update(stableStringify(redactConfigForSkillSnapshotCache(config))) - .digest("hex"); -} - -function cacheResolvedSkills(cacheKey: string, snapshot: SkillSnapshot): SkillSnapshot { - resolvedSkillsCache.set(cacheKey, snapshot.resolvedSkills); - if (resolvedSkillsCache.size > RESOLVED_SKILLS_CACHE_MAX) { - const oldest = resolvedSkillsCache.keys().next().value; - if (oldest !== undefined) { - resolvedSkillsCache.delete(oldest); - } - } - return snapshot; -} - -// nextEntry.skillsSnapshot may carry resolvedSkills (full Skill[] with -// SKILL.md bodies) for in-turn use. The persistence layer in -// src/config/sessions/store-load.ts strips resolvedSkills before serializing, -// so the on-disk sessions.json stays small. The in-memory params.sessionStore -// reference still carries the runtime cache for the rest of this turn. async function persistSessionEntryUpdate(params: { sessionStore?: Record; sessionKey?: string; @@ -260,39 +162,17 @@ export async function ensureSkillSnapshot(params: { }), }); const existingSnapshot = nextEntry?.skillsSnapshot; - ensureSkillsWatcher({ workspaceDir, config: cfg }); - const snapshotVersion = getSkillsSnapshotVersion(workspaceDir); - const shouldRefreshSnapshot = - shouldRefreshSnapshotForVersion(existingSnapshot?.version, snapshotVersion) || - !matchesSkillFilter(existingSnapshot?.skillFilter, skillFilter); - const buildSnapshot = () => { - return buildWorkspaceSkillSnapshot(workspaceDir, { + const resolveSnapshot = (snapshot: SessionEntry["skillsSnapshot"]) => + resolveReusableWorkspaceSkillSnapshot({ + workspaceDir, config: cfg, agentId: sessionAgentId, skillFilter, eligibility: { remote: remoteEligibility }, - snapshotVersion, + existingSnapshot: snapshot, }); - }; - - const configFingerprint = fingerprintSkillSnapshotConfig(cfg); - const snapshotCacheKey = JSON.stringify([ - workspaceDir, - snapshotVersion, - skillFilter, - sessionAgentId, - remoteEligibility, - configFingerprint, - ]); - - const cachedRebuild = (): SkillSnapshot => { - if (resolvedSkillsCache.has(snapshotCacheKey)) { - return { resolvedSkills: resolvedSkillsCache.get(snapshotCacheKey) } as SkillSnapshot; - } - return cacheResolvedSkills(snapshotCacheKey, buildSnapshot()); - }; - - const buildAndCache = (): SkillSnapshot => cacheResolvedSkills(snapshotCacheKey, buildSnapshot()); + const initialSnapshotState = resolveSnapshot(existingSnapshot); + const shouldRefreshSnapshot = initialSnapshotState.shouldRefresh; if (isFirstTurnInSession && sessionStore && sessionKey) { const current = nextEntry ?? @@ -302,8 +182,8 @@ export async function ensureSkillSnapshot(params: { }; const skillSnapshot = !current.skillsSnapshot || shouldRefreshSnapshot - ? buildAndCache() - : hydrateResolvedSkills(current.skillsSnapshot, cachedRebuild); + ? initialSnapshotState.snapshot + : resolveSnapshot(current.skillsSnapshot).snapshot; nextEntry = { ...current, sessionId: sessionId ?? current.sessionId ?? crypto.randomUUID(), @@ -320,10 +200,10 @@ export async function ensureSkillSnapshot(params: { (nextEntry?.skillsSnapshot !== existingSnapshot || !shouldRefreshSnapshot); const skillsSnapshot = hasFreshSnapshotInEntry && nextEntry?.skillsSnapshot - ? hydrateResolvedSkills(nextEntry.skillsSnapshot, cachedRebuild) + ? resolveSnapshot(nextEntry.skillsSnapshot).snapshot : shouldRefreshSnapshot || !nextEntry?.skillsSnapshot - ? buildAndCache() - : hydrateResolvedSkills(nextEntry.skillsSnapshot, cachedRebuild); + ? initialSnapshotState.snapshot + : resolveSnapshot(nextEntry.skillsSnapshot).snapshot; if ( skillsSnapshot && sessionStore && diff --git a/src/auto-reply/skill-commands.test.ts b/src/auto-reply/skill-commands.test.ts index 131cc7cc424..3b81f806dd9 100644 --- a/src/auto-reply/skill-commands.test.ts +++ b/src/auto-reply/skill-commands.test.ts @@ -127,12 +127,30 @@ vi.mock("./commands-registry.js", () => ({ listChatCommands: () => [], })); -vi.mock("../infra/skills-remote.js", () => ({ +vi.mock("../skills/command-specs.js", () => ({ + buildWorkspaceSkillCommandSpecs, +})); + +vi.mock("../skills/remote.js", () => ({ getRemoteSkillEligibility: () => ({}), })); -vi.mock("../skills/index.js", () => ({ - buildWorkspaceSkillCommandSpecs, +vi.mock("../skills/agent-filter.js", () => ({ + resolveEffectiveAgentSkillFilter: ( + cfg: { + agents?: { + defaults?: { skills?: string[] }; + list?: Array<{ id?: string; skills?: string[] }>; + }; + }, + agentId: string, + ) => { + const agent = cfg.agents?.list?.find((entry) => entry.id === agentId); + if (agent && Object.hasOwn(agent, "skills")) { + return agent.skills; + } + return cfg.agents?.defaults?.skills; + }, })); beforeAll(async () => { diff --git a/src/auto-reply/skill-commands.ts b/src/auto-reply/skill-commands.ts index 455dd69ccea..64dda820a33 100644 --- a/src/auto-reply/skill-commands.ts +++ b/src/auto-reply/skill-commands.ts @@ -1,19 +1,17 @@ import fs from "node:fs"; -import { - listAgentIds, - resolveAgentSkillsFilter, - resolveAgentWorkspaceDir, -} from "../agents/agent-scope.js"; +import { listAgentIds, resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { canExecRequestNode } from "../agents/exec-defaults.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { logVerbose } from "../globals.js"; -import { getRemoteSkillEligibility } from "../infra/skills-remote.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, } from "../shared/string-coerce.js"; import { uniqueStrings } from "../shared/string-normalization.js"; -import { buildWorkspaceSkillCommandSpecs, type SkillCommandSpec } from "../skills/index.js"; +import { resolveEffectiveAgentSkillFilter } from "../skills/agent-filter.js"; +import { buildWorkspaceSkillCommandSpecs } from "../skills/command-specs.js"; +import { getRemoteSkillEligibility } from "../skills/remote.js"; +import type { SkillCommandSpec } from "../skills/types.js"; import { listReservedChatSlashCommandNames } from "./skill-commands-base.js"; export { listReservedChatSlashCommandNames, @@ -97,7 +95,7 @@ export function listSkillCommandsForAgents(params: { logVerbose(`Skipping agent "${agentId}": cannot resolve workspace: ${workspaceDir}`); continue; } - const skillFilter = resolveAgentSkillsFilter(params.cfg, agentId); + const skillFilter = resolveEffectiveAgentSkillFilter(params.cfg, agentId); const existing = workspaceFilters.get(canonicalDir); if (existing) { existing.skillFilter = mergeSkillFilters(existing.skillFilter, skillFilter); diff --git a/src/commands/agent-command.test-mocks.ts b/src/commands/agent-command.test-mocks.ts index 1951f0ae5bf..a4ca0243e7c 100644 --- a/src/commands/agent-command.test-mocks.ts +++ b/src/commands/agent-command.test-mocks.ts @@ -1,5 +1,4 @@ import { vi } from "vitest"; -import { normalizeStringEntries } from "../shared/string-normalization.js"; vi.mock("../logging/subsystem.js", () => { const createMockLogger = () => ({ @@ -245,32 +244,30 @@ vi.mock("../skills/index.js", () => ({ loadWorkspaceSkillEntries: vi.fn(() => []), })); -vi.mock("../skills/refresh.js", () => ({ - getSkillsSnapshotVersion: vi.fn(() => 0), +vi.mock("../skills/remote.js", () => ({ + getRemoteSkillEligibility: vi.fn(() => undefined), })); -vi.mock("../skills/refresh-state.js", () => ({ - getSkillsSnapshotVersion: vi.fn(() => 0), - shouldRefreshSnapshotForVersion: vi.fn(() => false), +vi.mock("../skills/agent-filter.js", () => ({ + resolveEffectiveAgentSkillFilter: vi.fn(() => undefined), })); -vi.mock("../skills/filter.js", () => ({ - normalizeSkillFilter: vi.fn((skillFilter?: ReadonlyArray) => - skillFilter ? normalizeStringEntries(skillFilter) : undefined, +vi.mock("../skills/session-snapshot.js", () => ({ + resolveReusableWorkspaceSkillSnapshot: vi.fn( + (params?: { existingSnapshot?: unknown; skillFilter?: string[] }) => ({ + snapshot: params?.existingSnapshot ?? { + prompt: "", + skills: [], + resolvedSkills: [], + ...(params?.skillFilter === undefined ? {} : { skillFilter: params.skillFilter }), + version: 0, + }, + shouldRefresh: !params?.existingSnapshot, + snapshotVersion: 0, + }), ), - normalizeSkillFilterForComparison: vi.fn((skillFilter?: ReadonlyArray) => - skillFilter - ?.map((entry) => String(entry).trim()) - .filter(Boolean) - .toSorted(), - ), - matchesSkillFilter: vi.fn(() => true), })); vi.mock("../agents/exec-defaults.js", () => ({ canExecRequestNode: vi.fn(() => false), })); - -vi.mock("../infra/skills-remote.js", () => ({ - getRemoteSkillEligibility: vi.fn(() => undefined), -})); diff --git a/src/commands/status-all/report-data.ts b/src/commands/status-all/report-data.ts index 221baef6b2f..07c9253ed1a 100644 --- a/src/commands/status-all/report-data.ts +++ b/src/commands/status-all/report-data.ts @@ -3,8 +3,8 @@ import { readConfigFileSnapshot, resolveGatewayPort } from "../../config/config. import { readLastGatewayErrorLine } from "../../daemon/diagnostics.js"; import { inspectPortUsage } from "../../infra/ports.js"; import { readRestartSentinel } from "../../infra/restart-sentinel.js"; -import { getRemoteSkillEligibility } from "../../infra/skills-remote.js"; import { buildPluginCompatibilityNotices } from "../../plugins/status.js"; +import { getRemoteSkillEligibility } from "../../skills/remote.js"; import { buildWorkspaceSkillStatus } from "../../skills/status.js"; import { buildStatusAllOverviewRows } from "../status-overview-rows.ts"; import { diff --git a/src/cron/isolated-agent/run.test-harness.ts b/src/cron/isolated-agent/run.test-harness.ts index 2c065b25c17..635137aa47a 100644 --- a/src/cron/isolated-agent/run.test-harness.ts +++ b/src/cron/isolated-agent/run.test-harness.ts @@ -157,11 +157,41 @@ vi.mock("../../plugins/runtime-plugins.runtime.js", () => ({ })); vi.mock("./skills-snapshot.runtime.js", () => ({ - buildWorkspaceSkillSnapshot: buildWorkspaceSkillSnapshotMock, canExecRequestNode: vi.fn(() => false), getRemoteSkillEligibility: getRemoteSkillEligibilityMock, - getSkillsSnapshotVersion: getSkillsSnapshotVersionMock, - resolveAgentSkillsFilter: resolveAgentSkillsFilterMock, + resolveEffectiveAgentSkillFilter: resolveAgentSkillsFilterMock, + resolveReusableWorkspaceSkillSnapshot: (params: { + workspaceDir: string; + config?: unknown; + agentId?: string; + existingSnapshot?: { version?: number; skillFilter?: string[] }; + skillFilter?: string[]; + eligibility?: unknown; + }) => { + const normalize = (skillFilter?: string[]) => + Array.from(new Set(skillFilter?.map((entry) => entry.trim()).filter(Boolean))).toSorted(); + const sameFilter = + JSON.stringify(normalize(params.existingSnapshot?.skillFilter)) === + JSON.stringify(normalize(params.skillFilter)); + const snapshotVersion = getSkillsSnapshotVersionMock(params.workspaceDir); + const shouldRefresh = + !params.existingSnapshot || + params.existingSnapshot.version !== snapshotVersion || + !sameFilter; + return { + snapshot: shouldRefresh + ? buildWorkspaceSkillSnapshotMock(params.workspaceDir, { + config: params.config, + agentId: params.agentId, + skillFilter: params.skillFilter, + eligibility: params.eligibility, + snapshotVersion, + }) + : params.existingSnapshot, + shouldRefresh, + snapshotVersion, + }; + }, })); vi.mock("./run-model-selection.runtime.js", () => ({ diff --git a/src/cron/isolated-agent/skills-snapshot.runtime.ts b/src/cron/isolated-agent/skills-snapshot.runtime.ts index 673c5771102..16baf5d8cf4 100644 --- a/src/cron/isolated-agent/skills-snapshot.runtime.ts +++ b/src/cron/isolated-agent/skills-snapshot.runtime.ts @@ -1,5 +1,4 @@ export { canExecRequestNode } from "../../agents/exec-defaults.js"; -export { resolveAgentSkillsFilter } from "../../agents/agent-scope.js"; -export { buildWorkspaceSkillSnapshot } from "../../skills/index.js"; -export { getSkillsSnapshotVersion } from "../../skills/refresh-state.js"; -export { getRemoteSkillEligibility } from "../../infra/skills-remote.js"; +export { resolveEffectiveAgentSkillFilter } from "../../skills/agent-filter.js"; +export { getRemoteSkillEligibility } from "../../skills/remote.js"; +export { resolveReusableWorkspaceSkillSnapshot } from "../../skills/session-snapshot.js"; diff --git a/src/cron/isolated-agent/skills-snapshot.test.ts b/src/cron/isolated-agent/skills-snapshot.test.ts index b64104fcf1f..f2dbb92fc4c 100644 --- a/src/cron/isolated-agent/skills-snapshot.test.ts +++ b/src/cron/isolated-agent/skills-snapshot.test.ts @@ -1,25 +1,22 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const { - buildWorkspaceSkillSnapshotMock, canExecRequestNodeMock, getRemoteSkillEligibilityMock, - getSkillsSnapshotVersionMock, - resolveAgentSkillsFilterMock, + resolveReusableWorkspaceSkillSnapshotMock, + resolveEffectiveAgentSkillFilterMock, } = vi.hoisted(() => ({ - buildWorkspaceSkillSnapshotMock: vi.fn(), canExecRequestNodeMock: vi.fn().mockReturnValue(false), getRemoteSkillEligibilityMock: vi.fn(), - getSkillsSnapshotVersionMock: vi.fn(), - resolveAgentSkillsFilterMock: vi.fn(), + resolveReusableWorkspaceSkillSnapshotMock: vi.fn(), + resolveEffectiveAgentSkillFilterMock: vi.fn(), })); vi.mock("./skills-snapshot.runtime.js", () => ({ - buildWorkspaceSkillSnapshot: buildWorkspaceSkillSnapshotMock, canExecRequestNode: canExecRequestNodeMock, getRemoteSkillEligibility: getRemoteSkillEligibilityMock, - getSkillsSnapshotVersion: getSkillsSnapshotVersionMock, - resolveAgentSkillsFilter: resolveAgentSkillsFilterMock, + resolveReusableWorkspaceSkillSnapshot: resolveReusableWorkspaceSkillSnapshotMock, + resolveEffectiveAgentSkillFilter: resolveEffectiveAgentSkillFilterMock, })); const { resolveCronSkillsSnapshot } = await import("./skills-snapshot.js"); @@ -27,18 +24,21 @@ const { resolveCronSkillsSnapshot } = await import("./skills-snapshot.js"); describe("resolveCronSkillsSnapshot", () => { beforeEach(() => { vi.clearAllMocks(); - getSkillsSnapshotVersionMock.mockReturnValue(0); - resolveAgentSkillsFilterMock.mockReturnValue(undefined); + resolveEffectiveAgentSkillFilterMock.mockReturnValue(undefined); getRemoteSkillEligibilityMock.mockReturnValue({ platforms: [], hasBin: () => false, hasAnyBin: () => false, }); - buildWorkspaceSkillSnapshotMock.mockReturnValue({ prompt: "fresh", skills: [] }); + resolveReusableWorkspaceSkillSnapshotMock.mockReturnValue({ + snapshot: { prompt: "fresh", skills: [] }, + shouldRefresh: true, + snapshotVersion: 0, + }); }); it("refreshes when the cached skill filter changes", async () => { - resolveAgentSkillsFilterMock.mockReturnValue(["docs-search", "github"]); + resolveEffectiveAgentSkillFilterMock.mockReturnValue(["docs-search", "github"]); const result = await resolveCronSkillsSnapshot({ workspaceDir: "/tmp/workspace", @@ -53,18 +53,17 @@ describe("resolveCronSkillsSnapshot", () => { isFastTestEnv: false, }); - expect(buildWorkspaceSkillSnapshotMock).toHaveBeenCalledOnce(); - const snapshotOptions = buildWorkspaceSkillSnapshotMock.mock.calls[0]?.[1] as - | { agentId?: string; snapshotVersion?: number } + expect(resolveReusableWorkspaceSkillSnapshotMock).toHaveBeenCalledOnce(); + const snapshotOptions = resolveReusableWorkspaceSkillSnapshotMock.mock.calls[0]?.[0] as + | { agentId?: string; watch?: boolean; hydrateExisting?: boolean } | undefined; expect(snapshotOptions?.agentId).toBe("writer"); - expect(snapshotOptions?.snapshotVersion).toBe(0); + expect(snapshotOptions?.watch).toBe(false); + expect(snapshotOptions?.hydrateExisting).toBe(false); expect(result).toEqual({ prompt: "fresh", skills: [] }); }); it("refreshes when the process version resets to 0 but the cached snapshot is stale", async () => { - getSkillsSnapshotVersionMock.mockReturnValue(0); - await resolveCronSkillsSnapshot({ workspaceDir: "/tmp/workspace", config: {} as never, @@ -77,6 +76,6 @@ describe("resolveCronSkillsSnapshot", () => { isFastTestEnv: false, }); - expect(buildWorkspaceSkillSnapshotMock).toHaveBeenCalledOnce(); + expect(resolveReusableWorkspaceSkillSnapshotMock).toHaveBeenCalledOnce(); }); }); diff --git a/src/cron/isolated-agent/skills-snapshot.ts b/src/cron/isolated-agent/skills-snapshot.ts index c917428bc38..dfeb719051a 100644 --- a/src/cron/isolated-agent/skills-snapshot.ts +++ b/src/cron/isolated-agent/skills-snapshot.ts @@ -1,7 +1,6 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { createLazyImportLoader } from "../../shared/lazy-promise.js"; -import { matchesSkillFilter } from "../../skills/filter.js"; -import type { SkillSnapshot } from "../../skills/index.js"; +import type { SkillSnapshot } from "../../skills/types.js"; const skillsSnapshotRuntimeLoader = createLazyImportLoader( () => import("./skills-snapshot.runtime.js"), @@ -24,20 +23,12 @@ export async function resolveCronSkillsSnapshot(params: { } const runtime = await loadSkillsSnapshotRuntime(); - const snapshotVersion = runtime.getSkillsSnapshotVersion(params.workspaceDir); - const skillFilter = runtime.resolveAgentSkillsFilter(params.config, params.agentId); - const existingSnapshot = params.existingSnapshot; - const shouldRefresh = - !existingSnapshot || - existingSnapshot.version !== snapshotVersion || - !matchesSkillFilter(existingSnapshot.skillFilter, skillFilter); - if (!shouldRefresh) { - return existingSnapshot; - } - - return runtime.buildWorkspaceSkillSnapshot(params.workspaceDir, { + const skillFilter = runtime.resolveEffectiveAgentSkillFilter(params.config, params.agentId); + return runtime.resolveReusableWorkspaceSkillSnapshot({ + workspaceDir: params.workspaceDir, config: params.config, agentId: params.agentId, + existingSnapshot: params.existingSnapshot, skillFilter, eligibility: { remote: runtime.getRemoteSkillEligibility({ @@ -47,6 +38,7 @@ export async function resolveCronSkillsSnapshot(params: { }), }), }, - snapshotVersion, - }); + watch: false, + hydrateExisting: false, + }).snapshot; } diff --git a/src/gateway/server-methods/nodes.ts b/src/gateway/server-methods/nodes.ts index 27bded09707..d2fb9bcd812 100644 --- a/src/gateway/server-methods/nodes.ts +++ b/src/gateway/server-methods/nodes.ts @@ -38,16 +38,16 @@ import { resolveApnsAuthConfigFromEnv, resolveApnsRelayConfigFromEnv, } from "../../infra/push-apns.js"; -import { - recordRemoteNodeInfo, - refreshRemoteNodeBins, - removeRemoteNodeInfo, -} from "../../infra/skills-remote.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "../../shared/string-coerce.js"; import { normalizeUniqueTrimmedStringList } from "../../shared/string-normalization.js"; +import { + recordRemoteNodeInfo, + refreshRemoteNodeBins, + removeRemoteNodeInfo, +} from "../../skills/remote.js"; import { createKnownNodeCatalog, getKnownNode, listKnownNodes } from "../node-catalog.js"; import { isForegroundRestrictedPluginNodeCommand, diff --git a/src/gateway/server-methods/skills-upload.ts b/src/gateway/server-methods/skills-upload.ts index 10b6902e858..8aad609136e 100644 --- a/src/gateway/server-methods/skills-upload.ts +++ b/src/gateway/server-methods/skills-upload.ts @@ -1,10 +1,3 @@ -import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { formatErrorMessage } from "../../infra/errors.js"; -import { - installSkillArchiveFromPath, - type SkillArchiveInstallFailureKind, - validateRequestedSkillSlug, -} from "../../skills/archive-install.js"; import { ErrorCodes, errorShape, @@ -15,41 +8,14 @@ import { validateSkillsUploadCommitParams, } from "../../../packages/gateway-protocol/src/index.js"; import type { ErrorShape } from "../../../packages/gateway-protocol/src/index.js"; +import { formatErrorMessage } from "../../infra/errors.js"; import { - defaultSkillUploadStore, - normalizeSkillUploadSha256, - SkillUploadRequestError, - type SkillUploadStore, -} from "./skills-upload-store.js"; -import type { GatewayRequestContext } from "./types.js"; + areUploadedSkillArchivesEnabled, + UPLOADED_SKILL_ARCHIVES_DISABLED_MESSAGE, +} from "../../skills/upload-install.js"; +import { defaultSkillUploadStore, SkillUploadRequestError } from "../../skills/upload-store.js"; import type { GatewayRequestHandlers } from "./types.js"; -type UploadInstallErrorCode = typeof ErrorCodes.INVALID_REQUEST | typeof ErrorCodes.UNAVAILABLE; - -const UPLOADED_SKILL_ARCHIVES_DISABLED_MESSAGE = - "Uploaded skill archive installs are disabled by skills.install.allowUploadedArchives"; - -export function areUploadedSkillArchivesEnabled(config: OpenClawConfig): boolean { - return config.skills?.install?.allowUploadedArchives === true; -} - -export type UploadedSkillInstallResult = - | { - ok: true; - message: string; - stdout: string; - stderr: string; - code: 0; - slug: string; - targetDir: string; - sha256: string; - } - | { - ok: false; - error: string; - errorCode: UploadInstallErrorCode; - }; - function uploadErrorShape( prefix: string, errors: Parameters[0], @@ -64,12 +30,6 @@ function mapUploadError(err: unknown): ErrorShape { return errorShape(ErrorCodes.UNAVAILABLE, formatErrorMessage(err)); } -function uploadInstallFailureErrorCode( - failureKind: SkillArchiveInstallFailureKind, -): UploadInstallErrorCode { - return failureKind === "invalid-request" ? ErrorCodes.INVALID_REQUEST : ErrorCodes.UNAVAILABLE; -} - export const skillsUploadHandlers: GatewayRequestHandlers = { "skills.upload.begin": makeUploadHandler( "skills.upload.begin", @@ -113,104 +73,3 @@ function makeUploadHandler( } }; } - -export async function installUploadedSkillArchive(params: { - uploadId: string; - slug: string; - force: boolean; - sha256?: string; - timeoutMs?: number; - workspaceDir: string; - context: GatewayRequestContext; - store?: SkillUploadStore; -}): Promise { - const store = params.store ?? defaultSkillUploadStore; - if (!areUploadedSkillArchivesEnabled(params.context.getRuntimeConfig())) { - return { - ok: false, - error: UPLOADED_SKILL_ARCHIVES_DISABLED_MESSAGE, - errorCode: ErrorCodes.UNAVAILABLE, - }; - } - try { - const requestedSlug = validateRequestedSkillSlug(params.slug); - const requestedSha = normalizeSkillUploadSha256(params.sha256); - return await store.withCommittedUpload(params.uploadId, async (record, upload) => { - const rejectInvalid = async (error: string): Promise => { - await upload.remove().catch(() => undefined); - return { ok: false, error, errorCode: ErrorCodes.INVALID_REQUEST }; - }; - if (record.kind !== "skill-archive") { - return await rejectInvalid("unsupported upload kind"); - } - if (record.slug !== requestedSlug) { - return await rejectInvalid("install slug does not match upload slug"); - } - if (record.force !== params.force) { - return await rejectInvalid("install force does not match upload force"); - } - if (requestedSha && requestedSha !== record.actualSha256) { - return await rejectInvalid("install sha256 does not match uploaded archive"); - } - if (!record.actualSha256) { - return await rejectInvalid("committed upload is missing sha256"); - } - - const install = await installSkillArchiveFromPath({ - archivePath: record.archivePath, - workspaceDir: params.workspaceDir, - slug: record.slug, - force: record.force, - timeoutMs: params.timeoutMs, - logger: params.context.logGateway, - scan: { - installId: "upload", - origin: "skill-upload", - }, - }); - if (!install.ok) { - const errorCode = uploadInstallFailureErrorCode(install.failureKind); - if (install.failureKind === "invalid-request") { - await upload.remove().catch(() => undefined); - } - return { - ok: false, - error: install.error, - errorCode, - }; - } - await upload.remove().catch(() => undefined); - return { - ok: true, - message: `Installed ${record.slug}`, - stdout: "", - stderr: "", - code: 0, - slug: record.slug, - targetDir: install.targetDir, - sha256: record.actualSha256, - }; - }); - } catch (err) { - if (err instanceof SkillUploadRequestError) { - return { - ok: false, - error: err.message, - errorCode: ErrorCodes.INVALID_REQUEST, - }; - } - const error = formatErrorMessage(err); - if (error.startsWith("Invalid skill slug")) { - return { - ok: false, - error, - errorCode: ErrorCodes.INVALID_REQUEST, - }; - } - return { - ok: false, - error, - errorCode: ErrorCodes.UNAVAILABLE, - }; - } -} diff --git a/src/gateway/server-methods/skills.ts b/src/gateway/server-methods/skills.ts index 8c2bd423219..3df2aa95484 100644 --- a/src/gateway/server-methods/skills.ts +++ b/src/gateway/server-methods/skills.ts @@ -26,7 +26,6 @@ import { type ClawHubSkillSecurityVerdictItem, } from "../../infra/clawhub.js"; import { formatErrorMessage } from "../../infra/errors.js"; -import { getRemoteSkillEligibility } from "../../infra/skills-remote.js"; import { normalizeAgentId } from "../../routing/session-key.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; import { @@ -35,11 +34,14 @@ import { searchSkillsFromClawHub, updateSkillsFromClawHub, } from "../../skills/clawhub.js"; -import { loadWorkspaceSkillEntries, type SkillEntry } from "../../skills/index.js"; import { installSkill } from "../../skills/install.js"; +import { getRemoteSkillEligibility } from "../../skills/remote.js"; import { buildWorkspaceSkillStatus } from "../../skills/status.js"; +import type { SkillEntry } from "../../skills/types.js"; +import { installUploadedSkillArchive } from "../../skills/upload-install.js"; +import { loadWorkspaceSkillEntries } from "../../skills/workspace.js"; import { updateSkillConfigEntry } from "./skills-config-mutations.js"; -import { installUploadedSkillArchive, skillsUploadHandlers } from "./skills-upload.js"; +import { skillsUploadHandlers } from "./skills-upload.js"; import type { GatewayRequestContext, GatewayRequestHandlers } from "./types.js"; function collectSkillBins(entries: SkillEntry[]): string[] { @@ -499,12 +501,24 @@ export const skillsHandlers: GatewayRequestHandlers = { sha256: p.sha256, timeoutMs: p.timeoutMs, workspaceDir: workspaceDirRaw, - context, + config: context.getRuntimeConfig(), + log: context.logGateway, }); + const errorCode = + !result.ok && result.errorKind === "invalid-request" + ? ErrorCodes.INVALID_REQUEST + : ErrorCodes.UNAVAILABLE; + const responseResult = result.ok + ? result + : { + ok: false, + error: result.error, + errorCode, + }; respond( result.ok, - result, - result.ok ? undefined : errorShape(result.errorCode, result.error), + responseResult, + result.ok ? undefined : errorShape(errorCode, result.error), ); return; } diff --git a/src/gateway/server-startup-early.ts b/src/gateway/server-startup-early.ts index bd96ca8bdd0..33f2eed6b59 100644 --- a/src/gateway/server-startup-early.ts +++ b/src/gateway/server-startup-early.ts @@ -78,7 +78,7 @@ export async function startGatewayEarlyRuntime(params: { info: (msg: string) => void; warn: (msg: string) => void; }; - nodeRegistry: Parameters[0]; + nodeRegistry: Parameters[0]; pluginRegistry?: PluginRegistry; broadcast: GatewayMaintenanceParams["broadcast"]; nodeSendToAllSubscribed: Parameters[0]["nodeSendToAllSubscribed"]; @@ -111,7 +111,7 @@ export async function startGatewayEarlyRuntime(params: { const [{ primeRemoteSkillsCache, setSkillsRemoteRegistry }, taskRegistryMaintenance] = await measureStartup(params.startupTrace, "runtime.early.lazy-runtime-imports", () => Promise.all([ - import("../infra/skills-remote.js"), + import("../skills/remote.js"), import("../tasks/task-registry.maintenance.js"), ]), ); @@ -130,7 +130,7 @@ export async function startGatewayEarlyRuntime(params: { ? () => {} : await measureStartup(params.startupTrace, "runtime.early.skills-listener", async () => { const [{ registerSkillsChangeListener }, { refreshRemoteBinsForConnectedNodes }] = - await Promise.all([import("../skills/refresh.js"), import("../infra/skills-remote.js")]); + await Promise.all([import("../skills/refresh.js"), import("../skills/remote.js")]); return registerSkillsChangeListener((event) => { if (event.reason === "remote-node") { return; diff --git a/src/gateway/server/ws-connection.ts b/src/gateway/server/ws-connection.ts index da77aad6183..ee4b2caa344 100644 --- a/src/gateway/server/ws-connection.ts +++ b/src/gateway/server/ws-connection.ts @@ -6,11 +6,11 @@ import { GATEWAY_STARTUP_PENDING_CLOSE_CAUSE, } from "../../../packages/gateway-protocol/src/startup-unavailable.js"; import { getRuntimeConfig } from "../../config/io.js"; -import { removeRemoteNodeInfo } from "../../infra/skills-remote.js"; import { upsertPresence } from "../../infra/system-presence.js"; import { logRejectedLargePayload } from "../../logging/diagnostic-payload.js"; import type { createSubsystemLogger } from "../../logging/subsystem.js"; import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js"; +import { removeRemoteNodeInfo } from "../../skills/remote.js"; import { truncateUtf16Safe } from "../../utils.js"; import { isWebchatClient } from "../../utils/message-channel.js"; import type { AuthRateLimiter } from "../auth-rate-limit.js"; diff --git a/src/gateway/server/ws-connection/message-handler.ts b/src/gateway/server/ws-connection/message-handler.ts index 421f3213f0e..fe7dc985dfc 100644 --- a/src/gateway/server/ws-connection/message-handler.ts +++ b/src/gateway/server/ws-connection/message-handler.ts @@ -70,7 +70,6 @@ import { requestNodePairing, updatePairedNodeMetadata, } from "../../../infra/node-pairing.js"; -import { recordRemoteNodeInfo, refreshRemoteNodeBins } from "../../../infra/skills-remote.js"; import { upsertPresence } from "../../../infra/system-presence.js"; import { loadVoiceWakeRoutingConfig } from "../../../infra/voicewake-routing.js"; import { loadVoiceWakeConfig } from "../../../infra/voicewake.js"; @@ -85,6 +84,7 @@ import { } from "../../../shared/device-bootstrap-profile.js"; import { roleScopesAllow } from "../../../shared/operator-scope-compat.js"; import { uniqueStrings } from "../../../shared/string-normalization.js"; +import { recordRemoteNodeInfo, refreshRemoteNodeBins } from "../../../skills/remote.js"; import { isBrowserOperatorUiClient, isGatewayCliClient, diff --git a/src/infra/skills-remote.test.ts b/src/skills/remote.test.ts similarity index 99% rename from src/infra/skills-remote.test.ts rename to src/skills/remote.test.ts index e154ff176e9..b6dacf4eaaf 100644 --- a/src/infra/skills-remote.test.ts +++ b/src/skills/remote.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { NodeRegistry } from "../gateway/node-registry.js"; -import { getSkillsSnapshotVersion, resetSkillsRefreshForTest } from "../skills/refresh.js"; +import { getSkillsSnapshotVersion, resetSkillsRefreshForTest } from "./refresh.js"; import { getRemoteSkillEligibility, recordRemoteNodeBins, @@ -13,7 +13,7 @@ import { removeRemoteNodeInfo, refreshRemoteNodeBins, setSkillsRemoteRegistry, -} from "./skills-remote.js"; +} from "./remote.js"; function createRemoteSkillWorkspace(bin: string): { cfg: OpenClawConfig; workspaceDir: string } { const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-remote-skills-")); diff --git a/src/infra/skills-remote.ts b/src/skills/remote.ts similarity index 98% rename from src/infra/skills-remote.ts rename to src/skills/remote.ts index 735ac48dd5b..29b2ac451b5 100644 --- a/src/infra/skills-remote.ts +++ b/src/skills/remote.ts @@ -1,16 +1,16 @@ import { listAgentWorkspaceDirs } from "../agents/workspace-dirs.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { NodeRegistry } from "../gateway/node-registry.js"; +import { listNodePairing, updatePairedNodeMetadata } from "../infra/node-pairing.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "../shared/string-coerce.js"; import { normalizeStringEntries } from "../shared/string-normalization.js"; -import { bumpSkillsSnapshotVersion } from "../skills/refresh-state.js"; -import type { SkillEligibilityContext, SkillEntry } from "../skills/types.js"; -import { loadWorkspaceSkillEntries } from "../skills/workspace.js"; -import { listNodePairing, updatePairedNodeMetadata } from "./node-pairing.js"; +import { bumpSkillsSnapshotVersion } from "./refresh-state.js"; +import type { SkillEligibilityContext, SkillEntry } from "./types.js"; +import { loadWorkspaceSkillEntries } from "./workspace.js"; type RemoteNodeRecord = { nodeId: string; diff --git a/src/skills/session-snapshot.ts b/src/skills/session-snapshot.ts new file mode 100644 index 00000000000..67a3e4a0c89 --- /dev/null +++ b/src/skills/session-snapshot.ts @@ -0,0 +1,99 @@ +import crypto from "node:crypto"; +import { stableStringify } from "../agents/stable-stringify.js"; +import { redactConfigObject } from "../config/redact-snapshot.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { matchesSkillFilter } from "./filter.js"; +import { getSkillsSnapshotVersion, shouldRefreshSnapshotForVersion } from "./refresh-state.js"; +import { ensureSkillsWatcher } from "./refresh.js"; +import { buildWorkspaceSkillSnapshot } from "./service.js"; +import { hydrateResolvedSkills } from "./snapshot-hydration.js"; +import type { SkillEligibilityContext, SkillSnapshot } from "./types.js"; + +const resolvedSkillsCache = new Map(); +const RESOLVED_SKILLS_CACHE_MAX = 10; + +export type ReusableSkillSnapshotParams = { + workspaceDir: string; + config: OpenClawConfig; + agentId?: string; + skillFilter?: string[]; + eligibility?: SkillEligibilityContext; + existingSnapshot?: SkillSnapshot; + snapshotVersion?: number; + watch?: boolean; + hydrateExisting?: boolean; +}; + +export type ReusableSkillSnapshotResult = { + snapshot: SkillSnapshot; + shouldRefresh: boolean; + snapshotVersion: number; +}; + +export function resetResolvedSkillsCacheForTests(): void { + resolvedSkillsCache.clear(); +} + +function fingerprintSkillSnapshotConfig(config: OpenClawConfig): string { + return crypto + .createHash("sha256") + .update(stableStringify(redactConfigObject(config))) + .digest("hex"); +} + +function cacheResolvedSkills(cacheKey: string, snapshot: SkillSnapshot): SkillSnapshot { + resolvedSkillsCache.set(cacheKey, snapshot.resolvedSkills); + if (resolvedSkillsCache.size > RESOLVED_SKILLS_CACHE_MAX) { + const oldest = resolvedSkillsCache.keys().next().value; + if (oldest !== undefined) { + resolvedSkillsCache.delete(oldest); + } + } + return snapshot; +} + +export function resolveReusableWorkspaceSkillSnapshot( + params: ReusableSkillSnapshotParams, +): ReusableSkillSnapshotResult { + if (params.watch !== false) { + ensureSkillsWatcher({ workspaceDir: params.workspaceDir, config: params.config }); + } + const snapshotVersion = params.snapshotVersion ?? getSkillsSnapshotVersion(params.workspaceDir); + const shouldRefresh = + shouldRefreshSnapshotForVersion(params.existingSnapshot?.version, snapshotVersion) || + !matchesSkillFilter(params.existingSnapshot?.skillFilter, params.skillFilter); + const buildSnapshot = () => { + return buildWorkspaceSkillSnapshot(params.workspaceDir, { + config: params.config, + agentId: params.agentId, + skillFilter: params.skillFilter, + eligibility: params.eligibility, + snapshotVersion, + }); + }; + + const configFingerprint = fingerprintSkillSnapshotConfig(params.config); + const snapshotCacheKey = JSON.stringify([ + params.workspaceDir, + snapshotVersion, + params.skillFilter, + params.agentId, + params.eligibility, + configFingerprint, + ]); + + const cachedRebuild = (): SkillSnapshot => { + if (resolvedSkillsCache.has(snapshotCacheKey)) { + return { resolvedSkills: resolvedSkillsCache.get(snapshotCacheKey) } as SkillSnapshot; + } + return cacheResolvedSkills(snapshotCacheKey, buildSnapshot()); + }; + + const snapshot = + !params.existingSnapshot || shouldRefresh + ? cacheResolvedSkills(snapshotCacheKey, buildSnapshot()) + : params.hydrateExisting === false + ? params.existingSnapshot + : hydrateResolvedSkills(params.existingSnapshot, cachedRebuild); + return { snapshot, shouldRefresh, snapshotVersion }; +} diff --git a/src/skills/upload-install.ts b/src/skills/upload-install.ts new file mode 100644 index 00000000000..184d4676392 --- /dev/null +++ b/src/skills/upload-install.ts @@ -0,0 +1,147 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { + installSkillArchiveFromPath, + type SkillArchiveInstallFailureKind, + validateRequestedSkillSlug, +} from "./archive-install.js"; +import { + defaultSkillUploadStore, + normalizeSkillUploadSha256, + SkillUploadRequestError, + type SkillUploadStore, +} from "./upload-store.js"; + +export type UploadedSkillInstallErrorKind = "invalid-request" | "unavailable"; + +export const UPLOADED_SKILL_ARCHIVES_DISABLED_MESSAGE = + "Uploaded skill archive installs are disabled by skills.install.allowUploadedArchives"; + +export function areUploadedSkillArchivesEnabled(config: OpenClawConfig): boolean { + return config.skills?.install?.allowUploadedArchives === true; +} + +export type UploadedSkillInstallResult = + | { + ok: true; + message: string; + stdout: string; + stderr: string; + code: 0; + slug: string; + targetDir: string; + sha256: string; + } + | { + ok: false; + error: string; + errorKind: UploadedSkillInstallErrorKind; + }; + +function uploadInstallFailureErrorKind( + failureKind: SkillArchiveInstallFailureKind, +): UploadedSkillInstallErrorKind { + return failureKind === "invalid-request" ? "invalid-request" : "unavailable"; +} + +export async function installUploadedSkillArchive(params: { + uploadId: string; + slug: string; + force: boolean; + sha256?: string; + timeoutMs?: number; + workspaceDir: string; + config: OpenClawConfig; + log?: (message: string) => void; + store?: SkillUploadStore; +}): Promise { + const store = params.store ?? defaultSkillUploadStore; + if (!areUploadedSkillArchivesEnabled(params.config)) { + return { + ok: false, + error: UPLOADED_SKILL_ARCHIVES_DISABLED_MESSAGE, + errorKind: "unavailable", + }; + } + try { + const requestedSlug = validateRequestedSkillSlug(params.slug); + const requestedSha = normalizeSkillUploadSha256(params.sha256); + return await store.withCommittedUpload(params.uploadId, async (record, upload) => { + const rejectInvalid = async (error: string): Promise => { + await upload.remove().catch(() => undefined); + return { ok: false, error, errorKind: "invalid-request" }; + }; + if (record.kind !== "skill-archive") { + return await rejectInvalid("unsupported upload kind"); + } + if (record.slug !== requestedSlug) { + return await rejectInvalid("install slug does not match upload slug"); + } + if (record.force !== params.force) { + return await rejectInvalid("install force does not match upload force"); + } + if (requestedSha && requestedSha !== record.actualSha256) { + return await rejectInvalid("install sha256 does not match uploaded archive"); + } + if (!record.actualSha256) { + return await rejectInvalid("committed upload is missing sha256"); + } + + const install = await installSkillArchiveFromPath({ + archivePath: record.archivePath, + workspaceDir: params.workspaceDir, + slug: record.slug, + force: record.force, + timeoutMs: params.timeoutMs, + logger: params.log, + scan: { + installId: "upload", + origin: "skill-upload", + }, + }); + if (!install.ok) { + const errorKind = uploadInstallFailureErrorKind(install.failureKind); + if (install.failureKind === "invalid-request") { + await upload.remove().catch(() => undefined); + } + return { + ok: false, + error: install.error, + errorKind, + }; + } + await upload.remove().catch(() => undefined); + return { + ok: true, + message: `Installed ${record.slug}`, + stdout: "", + stderr: "", + code: 0, + slug: record.slug, + targetDir: install.targetDir, + sha256: record.actualSha256, + }; + }); + } catch (err) { + if (err instanceof SkillUploadRequestError) { + return { + ok: false, + error: err.message, + errorKind: "invalid-request", + }; + } + const error = formatErrorMessage(err); + if (error.startsWith("Invalid skill slug")) { + return { + ok: false, + error, + errorKind: "invalid-request", + }; + } + return { + ok: false, + error, + errorKind: "unavailable", + }; + } +} diff --git a/src/gateway/server-methods/skills-upload-store.test.ts b/src/skills/upload-store.test.ts similarity index 99% rename from src/gateway/server-methods/skills-upload-store.test.ts rename to src/skills/upload-store.test.ts index f2e3e7f2bb0..780c117440d 100644 --- a/src/gateway/server-methods/skills-upload-store.test.ts +++ b/src/skills/upload-store.test.ts @@ -7,7 +7,7 @@ import { createSkillUploadStore, MAX_ACTIVE_SKILL_UPLOADS, SkillUploadRequestError, -} from "./skills-upload-store.js"; +} from "./upload-store.js"; let tempDirs: string[] = []; diff --git a/src/gateway/server-methods/skills-upload-store.ts b/src/skills/upload-store.ts similarity index 98% rename from src/gateway/server-methods/skills-upload-store.ts rename to src/skills/upload-store.ts index d88f9de8b52..b772beff970 100644 --- a/src/gateway/server-methods/skills-upload-store.ts +++ b/src/skills/upload-store.ts @@ -2,11 +2,11 @@ import { createHash, randomUUID } from "node:crypto"; import { createReadStream } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; -import { resolveStateDir } from "../../config/paths.js"; -import { DEFAULT_MAX_ARCHIVE_BYTES_ZIP } from "../../infra/archive.js"; -import { formatErrorMessage } from "../../infra/errors.js"; -import { createAsyncLock, readDurableJsonFile, writeJsonAtomic } from "../../infra/json-files.js"; -import { validateRequestedSkillSlug } from "../../skills/archive-install.js"; +import { resolveStateDir } from "../config/paths.js"; +import { DEFAULT_MAX_ARCHIVE_BYTES_ZIP } from "../infra/archive.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { createAsyncLock, readDurableJsonFile, writeJsonAtomic } from "../infra/json-files.js"; +import { validateRequestedSkillSlug } from "./archive-install.js"; export const SKILL_UPLOAD_TTL_MS = 60 * 60 * 1000; export const MAX_SKILL_UPLOAD_CHUNK_BYTES = 4 * 1024 * 1024;