From 133ca4b46990ea9072b19bcb2635b568e4cd18e8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Jul 2026 07:21:08 +0100 Subject: [PATCH] feat(skills): diagnose skill_workshop hidden by tool policy (#100654) * feat(skills): diagnose skill_workshop hidden by tool policy (#87570) Workshop can be enabled and auto-capturing while tools.profile hides the skill_workshop tool; every inspection surface looked healthy. plugins inspect and openclaw doctor now name the excluding policy layer (global/ agent/provider profile, allowlist, denylist) and the exact alsoAllow grant to add, via a shared resolveSkillWorkshopToolPolicyAvailability helper that /learn's guard now reuses instead of composing policy itself. Diagnosis only; no policy behavior change. * ci: retrigger --- docs/tools/skill-workshop.md | 9 + .../effective-tool-policy.ts | 3 + src/agents/tool-policy-pipeline.ts | 10 + src/auto-reply/reply/commands-learn.ts | 16 +- src/cli/plugins-cli.list.test.ts | 47 +++- src/cli/plugins-inspect-command.ts | 18 ++ src/flows/doctor-core-checks.test.ts | 41 +++ src/flows/doctor-core-checks.ts | 31 +++ src/flows/doctor-health-contributions.test.ts | 12 + src/flows/doctor-health-contributions.ts | 44 ++- .../workshop/tool-policy-diagnostic.test.ts | 125 +++++++++ src/skills/workshop/tool-policy-diagnostic.ts | 252 ++++++++++++++++++ 12 files changed, 570 insertions(+), 38 deletions(-) create mode 100644 src/skills/workshop/tool-policy-diagnostic.test.ts create mode 100644 src/skills/workshop/tool-policy-diagnostic.ts diff --git a/docs/tools/skill-workshop.md b/docs/tools/skill-workshop.md index e8234da473be..4a3f99ae2b1f 100644 --- a/docs/tools/skill-workshop.md +++ b/docs/tools/skill-workshop.md @@ -296,6 +296,15 @@ Default state directory: `~/.openclaw`. | Proposal does not show in list | Check the selected `--agent` workspace and `OPENCLAW_STATE_DIR`. | | Agent cannot call `skill_workshop` | Check the active tool policy and run mode. `coding` includes the tool; restrictive `tools.allow` policies must list it explicitly, and sandboxed runs must use a normal host-side agent session or the CLI. | +### Tool-policy diagnostic + +When autonomous capture is enabled, `openclaw doctor` runs the +`core/doctor/skill-workshop-tool-policy` check for the default agent. If policy +hides `skill_workshop`, the warning names the first excluding config layer and +the exact `allow` or `alsoAllow` change to make. Older runbooks may still use +`openclaw plugins inspect skill-workshop`; that command now explains that Skill +Workshop is built in and prints the same policy hint when applicable. + ## Related - [Skills](/tools/skills) for load order, precedence, and visibility diff --git a/src/agents/embedded-agent-runner/effective-tool-policy.ts b/src/agents/embedded-agent-runner/effective-tool-policy.ts index 3352f0756175..3b5369726ab8 100644 --- a/src/agents/embedded-agent-runner/effective-tool-policy.ts +++ b/src/agents/embedded-agent-runner/effective-tool-policy.ts @@ -8,6 +8,7 @@ import { buildDeclaredToolAllowlistContext } from "../tool-policy-declared-conte import { applyToolPolicyPipeline, buildDefaultToolPolicyPipelineSteps, + type ToolPolicyFilterEvent, type ToolPolicyPipelineStep, } from "../tool-policy-pipeline.js"; import { collectExplicitDenylist, mergeAlsoAllowPolicy } from "../tool-policy.js"; @@ -34,6 +35,7 @@ type FinalEffectiveToolPolicyParams = { conversationCapabilityProfile: ResolvedConversationCapabilityProfile; warn: (message: string) => void; toolPolicyAuditLogLevel?: "info" | "debug"; + onFilter?: (event: ToolPolicyFilterEvent) => void; }; export function applyFinalEffectiveToolPolicy( @@ -110,6 +112,7 @@ export function applyFinalEffectiveToolPolicy( warn: params.warn, steps: pipelineSteps, auditLogLevel: params.toolPolicyAuditLogLevel, + onFilter: params.onFilter, declaredToolAllowlist: buildDeclaredToolAllowlistContext({ config: params.config, toolDenylist: collectExplicitDenylist(pipelineSteps.map((step) => step.policy)), diff --git a/src/agents/tool-policy-pipeline.ts b/src/agents/tool-policy-pipeline.ts index 17401331c788..8913a201ffd1 100644 --- a/src/agents/tool-policy-pipeline.ts +++ b/src/agents/tool-policy-pipeline.ts @@ -45,6 +45,14 @@ export type ToolPolicyPipelineStep = { unavailableCoreToolReason?: string; }; +/** One policy application, exposed for diagnostics that need exclusion provenance. */ +export type ToolPolicyFilterEvent = { + step: ToolPolicyPipelineStep; + policy: ToolPolicyLike; + before: readonly AnyAgentTool[]; + after: readonly AnyAgentTool[]; +}; + /** Builds the default profile, provider, agent, group, and sender policy layers. */ export function buildDefaultToolPolicyPipelineSteps(params: { profilePolicy?: ToolPolicyLike; @@ -131,6 +139,7 @@ export function applyToolPolicyPipeline(params: { steps: ToolPolicyPipelineStep[]; auditLogLevel?: ToolPolicyAuditLogLevel; declaredToolAllowlist?: DeclaredToolAllowlistContext; + onFilter?: (event: ToolPolicyFilterEvent) => void; }): AnyAgentTool[] { const coreToolNames = new Set( params.tools @@ -203,6 +212,7 @@ export function applyToolPolicyPipeline(params: { } const before = filtered; filtered = filterToolsByPolicy(before, expanded); + params.onFilter?.({ step, policy: expanded, before, after: filtered }); auditToolPolicyFilter({ stepLabel: step.label, policy: expanded, diff --git a/src/auto-reply/reply/commands-learn.ts b/src/auto-reply/reply/commands-learn.ts index f8cb495db9f5..8212a3534545 100644 --- a/src/auto-reply/reply/commands-learn.ts +++ b/src/auto-reply/reply/commands-learn.ts @@ -1,7 +1,6 @@ // Handles /learn by turning the command into a Skill Workshop authoring turn. import { resolveCliBackendConfig } from "../../agents/cli-backends.js"; import { resolveConversationCapabilityProfile } from "../../agents/conversation-capability-profile.js"; -import { applyFinalEffectiveToolPolicy } from "../../agents/embedded-agent-runner/effective-tool-policy.js"; import { agentHarnessExposesOpenClawTools, selectAgentHarness, @@ -14,8 +13,8 @@ import { supportsModelTools } from "../../agents/model-tool-support.js"; import { resolveSandboxRuntimeStatus } from "../../agents/sandbox.js"; import { isToolAllowedByPolicyName } from "../../agents/tool-policy-match.js"; import { resolveConfiguredModelCompat } from "../../agents/tools-effective-inventory.js"; -import { createSkillWorkshopTool } from "../../agents/tools/skill-workshop-tool.js"; import { buildLearnPrompt, DEFAULT_LEARN_REQUEST } from "../../skills/workshop/learn-prompt.js"; +import { resolveSkillWorkshopToolPolicyAvailability } from "../../skills/workshop/tool-policy-diagnostic.js"; import { rejectUnauthorizedCommand } from "./command-gates.js"; import type { CommandHandler, @@ -156,19 +155,10 @@ function workshopIsAvailable(params: HandleCommandsParams): boolean { groupChannel: params.sessionEntry?.groupChannel ?? params.ctx.GroupChannel, groupSpace: params.sessionEntry?.space ?? params.ctx.GroupSpace, }); - const tools = applyFinalEffectiveToolPolicy({ - bundledTools: [ - createSkillWorkshopTool({ - workspaceDir: params.workspaceDir, - config: params.cfg, - agentId: capabilityProfile.agentId, - }), - ], + return resolveSkillWorkshopToolPolicyAvailability({ config: params.cfg, conversationCapabilityProfile: capabilityProfile, - warn: () => undefined, - }); - return tools.some((tool) => tool.name === SKILL_WORKSHOP_TOOL_NAME); + }).available; } catch { return false; } diff --git a/src/cli/plugins-cli.list.test.ts b/src/cli/plugins-cli.list.test.ts index 3d7fd395e90a..23011ec45d98 100644 --- a/src/cli/plugins-cli.list.test.ts +++ b/src/cli/plugins-cli.list.test.ts @@ -1,5 +1,6 @@ // Plugins CLI list tests cover plugin listing output and installed-state formatting. -import { beforeEach, describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createPluginRecord } from "../plugins/status.test-helpers.js"; import { buildPluginDiagnosticsReport, @@ -17,9 +18,18 @@ import { setInstalledPluginIndexInstallRecords, } from "./plugins-cli-test-helpers.js"; +const workshopMocks = vi.hoisted(() => ({ + detectToolPolicyDiagnostic: vi.fn(), +})); + +vi.mock("../skills/workshop/tool-policy-diagnostic.js", () => ({ + detectSkillWorkshopToolPolicyDiagnostic: workshopMocks.detectToolPolicyDiagnostic, +})); + describe("plugins cli list", () => { beforeEach(() => { resetPluginsCliTestState(); + workshopMocks.detectToolPolicyDiagnostic.mockReset(); }); it("includes imported state in JSON output", async () => { @@ -620,4 +630,39 @@ describe("plugins cli list", () => { expect(buildPluginDiagnosticsReport).not.toHaveBeenCalled(); expect(runtimeErrors.at(-1)).toContain("Plugin not found: missing-plugin"); }); + + it("explains a policy-hidden built-in Skill Workshop at the legacy inspect surface", async () => { + const config: OpenClawConfig = { + tools: { profile: "messaging" }, + }; + loadConfig.mockReturnValue(config); + workshopMocks.detectToolPolicyDiagnostic.mockReturnValue({ + agentId: "main", + source: "tools.profile", + detail: 'tools.profile: "messaging" does not include "skill_workshop".', + fix: 'Add tools.alsoAllow: ["skill_workshop"].', + message: + 'Skill Workshop is active, but "skill_workshop" is hidden for agent "main": tools.profile: "messaging" does not include "skill_workshop". Add tools.alsoAllow: ["skill_workshop"].', + }); + buildPluginSnapshotReport.mockReturnValue({ + plugins: [], + diagnostics: [], + }); + + await expect(runPluginsCommand(["plugins", "inspect", "skill-workshop"])).rejects.toThrow( + "__exit__:1", + ); + + expect(runtimeErrors.at(-1)).toContain( + "Skill Workshop is built into OpenClaw, not a plugin; configure it under skills.workshop.", + ); + expect(workshopMocks.detectToolPolicyDiagnostic).toHaveBeenCalledWith({ + config, + workshopEnabled: true, + }); + expect(runtimeErrors.at(-1)).toContain( + 'tools.profile: "messaging" does not include "skill_workshop".', + ); + expect(runtimeErrors.at(-1)).toContain('Add tools.alsoAllow: ["skill_workshop"].'); + }); }); diff --git a/src/cli/plugins-inspect-command.ts b/src/cli/plugins-inspect-command.ts index c52e6e74127b..1548fc7b1ba1 100644 --- a/src/cli/plugins-inspect-command.ts +++ b/src/cli/plugins-inspect-command.ts @@ -236,6 +236,24 @@ export async function runPluginsInspectCommand( ); const targetPlugin = snapshotReport.plugins.find((entry) => entry.id === id || entry.name === id); if (!targetPlugin) { + if (id === "skill-workshop") { + const { detectSkillWorkshopToolPolicyDiagnostic } = + await import("../skills/workshop/tool-policy-diagnostic.js"); + const diagnostic = detectSkillWorkshopToolPolicyDiagnostic({ + config: cfg, + // Invoking the legacy inspect id is explicit Workshop intent even when + // autonomous capture is off; report manual-tool availability too. + workshopEnabled: true, + }); + const lines = [ + "Skill Workshop is built into OpenClaw, not a plugin; configure it under skills.workshop.", + ]; + if (diagnostic) { + lines.push(diagnostic.message); + } + defaultRuntime.error(lines.join("\n")); + return defaultRuntime.exit(1); + } defaultRuntime.error(formatMissingPluginMessage({ id, includeSearch: true })); return defaultRuntime.exit(1); } diff --git a/src/flows/doctor-core-checks.test.ts b/src/flows/doctor-core-checks.test.ts index 0eed125eb14d..7c50443452ab 100644 --- a/src/flows/doctor-core-checks.test.ts +++ b/src/flows/doctor-core-checks.test.ts @@ -170,6 +170,47 @@ describe("CORE_HEALTH_CHECKS", () => { ).toBe(false); }); + it("warns when autonomous Skill Workshop capture is enabled but policy hides its tool", async () => { + const check = getCheck( + createCoreHealthChecks(createDeps()), + "core/doctor/skill-workshop-tool-policy", + ); + + const findings = await check.detect({ + mode: "doctor", + runtime, + cfg: { + skills: { workshop: { autonomous: { enabled: true } } }, + tools: { profile: "messaging" }, + }, + }); + + expect(findings).toEqual([ + expect.objectContaining({ + checkId: "core/doctor/skill-workshop-tool-policy", + severity: "warning", + message: 'tools.profile: "messaging" does not include "skill_workshop".', + path: "tools.profile", + fixHint: 'Add tools.alsoAllow: ["skill_workshop"].', + }), + ]); + }); + + it("does not warn when autonomous Skill Workshop capture is disabled", async () => { + const check = getCheck( + createCoreHealthChecks(createDeps()), + "core/doctor/skill-workshop-tool-policy", + ); + + await expect( + check.detect({ + mode: "doctor", + runtime, + cfg: { tools: { profile: "messaging" } }, + }), + ).resolves.toEqual([]); + }); + it("threads deep mode into structured extra gateway service detection", async () => { const check = getCheck( createCoreHealthChecks(createDeps()), diff --git a/src/flows/doctor-core-checks.ts b/src/flows/doctor-core-checks.ts index e1fedf01cbd1..0f1d428fb291 100644 --- a/src/flows/doctor-core-checks.ts +++ b/src/flows/doctor-core-checks.ts @@ -35,6 +35,8 @@ import { resolveGatewayAuthToken } from "../gateway/auth-token-resolution.js"; import { resolveGatewayAuth } from "../gateway/auth.js"; import { getSkippedExecRefStaticError } from "../secrets/exec-resolution-policy.js"; import type { SkillStatusEntry } from "../skills/discovery/status.js"; +import { resolveSkillWorkshopConfig } from "../skills/workshop/config.js"; +import { detectSkillWorkshopToolPolicyDiagnostic } from "../skills/workshop/tool-policy-diagnostic.js"; import { registerHealthCheck } from "./health-check-registry.js"; import type { SplitHealthCheckInput } from "./health-check-runner-types.js"; import type { @@ -51,6 +53,7 @@ const GATEWAY_DAEMON_CHECK_ID = "core/doctor/gateway-daemon"; const GATEWAY_HEALTH_CHECK_ID = "core/doctor/gateway-health"; const GATEWAY_SERVICES_EXTRA_CHECK_ID = "core/doctor/gateway-services/extra"; const SESSION_LOCKS_CHECK_ID = "core/doctor/session-locks"; +const SKILL_WORKSHOP_TOOL_POLICY_CHECK_ID = "core/doctor/skill-workshop-tool-policy"; type CoreHealthCheckContext = HealthCheckContext & { readonly deep?: boolean; @@ -216,6 +219,33 @@ const commandOwnerCheck: HealthCheck = { }, }; +const skillWorkshopToolPolicyCheck: HealthCheck = { + id: SKILL_WORKSHOP_TOOL_POLICY_CHECK_ID, + kind: "core", + description: "Autonomous Skill Workshop capture has a callable review tool.", + source: "doctor", + async detect(ctx) { + const diagnostic = detectSkillWorkshopToolPolicyDiagnostic({ + config: ctx.cfg, + workshopEnabled: resolveSkillWorkshopConfig(ctx.cfg).autonomous.enabled, + }); + if (!diagnostic) { + return []; + } + return [ + { + checkId: SKILL_WORKSHOP_TOOL_POLICY_CHECK_ID, + severity: "warning", + message: diagnostic.detail, + path: diagnostic.source, + target: diagnostic.agentId, + requirement: "Autonomous Skill Workshop review requires the skill_workshop tool.", + fixHint: diagnostic.fix, + }, + ]; + }, +}; + function resolveDoctorMode(cfg: OpenClawConfig): "local" | "remote" { return cfg.gateway?.mode === "remote" ? "remote" : "local"; } @@ -1100,6 +1130,7 @@ function createConvertedWorkflowChecks( createProviderCatalogProjectionCheck(deps), createRuntimeToolSchemaCheck(deps), createWorkspaceSuggestionsCheck(deps), + skillWorkshopToolPolicyCheck, ]; } diff --git a/src/flows/doctor-health-contributions.test.ts b/src/flows/doctor-health-contributions.test.ts index 04d79b2abb48..c8b03f818af2 100644 --- a/src/flows/doctor-health-contributions.test.ts +++ b/src/flows/doctor-health-contributions.test.ts @@ -1264,6 +1264,18 @@ describe("doctor health contributions", () => { expect(mocks.collectHeartbeatTemplateHealthFindings).toHaveBeenCalledWith(ctx.cfg); }); + it("exposes the Skill Workshop tool-policy check to doctor lint", async () => { + const contributionChecks = await resolveDoctorContributionHealthChecks(); + const check = contributionChecks.find( + (entry) => entry.id === "core/doctor/skill-workshop-tool-policy", + ); + + expect(check).toMatchObject({ + id: "core/doctor/skill-workshop-tool-policy", + kind: "core", + }); + }); + it("preserves allow-exec Gateway SecretRef resolution in auth health", async () => { const contribution = requireDoctorContribution("doctor:gateway-auth"); const ctx = { diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index e1fc2badfa70..a69f3446006d 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -1446,14 +1446,17 @@ function formatHealthFindings(findings: readonly HealthFinding[]): string { .join("\n"); } -async function runProviderCatalogProjectionHealth(ctx: DoctorHealthFlowContext): Promise { +async function runCoreHealthFindingNote( + ctx: DoctorHealthFlowContext, + checkId: string, +): Promise { const { registerCoreHealthChecks } = await loadDoctorCoreChecksModule(); const { getHealthCheck } = await loadHealthCheckRegistryModule(); const { resolveAgentWorkspaceDir, resolveDefaultAgentId } = await loadAgentScopeModule(); const { note } = await loadNoteModule(); registerCoreHealthChecks(); - const check = getHealthCheck("core/doctor/provider-catalog-projection"); + const check = getHealthCheck(checkId); if (!check) { return; } @@ -1471,29 +1474,16 @@ async function runProviderCatalogProjectionHealth(ctx: DoctorHealthFlowContext): note(formatHealthFindings(findings), "Doctor warnings"); } -async function runRuntimeToolSchemasHealth(ctx: DoctorHealthFlowContext): Promise { - const { registerCoreHealthChecks } = await loadDoctorCoreChecksModule(); - const { getHealthCheck } = await loadHealthCheckRegistryModule(); - const { resolveAgentWorkspaceDir, resolveDefaultAgentId } = await loadAgentScopeModule(); - const { note } = await loadNoteModule(); +async function runProviderCatalogProjectionHealth(ctx: DoctorHealthFlowContext): Promise { + await runCoreHealthFindingNote(ctx, "core/doctor/provider-catalog-projection"); +} - registerCoreHealthChecks(); - const check = getHealthCheck("core/doctor/runtime-tool-schemas"); - if (!check) { - return; - } - const findings = await check.detect({ - mode: "doctor", - runtime: ctx.runtime, - cfg: ctx.cfg, - cwd: resolveAgentWorkspaceDir(ctx.cfg, resolveDefaultAgentId(ctx.cfg)), - configPath: ctx.configPath, - }); - if (findings.length === 0) { - return; - } - ctx.healthOk = false; - note(formatHealthFindings(findings), "Doctor warnings"); +async function runRuntimeToolSchemasHealth(ctx: DoctorHealthFlowContext): Promise { + await runCoreHealthFindingNote(ctx, "core/doctor/runtime-tool-schemas"); +} + +async function runSkillWorkshopToolPolicyHealth(ctx: DoctorHealthFlowContext): Promise { + await runCoreHealthFindingNote(ctx, "core/doctor/skill-workshop-tool-policy"); } export function resolveDoctorHealthContributions(): DoctorHealthContribution[] { @@ -1993,6 +1983,12 @@ export function resolveDoctorHealthContributions(): DoctorHealthContribution[] { healthCheckIds: ["core/doctor/runtime-tool-schemas"], run: runRuntimeToolSchemasHealth, }), + createDoctorHealthContribution({ + id: "doctor:skill-workshop-tool-policy", + label: "Skill Workshop tool policy", + healthCheckIds: ["core/doctor/skill-workshop-tool-policy"], + run: runSkillWorkshopToolPolicyHealth, + }), createDoctorHealthContribution({ id: "doctor:systemd-linger", label: "systemd linger", diff --git a/src/skills/workshop/tool-policy-diagnostic.test.ts b/src/skills/workshop/tool-policy-diagnostic.test.ts new file mode 100644 index 000000000000..41b2591b9121 --- /dev/null +++ b/src/skills/workshop/tool-policy-diagnostic.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { detectSkillWorkshopToolPolicyDiagnostic } from "./tool-policy-diagnostic.js"; + +function detect(config: OpenClawConfig, workshopEnabled = true) { + return detectSkillWorkshopToolPolicyDiagnostic({ config, workshopEnabled }); +} + +describe("detectSkillWorkshopToolPolicyDiagnostic", () => { + it("names the profile and exact additive grant when policy excludes the tool", () => { + expect(detect({ tools: { profile: "messaging" } })).toMatchObject({ + source: "tools.profile", + detail: 'tools.profile: "messaging" does not include "skill_workshop".', + fix: 'Add tools.alsoAllow: ["skill_workshop"].', + }); + }); + + it("returns no diagnostic when policy includes the tool", () => { + expect(detect({ tools: { profile: "coding" } })).toBeNull(); + expect(detect({ tools: { profile: "messaging", alsoAllow: ["skill_workshop"] } })).toBeNull(); + }); + + it("returns no diagnostic when Workshop capture is disabled", () => { + expect(detect({ tools: { profile: "messaging" } }, false)).toBeNull(); + }); + + it("names a restrictive allowlist that excludes the tool", () => { + expect(detect({ tools: { profile: "coding", allow: ["read", "write"] } })).toMatchObject({ + source: "tools.allow", + detail: 'tools.allow does not include "skill_workshop".', + fix: 'Add "skill_workshop" to tools.allow.', + }); + }); + + it("names agent-scoped profile and allowlist sources", () => { + expect( + detect({ + agents: { list: [{ id: "main", tools: { profile: "messaging" } }] }, + }), + ).toMatchObject({ + source: "agents.list[0].tools.profile", + fix: 'Add agents.list[0].tools.alsoAllow: ["skill_workshop"].', + }); + + expect( + detect({ + agents: { list: [{ id: "main", tools: { allow: ["read"] } }] }, + }), + ).toMatchObject({ + source: "agents.list[0].tools.allow", + fix: 'Add "skill_workshop" to agents.list[0].tools.allow.', + }); + }); + + it("targets the effective agent-scoped profile grant owner", () => { + expect( + detect({ + tools: { profile: "messaging" }, + agents: { list: [{ id: "main", tools: { alsoAllow: ["read"] } }] }, + }), + ).toMatchObject({ + source: "tools.profile", + fix: 'Add agents.list[0].tools.alsoAllow: ["skill_workshop"].', + }); + }); + + it("names the matching provider profile source", () => { + expect( + detect({ + agents: { defaults: { model: { primary: "openai/gpt-5.5" } } }, + tools: { byProvider: { openai: { profile: "messaging" } } }, + }), + ).toMatchObject({ + source: 'tools.byProvider["openai"].profile', + fix: 'Add tools.byProvider["openai"].alsoAllow: ["skill_workshop"].', + }); + }); + + it("targets the effective agent-scoped provider profile grant owner", () => { + expect( + detect({ + agents: { + defaults: { model: { primary: "openai/gpt-5.5" } }, + list: [ + { + id: "main", + tools: { byProvider: { openai: { alsoAllow: ["read"] } } }, + }, + ], + }, + tools: { byProvider: { openai: { profile: "messaging" } } }, + }), + ).toMatchObject({ + source: 'tools.byProvider["openai"].profile', + fix: 'Add agents.list[0].tools.byProvider["openai"].alsoAllow: ["skill_workshop"].', + }); + }); + + it("names the matching agent provider allowlist source", () => { + expect( + detect({ + agents: { + defaults: { model: { primary: "openai/gpt-5.5" } }, + list: [ + { + id: "main", + tools: { byProvider: { openai: { allow: ["read"] } } }, + }, + ], + }, + }), + ).toMatchObject({ + source: 'agents.list[0].tools.byProvider["openai"].allow', + fix: 'Add "skill_workshop" to agents.list[0].tools.byProvider["openai"].allow.', + }); + }); + + it("names an explicit deny and its removal", () => { + expect(detect({ tools: { deny: ["skill_workshop"] } })).toMatchObject({ + source: "tools.deny", + detail: 'tools.deny denies "skill_workshop".', + fix: 'Remove the matching "skill_workshop" deny entry from tools.deny.', + }); + }); +}); diff --git a/src/skills/workshop/tool-policy-diagnostic.ts b/src/skills/workshop/tool-policy-diagnostic.ts new file mode 100644 index 000000000000..26f2637430f4 --- /dev/null +++ b/src/skills/workshop/tool-policy-diagnostic.ts @@ -0,0 +1,252 @@ +// Skill Workshop diagnostics explain which effective policy layer hides its agent tool. +import { resolveDefaultAgentId } from "../../agents/agent-scope.js"; +import { + resolveConversationCapabilityProfile, + type ResolvedConversationCapabilityProfile, +} from "../../agents/conversation-capability-profile.js"; +import { applyFinalEffectiveToolPolicy } from "../../agents/embedded-agent-runner/effective-tool-policy.js"; +import { resolveDefaultModelForAgent } from "../../agents/model-selection.js"; +import { resolveProviderToolPolicyEntry } from "../../agents/provider-tool-policy.js"; +import { isToolAllowedByPolicyName } from "../../agents/tool-policy-match.js"; +import type { ToolPolicyFilterEvent } from "../../agents/tool-policy-pipeline.js"; +import type { AnyAgentTool } from "../../agents/tools/common.js"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import type { AgentToolsConfig } from "../../config/types.tools.js"; +import { normalizeAgentId } from "../../routing/session-key.js"; + +const SKILL_WORKSHOP_TOOL_NAME = "skill_workshop"; + +export type SkillWorkshopToolPolicyDiagnostic = { + agentId: string; + source: string; + detail: string; + fix: string; + message: string; +}; + +type AgentToolsLocation = { + path: string; + tools: AgentToolsConfig; +}; + +function findAgentTools(config: OpenClawConfig, agentId: string): AgentToolsLocation | undefined { + const index = config.agents?.list?.findIndex( + (entry) => normalizeAgentId(entry.id) === normalizeAgentId(agentId), + ); + const tools = index !== undefined && index >= 0 ? config.agents?.list?.[index]?.tools : undefined; + return index !== undefined && index >= 0 && tools + ? { path: `agents.list[${index}].tools`, tools } + : undefined; +} + +function providerPolicyPath(params: { + tools: AgentToolsConfig | OpenClawConfig["tools"] | undefined; + basePath: string; + capabilityProfile: ResolvedConversationCapabilityProfile; +}): { path: string; profile?: string; ownsAlsoAllow: boolean } | undefined { + const entry = resolveProviderToolPolicyEntry({ + byProvider: params.tools?.byProvider, + modelProvider: params.capabilityProfile.model.provider, + modelId: params.capabilityProfile.model.id, + }); + return entry + ? { + path: `${params.basePath}.byProvider[${JSON.stringify(entry.key)}]`, + profile: entry.policy.profile, + ownsAlsoAllow: Array.isArray(entry.policy.alsoAllow), + } + : undefined; +} + +function profileAlsoAllowPath(params: { + config: OpenClawConfig; + agent: AgentToolsLocation | undefined; + profileOwnerPath: string; +}): string { + if (Array.isArray(params.agent?.tools.alsoAllow)) { + return `${params.agent.path}.alsoAllow`; + } + if (Array.isArray(params.config.tools?.alsoAllow)) { + return "tools.alsoAllow"; + } + return `${params.profileOwnerPath}.alsoAllow`; +} + +function providerProfileAlsoAllowPath(params: { + globalProvider: ReturnType; + agentProvider: ReturnType; + profileOwnerPath: string; +}): string { + if (params.agentProvider?.ownsAlsoAllow) { + return `${params.agentProvider.path}.alsoAllow`; + } + if (params.globalProvider?.ownsAlsoAllow) { + return `${params.globalProvider.path}.alsoAllow`; + } + return `${params.profileOwnerPath}.alsoAllow`; +} + +function policyDeniesWorkshop(event: ToolPolicyFilterEvent): boolean { + return !isToolAllowedByPolicyName(SKILL_WORKSHOP_TOOL_NAME, { + deny: event.policy.deny, + }); +} + +function describeExclusion(params: { + config: OpenClawConfig; + agentId: string; + capabilityProfile: ResolvedConversationCapabilityProfile; + event: ToolPolicyFilterEvent; +}): Pick { + const label = params.event.step.label; + const agent = findAgentTools(params.config, params.agentId); + const globalProvider = providerPolicyPath({ + tools: params.config.tools, + basePath: "tools", + capabilityProfile: params.capabilityProfile, + }); + const agentProvider = providerPolicyPath({ + tools: agent?.tools, + basePath: agent?.path ?? "agents.list[].tools", + capabilityProfile: params.capabilityProfile, + }); + + if (label.startsWith("tools.profile")) { + const policyPath = agent?.tools.profile ? agent.path : "tools"; + const source = `${policyPath}.profile`; + const grant = profileAlsoAllowPath({ + config: params.config, + agent, + profileOwnerPath: policyPath, + }); + return { + source, + detail: `${source}: ${JSON.stringify(params.capabilityProfile.policy.profile ?? "unknown")} does not include ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}.`, + fix: `Add ${grant}: [${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}].`, + }; + } + + if (label.startsWith("tools.byProvider.profile")) { + const policyPath = agentProvider?.profile ? agentProvider.path : globalProvider?.path; + const source = policyPath ? `${policyPath}.profile` : "tools.byProvider.profile"; + const grant = policyPath + ? providerProfileAlsoAllowPath({ + globalProvider, + agentProvider, + profileOwnerPath: policyPath, + }) + : "the matching tools.byProvider alsoAllow"; + return { + source, + detail: `${source}: ${JSON.stringify(params.capabilityProfile.policy.providerProfile ?? "unknown")} does not include ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}.`, + fix: policyPath + ? `Add ${grant}: [${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}].` + : `Add ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)} to ${grant} list.`, + }; + } + + const normalizedLabel = label.startsWith(`agents.${params.agentId}.tools.byProvider`) + ? label.replace( + `agents.${params.agentId}.tools.byProvider`, + agentProvider?.path ?? `${agent?.path ?? "agents.list[].tools"}.byProvider`, + ) + : label.startsWith("tools.byProvider") + ? label.replace("tools.byProvider", globalProvider?.path ?? "tools.byProvider") + : label + .replace(`agents.${params.agentId}.tools`, agent?.path ?? "agents.list[].tools") + .replace("agent tools", agent?.path ?? "agents.list[].tools"); + if (policyDeniesWorkshop(params.event)) { + const source = normalizedLabel.replace(/\.allow$/, ".deny"); + return { + source, + detail: `${source} denies ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}.`, + fix: `Remove the matching ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)} deny entry from ${source}.`, + }; + } + return { + source: normalizedLabel, + detail: `${normalizedLabel} does not include ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)}.`, + fix: `Add ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)} to ${normalizedLabel}.`, + }; +} + +export type SkillWorkshopToolPolicyAvailability = { + available: boolean; + exclusion?: ToolPolicyFilterEvent; +}; + +function makeSkillWorkshopPolicyProbe(): AnyAgentTool { + return { + name: SKILL_WORKSHOP_TOOL_NAME, + label: SKILL_WORKSHOP_TOOL_NAME, + description: "Skill Workshop policy availability probe.", + parameters: { type: "object", properties: {} }, + execute: async () => ({ content: [], details: {} }), + } as AnyAgentTool; +} + +/** Applies the real final tool-policy composition used by agent sessions and /learn. */ +export function resolveSkillWorkshopToolPolicyAvailability(params: { + config: OpenClawConfig; + conversationCapabilityProfile: ResolvedConversationCapabilityProfile; +}): SkillWorkshopToolPolicyAvailability { + let exclusion: ToolPolicyFilterEvent | undefined; + const tools = applyFinalEffectiveToolPolicy({ + bundledTools: [makeSkillWorkshopPolicyProbe()], + config: params.config, + conversationCapabilityProfile: params.conversationCapabilityProfile, + warn: () => {}, + toolPolicyAuditLogLevel: "debug", + onFilter: (event) => { + if ( + !exclusion && + event.before.some((tool) => tool.name === SKILL_WORKSHOP_TOOL_NAME) && + !event.after.some((tool) => tool.name === SKILL_WORKSHOP_TOOL_NAME) + ) { + exclusion = event; + } + }, + }); + return { + available: tools.some((tool) => tool.name === SKILL_WORKSHOP_TOOL_NAME), + ...(exclusion ? { exclusion } : {}), + }; +} + +/** Returns an actionable diagnostic when an active Workshop tool is policy-hidden. */ +export function detectSkillWorkshopToolPolicyDiagnostic(params: { + config: OpenClawConfig; + workshopEnabled: boolean; + agentId?: string; +}): SkillWorkshopToolPolicyDiagnostic | null { + if (!params.workshopEnabled) { + return null; + } + const agentId = normalizeAgentId(params.agentId ?? resolveDefaultAgentId(params.config)); + const model = resolveDefaultModelForAgent({ cfg: params.config, agentId }); + const capabilityProfile = resolveConversationCapabilityProfile({ + config: params.config, + agentId, + modelProvider: model.provider, + modelId: model.model, + }); + const availability = resolveSkillWorkshopToolPolicyAvailability({ + config: params.config, + conversationCapabilityProfile: capabilityProfile, + }); + if (availability.available || !availability.exclusion) { + return null; + } + const explanation = describeExclusion({ + config: params.config, + agentId, + capabilityProfile, + event: availability.exclusion, + }); + const prefix = `Skill Workshop is active, but ${JSON.stringify(SKILL_WORKSHOP_TOOL_NAME)} is hidden for agent ${JSON.stringify(agentId)}:`; + return { + agentId, + ...explanation, + message: `${prefix} ${explanation.detail} ${explanation.fix}`, + }; +}