test: stabilize trigger handling and hook e2e tests

This commit is contained in:
Peter Steinberger
2026-03-23 11:05:43 +00:00
parent b9efba1faf
commit 6f048f59cb
8 changed files with 508 additions and 405 deletions

View File

@@ -1,7 +1,6 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { resolveSessionKey } from "../config/sessions.js";
import {
getProviderUsageMocks,
getRunEmbeddedPiAgentMock,
@@ -28,191 +27,132 @@ function getReplyFromConfigNow(getReplyFromConfig: () => GetReplyFromConfig): Ge
return getReplyFromConfig();
}
function seedUsageSummary(): void {
usageMocks.loadProviderUsageSummary.mockClear();
usageMocks.loadProviderUsageSummary.mockResolvedValue({
updatedAt: 0,
providers: [
{
provider: "anthropic",
displayName: "Anthropic",
windows: [
{
label: "5h",
usedPercent: 20,
},
],
},
],
});
}
export function registerTriggerHandlingUsageSummaryCases(params: {
getReplyFromConfig: () => GetReplyFromConfig;
}): void {
describe("usage and status command handling", () => {
it("handles status, usage cycles, and auth-profile status details", async () => {
it("shows status without invoking the agent", async () => {
await withTempHome(async (home) => {
const runEmbeddedPiAgentMock = getRunEmbeddedPiAgentMock();
const getReplyFromConfig = getReplyFromConfigNow(params.getReplyFromConfig);
usageMocks.loadProviderUsageSummary.mockClear();
usageMocks.loadProviderUsageSummary.mockResolvedValue({
updatedAt: 0,
providers: [
{
provider: "anthropic",
displayName: "Anthropic",
windows: [
{
label: "5h",
usedPercent: 20,
},
],
},
],
});
seedUsageSummary();
{
const res = await getReplyFromConfig(
{
Body: "/status",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
{},
makeCfg(home),
);
const text = Array.isArray(res) ? res[0]?.text : res?.text;
expect(text).toContain("Model:");
expect(text).toContain("OpenClaw");
expect(runEmbeddedPiAgentMock).not.toHaveBeenCalled();
}
{
const cfg = makeCfg(home);
cfg.session = { ...cfg.session, store: join(home, "usage-cycle.sessions.json") };
const usageStorePath = requireSessionStorePath(cfg);
const r0 = await getReplyFromConfig(
{
Body: "/usage on",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r0) ? r0[0]?.text : r0?.text) ?? "")).toContain(
"Usage footer: tokens",
);
const r1 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r1) ? r1[0]?.text : r1?.text) ?? "")).toContain(
"Usage footer: full",
);
const r2 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r2) ? r2[0]?.text : r2?.text) ?? "")).toContain(
"Usage footer: off",
);
const r3 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r3) ? r3[0]?.text : r3?.text) ?? "")).toContain(
"Usage footer: tokens",
);
const finalStore = await readSessionStore(usageStorePath);
expect(pickFirstStoreEntry<{ responseUsage?: string }>(finalStore)?.responseUsage).toBe(
"tokens",
);
expect(runEmbeddedPiAgentMock).not.toHaveBeenCalled();
}
{
runEmbeddedPiAgentMock.mockClear();
const cfg = makeCfg(home);
cfg.session = { ...cfg.session, store: join(home, "auth-profile-status.sessions.json") };
const agentDir = join(home, ".openclaw", "agents", "main", "agent");
await mkdir(agentDir, { recursive: true });
await writeFile(
join(agentDir, "auth-profiles.json"),
JSON.stringify(
{
version: 1,
profiles: {
"anthropic:work": {
type: "api_key",
provider: "anthropic",
key: "sk-test-1234567890abcdef",
},
},
lastGood: { anthropic: "anthropic:work" },
},
null,
2,
),
);
const sessionKey = resolveSessionKey("per-sender", {
From: "+1002",
const res = await getReplyFromConfig(
{
Body: "/status",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
} as Parameters<typeof resolveSessionKey>[1]);
await writeFile(
requireSessionStorePath(cfg),
JSON.stringify(
{
[sessionKey]: {
sessionId: "session-auth",
updatedAt: Date.now(),
authProfileOverride: "anthropic:work",
},
},
null,
2,
),
);
SenderE164: "+1000",
CommandAuthorized: true,
},
{},
makeCfg(home),
);
const res = await getReplyFromConfig(
{
Body: "/status",
From: "+1002",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1002",
CommandAuthorized: true,
},
{},
cfg,
);
const text = Array.isArray(res) ? res[0]?.text : res?.text;
expect(text).toContain("api-key");
expect(text).not.toContain("sk-test");
expect(text).not.toContain("abcdef");
expect(text).not.toContain("1234567890abcdef"); // pragma: allowlist secret
expect(text).toContain("(anthropic:work)");
expect(text).not.toContain("mixed");
expect(runEmbeddedPiAgentMock).not.toHaveBeenCalled();
}
const text = Array.isArray(res) ? res[0]?.text : res?.text;
expect(text).toContain("Model:");
expect(text).toContain("OpenClaw");
expect(runEmbeddedPiAgentMock).not.toHaveBeenCalled();
});
});
it("cycles usage footer modes and persists the final selection", async () => {
await withTempHome(async (home) => {
const runEmbeddedPiAgentMock = getRunEmbeddedPiAgentMock();
const getReplyFromConfig = getReplyFromConfigNow(params.getReplyFromConfig);
const cfg = makeCfg(home);
cfg.session = { ...cfg.session, store: join(home, "usage-cycle.sessions.json") };
const usageStorePath = requireSessionStorePath(cfg);
const r0 = await getReplyFromConfig(
{
Body: "/usage on",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r0) ? r0[0]?.text : r0?.text) ?? "")).toContain(
"Usage footer: tokens",
);
const r1 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r1) ? r1[0]?.text : r1?.text) ?? "")).toContain(
"Usage footer: full",
);
const r2 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r2) ? r2[0]?.text : r2?.text) ?? "")).toContain(
"Usage footer: off",
);
const r3 = await getReplyFromConfig(
{
Body: "/usage",
From: "+1000",
To: "+2000",
Provider: "whatsapp",
SenderE164: "+1000",
CommandAuthorized: true,
},
undefined,
cfg,
);
expect(String((Array.isArray(r3) ? r3[0]?.text : r3?.text) ?? "")).toContain(
"Usage footer: tokens",
);
const finalStore = await readSessionStore(usageStorePath);
expect(pickFirstStoreEntry<{ responseUsage?: string }>(finalStore)?.responseUsage).toBe(
"tokens",
);
expect(runEmbeddedPiAgentMock).not.toHaveBeenCalled();
});
});
});

View File

@@ -2,7 +2,6 @@ import fs from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { loadSessionStore, resolveSessionKey } from "../config/sessions.js";
import { getReplyFromConfig } from "./reply.js";
import { registerGroupIntroPromptCases } from "./reply.triggers.group-intro-prompts.cases.js";
import { registerTriggerHandlingUsageSummaryCases } from "./reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.cases.js";
import {
@@ -10,7 +9,7 @@ import {
getAbortEmbeddedPiRunMock,
getCompactEmbeddedPiSessionMock,
getRunEmbeddedPiAgentMock,
installTriggerHandlingE2eTestHooks,
installTriggerHandlingReplyHarness,
MAIN_SESSION_KEY,
makeCfg,
mockRunEmbeddedPiAgentOk,
@@ -21,6 +20,8 @@ import {
import { enqueueFollowupRun, getFollowupQueueDepth, type FollowupRun } from "./reply/queue.js";
import { HEARTBEAT_TOKEN } from "./tokens.js";
type GetReplyFromConfig = typeof import("./reply.js").getReplyFromConfig;
vi.mock("./reply/agent-runner.runtime.js", () => ({
runReplyAgent: async (params: {
commandBody: string;
@@ -75,7 +76,10 @@ vi.mock("./reply/agent-runner.runtime.js", () => ({
},
}));
installTriggerHandlingE2eTestHooks();
let getReplyFromConfig!: GetReplyFromConfig;
installTriggerHandlingReplyHarness((impl) => {
getReplyFromConfig = impl;
});
const BASE_MESSAGE = {
Body: "hello",
@@ -83,7 +87,7 @@ const BASE_MESSAGE = {
To: "+2000",
} as const;
function maybeReplyText(reply: Awaited<ReturnType<typeof getReplyFromConfig>>) {
function maybeReplyText(reply: Awaited<ReturnType<GetReplyFromConfig>>) {
return Array.isArray(reply) ? reply[0]?.text : reply?.text;
}

View File

@@ -3,7 +3,10 @@ import fs from "node:fs/promises";
import os from "node:os";
import { join } from "node:path";
import { afterAll, afterEach, beforeAll, expect, vi } from "vitest";
import { clearRuntimeAuthProfileStoreSnapshots } from "../agents/auth-profiles.js";
import { resetCliCredentialCachesForTest } from "../agents/cli-credentials.js";
import type { OpenClawConfig } from "../config/config.js";
import { resetProviderRuntimeHookCacheForTest } from "../plugins/provider-runtime.js";
// Avoid exporting vitest mock types (TS2742 under pnpm + d.ts emit).
// oxlint-disable-next-line typescript/no-explicit-any
@@ -107,6 +110,20 @@ const installModelCatalogMock = () =>
installModelCatalogMock();
vi.doMock("../agents/model-catalog.runtime.js", () => ({
loadModelCatalog: (...args: unknown[]) => modelCatalogMocks.loadModelCatalog(...args),
}));
vi.doMock("../plugins/provider-runtime.runtime.js", () => ({
augmentModelCatalogWithProviderPlugins: async (params: { catalog?: unknown[] }) =>
params.catalog ?? [],
buildProviderAuthDoctorHintWithPlugin: () => undefined,
buildProviderMissingAuthMessageWithPlugin: () => undefined,
formatProviderAuthProfileApiKeyWithPlugin: (params: { apiKey?: string }) => params.apiKey,
prepareProviderRuntimeAuth: async () => undefined,
refreshProviderOAuthCredentialWithPlugin: async () => undefined,
}));
const modelFallbackMocks = getSharedMocks("openclaw.trigger-handling.model-fallback-mocks", () => ({
runWithModelFallback: vi.fn(
async (params: {
@@ -131,6 +148,10 @@ const installModelFallbackMock = () =>
installModelFallbackMock();
vi.doMock("../infra/git-commit.js", () => ({
resolveCommitHash: vi.fn(() => "abcdef0"),
}));
const webSessionMocks = getSharedMocks("openclaw.trigger-handling.web-session-mocks", () => ({
webAuthExists: vi.fn().mockResolvedValue(true),
getWebAuthAgeMs: vi.fn().mockReturnValue(120_000),
@@ -419,6 +440,9 @@ export async function runGreetingPromptForBareNewOrReset(params: {
export function installTriggerHandlingE2eTestHooks() {
afterEach(() => {
clearRuntimeAuthProfileStoreSnapshots();
resetCliCredentialCachesForTest();
resetProviderRuntimeHookCacheForTest();
vi.clearAllMocks();
});
}

View File

@@ -29,6 +29,29 @@ import type { CommandContext } from "./commands-types.js";
import { getFollowupQueueDepth, resolveQueueSettings } from "./queue.js";
import { resolveSubagentLabel } from "./subagents-utils.js";
// Some usage endpoints only work with CLI/session OAuth tokens, not API keys.
// Skip those probes when the active auth mode cannot satisfy the endpoint.
const USAGE_OAUTH_ONLY_PROVIDERS = new Set([
"anthropic",
"github-copilot",
"google-gemini-cli",
"openai-codex",
]);
function shouldLoadUsageSummary(params: {
provider?: string;
selectedModelAuth?: string;
}): boolean {
if (!params.provider) {
return false;
}
if (!USAGE_OAUTH_ONLY_PROVIDERS.has(params.provider)) {
return true;
}
const auth = params.selectedModelAuth?.trim().toLowerCase();
return Boolean(auth?.startsWith("oauth") || auth?.startsWith("token"));
}
export async function buildStatusReply(params: {
cfg: OpenClawConfig;
command: CommandContext;
@@ -78,6 +101,25 @@ export async function buildStatusReply(params: {
? resolveSessionAgentId({ sessionKey, config: cfg })
: resolveDefaultAgentId(cfg);
const statusAgentDir = resolveAgentDir(cfg, statusAgentId);
const modelRefs = resolveSelectedAndActiveModel({
selectedProvider: provider,
selectedModel: model,
sessionEntry,
});
const selectedModelAuth = resolveModelAuthLabel({
provider,
cfg,
sessionEntry,
agentDir: statusAgentDir,
});
const activeModelAuth = modelRefs.activeDiffers
? resolveModelAuthLabel({
provider: modelRefs.active.provider,
cfg,
sessionEntry,
agentDir: statusAgentDir,
})
: selectedModelAuth;
const currentUsageProvider = (() => {
try {
return resolveUsageProviderId(provider);
@@ -86,12 +128,32 @@ export async function buildStatusReply(params: {
}
})();
let usageLine: string | null = null;
if (currentUsageProvider) {
if (
currentUsageProvider &&
shouldLoadUsageSummary({
provider: currentUsageProvider,
selectedModelAuth,
})
) {
try {
const usageSummary = await loadProviderUsageSummary({
timeoutMs: 3500,
providers: [currentUsageProvider],
agentDir: statusAgentDir,
const usageSummaryTimeoutMs = 3500;
let usageTimeout: NodeJS.Timeout | undefined;
const usageSummary = await Promise.race([
loadProviderUsageSummary({
timeoutMs: usageSummaryTimeoutMs,
providers: [currentUsageProvider],
agentDir: statusAgentDir,
}),
new Promise<never>((_, reject) => {
usageTimeout = setTimeout(
() => reject(new Error("usage summary timeout")),
usageSummaryTimeoutMs,
);
}),
]).finally(() => {
if (usageTimeout) {
clearTimeout(usageTimeout);
}
});
const usageEntry = usageSummary.providers[0];
if (usageEntry && !usageEntry.error && usageEntry.windows.length > 0) {
@@ -143,25 +205,6 @@ export async function buildStatusReply(params: {
const groupActivation = isGroup
? (normalizeGroupActivation(sessionEntry?.groupActivation) ?? defaultGroupActivation())
: undefined;
const modelRefs = resolveSelectedAndActiveModel({
selectedProvider: provider,
selectedModel: model,
sessionEntry,
});
const selectedModelAuth = resolveModelAuthLabel({
provider,
cfg,
sessionEntry,
agentDir: statusAgentDir,
});
const activeModelAuth = modelRefs.activeDiffers
? resolveModelAuthLabel({
provider: modelRefs.active.provider,
cfg,
sessionEntry,
agentDir: statusAgentDir,
})
: selectedModelAuth;
const agentDefaults = cfg.agents?.defaults ?? {};
const effectiveFastMode =
resolvedFastMode ??

View File

@@ -314,88 +314,100 @@ export async function handleDirectiveOnly(
directives.elevatedLevel !== undefined &&
elevatedEnabled &&
elevatedAllowed;
const shouldPersistSessionEntry =
(directives.hasThinkDirective && Boolean(directives.thinkLevel)) ||
(directives.hasFastDirective && directives.fastMode !== undefined) ||
(directives.hasVerboseDirective && Boolean(directives.verboseLevel)) ||
(directives.hasReasoningDirective && Boolean(directives.reasoningLevel)) ||
(directives.hasElevatedDirective && Boolean(directives.elevatedLevel)) ||
(directives.hasExecDirective && directives.hasExecOptions && allowInternalExecPersistence) ||
Boolean(modelSelection) ||
directives.hasQueueDirective ||
shouldDowngradeXHigh;
const fastModeChanged =
directives.hasFastDirective &&
directives.fastMode !== undefined &&
directives.fastMode !== currentFastMode;
let reasoningChanged =
directives.hasReasoningDirective && directives.reasoningLevel !== undefined;
if (directives.hasThinkDirective && directives.thinkLevel) {
sessionEntry.thinkingLevel = directives.thinkLevel;
}
if (directives.hasFastDirective && directives.fastMode !== undefined) {
sessionEntry.fastMode = directives.fastMode;
}
if (shouldDowngradeXHigh) {
sessionEntry.thinkingLevel = "high";
}
if (directives.hasVerboseDirective && directives.verboseLevel) {
applyVerboseOverride(sessionEntry, directives.verboseLevel);
}
if (directives.hasReasoningDirective && directives.reasoningLevel) {
if (directives.reasoningLevel === "off") {
// Persist explicit off so it overrides model-capability defaults.
sessionEntry.reasoningLevel = "off";
} else {
sessionEntry.reasoningLevel = directives.reasoningLevel;
if (shouldPersistSessionEntry) {
if (directives.hasThinkDirective && directives.thinkLevel) {
sessionEntry.thinkingLevel = directives.thinkLevel;
}
reasoningChanged =
directives.reasoningLevel !== prevReasoningLevel && directives.reasoningLevel !== undefined;
}
if (directives.hasElevatedDirective && directives.elevatedLevel) {
// Unlike other toggles, elevated defaults can be "on".
// Persist "off" explicitly so `/elevated off` actually overrides defaults.
sessionEntry.elevatedLevel = directives.elevatedLevel;
elevatedChanged =
elevatedChanged ||
(directives.elevatedLevel !== prevElevatedLevel && directives.elevatedLevel !== undefined);
}
if (directives.hasExecDirective && directives.hasExecOptions && allowInternalExecPersistence) {
if (directives.execHost) {
sessionEntry.execHost = directives.execHost;
if (directives.hasFastDirective && directives.fastMode !== undefined) {
sessionEntry.fastMode = directives.fastMode;
}
if (directives.execSecurity) {
sessionEntry.execSecurity = directives.execSecurity;
if (shouldDowngradeXHigh) {
sessionEntry.thinkingLevel = "high";
}
if (directives.execAsk) {
sessionEntry.execAsk = directives.execAsk;
if (directives.hasVerboseDirective && directives.verboseLevel) {
applyVerboseOverride(sessionEntry, directives.verboseLevel);
}
if (directives.execNode) {
sessionEntry.execNode = directives.execNode;
if (directives.hasReasoningDirective && directives.reasoningLevel) {
if (directives.reasoningLevel === "off") {
// Persist explicit off so it overrides model-capability defaults.
sessionEntry.reasoningLevel = "off";
} else {
sessionEntry.reasoningLevel = directives.reasoningLevel;
}
reasoningChanged =
directives.reasoningLevel !== prevReasoningLevel && directives.reasoningLevel !== undefined;
}
}
if (modelSelection) {
applyModelOverrideToSessionEntry({
entry: sessionEntry,
selection: modelSelection,
profileOverride,
});
}
if (directives.hasQueueDirective && directives.queueReset) {
delete sessionEntry.queueMode;
delete sessionEntry.queueDebounceMs;
delete sessionEntry.queueCap;
delete sessionEntry.queueDrop;
} else if (directives.hasQueueDirective) {
if (directives.queueMode) {
sessionEntry.queueMode = directives.queueMode;
if (directives.hasElevatedDirective && directives.elevatedLevel) {
// Unlike other toggles, elevated defaults can be "on".
// Persist "off" explicitly so `/elevated off` actually overrides defaults.
sessionEntry.elevatedLevel = directives.elevatedLevel;
elevatedChanged =
elevatedChanged ||
(directives.elevatedLevel !== prevElevatedLevel && directives.elevatedLevel !== undefined);
}
if (typeof directives.debounceMs === "number") {
sessionEntry.queueDebounceMs = directives.debounceMs;
if (directives.hasExecDirective && directives.hasExecOptions && allowInternalExecPersistence) {
if (directives.execHost) {
sessionEntry.execHost = directives.execHost;
}
if (directives.execSecurity) {
sessionEntry.execSecurity = directives.execSecurity;
}
if (directives.execAsk) {
sessionEntry.execAsk = directives.execAsk;
}
if (directives.execNode) {
sessionEntry.execNode = directives.execNode;
}
}
if (typeof directives.cap === "number") {
sessionEntry.queueCap = directives.cap;
if (modelSelection) {
applyModelOverrideToSessionEntry({
entry: sessionEntry,
selection: modelSelection,
profileOverride,
});
}
if (directives.dropPolicy) {
sessionEntry.queueDrop = directives.dropPolicy;
if (directives.hasQueueDirective && directives.queueReset) {
delete sessionEntry.queueMode;
delete sessionEntry.queueDebounceMs;
delete sessionEntry.queueCap;
delete sessionEntry.queueDrop;
} else if (directives.hasQueueDirective) {
if (directives.queueMode) {
sessionEntry.queueMode = directives.queueMode;
}
if (typeof directives.debounceMs === "number") {
sessionEntry.queueDebounceMs = directives.debounceMs;
}
if (typeof directives.cap === "number") {
sessionEntry.queueCap = directives.cap;
}
if (directives.dropPolicy) {
sessionEntry.queueDrop = directives.dropPolicy;
}
}
sessionEntry.updatedAt = Date.now();
sessionStore[sessionKey] = sessionEntry;
if (storePath) {
await updateSessionStore(storePath, (store) => {
store[sessionKey] = sessionEntry;
});
}
}
sessionEntry.updatedAt = Date.now();
sessionStore[sessionKey] = sessionEntry;
if (storePath) {
await updateSessionStore(storePath, (store) => {
store[sessionKey] = sessionEntry;
});
}
if (modelSelection) {
const nextLabel = `${modelSelection.provider}/${modelSelection.model}`;