mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 21:31:35 +00:00
300 lines
9.5 KiB
TypeScript
300 lines
9.5 KiB
TypeScript
// Verifies commitment extraction prompts and parsed model results.
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import type { OpenClawConfig } from "../config/config.js";
|
|
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
|
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
|
import {
|
|
buildCommitmentExtractionPrompt,
|
|
parseCommitmentExtractionOutput,
|
|
persistCommitmentExtractionResult,
|
|
} from "./extraction.js";
|
|
import { validateCommitmentCandidates } from "./extraction.test-support.js";
|
|
import { readCommitmentsForTest } from "./store.test-utils.js";
|
|
import type { CommitmentCandidate, CommitmentExtractionItem } from "./types.js";
|
|
|
|
vi.mock("./config.js", async (importOriginal) => ({
|
|
...(await importOriginal<typeof import("./config.js")>()),
|
|
resolveCommitmentsConfig: () => ({
|
|
enabled: true,
|
|
maxPerDay: 3,
|
|
extraction: {
|
|
debounceMs: 15_000,
|
|
batchMaxItems: 8,
|
|
queueMaxItems: 64,
|
|
confidenceThreshold: 0.72,
|
|
careConfidenceThreshold: 0.86,
|
|
timeoutSeconds: 45,
|
|
},
|
|
}),
|
|
}));
|
|
|
|
describe("commitment extraction", () => {
|
|
const tmpDirs: string[] = [];
|
|
let stateDirEnvSnapshot: ReturnType<typeof captureEnv> | undefined;
|
|
const nowMs = Date.parse("2026-04-29T16:00:00.000Z");
|
|
|
|
afterEach(async () => {
|
|
closeOpenClawStateDatabaseForTest();
|
|
vi.restoreAllMocks();
|
|
vi.unstubAllEnvs();
|
|
stateDirEnvSnapshot?.restore();
|
|
stateDirEnvSnapshot = undefined;
|
|
await Promise.all(tmpDirs.map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
|
tmpDirs.length = 0;
|
|
});
|
|
|
|
async function createConfig(): Promise<OpenClawConfig> {
|
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-commitments-"));
|
|
tmpDirs.push(tmpDir);
|
|
stateDirEnvSnapshot ??= captureEnv(["OPENCLAW_STATE_DIR"]);
|
|
setTestEnvValue("OPENCLAW_STATE_DIR", tmpDir);
|
|
return {};
|
|
}
|
|
|
|
function item(overrides?: Partial<CommitmentExtractionItem>): CommitmentExtractionItem {
|
|
return {
|
|
itemId: "turn-1",
|
|
nowMs,
|
|
timezone: "America/Los_Angeles",
|
|
agentId: "main",
|
|
sessionKey: "agent:main:telegram:user-1",
|
|
channel: "telegram",
|
|
to: "15551234567",
|
|
userText: "I have an interview tomorrow.",
|
|
assistantText: "Good luck. I hope it goes well.",
|
|
existingPending: [],
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function candidate(overrides?: Partial<CommitmentCandidate>): CommitmentCandidate {
|
|
return {
|
|
itemId: "turn-1",
|
|
kind: "event_check_in",
|
|
sensitivity: "routine",
|
|
source: "inferred_user_context",
|
|
reason: "The user said they had an interview tomorrow.",
|
|
suggestedText: "How did the interview go?",
|
|
dedupeKey: "interview:2026-04-30",
|
|
confidence: 0.91,
|
|
dueWindow: {
|
|
earliest: "2026-04-30T17:00:00.000Z",
|
|
latest: "2026-04-30T23:00:00.000Z",
|
|
timezone: "America/Los_Angeles",
|
|
},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function expectSingleValidCandidate(
|
|
valid: ReturnType<typeof validateCommitmentCandidates>,
|
|
): ReturnType<typeof validateCommitmentCandidates>[number] {
|
|
expect(valid).toHaveLength(1);
|
|
const [entry] = valid;
|
|
if (!entry) {
|
|
throw new Error("Expected one valid commitment candidate");
|
|
}
|
|
return entry;
|
|
}
|
|
|
|
it("parses valid candidates from JSON output with surrounding text", () => {
|
|
const parsed = parseCommitmentExtractionOutput(
|
|
`noise {"candidates":[${JSON.stringify(candidate())}]} trailing`,
|
|
);
|
|
|
|
expect(parsed.candidates).toHaveLength(1);
|
|
expect(parsed.candidates[0]?.kind).toBe("event_check_in");
|
|
expect(parsed.candidates[0]?.suggestedText).toBe("How did the interview go?");
|
|
});
|
|
|
|
it("omits routing scope identifiers from extractor prompts", () => {
|
|
const prompt = buildCommitmentExtractionPrompt({
|
|
items: [
|
|
item({
|
|
itemId: "public-item-1",
|
|
agentId: "agent-secret",
|
|
sessionKey: "session-secret",
|
|
channel: "channel-secret",
|
|
accountId: "account-secret",
|
|
to: "+15551234567",
|
|
threadId: "thread-secret",
|
|
}),
|
|
],
|
|
});
|
|
|
|
expect(prompt).toContain("public-item-1");
|
|
expect(prompt).not.toContain("agent-secret");
|
|
expect(prompt).not.toContain("session-secret");
|
|
expect(prompt).not.toContain("channel-secret");
|
|
expect(prompt).not.toContain("account-secret");
|
|
expect(prompt).not.toContain("+15551234567");
|
|
expect(prompt).not.toContain("thread-secret");
|
|
});
|
|
|
|
it("does not throw on out-of-range extraction prompt timestamps", () => {
|
|
vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-05-30T12:00:00.000Z"));
|
|
|
|
const prompt = buildCommitmentExtractionPrompt({
|
|
items: [
|
|
item({
|
|
nowMs: 8_640_000_000_000_001,
|
|
existingPending: [
|
|
{
|
|
kind: "event_check_in",
|
|
reason: "valid pending",
|
|
dedupeKey: "valid",
|
|
earliestMs: Date.parse("2026-05-31T12:00:00.000Z"),
|
|
latestMs: Date.parse("2026-05-31T13:00:00.000Z"),
|
|
},
|
|
{
|
|
kind: "open_loop",
|
|
reason: "invalid pending",
|
|
dedupeKey: "invalid",
|
|
earliestMs: 8_640_000_000_000_001,
|
|
latestMs: 8_640_000_000_000_001,
|
|
},
|
|
],
|
|
}),
|
|
],
|
|
});
|
|
|
|
expect(prompt).toContain('"now":"2026-05-30T12:00:00.000Z"');
|
|
expect(prompt).toContain('"dedupeKey":"valid"');
|
|
expect(prompt).not.toContain('"dedupeKey":"invalid"');
|
|
});
|
|
|
|
it("rejects disabled, low-confidence, and non-future candidates", () => {
|
|
const cfg: OpenClawConfig = {};
|
|
const valid = validateCommitmentCandidates({
|
|
cfg,
|
|
items: [item()],
|
|
result: {
|
|
candidates: [
|
|
candidate(),
|
|
candidate({ dedupeKey: "low-confidence", confidence: 0.5 }),
|
|
candidate({
|
|
dedupeKey: "past",
|
|
dueWindow: { earliest: "2026-04-29T15:00:00.000Z" },
|
|
}),
|
|
],
|
|
},
|
|
});
|
|
|
|
expect(valid.map((entry) => entry.candidate.dedupeKey)).toEqual(["interview:2026-04-30"]);
|
|
});
|
|
|
|
it("rejects calendar-invalid due timestamps", () => {
|
|
const valid = validateCommitmentCandidates({
|
|
items: [item()],
|
|
result: {
|
|
candidates: [
|
|
candidate({
|
|
dedupeKey: "invalid-earliest",
|
|
dueWindow: { earliest: "2026-04-31T17:00:00.000Z" },
|
|
}),
|
|
candidate({
|
|
dedupeKey: "invalid-latest",
|
|
dueWindow: {
|
|
earliest: "2026-04-30T17:00:00.000Z",
|
|
latest: "2026-04-31T23:00:00.000Z",
|
|
},
|
|
}),
|
|
],
|
|
},
|
|
nowMs,
|
|
});
|
|
|
|
const validCandidate = expectSingleValidCandidate(valid);
|
|
expect(validCandidate.candidate.dedupeKey).toBe("invalid-latest");
|
|
expect(validCandidate.earliestMs).toBe(Date.parse("2026-04-30T17:00:00.000Z"));
|
|
expect(validCandidate.latestMs).toBe(validCandidate.earliestMs + 12 * 60 * 60 * 1000);
|
|
});
|
|
|
|
it("accepts calendar-valid leap-day, offset, and lowercase RFC 3339 timestamps", () => {
|
|
const valid = validateCommitmentCandidates({
|
|
items: [item()],
|
|
result: {
|
|
candidates: [
|
|
candidate({
|
|
dedupeKey: "leap-day-offset",
|
|
dueWindow: {
|
|
earliest: "2028-02-29t09:00:00-08:00",
|
|
latest: "2028-02-29t20:00:00z",
|
|
},
|
|
}),
|
|
],
|
|
},
|
|
nowMs,
|
|
});
|
|
|
|
const validCandidate = expectSingleValidCandidate(valid);
|
|
expect(validCandidate.earliestMs).toBe(Date.parse("2028-02-29t09:00:00-08:00"));
|
|
expect(validCandidate.latestMs).toBe(Date.parse("2028-02-29t20:00:00z"));
|
|
});
|
|
|
|
it("clamps inferred due time to at least one heartbeat interval after write time", () => {
|
|
const writeMs = nowMs + 5_000;
|
|
const valid = validateCommitmentCandidates({
|
|
cfg: {
|
|
agents: {
|
|
defaults: {
|
|
heartbeat: { every: "10m" },
|
|
},
|
|
},
|
|
},
|
|
items: [item()],
|
|
result: {
|
|
candidates: [
|
|
candidate({
|
|
dedupeKey: "too-soon",
|
|
dueWindow: {
|
|
earliest: new Date(nowMs + 60_000).toISOString(),
|
|
latest: new Date(nowMs + 120_000).toISOString(),
|
|
},
|
|
}),
|
|
],
|
|
},
|
|
nowMs: writeMs,
|
|
});
|
|
|
|
const validCandidate = expectSingleValidCandidate(valid);
|
|
expect(validCandidate.earliestMs).toBe(writeMs + 10 * 60_000);
|
|
expect(validCandidate.latestMs).toBe(writeMs + 10 * 60_000 + 12 * 60 * 60_000);
|
|
});
|
|
|
|
it("persists inferred commitments and dedupes by scope and dedupe key", async () => {
|
|
const cfg = await createConfig();
|
|
const created = await persistCommitmentExtractionResult({
|
|
cfg,
|
|
items: [item()],
|
|
result: { candidates: [candidate()] },
|
|
nowMs,
|
|
});
|
|
const deduped = await persistCommitmentExtractionResult({
|
|
cfg,
|
|
items: [item()],
|
|
result: {
|
|
candidates: [
|
|
candidate({
|
|
reason: "Updated reason",
|
|
confidence: 0.97,
|
|
dueWindow: { earliest: "2026-04-30T18:00:00.000Z" },
|
|
}),
|
|
],
|
|
},
|
|
nowMs: nowMs + 1_000,
|
|
});
|
|
const commitments = readCommitmentsForTest();
|
|
|
|
expect(created).toHaveLength(1);
|
|
expect(deduped).toHaveLength(0);
|
|
expect(commitments).toHaveLength(1);
|
|
expect(commitments[0]?.reason).toBe("Updated reason");
|
|
expect(commitments[0]?.confidence).toBe(0.97);
|
|
expect(commitments[0]?.status).toBe("pending");
|
|
});
|
|
});
|