mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-22 16:41:15 +00:00
409d13f94f
* refactor(active-memory): split runtime modules * fix(active-memory): annotate hook deadline result * refactor(active-memory): keep runtime modules bounded * fix(active-memory): use type-only recall imports * refactor(active-memory): supersede partial split * fix(gateway): type pruned worker manifest
387 lines
13 KiB
TypeScript
387 lines
13 KiB
TypeScript
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
|
|
import { getModelRef } from "./query.js";
|
|
import { runRecallSubagent } from "./recall-run.js";
|
|
import {
|
|
buildCacheKey,
|
|
buildCircuitBreakerKey,
|
|
getCachedResult,
|
|
isCircuitBreakerOpen,
|
|
recordCircuitBreakerTimeout,
|
|
resetCircuitBreaker,
|
|
scheduleMemorySearchCleanupAfterTimeout,
|
|
setCachedResult,
|
|
shouldCacheResult,
|
|
toSingleLineLogValue,
|
|
} from "./recall-state.js";
|
|
import {
|
|
buildPersistedDebugSummary,
|
|
buildPluginStatusLine,
|
|
persistPluginStatusLines,
|
|
} from "./session.js";
|
|
import {
|
|
buildSubagentRecallResult,
|
|
buildTimeoutRecallResult,
|
|
readPartialTimeoutData,
|
|
} from "./transcript-result.js";
|
|
import { watchTerminalMemorySearchResult } from "./transcript-watch.js";
|
|
import type {
|
|
ActiveMemorySearchDebug,
|
|
ActiveMemoryTranscriptSource,
|
|
ActiveRecallResult,
|
|
ResolvedActiveRecallPluginConfig,
|
|
TerminalMemorySearchWatch,
|
|
} from "./types.js";
|
|
|
|
async function maybeResolveActiveRecall(params: {
|
|
api: OpenClawPluginApi;
|
|
config: ResolvedActiveRecallPluginConfig;
|
|
agentId: string;
|
|
sessionKey?: string;
|
|
sessionId?: string;
|
|
messageProvider?: string;
|
|
channelId?: string;
|
|
query: string;
|
|
searchQuery: string;
|
|
currentModelProviderId?: string;
|
|
currentModelId?: string;
|
|
abortSignal?: AbortSignal;
|
|
}): Promise<ActiveRecallResult> {
|
|
params.abortSignal?.throwIfAborted();
|
|
const startedAt = Date.now();
|
|
const cacheKey = buildCacheKey({
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
sessionId: params.sessionId,
|
|
query: params.query,
|
|
});
|
|
const cached = getCachedResult(cacheKey);
|
|
const resolvedModelRef = getModelRef(params.api, params.agentId, params.config, {
|
|
modelProviderId: params.currentModelProviderId,
|
|
modelId: params.currentModelId,
|
|
});
|
|
const logPrefix = [
|
|
`active-memory: agent=${toSingleLineLogValue(params.agentId)}`,
|
|
`session=${toSingleLineLogValue(params.sessionKey ?? params.sessionId ?? "none")}`,
|
|
...(resolvedModelRef?.provider
|
|
? [`activeProvider=${toSingleLineLogValue(resolvedModelRef.provider)}`]
|
|
: []),
|
|
...(resolvedModelRef?.model
|
|
? [`activeModel=${toSingleLineLogValue(resolvedModelRef.model)}`]
|
|
: []),
|
|
].join(" ");
|
|
if (cached) {
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: `${buildPluginStatusLine({ result: cached, config: params.config })} cached`,
|
|
debugSummary: buildPersistedDebugSummary(cached),
|
|
searchDebug: cached.searchDebug,
|
|
});
|
|
params.abortSignal?.throwIfAborted();
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} cached status=${cached.status} summaryChars=${String(cached.summary?.length ?? 0)} queryChars=${String(params.query.length)}`,
|
|
);
|
|
}
|
|
return cached;
|
|
}
|
|
|
|
// Circuit breaker: skip recall when the same agent/model has timed out
|
|
// too many times in a row (#74054).
|
|
const cbKey = buildCircuitBreakerKey(
|
|
params.agentId,
|
|
resolvedModelRef?.provider,
|
|
resolvedModelRef?.model,
|
|
);
|
|
let timeoutCleanupScheduled = false;
|
|
const scheduleTimeoutCleanup = () => {
|
|
if (timeoutCleanupScheduled) {
|
|
return;
|
|
}
|
|
timeoutCleanupScheduled = true;
|
|
scheduleMemorySearchCleanupAfterTimeout(params.api, logPrefix, params.agentId);
|
|
};
|
|
let circuitBreakerTimeoutRecorded = false;
|
|
const recordRecallTimeout = () => {
|
|
if (!circuitBreakerTimeoutRecorded) {
|
|
circuitBreakerTimeoutRecorded = true;
|
|
recordCircuitBreakerTimeout(cbKey);
|
|
}
|
|
scheduleTimeoutCleanup();
|
|
};
|
|
if (
|
|
isCircuitBreakerOpen(
|
|
cbKey,
|
|
params.config.circuitBreakerMaxTimeouts,
|
|
params.config.circuitBreakerCooldownMs,
|
|
)
|
|
) {
|
|
const result: ActiveRecallResult = {
|
|
status: "timeout",
|
|
elapsedMs: 0,
|
|
summary: null,
|
|
};
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} skipped (circuit breaker open after consecutive timeouts)`,
|
|
);
|
|
}
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: `${buildPluginStatusLine({ result, config: params.config })} circuit-breaker`,
|
|
});
|
|
return result;
|
|
}
|
|
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} start timeoutMs=${String(params.config.timeoutMs)} queryChars=${String(
|
|
params.query.length,
|
|
)} searchQueryChars=${String(params.searchQuery.length)}`,
|
|
);
|
|
}
|
|
|
|
const controller = new AbortController();
|
|
const abortFromParent = () => controller.abort(params.abortSignal?.reason);
|
|
params.abortSignal?.addEventListener("abort", abortFromParent, { once: true });
|
|
if (params.abortSignal?.aborted) {
|
|
abortFromParent();
|
|
}
|
|
const TIMEOUT_SENTINEL = Symbol("timeout");
|
|
let transcriptSources: readonly ActiveMemoryTranscriptSource[] = [];
|
|
let recallTimedOut = false;
|
|
const watchdogTimeoutMs = params.config.timeoutMs + params.config.setupGraceTimeoutMs;
|
|
const timeoutId = setTimeout(() => {
|
|
if (params.abortSignal?.aborted) {
|
|
return;
|
|
}
|
|
recallTimedOut = true;
|
|
controller.abort(new Error(`active-memory timeout after ${watchdogTimeoutMs}ms`));
|
|
}, watchdogTimeoutMs);
|
|
timeoutId.unref?.();
|
|
|
|
const timeoutPromise = new Promise<typeof TIMEOUT_SENTINEL>((resolve) => {
|
|
controller.signal.addEventListener(
|
|
"abort",
|
|
() => {
|
|
resolve(TIMEOUT_SENTINEL);
|
|
},
|
|
{ once: true },
|
|
);
|
|
});
|
|
|
|
let terminalMemorySearchWatch: TerminalMemorySearchWatch | undefined;
|
|
let recallInFlight = false;
|
|
try {
|
|
recallInFlight = true;
|
|
const subagentPromise = runRecallSubagent({
|
|
...params,
|
|
modelRef: resolvedModelRef,
|
|
abortSignal: controller.signal,
|
|
onTranscriptSources: (sources) => {
|
|
transcriptSources = sources;
|
|
},
|
|
});
|
|
terminalMemorySearchWatch = watchTerminalMemorySearchResult({
|
|
getTranscriptSources: () => transcriptSources,
|
|
abortSignal: controller.signal,
|
|
toolsAllow: params.config.toolsAllow,
|
|
});
|
|
// Silently catch late rejections after timeout so they don't become
|
|
// unhandled promise rejections.
|
|
subagentPromise.catch(() => undefined);
|
|
|
|
let raceResult = await Promise.race([
|
|
subagentPromise,
|
|
timeoutPromise,
|
|
terminalMemorySearchWatch.promise,
|
|
]);
|
|
terminalMemorySearchWatch.stop();
|
|
let fallbackSearchDebug: ActiveMemorySearchDebug | undefined;
|
|
let fallbackHasUsableMemoryResult = false;
|
|
if (
|
|
raceResult !== TIMEOUT_SENTINEL &&
|
|
"status" in raceResult &&
|
|
raceResult.hasUsableMemoryResult
|
|
) {
|
|
// A later unavailable call must not discard a summary grounded in an
|
|
// earlier successful recall. The existing watchdog remains the deadline.
|
|
fallbackSearchDebug = raceResult.searchDebug;
|
|
fallbackHasUsableMemoryResult = true;
|
|
raceResult = await Promise.race([subagentPromise, timeoutPromise]);
|
|
}
|
|
if (raceResult !== TIMEOUT_SENTINEL) {
|
|
recallInFlight = false;
|
|
}
|
|
|
|
if (raceResult === TIMEOUT_SENTINEL) {
|
|
if (recallTimedOut) {
|
|
recordRecallTimeout();
|
|
} else if (params.abortSignal?.aborted && recallInFlight) {
|
|
scheduleTimeoutCleanup();
|
|
}
|
|
const elapsedMs = Date.now() - startedAt;
|
|
const result: ActiveRecallResult = fallbackHasUsableMemoryResult
|
|
? {
|
|
status: "timeout",
|
|
elapsedMs,
|
|
summary: null,
|
|
searchDebug: fallbackSearchDebug,
|
|
}
|
|
: await buildTimeoutRecallResult({
|
|
elapsedMs,
|
|
maxSummaryChars: params.config.maxSummaryChars,
|
|
transcriptSources,
|
|
subagentPromise,
|
|
toolsAllow: params.config.toolsAllow,
|
|
});
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`,
|
|
);
|
|
}
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: buildPluginStatusLine({ result, config: params.config }),
|
|
debugSummary: buildPersistedDebugSummary(result),
|
|
searchDebug: result.searchDebug,
|
|
});
|
|
params.abortSignal?.throwIfAborted();
|
|
return result;
|
|
}
|
|
|
|
if ("status" in raceResult) {
|
|
controller.abort(new Error("active-memory terminal memory search result"));
|
|
const result: ActiveRecallResult = {
|
|
status: raceResult.status,
|
|
elapsedMs: Date.now() - startedAt,
|
|
summary: null,
|
|
searchDebug: raceResult.searchDebug,
|
|
};
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`,
|
|
);
|
|
}
|
|
resetCircuitBreaker(cbKey);
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: buildPluginStatusLine({ result, config: params.config }),
|
|
searchDebug: result.searchDebug,
|
|
});
|
|
params.abortSignal?.throwIfAborted();
|
|
if (shouldCacheResult(result)) {
|
|
setCachedResult(cacheKey, result, params.config.cacheTtlMs);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
const { transcriptPath } = raceResult;
|
|
if (params.config.logging && transcriptPath) {
|
|
params.api.logger.info?.(`${logPrefix} transcript=${transcriptPath}`);
|
|
}
|
|
const result = buildSubagentRecallResult({
|
|
subagentResult: raceResult,
|
|
fallbackSearchDebug,
|
|
fallbackHasUsableMemoryResult,
|
|
elapsedMs: Date.now() - startedAt,
|
|
maxSummaryChars: params.config.maxSummaryChars,
|
|
});
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`,
|
|
);
|
|
}
|
|
resetCircuitBreaker(cbKey);
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: buildPluginStatusLine({ result, config: params.config }),
|
|
debugSummary: buildPersistedDebugSummary(result),
|
|
searchDebug: result.searchDebug,
|
|
});
|
|
params.abortSignal?.throwIfAborted();
|
|
if (shouldCacheResult(result)) {
|
|
setCachedResult(cacheKey, result, params.config.cacheTtlMs);
|
|
}
|
|
return result;
|
|
} catch (error) {
|
|
if (params.abortSignal?.aborted) {
|
|
if (recallTimedOut) {
|
|
recordRecallTimeout();
|
|
} else if (recallInFlight) {
|
|
scheduleTimeoutCleanup();
|
|
}
|
|
params.abortSignal.throwIfAborted();
|
|
}
|
|
if (controller.signal.aborted) {
|
|
if (recallTimedOut) {
|
|
recordRecallTimeout();
|
|
}
|
|
const partialTimeoutData = readPartialTimeoutData(error);
|
|
const result = await buildTimeoutRecallResult({
|
|
elapsedMs: Date.now() - startedAt,
|
|
maxSummaryChars: params.config.maxSummaryChars,
|
|
transcriptSources,
|
|
rawReply: partialTimeoutData.rawReply,
|
|
searchDebug: partialTimeoutData.searchDebug,
|
|
hasUnavailableMemorySearchResult: partialTimeoutData.hasUnavailableMemorySearchResult,
|
|
toolsAllow: params.config.toolsAllow,
|
|
});
|
|
if (params.config.logging) {
|
|
params.api.logger.info?.(
|
|
`${logPrefix} done status=${result.status} elapsedMs=${String(result.elapsedMs)} summaryChars=${String(result.summary?.length ?? 0)}`,
|
|
);
|
|
}
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: buildPluginStatusLine({ result, config: params.config }),
|
|
debugSummary: buildPersistedDebugSummary(result),
|
|
searchDebug: result.searchDebug,
|
|
});
|
|
params.abortSignal?.throwIfAborted();
|
|
return result;
|
|
}
|
|
const message = toSingleLineLogValue(error instanceof Error ? error.message : String(error));
|
|
if (params.config.logging) {
|
|
params.api.logger.warn?.(`${logPrefix} failed error=${message}; skipping recall`);
|
|
}
|
|
const result: ActiveRecallResult = {
|
|
status: "failed",
|
|
elapsedMs: Date.now() - startedAt,
|
|
summary: null,
|
|
};
|
|
params.abortSignal?.throwIfAborted();
|
|
await persistPluginStatusLines({
|
|
api: params.api,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
statusLine: buildPluginStatusLine({ result, config: params.config }),
|
|
searchDebug: result.searchDebug,
|
|
});
|
|
return result;
|
|
} finally {
|
|
params.abortSignal?.removeEventListener("abort", abortFromParent);
|
|
terminalMemorySearchWatch?.stop();
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
export { maybeResolveActiveRecall };
|