fix: probe gateway memory status in doctor

This commit is contained in:
Gustavo Madeira Santana
2026-02-23 13:32:19 -05:00
parent c3eaa5a361
commit 2816f69d8b
11 changed files with 285 additions and 12 deletions

View File

@@ -17,6 +17,7 @@ Docs: https://docs.openclaw.ai
- Agents/Context pruning: extend `cache-ttl` eligibility to Moonshot/Kimi and ZAI/GLM providers (including OpenRouter model refs), so `contextPruning.mode: "cache-ttl"` is no longer silently skipped for those sessions. (#24497) Thanks @lailoo.
- Tools/web_search: add `provider: "kimi"` (Moonshot) support with key/config schema wiring and a corrected two-step `$web_search` tool flow that echoes tool results before final synthesis, including citation extraction from search results. (#16616, #18822) Thanks @adshine.
- Media understanding/Video: add a native Moonshot video provider and include Moonshot in auto video key detection, plus refactor video execution to honor `entry/config/provider` baseUrl+header precedence (matching audio behavior). (#12063) Thanks @xiaoyaner0201.
- Doctor/Memory: query gateway-side default-agent memory embedding readiness during `openclaw doctor` (instead of inferring from generic gateway health), and warn when the gateway memory probe is unavailable or not ready while keeping `openclaw configure` remediation guidance. (#22327) thanks @therk.
- Sessions/Store: canonicalize inbound mixed-case session keys for metadata and route updates, and migrate legacy case-variant entries to a single lowercase key to prevent duplicate sessions and missing TUI/WebUI history. (#9561) Thanks @hillghost86.
- Telegram/Reactions: soft-fail reaction action errors (policy/token/emoji/API), accept snake_case `message_id`, and fallback to inbound message-id context when explicit `messageId` is omitted so DM reactions stay stable without regeneration loops. (#20236, #21001) Thanks @PeterShanxin and @vincentkoc.
- Telegram/Polling: scope persisted polling offsets to bot identity and reuse a single awaited runner-stop path on abort/retry, preventing cross-token offset bleed and overlapping pollers during restart/error recovery. (#10850, #11347) Thanks @talhaorak, @anooprdawar, and @vincentkoc.

View File

@@ -1,11 +1,18 @@
import type { OpenClawConfig } from "../config/config.js";
import { buildGatewayConnectionDetails, callGateway } from "../gateway/call.js";
import type { DoctorMemoryStatusPayload } from "../gateway/server-methods/doctor.js";
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
import type { RuntimeEnv } from "../runtime.js";
import { note } from "../terminal/note.js";
import { formatHealthCheckFailure } from "./health-format.js";
import { healthCommand } from "./health.js";
export type GatewayMemoryProbe = {
checked: boolean;
ready: boolean;
error?: string;
};
export async function checkGatewayHealth(params: {
runtime: RuntimeEnv;
cfg: OpenClawConfig;
@@ -56,3 +63,30 @@ export async function checkGatewayHealth(params: {
return { healthOk };
}
export async function probeGatewayMemoryStatus(params: {
cfg: OpenClawConfig;
timeoutMs?: number;
}): Promise<GatewayMemoryProbe> {
const timeoutMs =
typeof params.timeoutMs === "number" && params.timeoutMs > 0 ? params.timeoutMs : 8_000;
try {
const payload = await callGateway<DoctorMemoryStatusPayload>({
method: "doctor.memory.status",
timeoutMs,
config: params.cfg,
});
return {
checked: true,
ready: payload.embedding.ok,
error: payload.embedding.error,
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
return {
checked: true,
ready: false,
error: `gateway memory probe unavailable: ${message}`,
};
}
}

View File

@@ -128,29 +128,38 @@ describe("noteMemorySearchHealth", () => {
expect(note).not.toHaveBeenCalled();
});
it("notes when gateway is healthy but CLI API key is missing", async () => {
it("notes when gateway probe reports embeddings ready and CLI API key is missing", async () => {
resolveMemorySearchConfig.mockReturnValue({
provider: "gemini",
local: {},
remote: {},
});
await noteMemorySearchHealth(cfg, { gatewayHealthOk: true });
await noteMemorySearchHealth(cfg, {
gatewayMemoryProbe: { checked: true, ready: true },
});
const message = note.mock.calls[0]?.[0] as string;
expect(message).toContain("may be loaded by the running gateway");
expect(message).toContain("reports memory embeddings are ready");
});
it("uses configure hint when gateway is down and API key is missing", async () => {
it("uses configure hint when gateway probe is unavailable and API key is missing", async () => {
resolveMemorySearchConfig.mockReturnValue({
provider: "gemini",
local: {},
remote: {},
});
await noteMemorySearchHealth(cfg, { gatewayHealthOk: false });
await noteMemorySearchHealth(cfg, {
gatewayMemoryProbe: {
checked: true,
ready: false,
error: "gateway memory probe unavailable: timeout",
},
});
const message = note.mock.calls[0]?.[0] as string;
expect(message).toContain("Gateway memory probe for default agent is not ready");
expect(message).toContain("openclaw configure");
expect(message).not.toContain("auth add");
});

View File

@@ -14,7 +14,13 @@ import { resolveUserPath } from "../utils.js";
*/
export async function noteMemorySearchHealth(
cfg: OpenClawConfig,
opts?: { gatewayHealthOk?: boolean },
opts?: {
gatewayMemoryProbe?: {
checked: boolean;
ready: boolean;
error?: string;
};
},
): Promise<void> {
const agentId = resolveDefaultAgentId(cfg);
const agentDir = resolveAgentDir(cfg, agentId);
@@ -57,22 +63,24 @@ export async function noteMemorySearchHealth(
if (hasRemoteApiKey || (await hasApiKeyForProvider(resolved.provider, cfg, agentDir))) {
return;
}
if (opts?.gatewayHealthOk) {
if (opts?.gatewayMemoryProbe?.checked && opts.gatewayMemoryProbe.ready) {
note(
[
`Memory search provider is set to "${resolved.provider}" but the API key was not found in the CLI environment.`,
"The key may be loaded by the running gateway (e.g. via secrets.env).",
"The running gateway reports memory embeddings are ready for the default agent.",
`Verify: ${formatCliCommand("openclaw memory status --deep")}`,
].join("\n"),
"Memory search",
);
return;
}
const gatewayProbeWarning = buildGatewayProbeWarning(opts?.gatewayMemoryProbe);
const envVar = providerEnvVar(resolved.provider);
note(
[
`Memory search provider is set to "${resolved.provider}" but no API key was found.`,
`Semantic recall will not work without a valid API key.`,
gatewayProbeWarning ? gatewayProbeWarning : null,
"",
"Fix (pick one):",
`- Set ${envVar} in your environment`,
@@ -96,22 +104,24 @@ export async function noteMemorySearchHealth(
}
}
if (opts?.gatewayHealthOk) {
if (opts?.gatewayMemoryProbe?.checked && opts.gatewayMemoryProbe.ready) {
note(
[
'Memory search provider is set to "auto" but the API key was not found in the CLI environment.',
"The key may be loaded by the running gateway (e.g. via secrets.env).",
"The running gateway reports memory embeddings are ready for the default agent.",
`Verify: ${formatCliCommand("openclaw memory status --deep")}`,
].join("\n"),
"Memory search",
);
return;
}
const gatewayProbeWarning = buildGatewayProbeWarning(opts?.gatewayMemoryProbe);
note(
[
"Memory search is enabled but no embedding provider is configured.",
"Semantic recall will not work without an embedding provider.",
gatewayProbeWarning ? gatewayProbeWarning : null,
"",
"Fix (pick one):",
"- Set OPENAI_API_KEY, GEMINI_API_KEY, VOYAGE_API_KEY, or MISTRAL_API_KEY in your environment",
@@ -171,3 +181,21 @@ function providerEnvVar(provider: string): string {
return `${provider.toUpperCase()}_API_KEY`;
}
}
function buildGatewayProbeWarning(
probe:
| {
checked: boolean;
ready: boolean;
error?: string;
}
| undefined,
): string | null {
if (!probe?.checked || probe.ready) {
return null;
}
const detail = probe.error?.trim();
return detail
? `Gateway memory probe for default agent is not ready: ${detail}`
: "Gateway memory probe for default agent is not ready.";
}

View File

@@ -10,6 +10,7 @@ vi.mock("./doctor-gateway-daemon-flow.js", () => ({
vi.mock("./doctor-gateway-health.js", () => ({
checkGatewayHealth: vi.fn().mockResolvedValue({ healthOk: false }),
probeGatewayMemoryStatus: vi.fn().mockResolvedValue({ checked: false, ready: false }),
}));
vi.mock("./doctor-memory-search.js", () => ({

View File

@@ -29,7 +29,7 @@ import {
import { doctorShellCompletion } from "./doctor-completion.js";
import { loadAndMaybeMigrateDoctorConfig } from "./doctor-config-flow.js";
import { maybeRepairGatewayDaemon } from "./doctor-gateway-daemon-flow.js";
import { checkGatewayHealth } from "./doctor-gateway-health.js";
import { checkGatewayHealth, probeGatewayMemoryStatus } from "./doctor-gateway-health.js";
import {
maybeRepairGatewayServiceConfig,
maybeScanExtraGatewayServices,
@@ -275,7 +275,13 @@ export async function doctorCommand(
cfg,
timeoutMs: options.nonInteractive === true ? 3000 : 10_000,
});
await noteMemorySearchHealth(cfg, { gatewayHealthOk: healthOk });
const gatewayMemoryProbe = healthOk
? await probeGatewayMemoryStatus({
cfg,
timeoutMs: options.nonInteractive === true ? 3000 : 10_000,
})
: { checked: false, ready: false };
await noteMemorySearchHealth(cfg, { gatewayMemoryProbe });
await maybeRepairGatewayDaemon({
cfg,
runtime,

View File

@@ -43,6 +43,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
],
[READ_SCOPE]: [
"health",
"doctor.memory.status",
"logs.tail",
"channels.status",
"status",

View File

@@ -3,6 +3,7 @@ import { GATEWAY_EVENT_UPDATE_AVAILABLE } from "./events.js";
const BASE_METHODS = [
"health",
"doctor.memory.status",
"logs.tail",
"channels.status",
"channels.logout",

View File

@@ -12,6 +12,7 @@ import { configHandlers } from "./server-methods/config.js";
import { connectHandlers } from "./server-methods/connect.js";
import { cronHandlers } from "./server-methods/cron.js";
import { deviceHandlers } from "./server-methods/devices.js";
import { doctorHandlers } from "./server-methods/doctor.js";
import { execApprovalsHandlers } from "./server-methods/exec-approvals.js";
import { healthHandlers } from "./server-methods/health.js";
import { logsHandlers } from "./server-methods/logs.js";
@@ -71,6 +72,7 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
...chatHandlers,
...cronHandlers,
...deviceHandlers,
...doctorHandlers,
...execApprovalsHandlers,
...webHandlers,
...modelsHandlers,

View File

@@ -0,0 +1,128 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
const loadConfig = vi.hoisted(() => vi.fn(() => ({}) as OpenClawConfig));
const resolveDefaultAgentId = vi.hoisted(() => vi.fn(() => "main"));
const getMemorySearchManager = vi.hoisted(() => vi.fn());
vi.mock("../../config/config.js", () => ({
loadConfig,
}));
vi.mock("../../agents/agent-scope.js", () => ({
resolveDefaultAgentId,
}));
vi.mock("../../memory/index.js", () => ({
getMemorySearchManager,
}));
import { doctorHandlers } from "./doctor.js";
describe("doctor.memory.status", () => {
beforeEach(() => {
loadConfig.mockClear();
resolveDefaultAgentId.mockClear();
getMemorySearchManager.mockReset();
});
it("returns gateway embedding probe status for the default agent", async () => {
const close = vi.fn().mockResolvedValue(undefined);
getMemorySearchManager.mockResolvedValue({
manager: {
status: () => ({ provider: "gemini" }),
probeEmbeddingAvailability: vi.fn().mockResolvedValue({ ok: true }),
close,
},
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(getMemorySearchManager).toHaveBeenCalledWith({
cfg: expect.any(Object),
agentId: "main",
purpose: "status",
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
provider: "gemini",
embedding: { ok: true },
},
undefined,
);
expect(close).toHaveBeenCalled();
});
it("returns unavailable when memory manager is missing", async () => {
getMemorySearchManager.mockResolvedValue({
manager: null,
error: "memory search unavailable",
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
embedding: {
ok: false,
error: "memory search unavailable",
},
},
undefined,
);
});
it("returns probe failure when manager probe throws", async () => {
const close = vi.fn().mockResolvedValue(undefined);
getMemorySearchManager.mockResolvedValue({
manager: {
status: () => ({ provider: "openai" }),
probeEmbeddingAvailability: vi.fn().mockRejectedValue(new Error("timeout")),
close,
},
});
const respond = vi.fn();
await doctorHandlers["doctor.memory.status"]({
req: {} as never,
params: {} as never,
respond: respond as never,
context: {} as never,
client: null,
isWebchatConnect: () => false,
});
expect(respond).toHaveBeenCalledWith(
true,
{
agentId: "main",
embedding: {
ok: false,
error: "gateway memory probe failed: timeout",
},
},
undefined,
);
expect(close).toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,62 @@
import { resolveDefaultAgentId } from "../../agents/agent-scope.js";
import { loadConfig } from "../../config/config.js";
import { getMemorySearchManager } from "../../memory/index.js";
import { formatError } from "../server-utils.js";
import type { GatewayRequestHandlers } from "./types.js";
export type DoctorMemoryStatusPayload = {
agentId: string;
provider?: string;
embedding: {
ok: boolean;
error?: string;
};
};
export const doctorHandlers: GatewayRequestHandlers = {
"doctor.memory.status": async ({ respond }) => {
const cfg = loadConfig();
const agentId = resolveDefaultAgentId(cfg);
const { manager, error } = await getMemorySearchManager({
cfg,
agentId,
purpose: "status",
});
if (!manager) {
const payload: DoctorMemoryStatusPayload = {
agentId,
embedding: {
ok: false,
error: error ?? "memory search unavailable",
},
};
respond(true, payload, undefined);
return;
}
try {
const status = manager.status();
let embedding = await manager.probeEmbeddingAvailability();
if (!embedding.ok && !embedding.error) {
embedding = { ok: false, error: "memory embeddings unavailable" };
}
const payload: DoctorMemoryStatusPayload = {
agentId,
provider: status.provider,
embedding,
};
respond(true, payload, undefined);
} catch (err) {
const payload: DoctorMemoryStatusPayload = {
agentId,
embedding: {
ok: false,
error: `gateway memory probe failed: ${formatError(err)}`,
},
};
respond(true, payload, undefined);
} finally {
await manager.close?.().catch(() => {});
}
},
};