mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-22 11:01:29 +00:00
refactor(whatsapp): use retryAsync for outbound delivery retries (#105579)
* refactor(whatsapp): reuse shared outbound retry runtime * refactor(whatsapp): narrow outbound retry classification * refactor(whatsapp): remove obsolete sleep export
This commit is contained in:
@@ -5,7 +5,6 @@ import {
|
||||
listMessageReceiptPlatformIds,
|
||||
} from "openclaw/plugin-sdk/channel-outbound";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { sleep } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { createAcceptedWhatsAppSendResult } from "../inbound/send-result.test-helper.js";
|
||||
import { createTestWebInboundMessage } from "../inbound/test-message.test-helper.js";
|
||||
@@ -39,16 +38,6 @@ vi.mock("openclaw/plugin-sdk/runtime-env", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/text-utility-runtime", async () => {
|
||||
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/text-utility-runtime")>(
|
||||
"openclaw/plugin-sdk/text-utility-runtime",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
sleep: vi.fn(async () => {}),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../media.js", () => ({
|
||||
loadWebMedia: vi.fn(),
|
||||
}));
|
||||
@@ -191,6 +180,18 @@ function mockSecondReplySuccess(msg: AdmittedWebInboundMessage) {
|
||||
).mockResolvedValueOnce(createAcceptedWhatsAppSendResult("text", "reply-retry-2"));
|
||||
}
|
||||
|
||||
async function runWithFakeTimers<T>(run: () => Promise<T>): Promise<T> {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const promise = run();
|
||||
await vi.runAllTimersAsync();
|
||||
return await promise;
|
||||
} finally {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
}
|
||||
|
||||
const replyLogger = {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
@@ -406,17 +407,18 @@ describe("deliverWebReply", () => {
|
||||
mockFirstReplyFailure(msg, errorMessage);
|
||||
mockSecondReplySuccess(msg);
|
||||
|
||||
await deliverWebReply({
|
||||
replyResult: { text: "hi" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
});
|
||||
await runWithFakeTimers(() =>
|
||||
deliverWebReply({
|
||||
replyResult: { text: "hi" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(msg.platform.reply).toHaveBeenCalledTimes(2);
|
||||
expect(sleep).toHaveBeenCalledWith(500);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -425,23 +427,23 @@ describe("deliverWebReply", () => {
|
||||
mockFirstReplyFailureWithWrappedError(msg, "connection closed");
|
||||
mockSecondReplySuccess(msg);
|
||||
|
||||
await deliverWebReply({
|
||||
replyResult: { text: "hi" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
});
|
||||
await runWithFakeTimers(() =>
|
||||
deliverWebReply({
|
||||
replyResult: { text: "hi" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(msg.platform.reply).toHaveBeenCalledTimes(2);
|
||||
expect(sleep).toHaveBeenCalledWith(500);
|
||||
});
|
||||
|
||||
it("does not retry terminal socket operation timeouts", async () => {
|
||||
const msg = makeMsg();
|
||||
const timeout = new WhatsAppSocketOperationTimeoutError("sendMessage", 60_000);
|
||||
(sleep as unknown as { mockClear: () => void }).mockClear();
|
||||
(
|
||||
msg.platform.reply as unknown as { mockRejectedValueOnce: (v: unknown) => void }
|
||||
).mockRejectedValueOnce(timeout);
|
||||
@@ -458,7 +460,6 @@ describe("deliverWebReply", () => {
|
||||
).rejects.toBe(timeout);
|
||||
|
||||
expect(msg.platform.reply).toHaveBeenCalledTimes(1);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("sends image media with caption and then remaining text", async () => {
|
||||
@@ -586,17 +587,18 @@ describe("deliverWebReply", () => {
|
||||
msg.platform.sendMedia as unknown as { mockResolvedValueOnce: (v: unknown) => void }
|
||||
).mockResolvedValueOnce(createAcceptedWhatsAppSendResult("media", "media-retry-2"));
|
||||
|
||||
await deliverWebReply({
|
||||
replyResult: { text: "caption", mediaUrl: "http://example.com/img.jpg" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
});
|
||||
await runWithFakeTimers(() =>
|
||||
deliverWebReply({
|
||||
replyResult: { text: "caption", mediaUrl: "http://example.com/img.jpg" },
|
||||
msg,
|
||||
maxMediaBytes: 1024 * 1024,
|
||||
textLimit: 200,
|
||||
replyLogger,
|
||||
skipLog: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(msg.platform.sendMedia).toHaveBeenCalledTimes(2);
|
||||
expect(sleep).toHaveBeenCalledWith(500);
|
||||
});
|
||||
|
||||
it("falls back to text-only when the first media send fails", async () => {
|
||||
|
||||
@@ -22,8 +22,8 @@ import {
|
||||
normalizeWhatsAppOutboundPayload,
|
||||
normalizeWhatsAppPayloadTextPreservingIndentation,
|
||||
prepareWhatsAppOutboundMedia,
|
||||
sendWhatsAppOutboundWithRetry,
|
||||
} from "../outbound-media-contract.js";
|
||||
import { sendWhatsAppOutboundWithRetry } from "../outbound-retry.js";
|
||||
import { buildQuotedMessageOptions, lookupInboundMessageMeta } from "../quoted-message.js";
|
||||
import { newConnectionId } from "../reconnect.js";
|
||||
import { formatError } from "../session.js";
|
||||
@@ -159,11 +159,10 @@ export async function deliverWebReply(params: {
|
||||
});
|
||||
};
|
||||
|
||||
const sendWithRetry = async <T>(fn: () => Promise<T>, label: string, maxAttempts = 3) => {
|
||||
const sendWithRetry = async <T>(fn: () => Promise<T>, label: string) => {
|
||||
try {
|
||||
return await sendWhatsAppOutboundWithRetry({
|
||||
send: fn,
|
||||
maxAttempts,
|
||||
onRetry: ({ attempt, maxAttempts: retryMaxAttempts, backoffMs, errorText }) => {
|
||||
logVerbose(
|
||||
`Retrying ${label} to ${conversationId} after failure (${attempt}/${retryMaxAttempts - 1}) in ${backoffMs}ms: ${errorText}`,
|
||||
|
||||
@@ -7,13 +7,10 @@ import { writeExternalFileWithinRoot } from "openclaw/plugin-sdk/security-runtim
|
||||
import { normalizeUniqueStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path";
|
||||
import { resolveWhatsAppDocumentFileName } from "./document-filename.js";
|
||||
import { formatError } from "./session-errors.js";
|
||||
import { isWhatsAppSocketOperationTimeoutError } from "./socket-timing.js";
|
||||
import {
|
||||
sanitizeAssistantVisibleText,
|
||||
sanitizeAssistantVisibleTextWithProfile,
|
||||
stripToolCallXmlTags,
|
||||
sleep,
|
||||
} from "./text-runtime.js";
|
||||
|
||||
type WhatsAppOutboundPayloadLike = {
|
||||
@@ -269,47 +266,3 @@ function deriveWhatsAppDocumentFileName(mediaUrl: string | undefined): string |
|
||||
return fileName || undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function isRetryableWhatsAppOutboundError(error: unknown): boolean {
|
||||
if (isWhatsAppSocketOperationTimeoutError(error)) {
|
||||
return false;
|
||||
}
|
||||
return /closed|reset|timed\s*out|disconnect/i.test(formatError(error));
|
||||
}
|
||||
|
||||
export async function sendWhatsAppOutboundWithRetry<T>(params: {
|
||||
send: () => Promise<T>;
|
||||
onRetry?: (params: {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
backoffMs: number;
|
||||
error: unknown;
|
||||
errorText: string;
|
||||
}) => Promise<void> | void;
|
||||
maxAttempts?: number;
|
||||
}): Promise<T> {
|
||||
const maxAttempts = params.maxAttempts ?? 3;
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await params.send();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
const errorText = formatError(error);
|
||||
const isLastAttempt = attempt === maxAttempts;
|
||||
if (!isRetryableWhatsAppOutboundError(error) || isLastAttempt) {
|
||||
throw error;
|
||||
}
|
||||
const backoffMs = 500 * attempt;
|
||||
await params.onRetry?.({
|
||||
attempt,
|
||||
maxAttempts,
|
||||
backoffMs,
|
||||
error,
|
||||
errorText,
|
||||
});
|
||||
await sleep(backoffMs);
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
110
extensions/whatsapp/src/outbound-retry.test.ts
Normal file
110
extensions/whatsapp/src/outbound-retry.test.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
// WhatsApp tests cover outbound retry behavior.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { sendWhatsAppOutboundWithRetry } from "./outbound-retry.js";
|
||||
import { WhatsAppSocketOperationTimeoutError } from "./socket-timing.js";
|
||||
|
||||
async function runWithFakeTimers<T>(run: () => Promise<T>): Promise<T> {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const promise = run();
|
||||
await vi.runAllTimersAsync();
|
||||
return await promise;
|
||||
} finally {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
}
|
||||
|
||||
describe("sendWhatsAppOutboundWithRetry", () => {
|
||||
it.each([new Error("connection closed"), { code: "ECONNRESET" }])(
|
||||
"retries a directly retryable error",
|
||||
async (error) => {
|
||||
const send = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(error)
|
||||
.mockResolvedValue("ok");
|
||||
|
||||
await expect(runWithFakeTimers(() => sendWhatsAppOutboundWithRetry({ send }))).resolves.toBe(
|
||||
"ok",
|
||||
);
|
||||
|
||||
expect(send).toHaveBeenCalledTimes(2);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ name: "a non-retryable direct error", error: new Error("invalid recipient") },
|
||||
{
|
||||
name: "a retryable signal only in the cause",
|
||||
error: new Error("request failed", { cause: new Error("socket disconnected") }),
|
||||
},
|
||||
])("does not retry $name", async ({ error }) => {
|
||||
const send = vi.fn<() => Promise<string>>().mockRejectedValue(error);
|
||||
const onRetry = vi.fn();
|
||||
|
||||
const failure = await sendWhatsAppOutboundWithRetry({ send, onRetry }).catch(
|
||||
(caught: unknown) => caught,
|
||||
);
|
||||
|
||||
expect(failure).toBe(error);
|
||||
expect(send).toHaveBeenCalledOnce();
|
||||
expect(onRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not retry a direct unknown-delivery socket timeout", async () => {
|
||||
const timeout = new WhatsAppSocketOperationTimeoutError("sendMessage", 60_000);
|
||||
const send = vi.fn<() => Promise<string>>().mockRejectedValue(timeout);
|
||||
const onRetry = vi.fn();
|
||||
|
||||
const failure = await sendWhatsAppOutboundWithRetry({ send, onRetry }).catch(
|
||||
(caught: unknown) => caught,
|
||||
);
|
||||
|
||||
expect(failure).toBe(timeout);
|
||||
expect(send).toHaveBeenCalledOnce();
|
||||
expect(onRetry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves attempts, delays, callback fields, and terminal error identity", async () => {
|
||||
const firstError = {
|
||||
output: {
|
||||
statusCode: 503,
|
||||
payload: {
|
||||
statusCode: 503,
|
||||
error: "Service Unavailable",
|
||||
message: "connection closed",
|
||||
},
|
||||
},
|
||||
};
|
||||
const secondError = new Error("socket reset");
|
||||
const terminalError = { code: "ECONNRESET", marker: "terminal" };
|
||||
const send = vi
|
||||
.fn<() => Promise<string>>()
|
||||
.mockRejectedValueOnce(firstError)
|
||||
.mockRejectedValueOnce(secondError)
|
||||
.mockRejectedValueOnce(terminalError);
|
||||
const onRetry = vi.fn();
|
||||
|
||||
const failure = await runWithFakeTimers(() =>
|
||||
sendWhatsAppOutboundWithRetry({ send, onRetry }).catch((caught: unknown) => caught),
|
||||
);
|
||||
|
||||
expect(failure).toBe(terminalError);
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
expect(onRetry).toHaveBeenCalledTimes(2);
|
||||
expect(onRetry).toHaveBeenNthCalledWith(1, {
|
||||
attempt: 1,
|
||||
maxAttempts: 3,
|
||||
backoffMs: 500,
|
||||
error: firstError,
|
||||
errorText: "status=503 Service Unavailable connection closed",
|
||||
});
|
||||
expect(onRetry).toHaveBeenNthCalledWith(2, {
|
||||
attempt: 2,
|
||||
maxAttempts: 3,
|
||||
backoffMs: 1_000,
|
||||
error: secondError,
|
||||
errorText: "socket reset",
|
||||
});
|
||||
});
|
||||
});
|
||||
77
extensions/whatsapp/src/outbound-retry.ts
Normal file
77
extensions/whatsapp/src/outbound-retry.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
// WhatsApp plugin module implements outbound retry behavior.
|
||||
import { retryAsync } from "openclaw/plugin-sdk/retry-runtime";
|
||||
import { formatError } from "./session-errors.js";
|
||||
import { isWhatsAppSocketOperationTimeoutError } from "./socket-timing.js";
|
||||
|
||||
const WHATSAPP_OUTBOUND_MAX_ATTEMPTS = 3;
|
||||
const WHATSAPP_OUTBOUND_MIN_DELAY_MS = 500;
|
||||
const WHATSAPP_OUTBOUND_MAX_DELAY_MS = 1_000;
|
||||
const WHATSAPP_RETRYABLE_OUTBOUND_ERROR_PATTERN = /closed|reset|timed\s*out|disconnect/i;
|
||||
|
||||
class WhatsAppOutboundRetryError extends Error {
|
||||
constructor(readonly original: unknown) {
|
||||
super(formatError(original), { cause: original });
|
||||
}
|
||||
}
|
||||
|
||||
function isRetryableWhatsAppOutboundError(error: unknown): boolean {
|
||||
// Outbound sends surface direct failures; inspecting wrappers or causes can
|
||||
// replay a non-idempotent send. A direct local timeout may have delivered it.
|
||||
if (isWhatsAppSocketOperationTimeoutError(error)) {
|
||||
return false;
|
||||
}
|
||||
return WHATSAPP_RETRYABLE_OUTBOUND_ERROR_PATTERN.test(formatError(error));
|
||||
}
|
||||
|
||||
type WhatsAppOutboundRetryInfo = {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
backoffMs: number;
|
||||
error: unknown;
|
||||
errorText: string;
|
||||
};
|
||||
|
||||
export async function sendWhatsAppOutboundWithRetry<T>(params: {
|
||||
send: () => Promise<T>;
|
||||
onRetry?: (info: WhatsAppOutboundRetryInfo) => void;
|
||||
}): Promise<T> {
|
||||
try {
|
||||
return await retryAsync(
|
||||
async () => {
|
||||
try {
|
||||
return await params.send();
|
||||
} catch (error) {
|
||||
// retryAsync normalizes non-Error throws. Keep the original value in
|
||||
// an Error wrapper so the WhatsApp adapter can restore exact identity.
|
||||
throw new WhatsAppOutboundRetryError(error);
|
||||
}
|
||||
},
|
||||
{
|
||||
attempts: WHATSAPP_OUTBOUND_MAX_ATTEMPTS,
|
||||
minDelayMs: WHATSAPP_OUTBOUND_MIN_DELAY_MS,
|
||||
maxDelayMs: WHATSAPP_OUTBOUND_MAX_DELAY_MS,
|
||||
jitter: 0,
|
||||
shouldRetry: (error) =>
|
||||
error instanceof WhatsAppOutboundRetryError &&
|
||||
isRetryableWhatsAppOutboundError(error.original),
|
||||
onRetry: ({ attempt, maxAttempts, delayMs, err }) => {
|
||||
if (!(err instanceof WhatsAppOutboundRetryError)) {
|
||||
return;
|
||||
}
|
||||
params.onRetry?.({
|
||||
attempt,
|
||||
maxAttempts,
|
||||
backoffMs: delayMs,
|
||||
error: err.original,
|
||||
errorText: formatError(err.original),
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof WhatsAppOutboundRetryError) {
|
||||
throw error.original;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ export {
|
||||
sanitizeAssistantVisibleTextWithProfile,
|
||||
stripToolCallXmlTags,
|
||||
} from "openclaw/plugin-sdk/text-chunking";
|
||||
export { normalizeE164, resolveUserPath, sleep } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
export { normalizeE164, resolveUserPath } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
export {
|
||||
assertWebChannel,
|
||||
isSelfChatMode,
|
||||
|
||||
Reference in New Issue
Block a user