From feb9a3b5b23cfaa58a8cf415eaef562e54072ff3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 19 Mar 2026 11:01:16 -0700 Subject: [PATCH] fix(ci): harden test gating under load --- scripts/test-parallel-utils.mjs | 12 ++++--- scripts/test-parallel.mjs | 13 ++++++-- src/channels/plugins/actions/actions.test.ts | 8 +++-- src/infra/fs-pinned-write-helper.ts | 35 +++++++++++++++++++- src/memory/batch-http.test.ts | 1 - src/memory/embeddings-remote-fetch.test.ts | 1 - src/memory/embeddings-voyage.test.ts | 1 - src/memory/manager.ts | 22 ++++++++++-- src/memory/post-json.test.ts | 1 - src/memory/search-manager.ts | 18 +++++++++- test/scripts/test-parallel.test.ts | 8 +++++ 11 files changed, 103 insertions(+), 17 deletions(-) diff --git a/scripts/test-parallel-utils.mjs b/scripts/test-parallel-utils.mjs index 96bf86a3c34..3f815a5036d 100644 --- a/scripts/test-parallel-utils.mjs +++ b/scripts/test-parallel-utils.mjs @@ -1,9 +1,10 @@ const DEFAULT_OUTPUT_CAPTURE_LIMIT = 200_000; const fatalOutputPatterns = [ - /FATAL ERROR:.*heap out of memory/i, - /Allocation failed - JavaScript heap out of memory/i, + /FATAL ERROR:/i, + /JavaScript heap out of memory/i, /node::OOMErrorHandler/i, + /ERR_WORKER_OUT_OF_MEMORY/i, ]; export function appendCapturedOutput(current, chunk, limit = DEFAULT_OUTPUT_CAPTURE_LIMIT) { @@ -21,14 +22,17 @@ export function hasFatalTestRunOutput(output) { return fatalOutputPatterns.some((pattern) => pattern.test(output)); } -export function resolveTestRunExitCode({ code, signal, output }) { +export function resolveTestRunExitCode({ code, signal, output, fatalSeen = false, childError }) { if (typeof code === "number" && code !== 0) { return code; } + if (childError) { + return 1; + } if (signal) { return 1; } - if (hasFatalTestRunOutput(output)) { + if (fatalSeen || hasFatalTestRunOutput(output)) { return 1; } return code ?? 0; diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index 9571668f3ba..78fcf35f6d1 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -4,7 +4,11 @@ import os from "node:os"; import path from "node:path"; import { channelTestPrefixes } from "../vitest.channel-paths.mjs"; import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs"; -import { appendCapturedOutput, resolveTestRunExitCode } from "./test-parallel-utils.mjs"; +import { + appendCapturedOutput, + hasFatalTestRunOutput, + resolveTestRunExitCode, +} from "./test-parallel-utils.mjs"; import { loadTestRunnerBehavior, loadUnitTimingManifest, @@ -742,6 +746,8 @@ const runOnce = (entry, extraArgs = []) => ? `${nextNodeOptions} ${heapFlag}`.trim() : nextNodeOptions; let output = ""; + let fatalSeen = false; + let childError = null; let child; try { child = spawn(pnpm, args, { @@ -757,20 +763,23 @@ const runOnce = (entry, extraArgs = []) => children.add(child); child.stdout?.on("data", (chunk) => { const text = chunk.toString(); + fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); output = appendCapturedOutput(output, text); process.stdout.write(chunk); }); child.stderr?.on("data", (chunk) => { const text = chunk.toString(); + fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); output = appendCapturedOutput(output, text); process.stderr.write(chunk); }); child.on("error", (err) => { + childError = err; console.error(`[test-parallel] child error: ${String(err)}`); }); child.on("close", (code, signal) => { children.delete(child); - const resolvedCode = resolveTestRunExitCode({ code, signal, output }); + const resolvedCode = resolveTestRunExitCode({ code, signal, output, fatalSeen, childError }); console.log( `[test-parallel] done ${entry.name} code=${String(resolvedCode)} elapsed=${formatElapsedMs(Date.now() - startedAt)}`, ); diff --git a/src/channels/plugins/actions/actions.test.ts b/src/channels/plugins/actions/actions.test.ts index 0752c1e7a4e..42d5224b93b 100644 --- a/src/channels/plugins/actions/actions.test.ts +++ b/src/channels/plugins/actions/actions.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../../../config/config.js"; import type { ChannelMessageActionAdapter } from "../types.js"; @@ -199,13 +199,15 @@ async function expectSlackSendRejected(params: Record, error: R expect(handleSlackAction).not.toHaveBeenCalled(); } -beforeEach(async () => { - vi.resetModules(); +beforeAll(async () => { ({ discordMessageActions } = await import("../../../../extensions/discord/runtime-api.js")); ({ handleDiscordMessageAction } = await import("./discord/handle-action.js")); ({ telegramMessageActions } = await import("../../../../extensions/telegram/runtime-api.js")); ({ signalMessageActions } = await import("../../../../extensions/signal/src/message-actions.js")); ({ createSlackActions } = await import("../../../../extensions/slack/src/channel-actions.js")); +}); + +beforeEach(() => { vi.clearAllMocks(); }); diff --git a/src/infra/fs-pinned-write-helper.ts b/src/infra/fs-pinned-write-helper.ts index b4bb5c56ed7..98a13aa9251 100644 --- a/src/infra/fs-pinned-write-helper.ts +++ b/src/infra/fs-pinned-write-helper.ts @@ -1,5 +1,6 @@ import { spawn } from "node:child_process"; import { once } from "node:events"; +import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { Readable } from "node:stream"; @@ -100,6 +101,38 @@ const LOCAL_PINNED_WRITE_PYTHON = [ " os.close(root_fd)", ].join("\n"); +const PINNED_WRITE_PYTHON_CANDIDATES = [ + process.env.OPENCLAW_PINNED_WRITE_PYTHON, + "/usr/bin/python3", + "/opt/homebrew/bin/python3", + "/usr/local/bin/python3", +].filter((value): value is string => Boolean(value)); + +let cachedPinnedWritePython = ""; + +function canExecute(binPath: string): boolean { + try { + fsSync.accessSync(binPath, fsSync.constants.X_OK); + return true; + } catch { + return false; + } +} + +function resolvePinnedWritePython(): string { + if (cachedPinnedWritePython) { + return cachedPinnedWritePython; + } + for (const candidate of PINNED_WRITE_PYTHON_CANDIDATES) { + if (canExecute(candidate)) { + cachedPinnedWritePython = candidate; + return cachedPinnedWritePython; + } + } + cachedPinnedWritePython = "python3"; + return cachedPinnedWritePython; +} + function parsePinnedIdentity(stdout: string): FileIdentityStat { const line = stdout .trim() @@ -128,7 +161,7 @@ export async function runPinnedWriteHelper(params: { input: PinnedWriteInput; }): Promise { const child = spawn( - "python3", + resolvePinnedWritePython(), [ "-c", LOCAL_PINNED_WRITE_PYTHON, diff --git a/src/memory/batch-http.test.ts b/src/memory/batch-http.test.ts index 275e3725eb9..7f5bf135e17 100644 --- a/src/memory/batch-http.test.ts +++ b/src/memory/batch-http.test.ts @@ -14,7 +14,6 @@ describe("postJsonWithRetry", () => { let postJsonWithRetry: typeof import("./batch-http.js").postJsonWithRetry; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); ({ postJsonWithRetry } = await import("./batch-http.js")); const retryModule = await import("../infra/retry.js"); diff --git a/src/memory/embeddings-remote-fetch.test.ts b/src/memory/embeddings-remote-fetch.test.ts index eeaa39e9277..f1b9cf7b19a 100644 --- a/src/memory/embeddings-remote-fetch.test.ts +++ b/src/memory/embeddings-remote-fetch.test.ts @@ -13,7 +13,6 @@ describe("fetchRemoteEmbeddingVectors", () => { const postJsonMock = vi.mocked(postJson); beforeEach(async () => { - vi.resetModules(); ({ fetchRemoteEmbeddingVectors } = await import("./embeddings-remote-fetch.js")); vi.clearAllMocks(); }); diff --git a/src/memory/embeddings-voyage.test.ts b/src/memory/embeddings-voyage.test.ts index 9dac8c04d75..5e759f2f629 100644 --- a/src/memory/embeddings-voyage.test.ts +++ b/src/memory/embeddings-voyage.test.ts @@ -23,7 +23,6 @@ let createVoyageEmbeddingProvider: typeof import("./embeddings-voyage.js").creat let normalizeVoyageModel: typeof import("./embeddings-voyage.js").normalizeVoyageModel; beforeEach(async () => { - vi.resetModules(); authModule = await import("../agents/model-auth.js"); ({ createVoyageEmbeddingProvider, normalizeVoyageModel } = await import("./embeddings-voyage.js")); diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 61e2cd71af8..93a2332c9a9 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -37,10 +37,28 @@ const FTS_TABLE = "chunks_fts"; const EMBEDDING_CACHE_TABLE = "embedding_cache"; const BATCH_FAILURE_LIMIT = 2; +const MEMORY_INDEX_MANAGER_CACHE_KEY = "__openclawMemoryIndexManagerCache"; +type MemoryIndexManagerCacheStore = { + indexCache: Map; + indexCachePending: Map>; +}; + +function getMemoryIndexManagerCacheStore(): MemoryIndexManagerCacheStore { + const globalCache = globalThis as typeof globalThis & { + [MEMORY_INDEX_MANAGER_CACHE_KEY]?: MemoryIndexManagerCacheStore; + }; + // Keep manager caches reachable across `vi.resetModules()` so cleanup still reaches older managers. + globalCache[MEMORY_INDEX_MANAGER_CACHE_KEY] ??= { + indexCache: new Map(), + indexCachePending: new Map>(), + }; + return globalCache[MEMORY_INDEX_MANAGER_CACHE_KEY]; +} + const log = createSubsystemLogger("memory"); -const INDEX_CACHE = new Map(); -const INDEX_CACHE_PENDING = new Map>(); +const { indexCache: INDEX_CACHE, indexCachePending: INDEX_CACHE_PENDING } = + getMemoryIndexManagerCacheStore(); export async function closeAllMemoryIndexManagers(): Promise { const pending = Array.from(INDEX_CACHE_PENDING.values()); diff --git a/src/memory/post-json.test.ts b/src/memory/post-json.test.ts index 1fd4210c111..cb5e2bc32bf 100644 --- a/src/memory/post-json.test.ts +++ b/src/memory/post-json.test.ts @@ -11,7 +11,6 @@ describe("postJson", () => { let remoteHttpMock: ReturnType>; beforeEach(async () => { - vi.resetModules(); vi.clearAllMocks(); ({ postJson } = await import("./post-json.js")); ({ withRemoteHttpResponse } = await import("./remote-http.js")); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index 6cc8d9f20a4..24cb901592f 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -8,8 +8,24 @@ import type { MemorySyncProgressUpdate, } from "./types.js"; +const MEMORY_SEARCH_MANAGER_CACHE_KEY = "__openclawMemorySearchManagerCache"; +type MemorySearchManagerCacheStore = { + qmdManagerCache: Map; +}; + +function getMemorySearchManagerCacheStore(): MemorySearchManagerCacheStore { + const globalCache = globalThis as typeof globalThis & { + [MEMORY_SEARCH_MANAGER_CACHE_KEY]?: MemorySearchManagerCacheStore; + }; + // Keep caches reachable across `vi.resetModules()` so later cleanup can close older instances. + globalCache[MEMORY_SEARCH_MANAGER_CACHE_KEY] ??= { + qmdManagerCache: new Map(), + }; + return globalCache[MEMORY_SEARCH_MANAGER_CACHE_KEY]; +} + const log = createSubsystemLogger("memory"); -const QMD_MANAGER_CACHE = new Map(); +const { qmdManagerCache: QMD_MANAGER_CACHE } = getMemorySearchManagerCacheStore(); let managerRuntimePromise: Promise | null = null; function loadManagerRuntime() { diff --git a/test/scripts/test-parallel.test.ts b/test/scripts/test-parallel.test.ts index d5826de5412..c2c217ad181 100644 --- a/test/scripts/test-parallel.test.ts +++ b/test/scripts/test-parallel.test.ts @@ -31,6 +31,14 @@ describe("scripts/test-parallel fatal output guard", () => { expect(resolveTestRunExitCode({ code: 2, signal: null, output: "" })).toBe(2); }); + it("fails even when the fatal line scrolls out of the retained tail", () => { + const fatalLine = "FATAL ERROR: Ineffective mark-compacts near heap limit"; + const output = appendCapturedOutput(fatalLine, "x".repeat(250_000), 200_000); + + expect(hasFatalTestRunOutput(output)).toBe(false); + expect(resolveTestRunExitCode({ code: 0, signal: null, output, fatalSeen: true })).toBe(1); + }); + it("keeps only the tail of captured output", () => { const output = appendCapturedOutput("", "abc", 5); expect(appendCapturedOutput(output, "defg", 5)).toBe("cdefg");