fix(ci): harden test gating under load

scripts/test-parallel-utils.mjs

@@ -1,9 +1,10 @@
 const DEFAULT_OUTPUT_CAPTURE_LIMIT = 200_000;
 
 const fatalOutputPatterns = [
-  /FATAL ERROR:.*heap out of memory/i,
-  /Allocation failed - JavaScript heap out of memory/i,
+  /FATAL ERROR:/i,
+  /JavaScript heap out of memory/i,
   /node::OOMErrorHandler/i,
+  /ERR_WORKER_OUT_OF_MEMORY/i,
 ];
 
 export function appendCapturedOutput(current, chunk, limit = DEFAULT_OUTPUT_CAPTURE_LIMIT) {
@@ -21,14 +22,17 @@ export function hasFatalTestRunOutput(output) {
   return fatalOutputPatterns.some((pattern) => pattern.test(output));
 }
 
-export function resolveTestRunExitCode({ code, signal, output }) {
+export function resolveTestRunExitCode({ code, signal, output, fatalSeen = false, childError }) {
   if (typeof code === "number" && code !== 0) {
     return code;
   }
+  if (childError) {
+    return 1;
+  }
   if (signal) {
     return 1;
   }
-  if (hasFatalTestRunOutput(output)) {
+  if (fatalSeen || hasFatalTestRunOutput(output)) {
     return 1;
   }
   return code ?? 0;

scripts/test-parallel.mjs

@@ -4,7 +4,11 @@ import os from "node:os";
 import path from "node:path";
 import { channelTestPrefixes } from "../vitest.channel-paths.mjs";
 import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs";
-import { appendCapturedOutput, resolveTestRunExitCode } from "./test-parallel-utils.mjs";
+import {
+  appendCapturedOutput,
+  hasFatalTestRunOutput,
+  resolveTestRunExitCode,
+} from "./test-parallel-utils.mjs";
 import {
   loadTestRunnerBehavior,
   loadUnitTimingManifest,
@@ -742,6 +746,8 @@ const runOnce = (entry, extraArgs = []) =>
       ? `${nextNodeOptions} ${heapFlag}`.trim()
       : nextNodeOptions;
     let output = "";
+    let fatalSeen = false;
+    let childError = null;
     let child;
     try {
       child = spawn(pnpm, args, {
@@ -757,20 +763,23 @@ const runOnce = (entry, extraArgs = []) =>
     children.add(child);
     child.stdout?.on("data", (chunk) => {
       const text = chunk.toString();
+      fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
       output = appendCapturedOutput(output, text);
       process.stdout.write(chunk);
     });
     child.stderr?.on("data", (chunk) => {
       const text = chunk.toString();
+      fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`);
       output = appendCapturedOutput(output, text);
       process.stderr.write(chunk);
     });
     child.on("error", (err) => {
+      childError = err;
       console.error(`[test-parallel] child error: ${String(err)}`);
     });
     child.on("close", (code, signal) => {
       children.delete(child);
-      const resolvedCode = resolveTestRunExitCode({ code, signal, output });
+      const resolvedCode = resolveTestRunExitCode({ code, signal, output, fatalSeen, childError });
       console.log(
         `[test-parallel] done ${entry.name} code=${String(resolvedCode)} elapsed=${formatElapsedMs(Date.now() - startedAt)}`,
       );

src/channels/plugins/actions/actions.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
 import type { OpenClawConfig } from "../../../config/config.js";
 import type { ChannelMessageActionAdapter } from "../types.js";
 
@@ -199,13 +199,15 @@ async function expectSlackSendRejected(params: Record<string, unknown>, error: R
   expect(handleSlackAction).not.toHaveBeenCalled();
 }
 
-beforeEach(async () => {
-  vi.resetModules();
+beforeAll(async () => {
   ({ discordMessageActions } = await import("../../../../extensions/discord/runtime-api.js"));
   ({ handleDiscordMessageAction } = await import("./discord/handle-action.js"));
   ({ telegramMessageActions } = await import("../../../../extensions/telegram/runtime-api.js"));
   ({ signalMessageActions } = await import("../../../../extensions/signal/src/message-actions.js"));
   ({ createSlackActions } = await import("../../../../extensions/slack/src/channel-actions.js"));
+});
+
+beforeEach(() => {
   vi.clearAllMocks();
 });
 

src/infra/fs-pinned-write-helper.ts

@@ -1,5 +1,6 @@
 import { spawn } from "node:child_process";
 import { once } from "node:events";
+import fsSync from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
 import type { Readable } from "node:stream";
@@ -100,6 +101,38 @@ const LOCAL_PINNED_WRITE_PYTHON = [
   "    os.close(root_fd)",
 ].join("\n");
 
+const PINNED_WRITE_PYTHON_CANDIDATES = [
+  process.env.OPENCLAW_PINNED_WRITE_PYTHON,
+  "/usr/bin/python3",
+  "/opt/homebrew/bin/python3",
+  "/usr/local/bin/python3",
+].filter((value): value is string => Boolean(value));
+
+let cachedPinnedWritePython = "";
+
+function canExecute(binPath: string): boolean {
+  try {
+    fsSync.accessSync(binPath, fsSync.constants.X_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function resolvePinnedWritePython(): string {
+  if (cachedPinnedWritePython) {
+    return cachedPinnedWritePython;
+  }
+  for (const candidate of PINNED_WRITE_PYTHON_CANDIDATES) {
+    if (canExecute(candidate)) {
+      cachedPinnedWritePython = candidate;
+      return cachedPinnedWritePython;
+    }
+  }
+  cachedPinnedWritePython = "python3";
+  return cachedPinnedWritePython;
+}
+
 function parsePinnedIdentity(stdout: string): FileIdentityStat {
   const line = stdout
     .trim()
@@ -128,7 +161,7 @@ export async function runPinnedWriteHelper(params: {
   input: PinnedWriteInput;
 }): Promise<FileIdentityStat> {
   const child = spawn(
-    "python3",
+    resolvePinnedWritePython(),
     [
       "-c",
       LOCAL_PINNED_WRITE_PYTHON,

src/memory/batch-http.test.ts

@@ -14,7 +14,6 @@ describe("postJsonWithRetry", () => {
   let postJsonWithRetry: typeof import("./batch-http.js").postJsonWithRetry;
 
   beforeEach(async () => {
-    vi.resetModules();
     vi.clearAllMocks();
     ({ postJsonWithRetry } = await import("./batch-http.js"));
     const retryModule = await import("../infra/retry.js");

src/memory/embeddings-remote-fetch.test.ts

@@ -13,7 +13,6 @@ describe("fetchRemoteEmbeddingVectors", () => {
   const postJsonMock = vi.mocked(postJson);
 
   beforeEach(async () => {
-    vi.resetModules();
     ({ fetchRemoteEmbeddingVectors } = await import("./embeddings-remote-fetch.js"));
     vi.clearAllMocks();
   });

src/memory/embeddings-voyage.test.ts

@@ -23,7 +23,6 @@ let createVoyageEmbeddingProvider: typeof import("./embeddings-voyage.js").creat
 let normalizeVoyageModel: typeof import("./embeddings-voyage.js").normalizeVoyageModel;
 
 beforeEach(async () => {
-  vi.resetModules();
   authModule = await import("../agents/model-auth.js");
   ({ createVoyageEmbeddingProvider, normalizeVoyageModel } =
     await import("./embeddings-voyage.js"));

src/memory/manager.ts

@@ -37,10 +37,28 @@ const FTS_TABLE = "chunks_fts";
 const EMBEDDING_CACHE_TABLE = "embedding_cache";
 const BATCH_FAILURE_LIMIT = 2;
 
+const MEMORY_INDEX_MANAGER_CACHE_KEY = "__openclawMemoryIndexManagerCache";
+type MemoryIndexManagerCacheStore = {
+  indexCache: Map<string, MemoryIndexManager>;
+  indexCachePending: Map<string, Promise<MemoryIndexManager>>;
+};
+
+function getMemoryIndexManagerCacheStore(): MemoryIndexManagerCacheStore {
+  const globalCache = globalThis as typeof globalThis & {
+    [MEMORY_INDEX_MANAGER_CACHE_KEY]?: MemoryIndexManagerCacheStore;
+  };
+  // Keep manager caches reachable across `vi.resetModules()` so cleanup still reaches older managers.
+  globalCache[MEMORY_INDEX_MANAGER_CACHE_KEY] ??= {
+    indexCache: new Map<string, MemoryIndexManager>(),
+    indexCachePending: new Map<string, Promise<MemoryIndexManager>>(),
+  };
+  return globalCache[MEMORY_INDEX_MANAGER_CACHE_KEY];
+}
+
 const log = createSubsystemLogger("memory");
 
-const INDEX_CACHE = new Map<string, MemoryIndexManager>();
-const INDEX_CACHE_PENDING = new Map<string, Promise<MemoryIndexManager>>();
+const { indexCache: INDEX_CACHE, indexCachePending: INDEX_CACHE_PENDING } =
+  getMemoryIndexManagerCacheStore();
 
 export async function closeAllMemoryIndexManagers(): Promise<void> {
   const pending = Array.from(INDEX_CACHE_PENDING.values());

src/memory/post-json.test.ts

@@ -11,7 +11,6 @@ describe("postJson", () => {
   let remoteHttpMock: ReturnType<typeof vi.mocked<typeof withRemoteHttpResponse>>;
 
   beforeEach(async () => {
-    vi.resetModules();
     vi.clearAllMocks();
     ({ postJson } = await import("./post-json.js"));
     ({ withRemoteHttpResponse } = await import("./remote-http.js"));

src/memory/search-manager.ts

@@ -8,8 +8,24 @@ import type {
   MemorySyncProgressUpdate,
 } from "./types.js";
 
+const MEMORY_SEARCH_MANAGER_CACHE_KEY = "__openclawMemorySearchManagerCache";
+type MemorySearchManagerCacheStore = {
+  qmdManagerCache: Map<string, MemorySearchManager>;
+};
+
+function getMemorySearchManagerCacheStore(): MemorySearchManagerCacheStore {
+  const globalCache = globalThis as typeof globalThis & {
+    [MEMORY_SEARCH_MANAGER_CACHE_KEY]?: MemorySearchManagerCacheStore;
+  };
+  // Keep caches reachable across `vi.resetModules()` so later cleanup can close older instances.
+  globalCache[MEMORY_SEARCH_MANAGER_CACHE_KEY] ??= {
+    qmdManagerCache: new Map<string, MemorySearchManager>(),
+  };
+  return globalCache[MEMORY_SEARCH_MANAGER_CACHE_KEY];
+}
+
 const log = createSubsystemLogger("memory");
-const QMD_MANAGER_CACHE = new Map<string, MemorySearchManager>();
+const { qmdManagerCache: QMD_MANAGER_CACHE } = getMemorySearchManagerCacheStore();
 let managerRuntimePromise: Promise<typeof import("./manager-runtime.js")> | null = null;
 
 function loadManagerRuntime() {

test/scripts/test-parallel.test.ts

@@ -31,6 +31,14 @@ describe("scripts/test-parallel fatal output guard", () => {
     expect(resolveTestRunExitCode({ code: 2, signal: null, output: "" })).toBe(2);
   });
 
+  it("fails even when the fatal line scrolls out of the retained tail", () => {
+    const fatalLine = "FATAL ERROR: Ineffective mark-compacts near heap limit";
+    const output = appendCapturedOutput(fatalLine, "x".repeat(250_000), 200_000);
+
+    expect(hasFatalTestRunOutput(output)).toBe(false);
+    expect(resolveTestRunExitCode({ code: 0, signal: null, output, fatalSeen: true })).toBe(1);
+  });
+
   it("keeps only the tail of captured output", () => {
     const output = appendCapturedOutput("", "abc", 5);
     expect(appendCapturedOutput(output, "defg", 5)).toBe("cdefg");