openclaw/scripts/run-oxlint-shards.mjs

import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
  acquireLocalHeavyCheckLockSync,
  resolveLocalHeavyCheckEnv,
  shouldAcquireLocalHeavyCheckLockForOxlint,
} from "./lib/local-heavy-check-runtime.mjs";

const DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE = 8;
const DEFAULT_SHARD_HEARTBEAT_MS = 30_000;
const DEFAULT_SHARD_TIMEOUT_MS = 15 * 60_000;
const DEFAULT_SHARD_KILL_GRACE_MS = 5_000;
const DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY = 4;
const FAST_LOCAL_CHECK_MIN_CPUS = 12;
const FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * 1024 ** 3;
const EXTENSION_TS_CONFIG = "config/tsconfig/oxlint.extensions.json";
const EXTENSIONS_DIR = "extensions";
const OXLINT_SOURCE_FILE_PATTERN = /\.[cm]?[jt]sx?$/;
const PARENT_TERMINATION_SIGNALS = ["SIGINT", "SIGTERM"];
const ACTIVE_SHARD_CHILDREN = new Set();
let parentTerminationSignal = null;
let parentTerminationForceKill = null;
let parentSignalForwardingInstalled = false;

const CORE_SHARD = {
  name: "core",
  args: ["--tsconfig", "config/tsconfig/oxlint.core.json", "src", "ui", "packages"],
};
const CORE_TS_CONFIG = "config/tsconfig/oxlint.core.json";
const CORE_SPLIT_TARGETS = ["ui", "packages"];
const EXTENSIONS_SHARD = {
  name: "extensions",
  args: ["--tsconfig", EXTENSION_TS_CONFIG, EXTENSIONS_DIR],
};
const SCRIPTS_SHARD = {
  name: "scripts",
  args: ["--tsconfig", "config/tsconfig/oxlint.scripts.json", "scripts"],
};

export function createOxlintShards({
  cwd = process.cwd(),
  env = process.env,
  platform = process.platform,
  readDir = fs.readdirSync,
  splitCore = false,
} = {}) {
  const coreShards = splitCore ? createCoreOxlintShards({ cwd, readDir }) : [CORE_SHARD];
  const extensionShards =
    platform === "win32" ? createWindowsExtensionShards({ cwd, env, readDir }) : [EXTENSIONS_SHARD];

  return [...coreShards, ...extensionShards, SCRIPTS_SHARD];
}

export function createCoreOxlintShards({ cwd = process.cwd(), readDir = fs.readdirSync } = {}) {
  const sourceShards = listSourceRootTargetGroups({ cwd, readDir }).map((targets) => ({
    name: targets.length === 1 ? `core:${targets[0].replaceAll("/", ":")}` : "core:src:root",
    args: ["--tsconfig", CORE_TS_CONFIG, ...targets],
  }));
  const sourceEntries = sourceShards.length > 0 ? sourceShards : [createCoreShard("src")];

  return [...sourceEntries, ...CORE_SPLIT_TARGETS.map((target) => createCoreShard(target))];
}

function createCoreShard(target) {
  return {
    name: `core:${target}`,
    args: ["--tsconfig", CORE_TS_CONFIG, target],
  };
}

export function createWindowsExtensionShards({
  cwd = process.cwd(),
  env = process.env,
  readDir = fs.readdirSync,
} = {}) {
  const entries = listExtensionEntries({ cwd, readDir });
  if (entries.dirs.length === 0 && entries.rootFiles.length === 0) {
    return [EXTENSIONS_SHARD];
  }

  const chunkSize = resolveWindowsExtensionChunkSize(env);
  const shards = [];

  if (entries.rootFiles.length > 0) {
    shards.push({
      name: "extensions:root",
      args: ["--tsconfig", EXTENSION_TS_CONFIG, ...entries.rootFiles],
    });
  }

  for (let index = 0; index < entries.dirs.length; index += chunkSize) {
    const chunk = entries.dirs.slice(index, index + chunkSize);
    const chunkNumber = String(index / chunkSize + 1).padStart(2, "0");
    shards.push({
      name: `extensions:${chunkNumber}`,
      args: ["--tsconfig", EXTENSION_TS_CONFIG, ...chunk],
    });
  }
  return shards;
}

export function resolveWindowsExtensionChunkSize(env = process.env) {
  const rawValue = env.OPENCLAW_OXLINT_WINDOWS_EXTENSION_CHUNK_SIZE;
  if (rawValue === undefined) {
    return DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE;
  }

  const parsedValue = Number.parseInt(rawValue, 10);
  return Number.isFinite(parsedValue) && parsedValue > 0
    ? parsedValue
    : DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE;
}

export function shouldRunOxlintShardsSerial({
  env = process.env,
  platform = process.platform,
  hostResources,
} = {}) {
  const explicitMode = env.OPENCLAW_OXLINT_SHARDS_SERIAL?.trim();
  if (explicitMode === "1") {
    return true;
  }
  if (platform === "win32") {
    return true;
  }
  if (explicitMode === "0") {
    return false;
  }
  const localCheckMode = env.OPENCLAW_LOCAL_CHECK_MODE?.trim().toLowerCase();
  if (!isRemoteChangedGateEnv(env)) {
    if (localCheckMode === "full" || localCheckMode === "fast") {
      return false;
    }
    if (localCheckMode === "throttled" || localCheckMode === "low-memory") {
      return true;
    }
  }
  const resources = resolveHostResources(hostResources);
  if (env.CI === "true" || env.GITHUB_ACTIONS === "true") {
    return (
      resources.totalMemoryBytes < FAST_LOCAL_CHECK_MIN_MEMORY_BYTES ||
      resources.logicalCpuCount < FAST_LOCAL_CHECK_MIN_CPUS
    );
  }
  return (
    resources.totalMemoryBytes < FAST_LOCAL_CHECK_MIN_MEMORY_BYTES ||
    resources.logicalCpuCount < FAST_LOCAL_CHECK_MIN_CPUS
  );
}

function isRemoteChangedGateEnv(env) {
  return (
    env.OPENCLAW_CHECK_CHANGED_REMOTE_CHILD === "1" ||
    env.OPENCLAW_CHANGED_LANES_RAW_SYNC === "1"
  );
}

function listExtensionEntries({ cwd, readDir }) {
  let entries;
  try {
    entries = readDir(path.join(cwd, EXTENSIONS_DIR), { withFileTypes: true });
  } catch {
    return {
      dirs: [],
      rootFiles: [],
    };
  }

  const dirs = entries
    .filter((entry) => entry.isDirectory())
    .map((entry) => `${EXTENSIONS_DIR}/${entry.name}`)
    .toSorted((left, right) => left.localeCompare(right));
  const rootFiles = entries
    .filter((entry) => entry.isFile() && OXLINT_SOURCE_FILE_PATTERN.test(entry.name))
    .map((entry) => `${EXTENSIONS_DIR}/${entry.name}`)
    .toSorted((left, right) => left.localeCompare(right));

  return {
    dirs,
    rootFiles,
  };
}

function listSourceRootTargetGroups({ cwd, readDir }) {
  let entries;
  try {
    entries = readDir(path.join(cwd, "src"), { withFileTypes: true });
  } catch {
    return [];
  }

  const dirs = entries
    .filter((entry) => entry.isDirectory())
    .map((entry) => `src/${entry.name}`)
    .toSorted((left, right) => left.localeCompare(right));
  const rootFiles = entries
    .filter((entry) => entry.isFile() && OXLINT_SOURCE_FILE_PATTERN.test(entry.name))
    .map((entry) => `src/${entry.name}`)
    .toSorted((left, right) => left.localeCompare(right));

  return [...dirs.map((target) => [target]), ...(rootFiles.length > 0 ? [rootFiles] : [])];
}

export async function main(extraArgs = process.argv.slice(2), runtimeEnv = process.env) {
  const runner = path.resolve("scripts", "run-oxlint.mjs");
  const shardArgs = parseShardRunnerArgs(extraArgs);
  const env = resolveLocalHeavyCheckEnv(runtimeEnv);
  const hasMetadataOnlyFlag = shardArgs.oxlintArgs.some((arg) =>
    ["--help", "-h", "--version", "-V", "--rules", "--print-config", "--init"].includes(arg),
  );
  const shouldAcquireParentLock =
    !hasMetadataOnlyFlag ||
    shouldAcquireLocalHeavyCheckLockForOxlint(shardArgs.oxlintArgs, {
      cwd: process.cwd(),
      env,
    });
  const releaseLock =
    env.OPENCLAW_OXLINT_SKIP_LOCK === "1"
      ? () => {}
      : shouldAcquireParentLock
        ? acquireLocalHeavyCheckLockSync({
            cwd: process.cwd(),
            env,
            toolName: "oxlint shards",
          })
        : () => {};

  const shards = createOxlintShards({
    cwd: process.cwd(),
    env,
    platform: process.platform,
    splitCore: shardArgs.splitCore,
  });
  const selectedShards = filterOxlintShards(shards, shardArgs.only);

  try {
    const prepareResult = spawnSync(
      process.execPath,
      [path.resolve("scripts", "prepare-extension-package-boundary-artifacts.mjs")],
      {
        stdio: "inherit",
        env,
      },
    );

    if (prepareResult.error) {
      throw prepareResult.error;
    }
    if ((prepareResult.status ?? 1) !== 0) {
      process.exitCode = prepareResult.status ?? 1;
    } else {
      const shardConcurrency = resolveOxlintShardConcurrency({
        env,
        platform: process.platform,
        splitCore: shardArgs.splitCore,
      });
      const results = shardConcurrency <= 1
        ? await runShardsSerial({
            entries: selectedShards,
            env,
            extraArgs: shardArgs.oxlintArgs,
            runner,
          })
        : await runShardsParallel({
            concurrency: Math.min(shardConcurrency, selectedShards.length),
            entries: selectedShards,
            env,
            extraArgs: shardArgs.oxlintArgs,
            runner,
          });
      process.exitCode = results.find((status) => status !== 0) ?? 0;
    }
  } finally {
    releaseLock();
  }
}

if (import.meta.main) {
  await main();
}

function resolveHostResources(hostResources) {
  if (hostResources) {
    return hostResources;
  }

  return {
    totalMemoryBytes: os.totalmem(),
    logicalCpuCount:
      typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length,
  };
}

export function parseShardRunnerArgs(args) {
  const only = new Set();
  const oxlintArgs = [];
  let splitCore = false;

  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--split-core") {
      splitCore = true;
      continue;
    }
    if (arg === "--only") {
      const value = args[index + 1];
      if (value) {
        only.add(value);
        index += 1;
      }
      continue;
    }
    if (arg.startsWith("--only=")) {
      const value = arg.slice("--only=".length);
      if (value) {
        only.add(value);
      }
      continue;
    }
    oxlintArgs.push(arg);
  }

  return { only, oxlintArgs, splitCore };
}

export function filterOxlintShards(shards, only) {
  if (only.size === 0) {
    return shards;
  }

  return shards.filter((shard) => only.has(shard.name) || only.has(shard.name.split(":")[0]));
}

export function resolveOxlintShardConcurrency({
  env = process.env,
  platform = process.platform,
  hostResources,
  splitCore = false,
} = {}) {
  if (shouldRunOxlintShardsSerial({ env, platform, hostResources })) {
    return 1;
  }

  const explicitConcurrency = resolvePositiveEnvInt(env, "OPENCLAW_OXLINT_SHARD_CONCURRENCY");
  if (explicitConcurrency !== null) {
    return explicitConcurrency;
  }

  if (!splitCore) {
    return Number.MAX_SAFE_INTEGER;
  }

  const resources = resolveHostResources(hostResources);
  return Math.max(
    1,
    Math.min(DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY, Math.floor(resources.logicalCpuCount / 4)),
  );
}

async function runShardsSerial({ entries, env, extraArgs, runner }) {
  const results = [];
  for (const shard of entries) {
    results.push(await runShard({ env, extraArgs, runner, shard }));
    if (isParentTerminationRequested()) {
      break;
    }
  }
  return results;
}

async function runShardsParallel({ concurrency, entries, env, extraArgs, runner }) {
  const results = [];
  results.length = entries.length;
  let nextIndex = 0;

  const workers = Array.from({ length: concurrency }, async () => {
    for (;;) {
      if (isParentTerminationRequested()) {
        return;
      }
      const currentIndex = nextIndex;
      nextIndex += 1;
      const shard = entries[currentIndex];
      if (!shard) {
        return;
      }
      results[currentIndex] = await runShard({ env, extraArgs, runner, shard });
    }
  });

  await Promise.all(workers);
  return results.filter((status) => status !== undefined);
}

export async function runShard({ env, extraArgs, runner, shard }) {
  console.error(`[oxlint:${shard.name}] starting`);
  const startedAt = Date.now();
  const heartbeatMs = resolveShardHeartbeatMs(env);
  const timeoutMs = resolveShardTimeoutMs(env);
  const killGraceMs = resolveShardKillGraceMs(env);
  const useProcessGroup = process.platform !== "win32";
  const child = spawn(process.execPath, [runner, ...shard.args, ...extraArgs], {
    stdio: "inherit",
    detached: useProcessGroup,
    env: {
      ...env,
      OPENCLAW_OXLINT_SKIP_LOCK: "1",
      OPENCLAW_OXLINT_SKIP_PREPARE: "1",
    },
  });
  const unregisterShardChild = registerShardChild({ child, killGraceMs, useProcessGroup });

  return await new Promise((resolve) => {
    let finished = false;
    let timedOut = false;
    let forceKill = null;
    const heartbeat =
      heartbeatMs > 0
        ? setInterval(() => {
            const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
            console.error(`[oxlint:${shard.name}] still running after ${elapsedSeconds}s`);
          }, heartbeatMs)
        : null;
    heartbeat?.unref();
    const timeout =
      timeoutMs > 0
        ? setTimeout(() => {
            timedOut = true;
            const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
            console.error(
              `[oxlint:${shard.name}] timed out after ${elapsedSeconds}s; terminating shard`,
            );
            signalChildProcess({ child, signal: "SIGTERM", useProcessGroup });
            if (killGraceMs > 0) {
              forceKill = setTimeout(() => {
                console.error(`[oxlint:${shard.name}] did not exit cleanly; killing shard`);
                signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
              }, killGraceMs);
              forceKill.unref();
            } else {
              signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
            }
          }, timeoutMs)
        : null;
    timeout?.unref();
    const finish = (status) => {
      if (finished) {
        return;
      }
      finished = true;
      if (heartbeat) {
        clearInterval(heartbeat);
      }
      if (timeout) {
        clearTimeout(timeout);
      }
      if (forceKill) {
        clearTimeout(forceKill);
      }
      unregisterShardChild();
      console.error(`[oxlint:${shard.name}] finished`);
      resolve(status);
    };
    child.once("error", (error) => {
      console.error(error);
      finish(1);
    });
    child.once("close", (status) => {
      finish(
        parentTerminationSignal
          ? getSignalExitCode(parentTerminationSignal)
          : timedOut
            ? 124
            : (status ?? 1),
      );
    });
  });
}

export function resolveShardHeartbeatMs(env) {
  return resolveNonNegativeEnvInt(
    env,
    "OPENCLAW_OXLINT_SHARD_HEARTBEAT_MS",
    DEFAULT_SHARD_HEARTBEAT_MS,
  );
}

export function resolveShardTimeoutMs(env) {
  return resolveNonNegativeEnvInt(
    env,
    "OPENCLAW_OXLINT_SHARD_TIMEOUT_MS",
    DEFAULT_SHARD_TIMEOUT_MS,
  );
}

export function resolveShardKillGraceMs(env) {
  return resolveNonNegativeEnvInt(
    env,
    "OPENCLAW_OXLINT_SHARD_KILL_GRACE_MS",
    DEFAULT_SHARD_KILL_GRACE_MS,
  );
}

function resolveNonNegativeEnvInt(env, key, defaultValue) {
  const rawValue = env[key];
  if (rawValue === undefined) {
    return defaultValue;
  }

  const parsedValue = Number.parseInt(rawValue, 10);
  return Number.isFinite(parsedValue) && parsedValue >= 0 ? parsedValue : defaultValue;
}

function resolvePositiveEnvInt(env, key) {
  const rawValue = env[key];
  if (rawValue === undefined) {
    return null;
  }

  const parsedValue = Number.parseInt(rawValue, 10);
  return Number.isFinite(parsedValue) && parsedValue > 0 ? parsedValue : null;
}

function signalChildProcess({ child, signal, useProcessGroup }) {
  if (!child.pid) {
    return;
  }

  try {
    if (useProcessGroup) {
      process.kill(-child.pid, signal);
    } else {
      child.kill(signal);
    }
  } catch (error) {
    if (error?.code !== "ESRCH") {
      console.error(error);
    }
  }
}

function registerShardChild(entry) {
  installParentSignalForwarding();
  ACTIVE_SHARD_CHILDREN.add(entry);
  return () => {
    ACTIVE_SHARD_CHILDREN.delete(entry);
    if (ACTIVE_SHARD_CHILDREN.size === 0 && parentTerminationForceKill) {
      clearTimeout(parentTerminationForceKill);
      parentTerminationForceKill = null;
    }
  };
}

function installParentSignalForwarding() {
  if (parentSignalForwardingInstalled) {
    return;
  }
  parentSignalForwardingInstalled = true;
  for (const signal of PARENT_TERMINATION_SIGNALS) {
    process.on(signal, () => {
      parentTerminationSignal = signal;
      process.exitCode = getSignalExitCode(signal);
      if (ACTIVE_SHARD_CHILDREN.size === 0) {
        process.exit(process.exitCode);
      }
      signalActiveShardChildren(signal);
      scheduleParentTerminationForceKill();
    });
  }
  process.once("exit", () => {
    signalActiveShardChildren("SIGTERM");
  });
}

function isParentTerminationRequested() {
  return parentTerminationSignal !== null;
}

function signalActiveShardChildren(signal) {
  for (const entry of ACTIVE_SHARD_CHILDREN) {
    signalChildProcess({ ...entry, signal });
  }
}

function scheduleParentTerminationForceKill() {
  if (parentTerminationForceKill) {
    return;
  }
  const killGraceMs = Math.max(
    0,
    ...Array.from(ACTIVE_SHARD_CHILDREN, (entry) => entry.killGraceMs),
  );
  if (killGraceMs === 0) {
    signalActiveShardChildren("SIGKILL");
    return;
  }
  parentTerminationForceKill = setTimeout(() => {
    parentTerminationForceKill = null;
    signalActiveShardChildren("SIGKILL");
  }, killGraceMs);
  parentTerminationForceKill.unref();
}

function getSignalExitCode(signal) {
  return signal === "SIGINT" ? 130 : 143;
}