From db0209ac5d8a7cdf2032e89135315f5c2a97eb67 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 31 May 2026 19:51:20 +0200 Subject: [PATCH] perf(scripts): parallelize remote core oxlint shards --- scripts/run-oxlint-shards.mjs | 105 ++++++++++++++++++++++++++------ test/scripts/run-oxlint.test.ts | 58 +++++++++++++++++- 2 files changed, 145 insertions(+), 18 deletions(-) diff --git a/scripts/run-oxlint-shards.mjs b/scripts/run-oxlint-shards.mjs index 324b6cf6085..c682716c9ec 100644 --- a/scripts/run-oxlint-shards.mjs +++ b/scripts/run-oxlint-shards.mjs @@ -12,6 +12,7 @@ const DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE = 8; const DEFAULT_SHARD_HEARTBEAT_MS = 30_000; const DEFAULT_SHARD_TIMEOUT_MS = 15 * 60_000; const DEFAULT_SHARD_KILL_GRACE_MS = 5_000; +const DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY = 4; const FAST_LOCAL_CHECK_MIN_CPUS = 12; const FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * 1024 ** 3; const EXTENSION_TS_CONFIG = "config/tsconfig/oxlint.extensions.json"; @@ -128,11 +129,13 @@ export function shouldRunOxlintShardsSerial({ return false; } const localCheckMode = env.OPENCLAW_LOCAL_CHECK_MODE?.trim().toLowerCase(); - if (localCheckMode === "full" || localCheckMode === "fast") { - return false; - } - if (localCheckMode === "throttled" || localCheckMode === "low-memory") { - return true; + if (!isRemoteChangedGateEnv(env)) { + if (localCheckMode === "full" || localCheckMode === "fast") { + return false; + } + if (localCheckMode === "throttled" || localCheckMode === "low-memory") { + return true; + } } const resources = resolveHostResources(hostResources); if (env.CI === "true" || env.GITHUB_ACTIONS === "true") { @@ -147,6 +150,13 @@ export function shouldRunOxlintShardsSerial({ ); } +function isRemoteChangedGateEnv(env) { + return ( + env.OPENCLAW_CHECK_CHANGED_REMOTE_CHILD === "1" || + env.OPENCLAW_CHANGED_LANES_RAW_SYNC === "1" + ); +} + function listExtensionEntries({ cwd, readDir }) { let entries; try { @@ -241,24 +251,25 @@ export async function main(extraArgs = process.argv.slice(2), runtimeEnv = proce if ((prepareResult.status ?? 1) !== 0) { process.exitCode = prepareResult.status ?? 1; } else { - const runSerial = - shardArgs.splitCore || - shouldRunOxlintShardsSerial({ - env, - platform: process.platform, - }); - const results = runSerial + const shardConcurrency = resolveOxlintShardConcurrency({ + env, + platform: process.platform, + splitCore: shardArgs.splitCore, + }); + const results = shardConcurrency <= 1 ? await runShardsSerial({ entries: selectedShards, env, extraArgs: shardArgs.oxlintArgs, runner, }) - : await Promise.all( - selectedShards.map((shard) => - runShard({ env, extraArgs: shardArgs.oxlintArgs, runner, shard }), - ), - ); + : await runShardsParallel({ + concurrency: Math.min(shardConcurrency, selectedShards.length), + entries: selectedShards, + env, + extraArgs: shardArgs.oxlintArgs, + runner, + }); process.exitCode = results.find((status) => status !== 0) ?? 0; } } finally { @@ -322,6 +333,32 @@ export function filterOxlintShards(shards, only) { return shards.filter((shard) => only.has(shard.name) || only.has(shard.name.split(":")[0])); } +export function resolveOxlintShardConcurrency({ + env = process.env, + platform = process.platform, + hostResources, + splitCore = false, +} = {}) { + if (shouldRunOxlintShardsSerial({ env, platform, hostResources })) { + return 1; + } + + const explicitConcurrency = resolvePositiveEnvInt(env, "OPENCLAW_OXLINT_SHARD_CONCURRENCY"); + if (explicitConcurrency !== null) { + return explicitConcurrency; + } + + if (!splitCore) { + return Number.MAX_SAFE_INTEGER; + } + + const resources = resolveHostResources(hostResources); + return Math.max( + 1, + Math.min(DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY, Math.floor(resources.logicalCpuCount / 4)), + ); +} + async function runShardsSerial({ entries, env, extraArgs, runner }) { const results = []; for (const shard of entries) { @@ -333,6 +370,30 @@ async function runShardsSerial({ entries, env, extraArgs, runner }) { return results; } +async function runShardsParallel({ concurrency, entries, env, extraArgs, runner }) { + const results = []; + results.length = entries.length; + let nextIndex = 0; + + const workers = Array.from({ length: concurrency }, async () => { + for (;;) { + if (isParentTerminationRequested()) { + return; + } + const currentIndex = nextIndex; + nextIndex += 1; + const shard = entries[currentIndex]; + if (!shard) { + return; + } + results[currentIndex] = await runShard({ env, extraArgs, runner, shard }); + } + }); + + await Promise.all(workers); + return results.filter((status) => status !== undefined); +} + export async function runShard({ env, extraArgs, runner, shard }) { console.error(`[oxlint:${shard.name}] starting`); const startedAt = Date.now(); @@ -452,6 +513,16 @@ function resolveNonNegativeEnvInt(env, key, defaultValue) { return Number.isFinite(parsedValue) && parsedValue >= 0 ? parsedValue : defaultValue; } +function resolvePositiveEnvInt(env, key) { + const rawValue = env[key]; + if (rawValue === undefined) { + return null; + } + + const parsedValue = Number.parseInt(rawValue, 10); + return Number.isFinite(parsedValue) && parsedValue > 0 ? parsedValue : null; +} + function signalChildProcess({ child, signal, useProcessGroup }) { if (!child.pid) { return; diff --git a/test/scripts/run-oxlint.test.ts b/test/scripts/run-oxlint.test.ts index e640d85a5ad..22032eb6ee0 100644 --- a/test/scripts/run-oxlint.test.ts +++ b/test/scripts/run-oxlint.test.ts @@ -12,6 +12,7 @@ import { resolveShardKillGraceMs, resolveShardHeartbeatMs, resolveShardTimeoutMs, + resolveOxlintShardConcurrency, resolveWindowsExtensionChunkSize, runShard, shouldRunOxlintShardsSerial, @@ -67,7 +68,7 @@ describe("run-oxlint", () => { expect(childSkipIndex).toBeGreaterThan(lockIndex); }); - it("lets dev update preflight run oxlint shards serially", () => { + it("keeps a serial oxlint shard path available", () => { const shardedLintRunner = readFileSync("scripts/run-oxlint-shards.mjs", "utf8"); expect(shardedLintRunner).toContain("OPENCLAW_OXLINT_SHARDS_SERIAL"); @@ -143,6 +144,61 @@ describe("run-oxlint", () => { ).toBe(false); }); + it("bounds split-core shard parallelism on roomy CI hosts", () => { + const roomyHost = { totalMemoryBytes: 64 * 1024 ** 3, logicalCpuCount: 16 }; + + expect( + resolveOxlintShardConcurrency({ + env: { CI: "true" }, + platform: "linux", + hostResources: roomyHost, + splitCore: true, + }), + ).toBe(4); + }); + + it("keeps split-core shard runs serial on constrained hosts", () => { + const constrainedHost = { totalMemoryBytes: 8 * 1024 ** 3, logicalCpuCount: 4 }; + + expect( + resolveOxlintShardConcurrency({ + env: { CI: "true" }, + platform: "linux", + hostResources: constrainedHost, + splitCore: true, + }), + ).toBe(1); + }); + + it("does not let local throttled mode serialize remote changed gates", () => { + const roomyHost = { totalMemoryBytes: 64 * 1024 ** 3, logicalCpuCount: 16 }; + + expect( + resolveOxlintShardConcurrency({ + env: { + OPENCLAW_CHECK_CHANGED_REMOTE_CHILD: "1", + OPENCLAW_LOCAL_CHECK_MODE: "throttled", + }, + platform: "linux", + hostResources: roomyHost, + splitCore: true, + }), + ).toBe(4); + }); + + it("honors explicit oxlint shard concurrency overrides", () => { + const roomyHost = { totalMemoryBytes: 64 * 1024 ** 3, logicalCpuCount: 16 }; + + expect( + resolveOxlintShardConcurrency({ + env: { CI: "true", OPENCLAW_OXLINT_SHARD_CONCURRENCY: "2" }, + platform: "linux", + hostResources: roomyHost, + splitCore: true, + }), + ).toBe(2); + }); + it("uses a bounded oxlint shard heartbeat by default", () => { expect(resolveShardHeartbeatMs({})).toBe(30_000); expect(resolveShardHeartbeatMs({ OPENCLAW_OXLINT_SHARD_HEARTBEAT_MS: "0" })).toBe(0);