From 017b8db616b94baa049f11db0c53b0daaf1c177d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 06:14:13 +0100 Subject: [PATCH] ci: speed up release validation shards --- .../openclaw-live-and-e2e-checks-reusable.yml | 7 +- scripts/lib/docker-e2e-plan.mjs | 6 +- scripts/lib/docker-e2e-scenarios.mjs | 23 +++-- scripts/test-docker-all.mjs | 84 ++++++++++++++--- .../live-cache-regression-runner.test.ts | 30 ++++++ src/agents/live-cache-regression-runner.ts | 94 +++++++++++++++---- test/scripts/docker-e2e-plan.test.ts | 23 ++++- 7 files changed, 223 insertions(+), 44 deletions(-) diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 4f046e3d6ff..d4745362224 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -471,10 +471,13 @@ jobs: timeout_minutes: 90 - chunk_id: bundled-channels-update-a label: bundled channels update A - timeout_minutes: 90 + timeout_minutes: 45 + - chunk_id: bundled-channels-update-discord + label: bundled channels update Discord + timeout_minutes: 30 - chunk_id: bundled-channels-update-b label: bundled channels update B - timeout_minutes: 90 + timeout_minutes: 45 - chunk_id: bundled-channels-contracts label: bundled channels contracts timeout_minutes: 90 diff --git a/scripts/lib/docker-e2e-plan.mjs b/scripts/lib/docker-e2e-plan.mjs index f6ca3deee94..07d1eefc727 100644 --- a/scripts/lib/docker-e2e-plan.mjs +++ b/scripts/lib/docker-e2e-plan.mjs @@ -122,10 +122,13 @@ export function laneResources(poolLane) { export function laneSummary(poolLane) { const resources = laneResources(poolLane).join(","); const timeout = poolLane.timeoutMs ? ` timeout=${Math.round(poolLane.timeoutMs / 1000)}s` : ""; + const noOutputTimeout = poolLane.noOutputTimeoutMs + ? ` no-output=${Math.round(poolLane.noOutputTimeoutMs / 1000)}s` + : ""; const retries = poolLane.retries > 0 ? ` retries=${poolLane.retries}` : ""; const cache = poolLane.cacheKey ? ` cache=${poolLane.cacheKey}` : ""; const image = poolLane.e2eImageKind ? ` image=${poolLane.e2eImageKind}` : ""; - return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources}${timeout}${retries}${cache}${image})`; + return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources}${timeout}${noOutputTimeout}${retries}${cache}${image})`; } export function lanesNeedE2eImageKind(poolLanes, kind) { @@ -179,6 +182,7 @@ export function buildPlanJson(params) { imageKind: poolLane.e2eImageKind, live: poolLane.live, name: poolLane.name, + noOutputTimeoutMs: poolLane.noOutputTimeoutMs, resources: laneResources(poolLane), timeoutMs: poolLane.timeoutMs, weight: laneWeight(poolLane), diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index 415f071c88c..87dc92d66c5 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -2,7 +2,8 @@ // Keep lane names, commands, image kind, timeout, resources, and release chunks // here. Planning and execution live in separate modules. -const BUNDLED_UPDATE_TIMEOUT_MS = 20 * 60 * 1000; +const BUNDLED_UPDATE_NO_OUTPUT_TIMEOUT_MS = 4 * 60 * 1000; +const BUNDLED_UPDATE_TIMEOUT_MS = 6 * 60 * 1000; export const DEFAULT_LIVE_RETRIES = 1; const LIVE_ACP_TIMEOUT_MS = 20 * 60 * 1000; const LIVE_CLI_TIMEOUT_MS = 20 * 60 * 1000; @@ -37,6 +38,7 @@ function lane(name, command, options = {}) { : (options.e2eImageKind ?? (options.live ? undefined : "functional")), estimateSeconds: options.estimateSeconds, live: options.live === true, + noOutputTimeoutMs: options.noOutputTimeoutMs, name, retryPatterns: options.retryPatterns ?? [], retries: options.retries ?? 0, @@ -131,7 +133,12 @@ const bundledChannelUpdateLanes = [ bundledChannelScenarioLane( `bundled-channel-update-${target}`, `OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=${target} OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0`, - { retryPatterns: LIVE_RETRY_PATTERNS, retries: 1, timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, + { + noOutputTimeoutMs: BUNDLED_UPDATE_NO_OUTPUT_TIMEOUT_MS, + retryPatterns: LIVE_RETRY_PATTERNS, + retries: 1, + timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS, + }, ), ); @@ -505,11 +512,8 @@ const primaryReleasePathChunks = { "plugins-runtime-install-a": bundledPluginInstallUninstallLanes.slice(0, 4), "plugins-runtime-install-b": bundledPluginInstallUninstallLanes.slice(4), "bundled-channels-core": [releasePathBundledChannelLanes[0], ...bundledChannelSmokeLanes], - "bundled-channels-update-a": [ - bundledChannelUpdateLanes[0], - bundledChannelUpdateLanes[1], - bundledChannelUpdateLanes[4], - ], + "bundled-channels-update-a": [bundledChannelUpdateLanes[0], bundledChannelUpdateLanes[4]], + "bundled-channels-update-discord": [bundledChannelUpdateLanes[1]], "bundled-channels-update-b": [ bundledChannelUpdateLanes[2], bundledChannelUpdateLanes[3], @@ -528,6 +532,11 @@ const legacyReleasePathChunks = { "plugins-runtime": releasePathPluginRuntimeLanes, "plugins-integrations": [...releasePathPluginRuntimeLanes, ...releasePathBundledChannelLanes], "bundled-channels": releasePathBundledChannelLanes, + "bundled-channels-update-a-legacy": [ + bundledChannelUpdateLanes[0], + bundledChannelUpdateLanes[1], + bundledChannelUpdateLanes[4], + ], }; function openWebUILane() { diff --git a/scripts/test-docker-all.mjs b/scripts/test-docker-all.mjs index f1cb6e8fc82..e6b33324429 100644 --- a/scripts/test-docker-all.mjs +++ b/scripts/test-docker-all.mjs @@ -367,6 +367,7 @@ async function writeFailureIndex(logDir, summary) { lane: failure.name, logFile: failure.logFile, name: failure.name, + noOutputTimedOut: failure.noOutputTimedOut, rerunCommand: failure.rerunCommand, status: failure.status, timedOut: failure.timedOut, @@ -441,54 +442,96 @@ function dockerPreflightContainerNames(raw) { ); } -function runShellCommand({ command, env, label, logFile, timeoutMs }) { +function runShellCommand({ command, env, label, logFile, timeoutMs, noOutputTimeoutMs }) { return new Promise((resolve) => { + const pipeOutput = Boolean(logFile || noOutputTimeoutMs > 0); const child = spawn("bash", ["-c", command], { cwd: ROOT_DIR, detached: process.platform !== "win32", env, - stdio: logFile ? ["ignore", "pipe", "pipe"] : "inherit", + stdio: pipeOutput ? ["ignore", "pipe", "pipe"] : "inherit", }); activeChildren.add(child); let timedOut = false; + let noOutputTimedOut = false; let killTimer; + let stream; + let noOutputTimer; + const terminateForTimeout = (message, options = {}) => { + if (timedOut) { + return; + } + timedOut = true; + noOutputTimedOut = options.noOutput === true; + if (stream) { + stream.write(`\n==> [${label}] ${message}; sending SIGTERM\n`); + } else { + console.error(`==> [${label}] ${message}; sending SIGTERM`); + } + terminateChild(child, "SIGTERM"); + killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), 10_000); + killTimer.unref?.(); + }; + const resetNoOutputTimer = () => { + if (!noOutputTimeoutMs || noOutputTimeoutMs <= 0 || timedOut) { + return; + } + if (noOutputTimer) { + clearTimeout(noOutputTimer); + } + noOutputTimer = setTimeout(() => { + terminateForTimeout(`no output for ${noOutputTimeoutMs}ms`, { noOutput: true }); + }, noOutputTimeoutMs); + noOutputTimer.unref?.(); + }; const timeoutTimer = timeoutMs > 0 ? setTimeout(() => { - timedOut = true; - if (stream) { - stream.write(`\n==> [${label}] timeout after ${timeoutMs}ms; sending SIGTERM\n`); - } - terminateChild(child, "SIGTERM"); - killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), 10_000); - killTimer.unref?.(); + terminateForTimeout(`timeout after ${timeoutMs}ms`); }, timeoutMs) : undefined; timeoutTimer?.unref?.(); - let stream; if (logFile) { stream = fs.createWriteStream(logFile, { flags: "a" }); stream.write(`==> [${label}] command: ${command}\n`); stream.write(`==> [${label}] started: ${utcStamp()}\n`); - child.stdout.pipe(stream, { end: false }); - child.stderr.pipe(stream, { end: false }); + } + if (pipeOutput) { + const writeOutput = (target, chunk) => { + resetNoOutputTimer(); + if (stream) { + stream.write(chunk); + } else { + target.write(chunk); + } + }; + child.stdout.on("data", (chunk) => writeOutput(process.stdout, chunk)); + child.stderr.on("data", (chunk) => writeOutput(process.stderr, chunk)); + resetNoOutputTimer(); } child.on("close", (status, signal) => { if (timeoutTimer) { clearTimeout(timeoutTimer); } + if (noOutputTimer) { + clearTimeout(noOutputTimer); + } if (killTimer) { clearTimeout(killTimer); } activeChildren.delete(child); const exitCode = typeof status === "number" ? status : signal ? 128 : 1; if (stream) { - stream.write(`\n==> [${label}] finished: ${utcStamp()} status=${exitCode}\n`); + stream.write( + `\n==> [${label}] finished: ${utcStamp()} status=${exitCode}${ + noOutputTimedOut ? " noOutputTimedOut=true" : "" + }\n`, + ); stream.end(); } - resolve({ signal, status: exitCode, timedOut }); + resolve({ signal, status: exitCode, timedOut, noOutputTimedOut }); }); }); } @@ -692,6 +735,7 @@ function laneEnv(poolLane, baseEnv, logDir, cacheKey) { async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) { const { name } = lane; const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs; + const noOutputTimeoutMs = lane.noOutputTimeoutMs; const logFile = path.join(logDir, `${name}.log`); const env = laneEnv(lane, baseEnv, logDir, lane.cacheKey); const command = withResolvedPnpmCommand(lane.command, env); @@ -703,6 +747,7 @@ async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) { `==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`, `==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`, `==> [${name}] timeout: ${timeoutMs}ms`, + `==> [${name}] no output timeout: ${noOutputTimeoutMs ?? 0}ms`, `==> [${name}] retries: ${lane.retries ?? 0}`, `==> [${name}] e2e image kind: ${lane.e2eImageKind ?? "none"}`, `==> [${name}] e2e image: ${env.OPENCLAW_DOCKER_E2E_IMAGE ?? ""}`, @@ -721,11 +766,19 @@ async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) { await fs.promises.appendFile(logFile, `\n==> [${name}] retry attempt ${attempt}\n`); console.log(`==> [${name}] retry ${attempt}/${maxAttempts}`); } - result = await runShellCommand({ command, env, label: name, logFile, timeoutMs }); + result = await runShellCommand({ + command, + env, + label: name, + logFile, + timeoutMs, + noOutputTimeoutMs, + }); attempts.push({ attempt, elapsedSeconds: phaseElapsedSeconds(attemptStartedAt), finishedAt: new Date().toISOString(), + noOutputTimedOut: result.noOutputTimedOut, startedAt: new Date(attemptStartedAt).toISOString(), status: result.status, timedOut: result.timedOut, @@ -760,6 +813,7 @@ async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) { rerunCommand: buildLaneRerunCommand(name, baseEnv), startedAt: startedAtIso, status: result.status, + noOutputTimedOut: result.noOutputTimedOut, timedOut: result.timedOut, }; } diff --git a/src/agents/live-cache-regression-runner.test.ts b/src/agents/live-cache-regression-runner.test.ts index 744336e5b3d..cac3c28ee65 100644 --- a/src/agents/live-cache-regression-runner.test.ts +++ b/src/agents/live-cache-regression-runner.test.ts @@ -53,4 +53,34 @@ describe("live cache regression runner", () => { ]); expect(warnings).toEqual([]); }); + + it("retries hard cache baseline misses once", () => { + expect( + __testing.shouldRetryBaselineFindings( + { + regressions: ["anthropic:image cacheRead=0 < min=4500"], + warnings: [], + }, + 1, + ), + ).toBe(true); + expect( + __testing.shouldRetryBaselineFindings( + { + regressions: ["anthropic:image cacheRead=0 < min=4500"], + warnings: [], + }, + 2, + ), + ).toBe(false); + expect( + __testing.shouldRetryBaselineFindings( + { + regressions: [], + warnings: ["openai:image cacheRead=0 < min=3840"], + }, + 1, + ), + ).toBe(false); + }); }); diff --git a/src/agents/live-cache-regression-runner.ts b/src/agents/live-cache-regression-runner.ts index 227fe229aea..fba9395aef0 100644 --- a/src/agents/live-cache-regression-runner.ts +++ b/src/agents/live-cache-regression-runner.ts @@ -19,6 +19,7 @@ import { const OPENAI_TIMEOUT_MS = 120_000; const ANTHROPIC_TIMEOUT_MS = 120_000; +const LIVE_CACHE_LANE_RETRIES = 1; const OPENAI_PREFIX = buildStableCachePrefix("openai"); const OPENAI_MCP_PREFIX = buildStableCachePrefix("openai-mcp-style"); const ANTHROPIC_PREFIX = buildStableCachePrefix("anthropic"); @@ -47,6 +48,10 @@ type LaneResult = { disabled?: CacheRun; warmup?: CacheRun; }; +type BaselineFindings = { + regressions: string[]; + warnings: string[]; +}; export type LiveCacheRegressionResult = { regressions: string[]; @@ -419,8 +424,71 @@ function assertAgainstBaseline(params: { } } +function evaluateAgainstBaseline(params: { + lane: BaselineLane; + provider: ProviderKey; + result: LaneResult; +}): BaselineFindings { + const regressions: string[] = []; + const warnings: string[] = []; + assertAgainstBaseline({ + ...params, + regressions, + warnings, + }); + return { regressions, warnings }; +} + +function shouldRetryBaselineFindings(findings: BaselineFindings, attempt: number): boolean { + return findings.regressions.length > 0 && attempt <= LIVE_CACHE_LANE_RETRIES; +} + +async function runRepeatedLaneWithBaselineRetry(params: { + lane: CacheLane; + providerTag: "anthropic" | "openai"; + fixture: LiveResolvedModel; + runToken: string; + pngBase64: string; +}): Promise<{ result: LaneResult; findings: BaselineFindings; attempts: number }> { + let result: LaneResult | undefined; + let findings: BaselineFindings = { regressions: [], warnings: [] }; + let attempts = 0; + for (let attempt = 1; attempt <= 1 + LIVE_CACHE_LANE_RETRIES; attempt += 1) { + attempts = attempt; + result = await runRepeatedLane({ + ...params, + sessionId: `live-cache-regression-${params.runToken}-${params.providerTag}-${params.lane}${ + attempt > 1 ? `-retry-${attempt}` : "" + }`, + }); + findings = evaluateAgainstBaseline({ + lane: params.lane, + provider: params.providerTag, + result, + }); + if (!shouldRetryBaselineFindings(findings, attempt)) { + break; + } + logLiveCache( + `${params.providerTag} ${params.lane} baseline miss; retrying lane once: ${JSON.stringify( + findings.regressions, + )}`, + ); + } + + assert(result, `expected ${params.providerTag} ${params.lane} cache lane result`); + return { result, findings, attempts }; +} + +function appendBaselineFindings(target: BaselineFindings, source: BaselineFindings) { + target.regressions.push(...source.regressions); + target.warnings.push(...source.warnings); +} + export const __testing = { assertAgainstBaseline, + evaluateAgainstBaseline, + shouldRetryBaselineFindings, }; export async function runLiveCacheRegression(): Promise { @@ -447,14 +515,14 @@ export async function runLiveCacheRegression(): Promise { profile: RELEASE_PATH_PROFILE, releaseChunk: "bundled-channels-update-a", }); + const bundledChannelsUpdateDiscord = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "bundled-channels-update-discord", + }); const bundledChannelsUpdateB = planFor({ includeOpenWebUI: true, profile: RELEASE_PATH_PROFILE, @@ -167,9 +172,15 @@ describe("scripts/lib/docker-e2e-plan", () => { ]); expect(bundledChannelsUpdateA.lanes.map((lane) => lane.name)).toEqual([ "bundled-channel-update-telegram", - "bundled-channel-update-discord", "bundled-channel-update-memory-lancedb", ]); + expect(bundledChannelsUpdateDiscord.lanes.map((lane) => lane.name)).toEqual([ + "bundled-channel-update-discord", + ]); + expect(bundledChannelsUpdateDiscord.lanes[0]).toMatchObject({ + noOutputTimeoutMs: 4 * 60 * 1000, + timeoutMs: 6 * 60 * 1000, + }); expect(bundledChannelsUpdateB.lanes.map((lane) => lane.name)).toEqual([ "bundled-channel-update-slack", "bundled-channel-update-feishu", @@ -196,6 +207,11 @@ describe("scripts/lib/docker-e2e-plan", () => { profile: RELEASE_PATH_PROFILE, releaseChunk: "plugins-runtime", }); + const bundledChannelsUpdateALegacy = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "bundled-channels-update-a-legacy", + }); const legacy = planFor({ includeOpenWebUI: true, profile: RELEASE_PATH_PROFILE, @@ -217,6 +233,11 @@ describe("scripts/lib/docker-e2e-plan", () => { "openwebui", ]), ); + expect(bundledChannelsUpdateALegacy.lanes.map((lane) => lane.name)).toEqual([ + "bundled-channel-update-telegram", + "bundled-channel-update-discord", + "bundled-channel-update-memory-lancedb", + ]); expect(legacy.lanes.map((lane) => lane.name)).toEqual( expect.arrayContaining([ "plugins",