From 2aa79e4aaa61acd0778dfee017bd84626c95ed32 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Jul 2026 13:23:35 -0400 Subject: [PATCH] fix: prevent cron and gateway stress-run timeouts (#115241) * fix: make cron and gateway stress tests deterministic * test: update focused agent project expectations * fix: preserve focused agent glob isolation --- scripts/test-projects.test-support.mjs | 51 +++++- .../server/ws-connection.startup.test.ts | 56 +++---- src/scripts/test-projects.test.ts | 8 +- test/scripts/test-projects.test.ts | 147 ++++++++++++++++++ 4 files changed, 224 insertions(+), 38 deletions(-) diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 83558a727cb3..3749d6efde62 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -8,6 +8,8 @@ import os from "node:os"; import path from "node:path"; import { agentsCoreIsolatedTestFiles, + agentsEmbeddedIncompleteTurnTestFiles, + agentsEmbeddedOverflowCompactionTestFiles, isAgentsCoreIsolatedTestFile, } from "../test/vitest/vitest.agents-paths.mjs"; import { isChannelSurfaceTestFile } from "../test/vitest/vitest.channel-paths.mjs"; @@ -88,6 +90,7 @@ import { } from "./run-vitest.mjs"; const DEFAULT_VITEST_CONFIG = "test/vitest/vitest.unit.config.ts"; +const AGENTS_EMBEDDED_AGENT_TEST_ROOT = "src/agents/embedded-agent-runner"; const AGENTS_CORE_ISOLATED_VITEST_CONFIG = "test/vitest/vitest.agents-core-isolated.config.ts"; const AGENTS_CORE_VITEST_CONFIG = "test/vitest/vitest.agents-core.config.ts"; const AGENTS_EMBEDDED_AGENT_VITEST_CONFIG = "test/vitest/vitest.agents-embedded-agent.config.ts"; @@ -4247,7 +4250,32 @@ function classifyTarget(arg, cwd) { return "autoReply"; } if (isPathAtOrUnder(relative, "src/agents")) { - return "agent"; + // Focused runs must preserve the full suite's isolated harness and hook-timeout contracts. + if (relative === "src/agents" || relative === AGENTS_EMBEDDED_AGENT_TEST_ROOT) { + return "agent"; + } + if (agentsEmbeddedIncompleteTurnTestFiles.includes(relative)) { + return "agentEmbeddedIncompleteTurn"; + } + if (agentsEmbeddedOverflowCompactionTestFiles.includes(relative)) { + return "agentEmbeddedOverflowCompaction"; + } + if (isPathAtOrUnder(relative, `${AGENTS_EMBEDDED_AGENT_TEST_ROOT}/run`)) { + return "agentEmbeddedRun"; + } + if (isPathAtOrUnder(relative, AGENTS_EMBEDDED_AGENT_TEST_ROOT)) { + return isGlobTarget(relative) ? "agent" : "agentEmbedded"; + } + if (isPathAtOrUnder(relative, "src/agents/tools")) { + return "agentTools"; + } + if (isGlobTarget(relative)) { + const owner = relative.slice("src/agents/".length).split("/", 1)[0]; + return isGlobTarget(owner) ? "agent" : "agentSupport"; + } + return isFileLikeTarget(relative) && path.posix.dirname(relative) === "src/agents" + ? "agentCore" + : "agentSupport"; } if (isPathAtOrUnder(relative, "src/plugins")) { return "plugin"; @@ -4409,6 +4437,27 @@ export function buildVitestRunPlans( const groupedTargets = new Map(); for (const targetArg of activeTargetArgs) { + if (!watchMode && toRepoRelativeTarget(targetArg, cwd) === AGENTS_EMBEDDED_AGENT_TEST_ROOT) { + // The recursive parent spans four harness owners; keep every isolated project intact. + const embeddedTargetsByKind = [ + ["agentEmbedded", [`${AGENTS_EMBEDDED_AGENT_TEST_ROOT}/*.test.ts`]], + ["agentEmbeddedIncompleteTurn", agentsEmbeddedIncompleteTurnTestFiles], + ["agentEmbeddedOverflowCompaction", agentsEmbeddedOverflowCompactionTestFiles], + ["agentEmbeddedRun", [`${AGENTS_EMBEDDED_AGENT_TEST_ROOT}/run`]], + ]; + + for (const [kind, targets] of embeddedTargetsByKind) { + const current = groupedTargets.get(kind) ?? []; + for (const target of targets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set(kind, current); + } + continue; + } + const kind = classifyTarget(targetArg, cwd); const current = groupedTargets.get(kind) ?? []; current.push(targetArg); diff --git a/src/gateway/server/ws-connection.startup.test.ts b/src/gateway/server/ws-connection.startup.test.ts index 15071a84d5ec..b968a44566cf 100644 --- a/src/gateway/server/ws-connection.startup.test.ts +++ b/src/gateway/server/ws-connection.startup.test.ts @@ -13,6 +13,7 @@ import { GATEWAY_STARTUP_PENDING_CLOSE_CAUSE, GATEWAY_STARTUP_UNAVAILABLE_REASON, } from "../../../packages/gateway-protocol/src/startup-unavailable.js"; +import { createDeferred } from "../../test-utils/deferred.js"; import { attachGatewayWsConnectionHandler } from "./ws-connection.js"; import { attachGatewayWsForTest, @@ -25,10 +26,28 @@ describe("attachGatewayWsConnectionHandler startup readiness", () => { it.each([GATEWAY_STARTUP_CLOSE_CODE, 1006])( "keeps startup-unavailable close code %i at debug level", async (observedCloseCode) => { - const sent: unknown[] = []; + const responseReceived = createDeferred<{ + type?: unknown; + id?: unknown; + ok?: unknown; + error?: { + code?: unknown; + retryable?: unknown; + retryAfterMs?: unknown; + details?: unknown; + }; + }>(); const socket = createGatewayWsTestSocket({ onSend: (data) => { - sent.push(JSON.parse(data)); + const frame = JSON.parse(data) as unknown; + if ( + typeof frame === "object" && + frame !== null && + (frame as { type?: unknown }).type === "res" && + (frame as { id?: unknown }).id === "connect-1" + ) { + responseReceived.resolve(frame); + } }, }); const logWsControl = createGatewayWsTestLogger(); @@ -65,37 +84,8 @@ describe("attachGatewayWsConnectionHandler startup readiness", () => { }), ); - await vi.waitFor(() => { - expect( - sent.some( - (frame) => - typeof frame === "object" && - frame !== null && - (frame as { type?: unknown; id?: unknown; ok?: unknown }).type === "res" && - (frame as { id?: unknown }).id === "connect-1", - ), - ).toBe(true); - }); - - const response = sent.find( - (frame) => - typeof frame === "object" && - frame !== null && - (frame as { type?: unknown; id?: unknown }).type === "res" && - (frame as { id?: unknown }).id === "connect-1", - ) as - | { - type?: unknown; - id?: unknown; - ok?: unknown; - error?: { - code?: unknown; - retryable?: unknown; - retryAfterMs?: unknown; - details?: unknown; - }; - } - | undefined; + // The handler is lazy-loaded; wait for its actual frame instead of a one-second poll. + const response = await responseReceived.promise; expect(response?.type).toBe("res"); expect(response?.id).toBe("connect-1"); expect(response?.ok).toBe(false); diff --git a/src/scripts/test-projects.test.ts b/src/scripts/test-projects.test.ts index dea74cdc413c..5382a3739fde 100644 --- a/src/scripts/test-projects.test.ts +++ b/src/scripts/test-projects.test.ts @@ -97,7 +97,7 @@ describe("test-projects args", () => { it("keeps split test entries in their owner configs", () => { expect(buildVitestRunPlans(["src/agents/openai-transport-stream.base.test.ts"])).toEqual([ { - config: "test/vitest/vitest.agents.config.ts", + config: "test/vitest/vitest.agents-core.config.ts", forwardedArgs: [], includePatterns: ["src/agents/openai-transport-stream.base.test.ts"], watchMode: false, @@ -116,7 +116,7 @@ describe("test-projects args", () => { it("expands a test filename prefix into standalone sibling suites", () => { expect(buildVitestRunPlans(["src/agents/openai-transport-stream"])).toEqual([ { - config: "test/vitest/vitest.agents.config.ts", + config: "test/vitest/vitest.agents-core.config.ts", forwardedArgs: [], includePatterns: [ "src/agents/openai-transport-stream.base.test.ts", @@ -373,10 +373,10 @@ describe("test-projects args", () => { ]); }); - it("routes agents targets to the agents config", () => { + it("routes agent tool targets to the agents-tools config", () => { expect(buildVitestRunPlans(["src/agents/tools/image-tool.test.ts"])).toEqual([ { - config: "test/vitest/vitest.agents.config.ts", + config: "test/vitest/vitest.agents-tools.config.ts", forwardedArgs: [], includePatterns: ["src/agents/tools/image-tool.test.ts"], watchMode: false, diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 33334977c933..c2cea5f2f37f 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -2631,6 +2631,153 @@ describe("scripts/test-projects changed-target routing", () => { }, ); + it.each([ + ["src/agents/agent-scope.test.ts", "test/vitest/vitest.agents-core.config.ts"], + [ + "src/agents/embedded-agent-runner/run.before-agent-reply-cron.test.ts", + "test/vitest/vitest.agents-embedded-agent.config.ts", + ], + [ + "src/agents/embedded-agent-runner/run.incomplete-turn.test.ts", + "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts", + ], + [ + "src/agents/embedded-agent-runner/run.overflow-compaction.test.ts", + "test/vitest/vitest.agents-embedded-agent-overflow-compaction.config.ts", + ], + [ + "src/agents/embedded-agent-runner/run/attempt.abort-race.test.ts", + "test/vitest/vitest.agents-embedded-agent-run.config.ts", + ], + ["src/agents/runtime-plan/tools.test.ts", "test/vitest/vitest.agents-support.config.ts"], + ["src/agents/tools/cron-tool.pacing.test.ts", "test/vitest/vitest.agents-tools.config.ts"], + ])("routes focused agent test %s to its owning shard", (testFile, config) => { + expect(buildVitestRunPlans([testFile])).toEqual([ + { + config, + forwardedArgs: [], + includePatterns: [testFile], + watchMode: false, + }, + ]); + }); + + it.each([ + [ + "src/agents/embedded-agent-runner/run", + "test/vitest/vitest.agents-embedded-agent-run.config.ts", + ], + ["src/agents/runtime-plan", "test/vitest/vitest.agents-support.config.ts"], + ["src/agents/tools", "test/vitest/vitest.agents-tools.config.ts"], + ])("routes focused agent directory %s to its owning shard", (directory, config) => { + const plans = buildVitestRunPlans([directory]); + + expect(plans).toEqual( + expect.arrayContaining([ + { + config, + forwardedArgs: [], + includePatterns: [`${directory}/**/*.test.ts`], + watchMode: false, + }, + ]), + ); + expect(plans.map((plan) => plan.config)).not.toContain( + "test/vitest/vitest.agents-core.config.ts", + ); + }); + + it("splits the embedded-agent parent directory across every isolated harness", () => { + const root = "src/agents/embedded-agent-runner"; + const plans = buildVitestRunPlans([root]); + + expect(plans).toEqual( + expect.arrayContaining([ + { + config: "test/vitest/vitest.agents-embedded-agent.config.ts", + forwardedArgs: [], + includePatterns: [`${root}/*.test.ts`], + watchMode: false, + }, + { + config: "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts", + forwardedArgs: [], + includePatterns: [`${root}/run.incomplete-turn.test.ts`], + watchMode: false, + }, + { + config: "test/vitest/vitest.agents-embedded-agent-overflow-compaction.config.ts", + forwardedArgs: [], + includePatterns: [`${root}/run.overflow-compaction.test.ts`], + watchMode: false, + }, + { + config: "test/vitest/vitest.agents-embedded-agent-run.config.ts", + forwardedArgs: [], + includePatterns: [`${root}/run/**/*.test.ts`], + watchMode: false, + }, + ]), + ); + expect(plans.map((plan) => plan.config)).not.toContain("test/vitest/vitest.agents.config.ts"); + }); + + it("keeps the broad agent test glob in the all-agents shard", () => { + const target = "src/agents/**/*.test.ts"; + + expect(buildVitestRunPlans([target])).toEqual([ + { + config: "test/vitest/vitest.agents.config.ts", + forwardedArgs: [], + includePatterns: [target], + watchMode: false, + }, + ]); + }); + + it.each([ + [ + "src/agents/embedded-agent-runner/run/*.test.ts", + "test/vitest/vitest.agents-embedded-agent-run.config.ts", + ], + ["src/agents/runtime-plan/**/*.test.ts", "test/vitest/vitest.agents-support.config.ts"], + ["src/agents/tools/**/*.test.ts", "test/vitest/vitest.agents-tools.config.ts"], + ])("routes focused agent glob %s to its owning shard", (target, config) => { + const plans = buildVitestRunPlans([target]); + + expect(plans).toEqual( + expect.arrayContaining([ + { + config, + forwardedArgs: [], + includePatterns: [target], + watchMode: false, + }, + ]), + ); + expect(plans.map((plan) => plan.config)).not.toContain("test/vitest/vitest.agents.config.ts"); + }); + + it("keeps mixed embedded-agent and cron-tool targets in their owning shards", () => { + const embeddedTest = "src/agents/embedded-agent-runner/run.before-agent-reply-cron.test.ts"; + const cronToolTest = "src/agents/tools/cron-tool.pacing.test.ts"; + + expect(buildVitestRunPlans([embeddedTest, cronToolTest])).toEqual([ + { + config: "test/vitest/vitest.agents-embedded-agent.config.ts", + forwardedArgs: [], + includePatterns: [embeddedTest], + watchMode: false, + }, + { + config: "test/vitest/vitest.agents-tools.config.ts", + forwardedArgs: [], + includePatterns: [cronToolTest], + watchMode: false, + }, + ]); + }); + it("routes Docker E2E script targets to their owner tooling tests", () => { const targets = [ "scripts/e2e/kitchen-sink-plugin-docker.sh",