diff --git a/scripts/test-parallel-memory.mjs b/scripts/test-parallel-memory.mjs index 3bf9eca4049..a3e071a7f8f 100644 --- a/scripts/test-parallel-memory.mjs +++ b/scripts/test-parallel-memory.mjs @@ -7,6 +7,8 @@ const ANSI_ESCAPE_PATTERN = new RegExp( `${ESCAPE}(?:\\][^${BELL}]*(?:${BELL}|${ESCAPE}\\\\)|\\[[0-?]*[ -/]*[@-~]|[@-Z\\\\-_])`, "g", ); +const GITHUB_CLI_LOG_PREFIX_PATTERN = + /^[^\t\r\n]+\t[^\t\r\n]+\t\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+/u; const GITHUB_ACTIONS_LOG_PREFIX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+/u; const COMPLETED_TEST_FILE_LINE_PATTERN = @@ -46,7 +48,9 @@ function stripAnsi(text) { } function normalizeLogLine(line) { - return line.replace(GITHUB_ACTIONS_LOG_PREFIX_PATTERN, ""); + return line + .replace(GITHUB_CLI_LOG_PREFIX_PATTERN, "") + .replace(GITHUB_ACTIONS_LOG_PREFIX_PATTERN, ""); } export function parseCompletedTestFileLines(text) { diff --git a/scripts/test-update-memory-hotspots-sources.mjs b/scripts/test-update-memory-hotspots-sources.mjs new file mode 100644 index 00000000000..19c7cfe5ac7 --- /dev/null +++ b/scripts/test-update-memory-hotspots-sources.mjs @@ -0,0 +1,28 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +export function loadHotspotInputTexts({ + logPaths = [], + ghJobs = [], + readFileSyncImpl = fs.readFileSync, + execFileSyncImpl = execFileSync, +}) { + const inputs = []; + for (const logPath of logPaths) { + inputs.push({ + sourceName: path.basename(logPath, path.extname(logPath)), + text: readFileSyncImpl(logPath, "utf8"), + }); + } + for (const ghJobId of ghJobs) { + inputs.push({ + sourceName: `gh-job-${String(ghJobId)}`, + text: execFileSyncImpl("gh", ["run", "view", "--job", String(ghJobId), "--log"], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }), + }); + } + return inputs; +} diff --git a/scripts/test-update-memory-hotspots.mjs b/scripts/test-update-memory-hotspots.mjs index 874e12e0ad9..2894c818a82 100644 --- a/scripts/test-update-memory-hotspots.mjs +++ b/scripts/test-update-memory-hotspots.mjs @@ -1,9 +1,8 @@ -import fs from "node:fs"; -import path from "node:path"; import { intFlag, parseFlagArgs, stringFlag, stringListFlag } from "./lib/arg-utils.mjs"; import { parseMemoryTraceSummaryLines } from "./test-parallel-memory.mjs"; import { normalizeTrackedRepoPath, tryReadJsonFile, writeJsonFile } from "./test-report-utils.mjs"; import { unitMemoryHotspotManifestPath } from "./test-runner-manifest.mjs"; +import { loadHotspotInputTexts } from "./test-update-memory-hotspots-sources.mjs"; import { matchesHotspotSummaryLane } from "./test-update-memory-hotspots-utils.mjs"; if (process.argv.slice(2).includes("--help")) { @@ -19,6 +18,7 @@ if (process.argv.slice(2).includes("--help")) { " --lane Primary lane name to match (default: unit-fast)", " --lane-prefix Additional lane prefixes to include (repeatable)", " --log Memory trace log to ingest (repeatable, required)", + " --gh-job GitHub Actions job id to ingest via gh (repeatable)", " --min-delta-kb Minimum RSS delta to retain (default: 262144)", " --limit Max hotspot entries to retain (default: 64)", " --help Show this help text", @@ -26,6 +26,7 @@ if (process.argv.slice(2).includes("--help")) { "Examples:", " node scripts/test-update-memory-hotspots.mjs --log /tmp/unit-fast.log", " node scripts/test-update-memory-hotspots.mjs --log a.log --log b.log --lane-prefix unit-fast-batch-", + " node scripts/test-update-memory-hotspots.mjs --gh-job 69804189668 --gh-job 69804189672", ].join("\n"), ); process.exit(0); @@ -40,6 +41,7 @@ function parseArgs(argv) { lane: "unit-fast", lanePrefixes: [], logs: [], + ghJobs: [], minDeltaKb: 256 * 1024, limit: 64, }, @@ -49,6 +51,7 @@ function parseArgs(argv) { stringFlag("--lane", "lane"), stringListFlag("--lane-prefix", "lanePrefixes"), stringListFlag("--log", "logs"), + stringListFlag("--gh-job", "ghJobs"), intFlag("--min-delta-kb", "minDeltaKb", { min: 1 }), intFlag("--limit", "limit", { min: 1 }), ], @@ -92,8 +95,8 @@ function mergeHotspotEntry(aggregated, file, value) { const opts = parseArgs(process.argv.slice(2)); -if (opts.logs.length === 0) { - console.error("[test-update-memory-hotspots] pass at least one --log ."); +if (opts.logs.length === 0 && opts.ghJobs.length === 0) { + console.error("[test-update-memory-hotspots] pass at least one --log or --gh-job ."); process.exit(2); } @@ -104,8 +107,8 @@ if (existing) { mergeHotspotEntry(aggregated, file, value); } } -for (const logPath of opts.logs) { - const text = fs.readFileSync(logPath, "utf8"); +for (const input of loadHotspotInputTexts({ logPaths: opts.logs, ghJobs: opts.ghJobs })) { + const text = input.text; const summaries = parseMemoryTraceSummaryLines(text).filter((summary) => matchesHotspotSummaryLane(summary.lane, opts.lane, opts.lanePrefixes), ); @@ -116,7 +119,7 @@ for (const logPath of opts.logs) { } mergeHotspotEntry(aggregated, record.file, { deltaKb: record.deltaKb, - sources: [`${path.basename(logPath, path.extname(logPath))}:${summary.lane}`], + sources: [`${input.sourceName}:${summary.lane}`], }); } } diff --git a/test/fixtures/test-memory-hotspots.extensions.json b/test/fixtures/test-memory-hotspots.extensions.json index 90518d2fbbb..7d193963eb3 100644 --- a/test/fixtures/test-memory-hotspots.extensions.json +++ b/test/fixtures/test-memory-hotspots.extensions.json @@ -1,6 +1,6 @@ { "config": "vitest.extensions.config.ts", - "generatedAt": "2026-04-03T00:00:00.000Z", + "generatedAt": "2026-04-03T04:18:33.578Z", "defaultMinDeltaKb": 1048576, "lane": "extensions, extensions-batch-*", "files": { @@ -36,6 +36,10 @@ "deltaKb": 1625293, "sources": ["checks-fast-extensions:2026-04-03"] }, + "extensions/bluebubbles/src/send.test.ts": { + "deltaKb": 1625293, + "sources": ["gh-job-69804189668:extensions-batch-19-shard-6"] + }, "extensions/googlechat/src/approval-auth.test.ts": { "deltaKb": 1614807, "sources": ["checks-fast-extensions:2026-04-03"] @@ -43,6 +47,42 @@ "extensions/diffs/src/tool.test.ts": { "deltaKb": 1604321, "sources": ["checks-fast-extensions:2026-04-03"] + }, + "extensions/memory-core/src/memory/mmr.test.ts": { + "deltaKb": 1572864, + "sources": ["gh-job-69804189668:extensions-batch-2-shard-6"] + }, + "extensions/diffs/src/config.test.ts": { + "deltaKb": 1572864, + "sources": ["gh-job-69804189668:extensions-batch-20-shard-6"] + }, + "extensions/voice-call/src/webhook-security.test.ts": { + "deltaKb": 1541407, + "sources": ["gh-job-69804189666:extensions-batch-5-shard-1"] + }, + "extensions/nostr/src/nostr-bus.fuzz.test.ts": { + "deltaKb": 1509949, + "sources": ["gh-job-69804189668:extensions-batch-11-shard-6"] + }, + "extensions/memory-lancedb/index.test.ts": { + "deltaKb": 1499464, + "sources": ["gh-job-69804189668:extensions-batch-12-shard-6"] + }, + "extensions/google/provider-models.test.ts": { + "deltaKb": 1478492, + "sources": ["gh-job-69804189681:extensions-batch-11-shard-5"] + }, + "extensions/fal/image-generation-provider.test.ts": { + "deltaKb": 1447035, + "sources": ["gh-job-69804189668:extensions-batch-13-shard-6"] + }, + "extensions/matrix/src/matrix/format.test.ts": { + "deltaKb": 1447035, + "sources": ["gh-job-69804189668:extensions-batch-23-shard-6"] + }, + "extensions/minimax/model-definitions.test.ts": { + "deltaKb": 1447035, + "sources": ["gh-job-69804189676:extensions-batch-15-shard-4"] } } } diff --git a/test/scripts/test-parallel.test.ts b/test/scripts/test-parallel.test.ts index 5b960e76d52..23f9527c828 100644 --- a/test/scripts/test-parallel.test.ts +++ b/test/scripts/test-parallel.test.ts @@ -253,6 +253,34 @@ describe("scripts/test-parallel memory trace parsing", () => { ], }); }); + + it("parses memory trace summaries from gh run job logs", () => { + const summaries = parseMemoryTraceSummaryLines( + [ + "checks-fast-extensions-6\tRun extensions (node)\t2026-04-03T04:07:10.5924943Z [test-parallel][mem] summary extensions-batch-22-shard-6 files=15 peak=2.66GiB totalDelta=+470.5MiB peakAt=poll top=extensions/microsoft-foundry/index.test.ts:+1.35GiB, extensions/acpx/src/service.test.ts:+212.1MiB", + ].join("\n"), + ); + + expect(summaries).toEqual([ + { + lane: "extensions-batch-22-shard-6", + files: 15, + peakRssKb: parseMemoryValueKb("2.66GiB"), + totalDeltaKb: parseMemoryValueKb("+470.5MiB"), + peakAt: "poll", + top: [ + { + file: "extensions/microsoft-foundry/index.test.ts", + deltaKb: parseMemoryValueKb("+1.35GiB"), + }, + { + file: "extensions/acpx/src/service.test.ts", + deltaKb: parseMemoryValueKb("+212.1MiB"), + }, + ], + }, + ]); + }); }); describe("scripts/test-parallel lane planning", () => { diff --git a/test/scripts/test-planner.test.ts b/test/scripts/test-planner.test.ts index 5b7596758d0..058b766fbd4 100644 --- a/test/scripts/test-planner.test.ts +++ b/test/scripts/test-planner.test.ts @@ -207,6 +207,38 @@ describe("test planner", () => { artifacts.cleanupTempArtifacts(); }); + it("auto-isolates newly-seeded extension memory survivors in CI", () => { + const env = { + CI: "true", + GITHUB_ACTIONS: "true", + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "4", + OPENCLAW_TEST_HOST_MEMORY_GIB: "16", + }; + const artifacts = createExecutionArtifacts(env); + const plan = buildExecutionPlan( + { + profile: null, + mode: "ci", + surfaces: ["extensions"], + passthroughArgs: [], + }, + { + env, + platform: "linux", + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + const hotspotFile = bundledPluginFile("bluebubbles", "src/send.test.ts"); + const hotspotUnit = plan.selectedUnits.find((unit) => unit.args.includes(hotspotFile)); + + expect(hotspotUnit).toBeTruthy(); + expect(hotspotUnit?.isolate).toBe(true); + expect(hotspotUnit?.reasons).toContain("extensions-memory-heavy"); + artifacts.cleanupTempArtifacts(); + }); + it("auto-isolates timed-heavy channel suites in CI", () => { const env = { CI: "true", diff --git a/test/scripts/test-update-memory-hotspots-sources.test.ts b/test/scripts/test-update-memory-hotspots-sources.test.ts new file mode 100644 index 00000000000..f121bbf2af2 --- /dev/null +++ b/test/scripts/test-update-memory-hotspots-sources.test.ts @@ -0,0 +1,48 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadHotspotInputTexts } from "../../scripts/test-update-memory-hotspots-sources.mjs"; + +const tempFiles = []; + +afterEach(() => { + for (const tempFile of tempFiles.splice(0)) { + try { + fs.unlinkSync(tempFile); + } catch { + // Ignore temp cleanup races in tests. + } + } +}); + +describe("test-update-memory-hotspots source loading", () => { + it("loads local log files with basename-derived source names", () => { + const tempLog = path.join(os.tmpdir(), `openclaw-hotspots-${Date.now()}.log`); + tempFiles.push(tempLog); + fs.writeFileSync(tempLog, "local log"); + + expect(loadHotspotInputTexts({ logPaths: [tempLog] })).toEqual([ + { sourceName: path.basename(tempLog, ".log"), text: "local log" }, + ]); + }); + + it("loads GitHub Actions job logs through gh", () => { + const execFileSyncImpl = vi.fn(() => "remote log"); + + expect( + loadHotspotInputTexts({ + ghJobs: ["69804189668"], + execFileSyncImpl, + }), + ).toEqual([{ sourceName: "gh-job-69804189668", text: "remote log" }]); + expect(execFileSyncImpl).toHaveBeenCalledWith( + "gh", + ["run", "view", "--job", "69804189668", "--log"], + { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }, + ); + }); +});