From 15adc741ffbc9b7ff2bb9fef6542b180e1d1dce5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Fri, 1 May 2026 02:58:26 -0700 Subject: [PATCH] test(package): expand upgrade survivor baselines --- .github/workflows/package-acceptance.yml | 4 ++ .../e2e/lib/upgrade-survivor/assertions.mjs | 16 +++-- .../lib/upgrade-survivor/config-recipe.mjs | 67 ++++++++++++++++++- .../config-recipe/agents.json | 3 +- scripts/e2e/upgrade-survivor-docker.sh | 2 + .../resolve-upgrade-survivor-baselines.mjs | 29 +++++++- .../upgrade-survivor-baselines.test.ts | 58 ++++++++++++++++ 7 files changed, 168 insertions(+), 11 deletions(-) diff --git a/.github/workflows/package-acceptance.yml b/.github/workflows/package-acceptance.yml index cefa1325631..2c8e6878930 100644 --- a/.github/workflows/package-acceptance.yml +++ b/.github/workflows/package-acceptance.yml @@ -441,10 +441,13 @@ jobs: exit 0 fi releases_json="" + npm_versions_json="" if [[ "$REQUESTED_BASELINES" == *"release-history"* ]]; then releases_json=".artifacts/package-candidate-input/openclaw-releases.json" + npm_versions_json=".artifacts/package-candidate-input/openclaw-npm-versions.json" mkdir -p "$(dirname "$releases_json")" gh release list --repo "$GITHUB_REPOSITORY" --limit 100 --json tagName,publishedAt,isPrerelease > "$releases_json" + npm view openclaw versions --json > "$npm_versions_json" fi args=( --requested "$REQUESTED_BASELINES" @@ -454,6 +457,7 @@ jobs: if [[ -n "$releases_json" ]]; then args+=( --releases-json "$releases_json" + --npm-versions-json "$npm_versions_json" --history-count 6 --include-version 2026.4.23 --pre-date 2026-03-15T00:00:00Z diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 0e152aa7afa..64e9f7b41a2 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -66,7 +66,11 @@ function acceptsIntent(coverage, id) { if (!coverage) { return true; } - return Array.isArray(coverage.acceptedIntents) && coverage.acceptedIntents.includes(id); + return ( + Array.isArray(coverage.acceptedIntents) && + coverage.acceptedIntents.includes(id) && + !coverage.skippedIntents?.includes(id) + ); } function hasCoverage(coverage) { @@ -189,10 +193,12 @@ function assertConfigSurvived() { "main agent contextTokens changed", ); } - assert( - agents.find((agent) => agent?.id === "ops")?.fastModeDefault === true, - "ops fastModeDefault changed", - ); + if (!hasCoverage(coverage) || !coverage.skippedIntents?.includes("agent-modern-preferences")) { + assert( + agents.find((agent) => agent?.id === "ops")?.fastModeDefault === true, + "ops fastModeDefault changed", + ); + } } if (acceptsIntent(coverage, "skills")) { diff --git a/scripts/e2e/lib/upgrade-survivor/config-recipe.mjs b/scripts/e2e/lib/upgrade-survivor/config-recipe.mjs index 28fe9db5c0f..e105bb23575 100644 --- a/scripts/e2e/lib/upgrade-survivor/config-recipe.mjs +++ b/scripts/e2e/lib/upgrade-survivor/config-recipe.mjs @@ -35,6 +35,28 @@ function readConfigSection(fileName) { return JSON.stringify(JSON.parse(fs.readFileSync(fileUrl, "utf8"))); } +function parseReleaseVersion(version) { + const match = /^([0-9]{4})\.([0-9]+)\.([0-9]+)/u.exec(String(version ?? "")); + if (!match) { + return null; + } + return match.slice(1).map((part) => Number.parseInt(part, 10)); +} + +function isReleaseBefore(version, minimum) { + const parsed = parseReleaseVersion(version); + const minimumParsed = parseReleaseVersion(minimum); + if (!parsed || !minimumParsed) { + return false; + } + for (let index = 0; index < parsed.length; index += 1) { + if (parsed[index] !== minimumParsed[index]) { + return parsed[index] < minimumParsed[index]; + } + } + return false; +} + function configSetJsonFile(id, intent, configPath, fileName) { return { id, @@ -112,6 +134,45 @@ function selectedScenario() { return process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO || "base"; } +function adaptStepForBaseline(step, baselineVersion, summary) { + if (!isReleaseBefore(baselineVersion, "2026.4.0")) { + return step; + } + if (step.id === "plugins-feishu" || step.id === "channels-feishu") { + if (!summary.skippedIntents.includes("feishu-channel")) { + summary.skippedIntents.push("feishu-channel"); + } + return null; + } + if (step.id === "agents") { + const agents = JSON.parse(step.argv[3]); + delete agents.defaults?.skills; + for (const agent of agents.list ?? []) { + delete agent.thinkingDefault; + delete agent.fastModeDefault; + delete agent.skills; + } + summary.skippedIntents.push("agent-modern-preferences"); + return { + ...step, + argv: [...step.argv.slice(0, 3), JSON.stringify(agents), ...step.argv.slice(4)], + }; + } + if (step.intent === "plugins") { + const plugins = JSON.parse(step.argv[3]); + plugins.allow = (plugins.allow ?? []).filter((id) => id !== "memory"); + delete plugins.entries?.memory; + if (!summary.skippedIntents.includes("memory-plugin-allow")) { + summary.skippedIntents.push("memory-plugin-allow"); + } + return { + ...step, + argv: [...step.argv.slice(0, 3), JSON.stringify(plugins), ...step.argv.slice(4)], + }; + } + return step; +} + function runOpenClaw(step) { const result = spawnSync("openclaw", step.argv, { encoding: "utf8", @@ -156,7 +217,11 @@ function applyRecipe() { }; for (const step of [...recipe.slice(0, -1), ...scenarioSteps, recipe.at(-1)]) { - const outcome = runOpenClaw(step); + const adaptedStep = adaptStepForBaseline(step, baselineVersion, summary); + if (!adaptedStep) { + continue; + } + const outcome = runOpenClaw(adaptedStep); summary.steps.push(outcome); writeJson(summaryPath, summary); if (!outcome.ok) { diff --git a/scripts/e2e/lib/upgrade-survivor/config-recipe/agents.json b/scripts/e2e/lib/upgrade-survivor/config-recipe/agents.json index 9cf0f87e2d1..cd9d479f513 100644 --- a/scripts/e2e/lib/upgrade-survivor/config-recipe/agents.json +++ b/scripts/e2e/lib/upgrade-survivor/config-recipe/agents.json @@ -3,8 +3,7 @@ "model": { "primary": "openai/gpt-4.1-mini" }, - "contextTokens": 64000, - "skills": ["memory"] + "contextTokens": 64000 }, "list": [ { diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index fadb899c087..27394737765 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -41,6 +41,7 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then fi mkdir -p "$ARTIFACT_DIR" + chmod -R a+rwX "$ARTIFACT_DIR" || true DOCKER_E2E_PACKAGE_ARGS=() CANDIDATE_RAW="${OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE:-current}" @@ -98,6 +99,7 @@ PACKAGE_TGZ="$(docker_e2e_prepare_package_tgz upgrade-survivor "${OPENCLAW_CURRE docker_e2e_package_mount_args "$PACKAGE_TGZ" OPENCLAW_TEST_STATE_SCRIPT_B64="$(docker_e2e_test_state_shell_b64 upgrade-survivor upgrade-survivor)" mkdir -p "$ARTIFACT_DIR" +chmod -R a+rwX "$ARTIFACT_DIR" || true docker_e2e_build_or_reuse "$IMAGE_NAME" upgrade-survivor "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "bare" "$SKIP_BUILD" diff --git a/scripts/resolve-upgrade-survivor-baselines.mjs b/scripts/resolve-upgrade-survivor-baselines.mjs index 441a3f9500a..af3e958d58b 100644 --- a/scripts/resolve-upgrade-survivor-baselines.mjs +++ b/scripts/resolve-upgrade-survivor-baselines.mjs @@ -31,6 +31,17 @@ function dedupeSpecs(specs) { return [...new Set(specs.map(normalizeUpgradeSurvivorBaselineSpec).filter(Boolean))]; } +function readPublishedVersions(file) { + if (!file) { + return undefined; + } + const parsed = JSON.parse(readFileSync(file, "utf8")); + if (!Array.isArray(parsed)) { + throw new Error(`npm versions list must be a JSON array: ${file}`); + } + return new Set(parsed.filter((version) => typeof version === "string")); +} + function stableVersionFromTag(tagName) { const version = String(tagName ?? "").replace(/^v/u, ""); if (!/^[0-9]{4}\.[0-9]+\.[0-9]+(?:-[0-9]+)?$/u.test(version)) { @@ -39,7 +50,18 @@ function stableVersionFromTag(tagName) { return version; } -function readStableReleases(file) { +function npmPublishedVersion(version, publishedVersions) { + if (!version || !publishedVersions) { + return version; + } + if (publishedVersions.has(version)) { + return version; + } + const baseVersion = version.replace(/-[0-9]+$/u, ""); + return publishedVersions.has(baseVersion) ? baseVersion : undefined; +} + +function readStableReleases(file, publishedVersions) { const ansiEscape = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, "g"); const raw = readFileSync(file, "utf8").replace(ansiEscape, ""); const parsed = JSON.parse(raw); @@ -50,7 +72,7 @@ function readStableReleases(file) { .filter((release) => !release.isPrerelease) .map((release) => ({ publishedAt: release.publishedAt, - version: stableVersionFromTag(release.tagName), + version: npmPublishedVersion(stableVersionFromTag(release.tagName), publishedVersions), })) .filter((release) => release.version && release.publishedAt) .toSorted((a, b) => String(b.publishedAt).localeCompare(String(a.publishedAt))); @@ -67,7 +89,8 @@ export function resolveReleaseHistory(args) { } const includeVersion = args.get("include-version") ?? "2026.4.23"; const preDate = args.get("pre-date") ?? "2026-03-15T00:00:00Z"; - const releases = readStableReleases(releasesJson); + const publishedVersions = readPublishedVersions(args.get("npm-versions-json")); + const releases = readStableReleases(releasesJson, publishedVersions); const versions = releases.slice(0, historyCount).map((release) => release.version); const exact = releases.find((release) => release.version === includeVersion); if (exact) { diff --git a/test/scripts/upgrade-survivor-baselines.test.ts b/test/scripts/upgrade-survivor-baselines.test.ts index fd340b15ab7..54035863013 100644 --- a/test/scripts/upgrade-survivor-baselines.test.ts +++ b/test/scripts/upgrade-survivor-baselines.test.ts @@ -15,6 +15,17 @@ function withReleaseFixture(releases: unknown[], fn: (file: string) => T): T } } +function withJsonFixture(name: string, contents: unknown, fn: (file: string) => T): T { + const dir = mkdtempSync(path.join(tmpdir(), "openclaw-upgrade-baselines-")); + try { + const file = path.join(dir, name); + writeFileSync(file, `${JSON.stringify(contents)}\n`); + return fn(file); + } finally { + rmSync(dir, { force: true, recursive: true }); + } +} + describe("scripts/resolve-upgrade-survivor-baselines", () => { it("keeps the single fallback baseline when no expanded request is provided", () => { expect(resolveBaselines(new Map([["fallback", "2026.4.23"]]))).toEqual(["openclaw@2026.4.23"]); @@ -63,4 +74,51 @@ describe("scripts/resolve-upgrade-survivor-baselines", () => { ]); }); }); + + it("maps release-history anchors to npm-published package versions when GitHub tags have republish suffixes", () => { + const releases = ( + [ + ["v2026.4.29", "2026-04-30T00:00:00Z"], + ["v2026.4.27", "2026-04-28T00:00:00Z"], + ["v2026.4.26", "2026-04-27T00:00:00Z"], + ["v2026.4.25", "2026-04-26T00:00:00Z"], + ["v2026.4.24", "2026-04-25T00:00:00Z"], + ["v2026.4.23", "2026-04-22T00:00:00Z"], + ["v2026.3.13-1", "2026-03-14T18:04:00Z"], + ] as const + ).map(([tagName, publishedAt]) => ({ + isPrerelease: false, + publishedAt, + tagName, + })); + + withReleaseFixture(releases, (releasesFile) => { + withJsonFixture( + "versions.json", + ["2026.4.29", "2026.4.27", "2026.4.26", "2026.4.25", "2026.4.24", "2026.4.23", "2026.3.13"], + (versionsFile) => { + expect( + resolveBaselines( + new Map([ + ["requested", "release-history"], + ["releases-json", releasesFile], + ["npm-versions-json", versionsFile], + ["history-count", "6"], + ["include-version", "2026.4.23"], + ["pre-date", "2026-03-15T00:00:00Z"], + ]), + ), + ).toEqual([ + "openclaw@2026.4.29", + "openclaw@2026.4.27", + "openclaw@2026.4.26", + "openclaw@2026.4.25", + "openclaw@2026.4.24", + "openclaw@2026.4.23", + "openclaw@2026.3.13", + ]); + }, + ); + }); + }); });