fix(release): preserve beta validation evidence (#103796)

* fix(release): proxy upgrade fixtures to npm

(cherry picked from commit 30f54a5fe1)

* fix(release): require installer doctor evidence

(cherry picked from commit 0782a94452)

* test(qa): scope aborted restart outcome to Codex

(cherry picked from commit ac54f80430)

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Vincent Koc
2026-07-10 09:05:14 -07:00
committed by GitHub
parent bd9b1b428c
commit 155e196e22
7 changed files with 110 additions and 4 deletions

View File

@@ -455,7 +455,9 @@ describe("qa scenario catalog", () => {
expect(interruptedStatusAssertion).toBeDefined();
const interruptedStatusContract = JSON.stringify(interruptedStatusAssertion);
expect(interruptedStatusContract).toContain("waited.stopReason === 'restart'");
expect(interruptedStatusContract).toContain("waited.stopReason === 'aborted'");
expect(interruptedStatusContract).toContain(
"env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex' && waited.stopReason === 'aborted'",
);
expect(interruptedStatusContract).toContain("EmbeddedAttemptSessionTakeoverError");
expect(interruptedStatusContract).toContain("AbortError");
expect(interruptedStatusContract).toContain("This operation was aborted");

View File

@@ -96,7 +96,7 @@ flow:
- expr: started.runId
- expr: liveTurnTimeoutMs(env, 180000)
- assert:
expr: "waited.status === 'ok' || waited.status === 'timeout' || (waited.status === 'error' && (waited.stopReason === 'restart' || waited.stopReason === 'aborted' || String(waited.error ?? '').includes('EmbeddedAttemptSessionTakeoverError') || String(waited.error ?? '').includes('AbortError') || String(waited.error ?? '').includes('This operation was aborted')))"
expr: "waited.status === 'ok' || waited.status === 'timeout' || (waited.status === 'error' && (waited.stopReason === 'restart' || (env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'codex' && waited.stopReason === 'aborted') || String(waited.error ?? '').includes('EmbeddedAttemptSessionTakeoverError') || String(waited.error ?? '').includes('AbortError') || String(waited.error ?? '').includes('This operation was aborted')))"
message:
expr: "`interrupted agent run ended with unexpected status: ${JSON.stringify(waited)}`"
- set: interruptedMatches

View File

@@ -481,6 +481,18 @@ if (Number(updateStep.exitCode ?? 1) !== 0) {
if (typeof updateStep.command !== "string" || !updateStep.command.includes(expectedUrl)) {
throw new Error(`global update step missing expected tgz URL: ${JSON.stringify(updateStep)}`);
}
const doctorStep = steps.find((step) => step?.name === "openclaw doctor");
if (!doctorStep) {
throw new Error("missing openclaw doctor step in update JSON");
}
// Exit 86 is the updater's explicit recoverable post-install doctor contract.
const doctorSucceeded = doctorStep.exitCode === 0;
const doctorWasAdvisory =
doctorStep.exitCode === 86 &&
doctorStep.advisory?.kind === "package-post-install-doctor";
if (!doctorSucceeded && !doctorWasAdvisory) {
throw new Error(`openclaw doctor step failed: ${JSON.stringify(doctorStep)}`);
}
NODE
echo "==> Verify updated version"

View File

@@ -425,7 +425,8 @@ fs.writeFileSync(
);
NODE
tar -czf "$tarball" -C "$fixture_root" package
node scripts/e2e/lib/plugins/npm-registry-server.mjs \
OPENCLAW_NPM_REGISTRY_UPSTREAM=https://registry.npmjs.org \
node scripts/e2e/lib/plugins/npm-registry-server.mjs \
"$port_file" \
"@openclaw/brave-plugin" \
"2026.5.2" \

View File

@@ -314,7 +314,8 @@ fs.writeFileSync(
);
NODE
tar -czf "$tarball" -C "$fixture_root" package
node scripts/e2e/lib/plugins/npm-registry-server.mjs \
OPENCLAW_NPM_REGISTRY_UPSTREAM=https://registry.npmjs.org \
node scripts/e2e/lib/plugins/npm-registry-server.mjs \
"$port_file" \
"@openclaw/brave-plugin" \
"2026.5.2" \

View File

@@ -2062,6 +2062,16 @@ grep -qx -- "OPENCLAW_E2E_COMMAND_TIMEOUT=23s" "$TMPDIR/package-args"
}
});
it("lets upgrade survivor fixture registries resolve transitive public packages", () => {
const runner = readFileSync(UPGRADE_SURVIVOR_DOCKER_E2E_PATH, "utf8");
const publishedRunner = readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8");
for (const script of [runner, publishedRunner]) {
expect(script).toContain("OPENCLAW_NPM_REGISTRY_UPSTREAM=https://registry.npmjs.org");
expect(script).toContain("node scripts/e2e/lib/plugins/npm-registry-server.mjs");
}
});
it("wraps package-backed scenario OpenClaw CLI calls with the shared timeout helper", () => {
const paths = [
CODEX_ON_DEMAND_DOCKER_E2E_PATH,

View File

@@ -149,6 +149,45 @@ function normalizeInstallE2eAgentOutput(output: string) {
}
}
function extractInstallSmokeUpdateJsonParser(): string {
const script = readFileSync(SMOKE_RUNNER_PATH, "utf8");
const match = script.match(
/UPDATE_JSON="\$UPDATE_JSON" \\\n[\s\S]*?node - <<'NODE'\n([\s\S]*?)\nNODE\n\n echo "==> Verify updated version"/u,
);
if (!match) {
throw new Error("install smoke update JSON parser was not found");
}
return match[1];
}
function validateInstallSmokeUpdateJson(doctorStep?: Record<string, unknown>) {
const updateUrl = "http://candidate.invalid/openclaw.tgz";
const payload = {
status: "ok",
before: { version: "2026.7.0" },
after: { version: "2026.7.1" },
steps: [
{
name: "global update",
exitCode: 0,
command: `npm install ${updateUrl}`,
},
...(doctorStep ? [doctorStep] : []),
],
};
return spawnSync(process.execPath, ["-"], {
encoding: "utf8",
input: extractInstallSmokeUpdateJsonParser(),
env: {
...process.env,
UPDATE_JSON: JSON.stringify(payload),
UPDATE_EXPECT_VERSION: payload.after.version,
UPDATE_BASELINE_VERSION: payload.before.version,
UPDATE_TAG_URL: updateUrl,
},
});
}
function expectInstallDockerfileContract(
dockerfilePath: string,
runnerPath: string,
@@ -1014,6 +1053,47 @@ describe("install-sh smoke runner", () => {
expect(script).toContain("openclaw infer image providers --json");
});
it.each([
["successful", { name: "openclaw doctor", exitCode: 0 }],
[
"recoverable advisory",
{
name: "openclaw doctor",
exitCode: 86,
advisory: { kind: "package-post-install-doctor", message: "repair deferred" },
},
],
])("accepts a %s package post-install doctor result", (_label, doctorStep) => {
const result = validateInstallSmokeUpdateJson(doctorStep);
expect(result.status, result.stderr).toBe(0);
});
it.each([
["missing", undefined, "missing openclaw doctor step"],
["fatal", { name: "openclaw doctor", exitCode: 1 }, "openclaw doctor step failed"],
["untyped advisory", { name: "openclaw doctor", exitCode: 86 }, "openclaw doctor step failed"],
[
"wrong advisory kind",
{ name: "openclaw doctor", exitCode: 86, advisory: { kind: "other" } },
"openclaw doctor step failed",
],
[
"wrong advisory exit",
{
name: "openclaw doctor",
exitCode: 1,
advisory: { kind: "package-post-install-doctor" },
},
"openclaw doctor step failed",
],
])("rejects a %s package post-install doctor result", (_label, doctorStep, error) => {
const result = validateInstallSmokeUpdateJson(doctorStep);
expect(result.status).not.toBe(0);
expect(result.stderr).toContain(error);
});
it.each([
["command timeout", "OPENCLAW_INSTALL_SMOKE_COMMAND_TIMEOUT", "900s"],
["heartbeat interval", "OPENCLAW_INSTALL_SMOKE_HEARTBEAT_INTERVAL", "60s"],