test(parallels): harden release VM smoke isolation

This commit is contained in:
Peter Steinberger
2026-05-24 05:49:56 +01:00
parent 0f83c93740
commit 3839b48615
11 changed files with 251 additions and 30 deletions

View File

@@ -4,6 +4,7 @@ export * from "./host-server.ts";
export * from "./lane-runner.ts";
export * from "./package-artifact.ts";
export * from "./parallels-vm.ts";
export * from "./plugin-isolation.ts";
export * from "./provider-auth.ts";
export * from "./snapshots.ts";
export * from "./types.ts";

View File

@@ -13,6 +13,7 @@ import {
parseMode,
parseProvider,
modelProviderConfigBatchJson,
posixProviderOnlyPluginIsolationScript,
repoRoot,
resolveParallelsModelTimeoutSeconds,
resolveHostIp,
@@ -126,7 +127,7 @@ const defaultOptions = (): LinuxOptions => ({
provider: "openai",
snapshotHint: "fresh",
targetPackageSpec: "",
vmName: "Ubuntu 24.04.3 ARM64",
vmName: "Ubuntu 26.04",
vmNameExplicit: false,
});
@@ -134,7 +135,7 @@ function usage(): string {
return `Usage: bash scripts/e2e/parallels-linux-smoke.sh [options]
Options:
--vm <name> Parallels VM name. Default: "Ubuntu 24.04.3 ARM64"
--vm <name> Parallels VM name. Default: "Ubuntu 26.04"
Falls back to the closest Ubuntu VM when omitted and unavailable.
--snapshot-hint <name> Snapshot name substring/fuzzy match. Default: "fresh"
--mode <fresh|upgrade|both>
@@ -758,6 +759,15 @@ PY
rm -rf /root/.openclaw/test-bad-plugin`);
}
private restrictAgentTurnPlugins(): void {
this.guestBash(
posixProviderOnlyPluginIsolationScript({
fallbackPluginId: this.options.provider,
modelId: this.auth.modelId,
}),
);
}
private verifyLocalTurn(): void {
this.guestExec(["openclaw", "models", "set", this.auth.modelId]);
const modelProviderConfigBatch = modelProviderConfigBatchJson(this.auth.modelId, "linux");
@@ -778,6 +788,7 @@ rm -f "$provider_config_batch"`);
"--strict-json",
]);
this.guestExec(["openclaw", "config", "set", "tools.profile", "minimal"]);
this.restrictAgentTurnPlugins();
this.prepareAgentWorkspace();
this.guestBash(
`agent_ok=false

View File

@@ -12,6 +12,7 @@ import {
parseMode,
parseProvider,
modelProviderConfigBatchJson,
posixProviderOnlyPluginIsolationScript,
resolveParallelsModelTimeoutSeconds,
resolveHostIp,
resolveHostPort,
@@ -115,7 +116,7 @@ const defaultOptions = (): MacosOptions => ({
modelId: undefined,
provider: "openai",
skipLatestRefCheck: false,
snapshotHint: "macOS 26.3.1 latest",
snapshotHint: "macOS 26.5 latest",
targetPackageSpec: "",
vmName: "macOS Tahoe",
});
@@ -126,7 +127,7 @@ function usage(): string {
Options:
--vm <name> Parallels VM name. Default: "macOS Tahoe"
--snapshot-hint <name> Snapshot name substring/fuzzy match.
Default: "macOS 26.3.1 latest"
Default: "macOS 26.5 latest"
--mode <fresh|upgrade|both>
--provider <openai|anthropic|minimax>
--model <provider/model> Override the model used for the agent-turn smoke.
@@ -977,6 +978,17 @@ echo "dashboard HTML did not become ready" >&2
exit 1`);
}
private restrictAgentTurnPlugins(): void {
this.guestSh(
posixProviderOnlyPluginIsolationScript({
fallbackPluginId: this.options.provider,
homeFallback: this.guestHome(),
modelId: this.auth.modelId,
nodeCommand: guestNode,
}),
);
}
private verifyTurn(): void {
this.guestExec([guestNode, guestOpenClawEntry, "models", "set", this.auth.modelId]);
const modelProviderConfigBatch = modelProviderConfigBatchJson(this.auth.modelId, "macos");
@@ -1000,6 +1012,7 @@ rm -f "$provider_config_batch"`);
"--strict-json",
]);
this.guestExec([guestNode, guestOpenClawEntry, "config", "set", "tools.profile", "minimal"]);
this.restrictAgentTurnPlugins();
this.guestSh(
`${posixAgentWorkspaceScript("Parallels macOS smoke test assistant.")}
agent_ok=false

View File

@@ -1,5 +1,6 @@
import { posixAgentWorkspaceScript, windowsAgentWorkspaceScript } from "./agent-workspace.ts";
import { shellQuote } from "./host-command.ts";
import { posixProviderOnlyPluginIsolationScript } from "./plugin-isolation.ts";
import {
psSingleQuote,
windowsAgentTurnConfigPatchScript,
@@ -42,7 +43,11 @@ if [ "$provider_config_exit" -ne 0 ]; then exit "$provider_config_exit"; fi`;
}
function posixAssertAgentOkScript(command: string, input: NpmUpdateScriptInput, sessionId: string) {
return `agent_ok=false
return `${posixProviderOnlyPluginIsolationScript({
fallbackPluginId: input.auth.modelId.split("/", 1)[0] || "openai",
modelId: input.auth.modelId,
})}
agent_ok=false
for attempt in 1 2; do
session_id=${shellQuote(sessionId)}
if [ "$attempt" -gt 1 ]; then session_id=${shellQuote(`${sessionId}-retry`)}"-$attempt"; fi

View File

@@ -94,7 +94,7 @@ interface NpmUpdateSummary {
const macosVm = "macOS Tahoe";
const windowsVm = "Windows 11";
const linuxVmDefault = "Ubuntu 24.04.3 ARM64";
const linuxVmDefault = "Ubuntu 26.04";
const updateTimeoutSeconds = Number(process.env.OPENCLAW_PARALLELS_NPM_UPDATE_TIMEOUT_S || 1200);
function usage(): string {

View File

@@ -68,7 +68,7 @@ export function resolveUbuntuVmName(requested: string, explicit = false): string
parts: item.version.split(".").map(Number),
}))
.filter((item) => item.parts[0] >= 24)
.toSorted((a, b) => compareVersions(a.parts, b.parts))[0]?.name ??
.toSorted((a, b) => compareVersions(b.parts, a.parts))[0]?.name ??
names.find((name) => /ubuntu/i.test(name));
if (!fallback) {
die(`VM not found: ${requested}`);

View File

@@ -0,0 +1,126 @@
import { shellQuote } from "./host-command.ts";
import { providerIdFromModelId } from "./provider-auth.ts";
interface PluginIsolationOptions {
fallbackPluginId: string;
homeFallback?: string;
modelId: string;
nodeCommand?: string;
}
export function providerOnlyPluginId(modelId: string, fallbackPluginId: string): string {
return providerIdFromModelId(modelId) || fallbackPluginId;
}
export function posixProviderOnlyPluginIsolationScript(options: PluginIsolationOptions): string {
const nodeCommand = shellQuote(options.nodeCommand ?? "node");
const homeEnv = options.homeFallback
? `OPENCLAW_PARALLELS_HOME=${shellQuote(options.homeFallback)} `
: "";
return `/usr/bin/env ${homeEnv}${nodeCommand} - <<'JS'
${providerOnlyPluginIsolationNodeScript(options)}
JS`;
}
export function windowsProviderOnlyPluginIsolationScript(options: PluginIsolationOptions): string {
const payloadJson = JSON.stringify({
modelId: options.modelId,
pluginId: providerOnlyPluginId(options.modelId, options.fallbackPluginId),
});
return `$env:OPENCLAW_PARALLELS_PLUGIN_ISOLATION = @'
${payloadJson}
'@
$isolationScriptPath = Join-Path ([System.IO.Path]::GetTempPath()) 'openclaw-parallels-plugin-isolation.cjs'
@'
${providerOnlyPluginIsolationNodeSource()}
'@ | Set-Content -Path $isolationScriptPath -Encoding UTF8
node.exe $isolationScriptPath
if ($LASTEXITCODE -ne 0) { throw "plugin isolation failed with exit code $LASTEXITCODE" }
Remove-Item $isolationScriptPath -Force -ErrorAction SilentlyContinue
Remove-Item Env:OPENCLAW_PARALLELS_PLUGIN_ISOLATION -Force -ErrorAction SilentlyContinue`;
}
function providerOnlyPluginIsolationNodeScript(options: PluginIsolationOptions): string {
const payloadJson = JSON.stringify({
homeFallback: options.homeFallback,
modelId: options.modelId,
pluginId: providerOnlyPluginId(options.modelId, options.fallbackPluginId),
});
return `process.env.OPENCLAW_PARALLELS_PLUGIN_ISOLATION = ${JSON.stringify(payloadJson)};
${providerOnlyPluginIsolationNodeSource()}`;
}
function providerOnlyPluginIsolationNodeSource(): string {
return String.raw`const fs = require("node:fs");
const path = require("node:path");
const payload = JSON.parse(process.env.OPENCLAW_PARALLELS_PLUGIN_ISOLATION || "{}");
const home =
process.env.OPENCLAW_PARALLELS_HOME ||
payload.homeFallback ||
process.env.HOME ||
process.env.USERPROFILE ||
"/root";
const configPath = path.join(home, ".openclaw", "openclaw.json");
const stateDir = path.dirname(configPath);
const modelId = String(payload.modelId || "");
const allowedPluginId = String(payload.pluginId || "").trim();
if (!allowedPluginId || !modelId) {
throw new Error("missing plugin isolation payload");
}
const readConfig = () => {
if (!fs.existsSync(configPath)) {
return {};
}
return JSON.parse(fs.readFileSync(configPath, "utf8"));
};
const objectRecord = (value) =>
value && typeof value === "object" && !Array.isArray(value) ? value : {};
const config = readConfig();
config.plugins = objectRecord(config.plugins);
config.plugins.entries = { [allowedPluginId]: { enabled: true } };
config.plugins.allow = [allowedPluginId];
config.agents = objectRecord(config.agents);
config.agents.defaults = objectRecord(config.agents.defaults);
config.agents.defaults.model = {
...objectRecord(config.agents.defaults.model),
primary: modelId,
};
config.agents.defaults.models = objectRecord(config.agents.defaults.models);
const selectedModelEntry = config.agents.defaults.models[modelId];
if (selectedModelEntry && typeof selectedModelEntry === "object" && !Array.isArray(selectedModelEntry)) {
delete selectedModelEntry.agentRuntime;
}
const providerId = modelId.split("/", 1)[0] || "";
const providerModelId = modelId.slice(providerId.length + 1);
const providers = objectRecord(objectRecord(config.models).providers);
const providerEntry = providers[providerId];
if (providerEntry && typeof providerEntry === "object" && !Array.isArray(providerEntry)) {
delete providerEntry.agentRuntime;
if (Array.isArray(providerEntry.models)) {
for (const model of providerEntry.models) {
if (
model &&
typeof model === "object" &&
(model.id === providerModelId ||
model.id === modelId ||
model.name === providerModelId ||
model.name === modelId)
) {
delete model.agentRuntime;
}
}
}
}
fs.rmSync(path.join(stateDir, "npm", "node_modules", "@openclaw", "codex"), {
recursive: true,
force: true,
});
fs.mkdirSync(stateDir, { recursive: true });
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");`;
}

View File

@@ -75,9 +75,11 @@ if ($providerTimeoutExit -ne 0) { throw "model provider timeout config set faile
export function windowsAgentTurnConfigPatchScript(modelId: string): string {
const batchJson = modelProviderConfigBatchJson(modelId, "windows");
const pluginId = providerIdFromModelId(modelId) || modelId.split("/", 1)[0] || "openai";
const payloadJson = JSON.stringify({
modelId,
operations: batchJson ? (JSON.parse(batchJson) as unknown) : [],
pluginId,
});
return `$agentTurnConfigPatchPath = $env:OPENCLAW_CONFIG_PATH
if (-not $agentTurnConfigPatchPath) { $agentTurnConfigPatchPath = Join-Path $env:USERPROFILE '.openclaw\\openclaw.json' }
@@ -113,6 +115,11 @@ cfg.agents.defaults.model = { ...existingModel, primary: payload.modelId };
cfg.agents.defaults.models = cfg.agents.defaults.models && typeof cfg.agents.defaults.models === "object" ? cfg.agents.defaults.models : {};
cfg.tools = cfg.tools && typeof cfg.tools === "object" ? cfg.tools : {};
cfg.tools.profile = "minimal";
cfg.plugins = cfg.plugins && typeof cfg.plugins === "object" && !Array.isArray(cfg.plugins) ? cfg.plugins : {};
cfg.plugins.entries = { [payload.pluginId]: { enabled: true } };
cfg.plugins.allow = [payload.pluginId];
const stateDir = path.dirname(configPath);
fs.rmSync(path.join(stateDir, "npm", "node_modules", "@openclaw", "codex"), { recursive: true, force: true });
for (const op of payload.operations || []) {
const segments = String(op.path || "").match(/(?:[^.[\\]]+)|(?:\\["((?:\\\\.|[^"\\\\])*)"\\])/g) || [];
let cursor = cfg;

View File

@@ -14,22 +14,30 @@ export function resolveSnapshot(vmName: string, hint: string): SnapshotInfo {
values.push(match[1]);
}
}
return values;
return values.flatMap((value) => {
const withoutLatest = value.replace(/\s+latest$/u, "").trim();
return withoutLatest && withoutLatest !== value ? [value, withoutLatest] : [value];
});
};
const normalizedHint = hint.trim().toLowerCase();
const normalizedHints = [normalizedHint, normalizedHint.replace(/\s+latest$/u, "").trim()].filter(
(value, index, values) => value && values.indexOf(value) === index,
);
for (const [id, meta] of Object.entries(payload)) {
const name = (meta.name ?? "").trim();
if (!name) {
continue;
}
let score = 0;
for (const alias of aliases(name.toLowerCase())) {
if (alias === normalizedHint) {
score = Math.max(score, 10);
} else if (normalizedHint && alias.includes(normalizedHint)) {
score = Math.max(score, 5 + normalizedHint.length / Math.max(alias.length, 1));
} else {
score = Math.max(score, stringSimilarity(normalizedHint, alias));
for (const hintAlias of normalizedHints) {
for (const alias of aliases(name.toLowerCase())) {
if (alias === hintAlias) {
score = Math.max(score, 10);
} else if (hintAlias && alias.includes(hintAlias)) {
score = Math.max(score, 5 + hintAlias.length / Math.max(alias.length, 1));
} else {
score = Math.max(score, stringSimilarity(hintAlias, alias));
}
}
}
if ((meta.state ?? "").toLowerCase() === "poweroff") {

View File

@@ -34,6 +34,7 @@ import { runWindowsBackgroundPowerShell, WindowsGuest } from "./guest-transports
import { runSmokeLane, type SmokeLane, type SmokeLaneStatus } from "./lane-runner.ts";
import { waitForVmStatus } from "./parallels-vm.ts";
import { PhaseRunner } from "./phase-runner.ts";
import { windowsProviderOnlyPluginIsolationScript } from "./plugin-isolation.ts";
import {
psSingleQuote,
windowsAgentTurnConfigPatchScript,
@@ -650,11 +651,19 @@ if ($LASTEXITCODE -ne 0) { throw "openclaw --version failed with exit code $LAST
$PSNativeCommandUseErrorActionPreference = $false
Set-Item -Path ('Env:' + ${psSingleQuote(this.auth.apiKeyEnv)}) -Value ${psSingleQuote(this.auth.apiKeyValue)}
Invoke-OpenClaw onboard --non-interactive --mode local --auth-choice ${psSingleQuote(this.auth.authChoice)} --secret-input-mode ref --gateway-port 18789 --gateway-bind loopback --install-daemon --skip-skills --skip-health --accept-risk --json
if ($LASTEXITCODE -ne 0) { throw "openclaw onboard failed with exit code $LASTEXITCODE" }`,
if ($LASTEXITCODE -ne 0) { throw "openclaw onboard failed with exit code $LASTEXITCODE" }
${this.windowsPluginIsolationScript()}`,
720_000,
);
}
private windowsPluginIsolationScript(): string {
return windowsProviderOnlyPluginIsolationScript({
fallbackPluginId: this.options.provider,
modelId: this.auth.modelId,
});
}
private async guestPowerShellBackground(
label: string,
script: string,

View File

@@ -1,11 +1,4 @@
import {
chmodSync,
copyFileSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { chmodSync, copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { delimiter, join, win32 } from "node:path";
import { pathToFileURL } from "node:url";
@@ -67,11 +60,7 @@ function fakePrlctlEnv(tempDir: string): Record<string, string> {
return { NODE_OPTIONS: nodeOptions, PATH: pathValue, Path: pathValue };
}
function writeFakePrlctl(
tempDir: string,
posixScript: string,
windowsBootstrap: string,
): void {
function writeFakePrlctl(tempDir: string, posixScript: string, windowsBootstrap: string): void {
const prlctlPath = join(tempDir, "prlctl");
writeFileSync(prlctlPath, posixScript);
chmodSync(prlctlPath, 0o755);
@@ -243,6 +232,56 @@ console.log([snapshot.id, snapshot.state, snapshot.name].join("\\t"));
}
});
it("resolves a latest snapshot hint to the matching version before older LATEST labels", () => {
const tempDir = mkdtempSync(join(tmpdir(), "openclaw-parallels-snapshot-latest-"));
writeFakePrlctl(
tempDir,
`#!/usr/bin/env bash
set -euo pipefail
if [[ "$1" == "snapshot-list" ]]; then
cat <<'JSON'
{
"{old}": {"name": "macOS 26.3.1 LATEST", "state": "poweron"},
"{wanted}": {"name": "macOS 26.5", "state": "poweron"}
}
JSON
exit 0
fi
exit 1
`,
`import { basename } from "node:path";
const isPrlctl = [process.argv0, process.execPath].some((value) =>
basename(value).toLowerCase() === "prlctl.exe",
);
if (isPrlctl) {
if (process.argv.some((arg) => arg.includes("snapshot-list"))) {
console.log(JSON.stringify({
"{old}": { name: "macOS 26.3.1 LATEST", state: "poweron" },
"{wanted}": { name: "macOS 26.5", state: "poweron" },
}));
process.exit(0);
}
process.exit(1);
}
`,
);
try {
const output = runTsEval(
`
import { resolveSnapshot } from "./${TS_PATHS.common}";
const snapshot = resolveSnapshot("vm", "macOS 26.5 latest");
console.log([snapshot.id, snapshot.state, snapshot.name].join("\\t"));
`,
fakePrlctlEnv(tempDir),
);
expect(output.trim()).toBe("{wanted}\tpoweron\tmacOS 26.5");
} finally {
rmSync(tempDir, { force: true, recursive: true });
}
});
it("uses one Ubuntu VM fallback resolver for Linux lanes", () => {
const tempDir = mkdtempSync(join(tmpdir(), "openclaw-parallels-vm-helper-"));
writeFakePrlctl(
@@ -252,6 +291,7 @@ set -euo pipefail
if [[ "$1" == "list" ]]; then
cat <<'JSON'
[
{"name": "Ubuntu 26.04"},
{"name": "Ubuntu 25.10"},
{"name": "Ubuntu 23.10"},
{"name": "Ubuntu 24.04.3 ARM64"}
@@ -268,6 +308,7 @@ const isPrlctl = [process.argv0, process.execPath].some((value) =>
if (isPrlctl) {
if (process.argv.some((arg) => arg.includes("list"))) {
console.log(JSON.stringify([
{ name: "Ubuntu 26.04" },
{ name: "Ubuntu 25.10" },
{ name: "Ubuntu 23.10" },
{ name: "Ubuntu 24.04.3 ARM64" },
@@ -288,7 +329,7 @@ console.log(resolveUbuntuVmName("Ubuntu missing"));
fakePrlctlEnv(tempDir),
);
expect(output.trim()).toBe("Ubuntu 24.04.3 ARM64");
expect(output.trim()).toBe("Ubuntu 26.04");
} finally {
rmSync(tempDir, { force: true, recursive: true });
}