From 3839b4861548dcb5822f4427eecbfdee77dd6ca6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 24 May 2026 05:49:56 +0100 Subject: [PATCH] test(parallels): harden release VM smoke isolation --- scripts/e2e/parallels/common.ts | 1 + scripts/e2e/parallels/linux-smoke.ts | 15 ++- scripts/e2e/parallels/macos-smoke.ts | 17 ++- scripts/e2e/parallels/npm-update-scripts.ts | 7 +- scripts/e2e/parallels/npm-update-smoke.ts | 2 +- scripts/e2e/parallels/parallels-vm.ts | 2 +- scripts/e2e/parallels/plugin-isolation.ts | 126 ++++++++++++++++++++ scripts/e2e/parallels/powershell.ts | 7 ++ scripts/e2e/parallels/snapshots.ts | 24 ++-- scripts/e2e/parallels/windows-smoke.ts | 11 +- test/scripts/parallels-smoke-model.test.ts | 69 ++++++++--- 11 files changed, 251 insertions(+), 30 deletions(-) create mode 100644 scripts/e2e/parallels/plugin-isolation.ts diff --git a/scripts/e2e/parallels/common.ts b/scripts/e2e/parallels/common.ts index 069b1ab2340..8fead72bf18 100644 --- a/scripts/e2e/parallels/common.ts +++ b/scripts/e2e/parallels/common.ts @@ -4,6 +4,7 @@ export * from "./host-server.ts"; export * from "./lane-runner.ts"; export * from "./package-artifact.ts"; export * from "./parallels-vm.ts"; +export * from "./plugin-isolation.ts"; export * from "./provider-auth.ts"; export * from "./snapshots.ts"; export * from "./types.ts"; diff --git a/scripts/e2e/parallels/linux-smoke.ts b/scripts/e2e/parallels/linux-smoke.ts index acaf1014784..f5af239b300 100755 --- a/scripts/e2e/parallels/linux-smoke.ts +++ b/scripts/e2e/parallels/linux-smoke.ts @@ -13,6 +13,7 @@ import { parseMode, parseProvider, modelProviderConfigBatchJson, + posixProviderOnlyPluginIsolationScript, repoRoot, resolveParallelsModelTimeoutSeconds, resolveHostIp, @@ -126,7 +127,7 @@ const defaultOptions = (): LinuxOptions => ({ provider: "openai", snapshotHint: "fresh", targetPackageSpec: "", - vmName: "Ubuntu 24.04.3 ARM64", + vmName: "Ubuntu 26.04", vmNameExplicit: false, }); @@ -134,7 +135,7 @@ function usage(): string { return `Usage: bash scripts/e2e/parallels-linux-smoke.sh [options] Options: - --vm Parallels VM name. Default: "Ubuntu 24.04.3 ARM64" + --vm Parallels VM name. Default: "Ubuntu 26.04" Falls back to the closest Ubuntu VM when omitted and unavailable. --snapshot-hint Snapshot name substring/fuzzy match. Default: "fresh" --mode @@ -758,6 +759,15 @@ PY rm -rf /root/.openclaw/test-bad-plugin`); } + private restrictAgentTurnPlugins(): void { + this.guestBash( + posixProviderOnlyPluginIsolationScript({ + fallbackPluginId: this.options.provider, + modelId: this.auth.modelId, + }), + ); + } + private verifyLocalTurn(): void { this.guestExec(["openclaw", "models", "set", this.auth.modelId]); const modelProviderConfigBatch = modelProviderConfigBatchJson(this.auth.modelId, "linux"); @@ -778,6 +788,7 @@ rm -f "$provider_config_batch"`); "--strict-json", ]); this.guestExec(["openclaw", "config", "set", "tools.profile", "minimal"]); + this.restrictAgentTurnPlugins(); this.prepareAgentWorkspace(); this.guestBash( `agent_ok=false diff --git a/scripts/e2e/parallels/macos-smoke.ts b/scripts/e2e/parallels/macos-smoke.ts index c6f6b117ff0..8c9fc159ee1 100755 --- a/scripts/e2e/parallels/macos-smoke.ts +++ b/scripts/e2e/parallels/macos-smoke.ts @@ -12,6 +12,7 @@ import { parseMode, parseProvider, modelProviderConfigBatchJson, + posixProviderOnlyPluginIsolationScript, resolveParallelsModelTimeoutSeconds, resolveHostIp, resolveHostPort, @@ -115,7 +116,7 @@ const defaultOptions = (): MacosOptions => ({ modelId: undefined, provider: "openai", skipLatestRefCheck: false, - snapshotHint: "macOS 26.3.1 latest", + snapshotHint: "macOS 26.5 latest", targetPackageSpec: "", vmName: "macOS Tahoe", }); @@ -126,7 +127,7 @@ function usage(): string { Options: --vm Parallels VM name. Default: "macOS Tahoe" --snapshot-hint Snapshot name substring/fuzzy match. - Default: "macOS 26.3.1 latest" + Default: "macOS 26.5 latest" --mode --provider --model Override the model used for the agent-turn smoke. @@ -977,6 +978,17 @@ echo "dashboard HTML did not become ready" >&2 exit 1`); } + private restrictAgentTurnPlugins(): void { + this.guestSh( + posixProviderOnlyPluginIsolationScript({ + fallbackPluginId: this.options.provider, + homeFallback: this.guestHome(), + modelId: this.auth.modelId, + nodeCommand: guestNode, + }), + ); + } + private verifyTurn(): void { this.guestExec([guestNode, guestOpenClawEntry, "models", "set", this.auth.modelId]); const modelProviderConfigBatch = modelProviderConfigBatchJson(this.auth.modelId, "macos"); @@ -1000,6 +1012,7 @@ rm -f "$provider_config_batch"`); "--strict-json", ]); this.guestExec([guestNode, guestOpenClawEntry, "config", "set", "tools.profile", "minimal"]); + this.restrictAgentTurnPlugins(); this.guestSh( `${posixAgentWorkspaceScript("Parallels macOS smoke test assistant.")} agent_ok=false diff --git a/scripts/e2e/parallels/npm-update-scripts.ts b/scripts/e2e/parallels/npm-update-scripts.ts index 743df96ab5c..458e6305bf0 100644 --- a/scripts/e2e/parallels/npm-update-scripts.ts +++ b/scripts/e2e/parallels/npm-update-scripts.ts @@ -1,5 +1,6 @@ import { posixAgentWorkspaceScript, windowsAgentWorkspaceScript } from "./agent-workspace.ts"; import { shellQuote } from "./host-command.ts"; +import { posixProviderOnlyPluginIsolationScript } from "./plugin-isolation.ts"; import { psSingleQuote, windowsAgentTurnConfigPatchScript, @@ -42,7 +43,11 @@ if [ "$provider_config_exit" -ne 0 ]; then exit "$provider_config_exit"; fi`; } function posixAssertAgentOkScript(command: string, input: NpmUpdateScriptInput, sessionId: string) { - return `agent_ok=false + return `${posixProviderOnlyPluginIsolationScript({ + fallbackPluginId: input.auth.modelId.split("/", 1)[0] || "openai", + modelId: input.auth.modelId, + })} +agent_ok=false for attempt in 1 2; do session_id=${shellQuote(sessionId)} if [ "$attempt" -gt 1 ]; then session_id=${shellQuote(`${sessionId}-retry`)}"-$attempt"; fi diff --git a/scripts/e2e/parallels/npm-update-smoke.ts b/scripts/e2e/parallels/npm-update-smoke.ts index fbf19fa1263..6ec45515584 100755 --- a/scripts/e2e/parallels/npm-update-smoke.ts +++ b/scripts/e2e/parallels/npm-update-smoke.ts @@ -94,7 +94,7 @@ interface NpmUpdateSummary { const macosVm = "macOS Tahoe"; const windowsVm = "Windows 11"; -const linuxVmDefault = "Ubuntu 24.04.3 ARM64"; +const linuxVmDefault = "Ubuntu 26.04"; const updateTimeoutSeconds = Number(process.env.OPENCLAW_PARALLELS_NPM_UPDATE_TIMEOUT_S || 1200); function usage(): string { diff --git a/scripts/e2e/parallels/parallels-vm.ts b/scripts/e2e/parallels/parallels-vm.ts index 7925e654d96..616346b4ff1 100644 --- a/scripts/e2e/parallels/parallels-vm.ts +++ b/scripts/e2e/parallels/parallels-vm.ts @@ -68,7 +68,7 @@ export function resolveUbuntuVmName(requested: string, explicit = false): string parts: item.version.split(".").map(Number), })) .filter((item) => item.parts[0] >= 24) - .toSorted((a, b) => compareVersions(a.parts, b.parts))[0]?.name ?? + .toSorted((a, b) => compareVersions(b.parts, a.parts))[0]?.name ?? names.find((name) => /ubuntu/i.test(name)); if (!fallback) { die(`VM not found: ${requested}`); diff --git a/scripts/e2e/parallels/plugin-isolation.ts b/scripts/e2e/parallels/plugin-isolation.ts new file mode 100644 index 00000000000..aac004153e1 --- /dev/null +++ b/scripts/e2e/parallels/plugin-isolation.ts @@ -0,0 +1,126 @@ +import { shellQuote } from "./host-command.ts"; +import { providerIdFromModelId } from "./provider-auth.ts"; + +interface PluginIsolationOptions { + fallbackPluginId: string; + homeFallback?: string; + modelId: string; + nodeCommand?: string; +} + +export function providerOnlyPluginId(modelId: string, fallbackPluginId: string): string { + return providerIdFromModelId(modelId) || fallbackPluginId; +} + +export function posixProviderOnlyPluginIsolationScript(options: PluginIsolationOptions): string { + const nodeCommand = shellQuote(options.nodeCommand ?? "node"); + const homeEnv = options.homeFallback + ? `OPENCLAW_PARALLELS_HOME=${shellQuote(options.homeFallback)} ` + : ""; + return `/usr/bin/env ${homeEnv}${nodeCommand} - <<'JS' +${providerOnlyPluginIsolationNodeScript(options)} +JS`; +} + +export function windowsProviderOnlyPluginIsolationScript(options: PluginIsolationOptions): string { + const payloadJson = JSON.stringify({ + modelId: options.modelId, + pluginId: providerOnlyPluginId(options.modelId, options.fallbackPluginId), + }); + return `$env:OPENCLAW_PARALLELS_PLUGIN_ISOLATION = @' +${payloadJson} +'@ +$isolationScriptPath = Join-Path ([System.IO.Path]::GetTempPath()) 'openclaw-parallels-plugin-isolation.cjs' +@' +${providerOnlyPluginIsolationNodeSource()} +'@ | Set-Content -Path $isolationScriptPath -Encoding UTF8 +node.exe $isolationScriptPath +if ($LASTEXITCODE -ne 0) { throw "plugin isolation failed with exit code $LASTEXITCODE" } +Remove-Item $isolationScriptPath -Force -ErrorAction SilentlyContinue +Remove-Item Env:OPENCLAW_PARALLELS_PLUGIN_ISOLATION -Force -ErrorAction SilentlyContinue`; +} + +function providerOnlyPluginIsolationNodeScript(options: PluginIsolationOptions): string { + const payloadJson = JSON.stringify({ + homeFallback: options.homeFallback, + modelId: options.modelId, + pluginId: providerOnlyPluginId(options.modelId, options.fallbackPluginId), + }); + return `process.env.OPENCLAW_PARALLELS_PLUGIN_ISOLATION = ${JSON.stringify(payloadJson)}; +${providerOnlyPluginIsolationNodeSource()}`; +} + +function providerOnlyPluginIsolationNodeSource(): string { + return String.raw`const fs = require("node:fs"); +const path = require("node:path"); + +const payload = JSON.parse(process.env.OPENCLAW_PARALLELS_PLUGIN_ISOLATION || "{}"); +const home = + process.env.OPENCLAW_PARALLELS_HOME || + payload.homeFallback || + process.env.HOME || + process.env.USERPROFILE || + "/root"; +const configPath = path.join(home, ".openclaw", "openclaw.json"); +const stateDir = path.dirname(configPath); +const modelId = String(payload.modelId || ""); +const allowedPluginId = String(payload.pluginId || "").trim(); +if (!allowedPluginId || !modelId) { + throw new Error("missing plugin isolation payload"); +} + +const readConfig = () => { + if (!fs.existsSync(configPath)) { + return {}; + } + return JSON.parse(fs.readFileSync(configPath, "utf8")); +}; +const objectRecord = (value) => + value && typeof value === "object" && !Array.isArray(value) ? value : {}; + +const config = readConfig(); +config.plugins = objectRecord(config.plugins); +config.plugins.entries = { [allowedPluginId]: { enabled: true } }; +config.plugins.allow = [allowedPluginId]; + +config.agents = objectRecord(config.agents); +config.agents.defaults = objectRecord(config.agents.defaults); +config.agents.defaults.model = { + ...objectRecord(config.agents.defaults.model), + primary: modelId, +}; +config.agents.defaults.models = objectRecord(config.agents.defaults.models); +const selectedModelEntry = config.agents.defaults.models[modelId]; +if (selectedModelEntry && typeof selectedModelEntry === "object" && !Array.isArray(selectedModelEntry)) { + delete selectedModelEntry.agentRuntime; +} + +const providerId = modelId.split("/", 1)[0] || ""; +const providerModelId = modelId.slice(providerId.length + 1); +const providers = objectRecord(objectRecord(config.models).providers); +const providerEntry = providers[providerId]; +if (providerEntry && typeof providerEntry === "object" && !Array.isArray(providerEntry)) { + delete providerEntry.agentRuntime; + if (Array.isArray(providerEntry.models)) { + for (const model of providerEntry.models) { + if ( + model && + typeof model === "object" && + (model.id === providerModelId || + model.id === modelId || + model.name === providerModelId || + model.name === modelId) + ) { + delete model.agentRuntime; + } + } + } +} + +fs.rmSync(path.join(stateDir, "npm", "node_modules", "@openclaw", "codex"), { + recursive: true, + force: true, +}); +fs.mkdirSync(stateDir, { recursive: true }); +fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n");`; +} diff --git a/scripts/e2e/parallels/powershell.ts b/scripts/e2e/parallels/powershell.ts index 6f3be9dbba6..3e7c0de0a1e 100644 --- a/scripts/e2e/parallels/powershell.ts +++ b/scripts/e2e/parallels/powershell.ts @@ -75,9 +75,11 @@ if ($providerTimeoutExit -ne 0) { throw "model provider timeout config set faile export function windowsAgentTurnConfigPatchScript(modelId: string): string { const batchJson = modelProviderConfigBatchJson(modelId, "windows"); + const pluginId = providerIdFromModelId(modelId) || modelId.split("/", 1)[0] || "openai"; const payloadJson = JSON.stringify({ modelId, operations: batchJson ? (JSON.parse(batchJson) as unknown) : [], + pluginId, }); return `$agentTurnConfigPatchPath = $env:OPENCLAW_CONFIG_PATH if (-not $agentTurnConfigPatchPath) { $agentTurnConfigPatchPath = Join-Path $env:USERPROFILE '.openclaw\\openclaw.json' } @@ -113,6 +115,11 @@ cfg.agents.defaults.model = { ...existingModel, primary: payload.modelId }; cfg.agents.defaults.models = cfg.agents.defaults.models && typeof cfg.agents.defaults.models === "object" ? cfg.agents.defaults.models : {}; cfg.tools = cfg.tools && typeof cfg.tools === "object" ? cfg.tools : {}; cfg.tools.profile = "minimal"; +cfg.plugins = cfg.plugins && typeof cfg.plugins === "object" && !Array.isArray(cfg.plugins) ? cfg.plugins : {}; +cfg.plugins.entries = { [payload.pluginId]: { enabled: true } }; +cfg.plugins.allow = [payload.pluginId]; +const stateDir = path.dirname(configPath); +fs.rmSync(path.join(stateDir, "npm", "node_modules", "@openclaw", "codex"), { recursive: true, force: true }); for (const op of payload.operations || []) { const segments = String(op.path || "").match(/(?:[^.[\\]]+)|(?:\\["((?:\\\\.|[^"\\\\])*)"\\])/g) || []; let cursor = cfg; diff --git a/scripts/e2e/parallels/snapshots.ts b/scripts/e2e/parallels/snapshots.ts index fd7c22667a8..6b1289bd5b1 100644 --- a/scripts/e2e/parallels/snapshots.ts +++ b/scripts/e2e/parallels/snapshots.ts @@ -14,22 +14,30 @@ export function resolveSnapshot(vmName: string, hint: string): SnapshotInfo { values.push(match[1]); } } - return values; + return values.flatMap((value) => { + const withoutLatest = value.replace(/\s+latest$/u, "").trim(); + return withoutLatest && withoutLatest !== value ? [value, withoutLatest] : [value]; + }); }; const normalizedHint = hint.trim().toLowerCase(); + const normalizedHints = [normalizedHint, normalizedHint.replace(/\s+latest$/u, "").trim()].filter( + (value, index, values) => value && values.indexOf(value) === index, + ); for (const [id, meta] of Object.entries(payload)) { const name = (meta.name ?? "").trim(); if (!name) { continue; } let score = 0; - for (const alias of aliases(name.toLowerCase())) { - if (alias === normalizedHint) { - score = Math.max(score, 10); - } else if (normalizedHint && alias.includes(normalizedHint)) { - score = Math.max(score, 5 + normalizedHint.length / Math.max(alias.length, 1)); - } else { - score = Math.max(score, stringSimilarity(normalizedHint, alias)); + for (const hintAlias of normalizedHints) { + for (const alias of aliases(name.toLowerCase())) { + if (alias === hintAlias) { + score = Math.max(score, 10); + } else if (hintAlias && alias.includes(hintAlias)) { + score = Math.max(score, 5 + hintAlias.length / Math.max(alias.length, 1)); + } else { + score = Math.max(score, stringSimilarity(hintAlias, alias)); + } } } if ((meta.state ?? "").toLowerCase() === "poweroff") { diff --git a/scripts/e2e/parallels/windows-smoke.ts b/scripts/e2e/parallels/windows-smoke.ts index ed3ff944326..0add30ab518 100755 --- a/scripts/e2e/parallels/windows-smoke.ts +++ b/scripts/e2e/parallels/windows-smoke.ts @@ -34,6 +34,7 @@ import { runWindowsBackgroundPowerShell, WindowsGuest } from "./guest-transports import { runSmokeLane, type SmokeLane, type SmokeLaneStatus } from "./lane-runner.ts"; import { waitForVmStatus } from "./parallels-vm.ts"; import { PhaseRunner } from "./phase-runner.ts"; +import { windowsProviderOnlyPluginIsolationScript } from "./plugin-isolation.ts"; import { psSingleQuote, windowsAgentTurnConfigPatchScript, @@ -650,11 +651,19 @@ if ($LASTEXITCODE -ne 0) { throw "openclaw --version failed with exit code $LAST $PSNativeCommandUseErrorActionPreference = $false Set-Item -Path ('Env:' + ${psSingleQuote(this.auth.apiKeyEnv)}) -Value ${psSingleQuote(this.auth.apiKeyValue)} Invoke-OpenClaw onboard --non-interactive --mode local --auth-choice ${psSingleQuote(this.auth.authChoice)} --secret-input-mode ref --gateway-port 18789 --gateway-bind loopback --install-daemon --skip-skills --skip-health --accept-risk --json -if ($LASTEXITCODE -ne 0) { throw "openclaw onboard failed with exit code $LASTEXITCODE" }`, +if ($LASTEXITCODE -ne 0) { throw "openclaw onboard failed with exit code $LASTEXITCODE" } +${this.windowsPluginIsolationScript()}`, 720_000, ); } + private windowsPluginIsolationScript(): string { + return windowsProviderOnlyPluginIsolationScript({ + fallbackPluginId: this.options.provider, + modelId: this.auth.modelId, + }); + } + private async guestPowerShellBackground( label: string, script: string, diff --git a/test/scripts/parallels-smoke-model.test.ts b/test/scripts/parallels-smoke-model.test.ts index 58228d7ce6c..c3a54c79601 100644 --- a/test/scripts/parallels-smoke-model.test.ts +++ b/test/scripts/parallels-smoke-model.test.ts @@ -1,11 +1,4 @@ -import { - chmodSync, - copyFileSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { chmodSync, copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, join, win32 } from "node:path"; import { pathToFileURL } from "node:url"; @@ -67,11 +60,7 @@ function fakePrlctlEnv(tempDir: string): Record { return { NODE_OPTIONS: nodeOptions, PATH: pathValue, Path: pathValue }; } -function writeFakePrlctl( - tempDir: string, - posixScript: string, - windowsBootstrap: string, -): void { +function writeFakePrlctl(tempDir: string, posixScript: string, windowsBootstrap: string): void { const prlctlPath = join(tempDir, "prlctl"); writeFileSync(prlctlPath, posixScript); chmodSync(prlctlPath, 0o755); @@ -243,6 +232,56 @@ console.log([snapshot.id, snapshot.state, snapshot.name].join("\\t")); } }); + it("resolves a latest snapshot hint to the matching version before older LATEST labels", () => { + const tempDir = mkdtempSync(join(tmpdir(), "openclaw-parallels-snapshot-latest-")); + writeFakePrlctl( + tempDir, + `#!/usr/bin/env bash +set -euo pipefail +if [[ "$1" == "snapshot-list" ]]; then + cat <<'JSON' +{ + "{old}": {"name": "macOS 26.3.1 LATEST", "state": "poweron"}, + "{wanted}": {"name": "macOS 26.5", "state": "poweron"} +} +JSON + exit 0 +fi +exit 1 +`, + `import { basename } from "node:path"; +const isPrlctl = [process.argv0, process.execPath].some((value) => + basename(value).toLowerCase() === "prlctl.exe", +); +if (isPrlctl) { + if (process.argv.some((arg) => arg.includes("snapshot-list"))) { + console.log(JSON.stringify({ + "{old}": { name: "macOS 26.3.1 LATEST", state: "poweron" }, + "{wanted}": { name: "macOS 26.5", state: "poweron" }, + })); + process.exit(0); + } + process.exit(1); +} +`, + ); + + try { + const output = runTsEval( + ` +import { resolveSnapshot } from "./${TS_PATHS.common}"; +const snapshot = resolveSnapshot("vm", "macOS 26.5 latest"); +console.log([snapshot.id, snapshot.state, snapshot.name].join("\\t")); +`, + fakePrlctlEnv(tempDir), + ); + + expect(output.trim()).toBe("{wanted}\tpoweron\tmacOS 26.5"); + } finally { + rmSync(tempDir, { force: true, recursive: true }); + } + }); + it("uses one Ubuntu VM fallback resolver for Linux lanes", () => { const tempDir = mkdtempSync(join(tmpdir(), "openclaw-parallels-vm-helper-")); writeFakePrlctl( @@ -252,6 +291,7 @@ set -euo pipefail if [[ "$1" == "list" ]]; then cat <<'JSON' [ + {"name": "Ubuntu 26.04"}, {"name": "Ubuntu 25.10"}, {"name": "Ubuntu 23.10"}, {"name": "Ubuntu 24.04.3 ARM64"} @@ -268,6 +308,7 @@ const isPrlctl = [process.argv0, process.execPath].some((value) => if (isPrlctl) { if (process.argv.some((arg) => arg.includes("list"))) { console.log(JSON.stringify([ + { name: "Ubuntu 26.04" }, { name: "Ubuntu 25.10" }, { name: "Ubuntu 23.10" }, { name: "Ubuntu 24.04.3 ARM64" }, @@ -288,7 +329,7 @@ console.log(resolveUbuntuVmName("Ubuntu missing")); fakePrlctlEnv(tempDir), ); - expect(output.trim()).toBe("Ubuntu 24.04.3 ARM64"); + expect(output.trim()).toBe("Ubuntu 26.04"); } finally { rmSync(tempDir, { force: true, recursive: true }); }