diff --git a/scripts/e2e/parallels/guest-transports.ts b/scripts/e2e/parallels/guest-transports.ts index ee63c9f765d..0018b8720e7 100644 --- a/scripts/e2e/parallels/guest-transports.ts +++ b/scripts/e2e/parallels/guest-transports.ts @@ -1,6 +1,6 @@ import { run } from "./host-command.ts"; import type { PhaseRunner } from "./phase-runner.ts"; -import { encodePowerShell } from "./powershell.ts"; +import { encodePowerShell, psSingleQuote } from "./powershell.ts"; import type { CommandResult } from "./types.ts"; export interface GuestExecOptions { @@ -9,6 +9,253 @@ export interface GuestExecOptions { timeoutMs?: number; } +export interface WindowsBackgroundPowerShellOptions { + append?: (chunk: string | Uint8Array) => void; + beforeLaunchAttempt?: () => void; + label: string; + onLaunchRetry?: (message: string) => void; + script: string; + timeoutMs: number; + vmName: string; +} + +function appendOutput( + append: ((chunk: string | Uint8Array) => void) | undefined, + result: CommandResult, +): void { + if (result.stdout) { + append?.(result.stdout); + } + if (result.stderr) { + append?.(result.stderr); + } +} + +function timeoutBefore(deadline: number, fallbackMs: number): number { + return Math.min(fallbackMs, Math.max(1_000, deadline - Date.now())); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function throwIfFailed(label: string, result: CommandResult, check: boolean | undefined): void { + if (check === false || result.status === 0) { + return; + } + throw new Error(`${label} failed with exit code ${result.status}`); +} + +export async function runWindowsBackgroundPowerShell( + options: WindowsBackgroundPowerShellOptions, +): Promise { + const append = options.append; + const safeLabel = options.label.replaceAll(/[^A-Za-z0-9_-]/g, "-"); + const nonce = `${safeLabel}-${Date.now()}-${Math.floor(Math.random() * 100000)}`; + const fileBase = `openclaw-parallels-${nonce}`; + const pathsScript = `$base = Join-Path $env:TEMP ${psSingleQuote(fileBase)} +$scriptPath = "$base.ps1" +$logPath = "$base.log" +$donePath = "$base.done" +$exitPath = "$base.exit"`; + const payload = `$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $false +${pathsScript} +try { + & { +${options.script} + } *>&1 | ForEach-Object { $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 } + Set-Content -Path $exitPath -Value '0' -Encoding UTF8 +} catch { + $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 + Set-Content -Path $exitPath -Value '1' -Encoding UTF8 +} finally { + Set-Content -Path $donePath -Value 'done' -Encoding UTF8 +}`; + const writeScript = run( + "prlctl", + [ + "exec", + options.vmName, + "--current-user", + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(`${pathsScript} +Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue +[System.IO.File]::WriteAllText($scriptPath, [Console]::In.ReadToEnd(), [System.Text.UTF8Encoding]::new($false)) +if (!(Test-Path $scriptPath)) { throw "${safeLabel} background script was not written" }`), + ], + { check: false, input: payload, timeoutMs: Math.min(options.timeoutMs, 120_000) }, + ); + appendOutput(append, writeScript); + if (writeScript.status !== 0) { + throw new Error( + `${options.label} background script write failed with exit code ${writeScript.status}`, + ); + } + + const deadline = Date.now() + options.timeoutMs; + let launched = false; + let lastLaunchStatus = 0; + for (let attempt = 1; attempt <= 5 && Date.now() < deadline; attempt++) { + options.beforeLaunchAttempt?.(); + const launch = run( + "prlctl", + [ + "exec", + options.vmName, + "--current-user", + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(`${pathsScript} +Start-Process -FilePath powershell.exe -WindowStyle Hidden -ArgumentList @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $scriptPath) +'started'`), + ], + { check: false, quiet: true, timeoutMs: timeoutBefore(deadline, 30_000) }, + ); + appendOutput(append, launch); + if (launch.status === 0 && launch.stdout.includes("started")) { + launched = true; + break; + } + lastLaunchStatus = launch.status; + if (launch.status === 0 || launch.status === 124) { + const materialized = waitForWindowsBackgroundMaterialized({ + append, + deadline, + pathsScript, + vmName: options.vmName, + }); + if (materialized) { + launched = true; + break; + } + options.onLaunchRetry?.( + `${options.label} launch retry ${attempt}: background log/done file did not materialize`, + ); + continue; + } + if (launch.stdout.includes("restoring") || launch.stderr.includes("restoring")) { + options.onLaunchRetry?.(`${options.label} launch retry ${attempt}: VM is still restoring`); + await sleep(5_000); + continue; + } + throw new Error(`${options.label} background launch failed with exit code ${launch.status}`); + } + if (!launched) { + throw new Error(`${options.label} background launch failed with exit code ${lastLaunchStatus}`); + } + + let lastLogOffset = 0; + while (Date.now() < deadline) { + const poll = run( + "prlctl", + [ + "exec", + options.vmName, + "--current-user", + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(`${pathsScript} +$offset = ${lastLogOffset} +if (Test-Path $logPath) { + $bytes = [System.IO.File]::ReadAllBytes($logPath) + if ($bytes.Length -gt $offset) { + "__OPENCLAW_LOG_OFFSET__:$($bytes.Length)" + [System.Text.Encoding]::UTF8.GetString($bytes, $offset, $bytes.Length - $offset) + } +} +if (Test-Path $donePath) { + $backgroundExit = if (Test-Path $exitPath) { (Get-Content -Path $exitPath -Raw).Trim() } else { '0' } + "__OPENCLAW_BACKGROUND_EXIT__:$backgroundExit" + '__OPENCLAW_BACKGROUND_DONE__' + if ($backgroundExit -ne '0') { exit 23 } + exit 0 +}`), + ], + { check: false, quiet: true, timeoutMs: timeoutBefore(deadline, 30_000) }, + ); + appendOutput(append, poll); + const offsetMatch = poll.stdout.match(/__OPENCLAW_LOG_OFFSET__:(\d+)/); + if (offsetMatch) { + lastLogOffset = Number(offsetMatch[1]); + } + if (poll.stdout.includes("__OPENCLAW_BACKGROUND_DONE__")) { + const exitMatch = poll.stdout.match(/__OPENCLAW_BACKGROUND_EXIT__:(\S+)/); + const backgroundExit = exitMatch?.[1] ?? "0"; + if (backgroundExit !== "0" || (poll.status !== 0 && poll.status !== 124)) { + throw new Error(`${options.label} failed`); + } + cleanupWindowsBackground(options.vmName, pathsScript); + return; + } + await sleep(5_000); + } + throw new Error(`${options.label} timed out`); +} + +function waitForWindowsBackgroundMaterialized(params: { + append?: (chunk: string | Uint8Array) => void; + deadline: number; + pathsScript: string; + vmName: string; +}): boolean { + const materializeDeadline = Math.min(Date.now() + 45_000, params.deadline); + while (Date.now() < materializeDeadline) { + const result = run( + "prlctl", + [ + "exec", + params.vmName, + "--current-user", + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(`${params.pathsScript} +if ((Test-Path $logPath) -or (Test-Path $donePath)) { + 'materialized' +}`), + ], + { check: false, quiet: true, timeoutMs: timeoutBefore(materializeDeadline, 15_000) }, + ); + appendOutput(params.append, result); + if (result.stdout.includes("materialized")) { + return true; + } + } + return false; +} + +function cleanupWindowsBackground(vmName: string, pathsScript: string): void { + run( + "prlctl", + [ + "exec", + vmName, + "--current-user", + "powershell.exe", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodePowerShell(`${pathsScript} +Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue`), + ], + { check: false, quiet: true, timeoutMs: 30_000 }, + ); +} + export class LinuxGuest { constructor( private vmName: string, @@ -17,13 +264,14 @@ export class LinuxGuest { exec(args: string[], options: GuestExecOptions = {}): string { const result = run("prlctl", ["exec", this.vmName, "/usr/bin/env", "HOME=/root", ...args], { - check: options.check, + check: false, input: options.input, quiet: true, timeoutMs: this.phases.remainingTimeoutMs(options.timeoutMs), }); this.phases.append(result.stdout); this.phases.append(result.stderr); + throwIfFailed("Linux guest command", result, options.check); return result.stdout.trim(); } @@ -91,13 +339,14 @@ export class MacosGuest { ] : ["exec", this.input.vmName, "--current-user", "/usr/bin/env", ...envArgs, ...args]; const result = run("prlctl", transportArgs, { - check: options.check, + check: false, input: options.input, quiet: true, timeoutMs: this.phases.remainingTimeoutMs(options.timeoutMs), }); this.phases.append(result.stdout); this.phases.append(result.stderr); + throwIfFailed("macOS guest command", result, options.check); return result; } @@ -124,13 +373,14 @@ export class WindowsGuest { run(args: string[], options: GuestExecOptions = {}): CommandResult { const result = run("prlctl", ["exec", this.vmName, "--current-user", ...args], { - check: options.check, + check: false, input: options.input, quiet: true, timeoutMs: this.phases.remainingTimeoutMs(options.timeoutMs), }); this.phases.append(result.stdout); this.phases.append(result.stderr); + throwIfFailed("Windows guest command", result, options.check); return result; } diff --git a/scripts/e2e/parallels/npm-update-scripts.ts b/scripts/e2e/parallels/npm-update-scripts.ts index 8fdb34652ff..1b189509003 100644 --- a/scripts/e2e/parallels/npm-update-scripts.ts +++ b/scripts/e2e/parallels/npm-update-scripts.ts @@ -4,6 +4,7 @@ import { psSingleQuote, windowsAgentTurnConfigPatchScript, windowsOpenClawResolver, + windowsScopedEnvFunction, } from "./powershell.ts"; import { modelProviderConfigBatchJson, @@ -73,6 +74,65 @@ if [ "$agent_ok" != true ]; then fi`; } +function windowsUpdateWithBundledPluginsDisabled(input: NpmUpdateScriptInput): string { + return `$script:OpenClawUpdateExit = 0 +$updateOutput = Invoke-WithScopedEnv @{ OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'; OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1' } { + Invoke-OpenClaw update --tag ${psSingleQuote(input.updateTarget)} --yes --json --no-restart 2>&1 + $script:OpenClawUpdateExit = $LASTEXITCODE +} +$updateExit = $script:OpenClawUpdateExit +$updateOutput`; +} + +function windowsGatewayReadyScript(): string { + return `function Wait-OpenClawGateway { + $deadline = (Get-Date).AddSeconds(180) + $attempt = 0 + while ((Get-Date) -lt $deadline) { + Invoke-OpenClaw gateway status --deep --require-rpc --timeout 15000 + if ($LASTEXITCODE -eq 0) { return } + $attempt += 1 + if ($attempt -eq 4) { + Invoke-OpenClaw gateway start *>&1 | Out-Host + } + Start-Sleep -Seconds 5 + } + throw "gateway did not become ready after update" +} +Invoke-OpenClaw gateway restart *>&1 | Out-Host +if ($LASTEXITCODE -ne 0) { + "gateway restart exited with code $LASTEXITCODE; probing readiness before failing" | Out-Host +} +Wait-OpenClawGateway`; +} + +function windowsAssertAgentOkScript(input: NpmUpdateScriptInput): string { + return `${windowsAgentTurnConfigPatchScript(input.auth.modelId)} +$sessionPath = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions\\parallels-npm-update-windows.jsonl' +Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue +${windowsAgentWorkspaceScript("Parallels npm update smoke test assistant.")} +Set-Item -Path ('Env:' + ${psSingleQuote(input.auth.apiKeyEnv)}) -Value ${psSingleQuote(input.auth.apiKeyValue)} +$agentOk = $false +for ($attempt = 1; $attempt -le 2; $attempt++) { + $sessionId = if ($attempt -eq 1) { 'parallels-npm-update-windows' } else { "parallels-npm-update-windows-retry-$attempt" } + $sessionsDir = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions' + $sessionPath = Join-Path $sessionsDir "$sessionId.jsonl" + Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue + $output = Invoke-OpenClaw agent --local --agent main --session-id $sessionId --model ${psSingleQuote(input.auth.modelId)} --message 'Reply with exact ASCII text OK only.' --thinking minimal --timeout ${resolveParallelsModelTimeoutSeconds("windows")} --json 2>&1 + if ($null -ne $output) { $output | ForEach-Object { $_ } } + if ($LASTEXITCODE -ne 0) { throw "agent failed with exit code $LASTEXITCODE" } + if (($output | Out-String) -match '"finalAssistant(Raw|Visible)Text":\\s*"OK"') { + $agentOk = $true + break + } + if ($attempt -lt 2) { + Write-Host "agent turn attempt $attempt finished without OK response; retrying" + Start-Sleep -Seconds 3 + } +} +if (-not $agentOk) { throw 'openclaw agent finished without OK response' }`; +} + export function macosUpdateScript(input: NpmUpdateScriptInput): string { return String.raw`set -euo pipefail export PATH=/opt/homebrew/bin:/opt/homebrew/opt/node/bin:/opt/homebrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin @@ -145,6 +205,7 @@ export function windowsUpdateScript(input: NpmUpdateScriptInput): string { return `$ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $false ${windowsOpenClawResolver} +${windowsScopedEnvFunction} function Remove-FuturePluginEntries { $configPath = Join-Path $env:USERPROFILE '.openclaw\\openclaw.json' if (-not (Test-Path $configPath)) { return } @@ -175,11 +236,7 @@ function Stop-OpenClawGatewayProcesses { } Remove-FuturePluginEntries Stop-OpenClawGatewayProcesses -$env:OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1' -$env:OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1' -$updateOutput = Invoke-OpenClaw update --tag ${psSingleQuote(input.updateTarget)} --yes --json --no-restart 2>&1 -$updateExit = $LASTEXITCODE -$updateOutput +${windowsUpdateWithBundledPluginsDisabled(input)} if ($updateExit -ne 0) { $updateText = $updateOutput | Out-String $stalePostSwapImport = $updateText -match 'ERR_MODULE_NOT_FOUND' -and $updateText -match ${psSingleQuote(windowsStalePostSwapImportRegex)} @@ -187,49 +244,8 @@ if ($updateExit -ne 0) { Write-Host "openclaw update returned a stale post-swap module import; continuing to post-update health checks" } ${windowsVersionCheck(input.expectedNeedle)} -function Wait-OpenClawGateway { - $deadline = (Get-Date).AddSeconds(180) - $attempt = 0 - while ((Get-Date) -lt $deadline) { - Invoke-OpenClaw gateway status --deep --require-rpc --timeout 15000 - if ($LASTEXITCODE -eq 0) { return } - $attempt += 1 - if ($attempt -eq 4) { - Invoke-OpenClaw gateway start *>&1 | Out-Host - } - Start-Sleep -Seconds 5 - } - throw "gateway did not become ready after update" -} -Invoke-OpenClaw gateway restart *>&1 | Out-Host -if ($LASTEXITCODE -ne 0) { - "gateway restart exited with code $LASTEXITCODE; probing readiness before failing" | Out-Host -} -Wait-OpenClawGateway -${windowsAgentTurnConfigPatchScript(input.auth.modelId)} -$sessionPath = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions\\parallels-npm-update-windows.jsonl' -Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue -${windowsAgentWorkspaceScript("Parallels npm update smoke test assistant.")} -Set-Item -Path ('Env:' + ${psSingleQuote(input.auth.apiKeyEnv)}) -Value ${psSingleQuote(input.auth.apiKeyValue)} -$agentOk = $false -for ($attempt = 1; $attempt -le 2; $attempt++) { - $sessionId = if ($attempt -eq 1) { 'parallels-npm-update-windows' } else { "parallels-npm-update-windows-retry-$attempt" } - $sessionsDir = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions' - $sessionPath = Join-Path $sessionsDir "$sessionId.jsonl" - Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue - $output = Invoke-OpenClaw agent --local --agent main --session-id $sessionId --model ${psSingleQuote(input.auth.modelId)} --message 'Reply with exact ASCII text OK only.' --thinking minimal --timeout ${resolveParallelsModelTimeoutSeconds("windows")} --json 2>&1 - if ($null -ne $output) { $output | ForEach-Object { $_ } } - if ($LASTEXITCODE -ne 0) { throw "agent failed with exit code $LASTEXITCODE" } - if (($output | Out-String) -match '"finalAssistant(Raw|Visible)Text":\\s*"OK"') { - $agentOk = $true - break - } - if ($attempt -lt 2) { - Write-Host "agent turn attempt $attempt finished without OK response; retrying" - Start-Sleep -Seconds 3 - } -} -if (-not $agentOk) { throw 'openclaw agent finished without OK response' }`; +${windowsGatewayReadyScript()} +${windowsAssertAgentOkScript(input)}`; } export function linuxUpdateScript(input: NpmUpdateScriptInput): string { diff --git a/scripts/e2e/parallels/npm-update-smoke.ts b/scripts/e2e/parallels/npm-update-smoke.ts index 97719bcdaa3..5e832d17f22 100755 --- a/scripts/e2e/parallels/npm-update-smoke.ts +++ b/scripts/e2e/parallels/npm-update-smoke.ts @@ -24,9 +24,9 @@ import { type Provider, type ProviderAuth, } from "./common.ts"; +import { runWindowsBackgroundPowerShell } from "./guest-transports.ts"; import { linuxUpdateScript, macosUpdateScript, windowsUpdateScript } from "./npm-update-scripts.ts"; import { ensureVmRunning, resolveUbuntuVmName } from "./parallels-vm.ts"; -import { encodePowerShell } from "./powershell.ts"; interface NpmUpdateOptions { packageSpec: string; @@ -40,9 +40,11 @@ interface NpmUpdateOptions { interface Job { done: boolean; + durationMs: number; label: string; logPath: string; promise: Promise; + startedAt: number; } interface UpdateJobContext { @@ -60,6 +62,13 @@ interface NpmUpdateSummary { runDir: string; fresh: Record; update: Record; + timings: Array<{ + durationMs: number; + label: string; + logPath: string; + phase: "fresh" | "update"; + status: string; + }>; } const macosVm = "macOS Tahoe"; @@ -163,6 +172,7 @@ class NpmUpdateSmoke { private freshStatus = platformRecord("skip"); private updateStatus = platformRecord("skip"); private updateVersion = platformRecord("skip"); + private timings: NpmUpdateSummary["timings"] = []; constructor(private options: NpmUpdateOptions) { this.auth = resolveProviderAuth({ @@ -236,6 +246,7 @@ class NpmUpdateSmoke { const status = (await job.promise) === 0 ? "pass" : "fail"; const platform = this.platformFromLabel(job.label); this.freshStatus[platform] = status; + this.recordTiming("fresh", job, status); if (status !== "pass") { this.dumpLogTail(job.logPath); die(`${job.label} fresh baseline failed`); @@ -270,11 +281,14 @@ class NpmUpdateSmoke { ]; const job: Job = { done: false, + durationMs: 0, label, logPath, promise: Promise.resolve(1), + startedAt: Date.now(), }; job.promise = this.spawnLogged("pnpm", args, logPath, env).finally(() => { + job.durationMs = Date.now() - job.startedAt; job.done = true; }); return job; @@ -323,6 +337,7 @@ class NpmUpdateSmoke { const status = (await job.promise) === 0 ? "pass" : "fail"; this.updateStatus[platform] = status; this.updateVersion[platform] = await this.extractLastVersion(job.logPath); + this.recordTiming("update", job, status); if (status !== "pass") { this.dumpLogTail(job.logPath); die(`${job.label} update failed`); @@ -338,9 +353,11 @@ class NpmUpdateSmoke { const logPath = path.join(this.runDir, `${platform}-update.log`); const job: Job = { done: false, + durationMs: 0, label, logPath, promise: Promise.resolve(1), + startedAt: Date.now(), }; job.promise = (async () => { let log = ""; @@ -364,6 +381,7 @@ class NpmUpdateSmoke { clearTimeout(timeout); } })().finally(() => { + job.durationMs = Date.now() - job.startedAt; job.done = true; }); return job; @@ -572,145 +590,13 @@ class NpmUpdateSmoke { timeoutMs: number, ctx: UpdateJobContext, ): Promise { - const fileBase = `openclaw-parallels-npm-update-windows-${process.pid}-${Date.now()}`; - const pathsScript = `$base = Join-Path $env:TEMP '${fileBase}' -$scriptPath = "$base.ps1" -$logPath = "$base.log" -$donePath = "$base.done" -$exitPath = "$base.exit"`; - const payload = `$ErrorActionPreference = 'Stop' -$PSNativeCommandUseErrorActionPreference = $false -${pathsScript} -try { - & { -${script} - } *>&1 | ForEach-Object { $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 } - Set-Content -Path $exitPath -Value '0' -Encoding UTF8 -} catch { - $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 - Set-Content -Path $exitPath -Value '1' -Encoding UTF8 -} finally { - Set-Content -Path $donePath -Value 'done' -Encoding UTF8 -}`; - const writeScript = run( - "prlctl", - [ - "exec", - windowsVm, - "--current-user", - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue -[System.IO.File]::WriteAllText($scriptPath, [Console]::In.ReadToEnd(), [System.Text.UTF8Encoding]::new($false)) -if (!(Test-Path $scriptPath)) { throw "background update script was not written" }`), - ], - { check: false, input: payload, timeoutMs: Math.min(timeoutMs, 120_000) }, - ); - if (writeScript.stdout) { - ctx.append(writeScript.stdout); - } - if (writeScript.stderr) { - ctx.append(writeScript.stderr); - } - if (writeScript.status !== 0) { - throw new Error( - `Windows update background script write failed with exit code ${writeScript.status}`, - ); - } - - const launchStatus = await this.runStreamingToJobLog( - "prlctl", - [ - "exec", - windowsVm, - "--current-user", - "cmd.exe", - "/d", - "/s", - "/c", - `start "" /min powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%TEMP%\\${fileBase}.ps1"`, - ], - 20_000, - ctx, - ); - if (launchStatus !== 0 && launchStatus !== 124) { - throw new Error(`Windows update background launch failed with exit code ${launchStatus}`); - } - - const deadline = Date.now() + timeoutMs; - let lastLogOffset = 0; - while (Date.now() < deadline) { - const poll = run( - "prlctl", - [ - "exec", - windowsVm, - "--current-user", - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -$offset = ${lastLogOffset} -if (Test-Path $logPath) { - $bytes = [System.IO.File]::ReadAllBytes($logPath) - if ($bytes.Length -gt $offset) { - "__OPENCLAW_LOG_OFFSET__:$($bytes.Length)" - [System.Text.Encoding]::UTF8.GetString($bytes, $offset, $bytes.Length - $offset) - } -} -if (Test-Path $donePath) { - $backgroundExit = if (Test-Path $exitPath) { (Get-Content -Path $exitPath -Raw).Trim() } else { '0' } - "__OPENCLAW_BACKGROUND_EXIT__:$backgroundExit" - '__OPENCLAW_BACKGROUND_DONE__' - if ($backgroundExit -ne '0') { exit 23 } - exit 0 -}`), - ], - { check: false, timeoutMs: Math.min(30_000, Math.max(1_000, deadline - Date.now())) }, - ); - if (poll.stdout) { - ctx.append(poll.stdout); - } - if (poll.stderr) { - ctx.append(poll.stderr); - } - const offsetMatch = poll.stdout.match(/__OPENCLAW_LOG_OFFSET__:(\d+)/); - if (offsetMatch) { - lastLogOffset = Number(offsetMatch[1]); - } - if (poll.stdout.includes("__OPENCLAW_BACKGROUND_DONE__")) { - const exitMatch = poll.stdout.match(/__OPENCLAW_BACKGROUND_EXIT__:(\S+)/); - const backgroundExit = exitMatch?.[1] ?? "0"; - if (backgroundExit !== "0" || (poll.status !== 0 && poll.status !== 124)) { - throw new Error("Windows update failed"); - } - run( - "prlctl", - [ - "exec", - windowsVm, - "--current-user", - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue`), - ], - { check: false, timeoutMs: 30_000 }, - ); - return; - } - await new Promise((resolve) => setTimeout(resolve, 5_000)); - } - throw new Error(`Windows update timed out after ${updateTimeoutSeconds}s`); + await runWindowsBackgroundPowerShell({ + append: (chunk) => ctx.append(chunk), + label: "Windows update", + script, + timeoutMs, + vmName: windowsVm, + }); } private async guestLinux( @@ -866,6 +752,16 @@ Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorActio } } + private recordTiming(phase: "fresh" | "update", job: Job, status: string): void { + this.timings.push({ + durationMs: job.durationMs || Date.now() - job.startedAt, + label: job.label, + logPath: job.logPath, + phase, + status, + }); + } + private async writeSummary(): Promise { const summary: NpmUpdateSummary = { currentHead: this.currentHeadShort, @@ -879,6 +775,7 @@ Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorActio macos: { status: this.updateStatus.macos, version: this.updateVersion.macos }, windows: { status: this.updateStatus.windows, version: this.updateVersion.windows }, }, + timings: this.timings, updateExpected: this.updateExpectedNeedle, updateTarget: this.updateTargetEffective, }; diff --git a/scripts/e2e/parallels/phase-runner.ts b/scripts/e2e/parallels/phase-runner.ts index c313c3e8094..b83b38e1a7c 100644 --- a/scripts/e2e/parallels/phase-runner.ts +++ b/scripts/e2e/parallels/phase-runner.ts @@ -5,6 +5,13 @@ import { say, warn } from "./host-command.ts"; export class PhaseRunner { private logText = ""; private deadlineMs = 0; + private timings: Array<{ + durationMs: number; + logPath: string; + name: string; + status: "pass" | "fail"; + timeoutSeconds: number; + }> = []; constructor(private runDir: string) {} @@ -13,6 +20,8 @@ export class PhaseRunner { say(name); this.logText = ""; this.deadlineMs = Date.now() + timeoutSeconds * 1000; + const startedAt = Date.now(); + let status: "pass" | "fail" = "fail"; let timer: NodeJS.Timeout | undefined; const timeout = new Promise((_, reject) => { timer = setTimeout( @@ -23,6 +32,7 @@ export class PhaseRunner { try { await Promise.race([Promise.resolve(fn()), timeout]); await writeFile(logPath, this.logText, "utf8"); + status = "pass"; } catch (error) { await writeFile(logPath, this.logText, "utf8").catch(() => undefined); warn(`${name} failed`); @@ -31,6 +41,14 @@ export class PhaseRunner { process.stderr.write("\n"); throw error; } finally { + this.timings.push({ + durationMs: Date.now() - startedAt, + logPath, + name, + status, + timeoutSeconds, + }); + await this.writeTimings().catch(() => undefined); if (timer) { clearTimeout(timer); } @@ -71,4 +89,13 @@ export class PhaseRunner { this.logText += "\n"; } } + + private async writeTimings(): Promise { + const slowest = this.timings.toSorted((a, b) => b.durationMs - a.durationMs)[0] ?? null; + await writeFile( + path.join(this.runDir, "phase-timings.json"), + `${JSON.stringify({ phases: this.timings, slowest }, null, 2)}\n`, + "utf8", + ); + } } diff --git a/scripts/e2e/parallels/powershell.ts b/scripts/e2e/parallels/powershell.ts index 06dd040d033..98000742663 100644 --- a/scripts/e2e/parallels/powershell.ts +++ b/scripts/e2e/parallels/powershell.ts @@ -19,6 +19,29 @@ export function encodePowerShell(script: string): string { ); } +export const windowsScopedEnvFunction = String.raw`function Invoke-WithScopedEnv { + param( + [Parameter(Mandatory = $true)][hashtable] $Values, + [Parameter(Mandatory = $true)][scriptblock] $Script + ) + $previous = @{} + foreach ($key in $Values.Keys) { + $previous[$key] = [Environment]::GetEnvironmentVariable([string]$key, 'Process') + Set-Item -Path ('Env:' + $key) -Value ([string]$Values[$key]) + } + try { + & $Script + } finally { + foreach ($key in $Values.Keys) { + if ($null -eq $previous[$key]) { + Remove-Item -Path ('Env:' + $key) -ErrorAction SilentlyContinue + } else { + Set-Item -Path ('Env:' + $key) -Value $previous[$key] + } + } + } +}`; + export function windowsModelProviderTimeoutScript(modelId: string): string { const providerId = providerIdFromModelId(modelId); const configJson = providerTimeoutConfigJson(modelId, "windows"); diff --git a/scripts/e2e/parallels/windows-smoke.ts b/scripts/e2e/parallels/windows-smoke.ts index 3d8b6d8cc49..ae7c1f2c06a 100755 --- a/scripts/e2e/parallels/windows-smoke.ts +++ b/scripts/e2e/parallels/windows-smoke.ts @@ -18,7 +18,6 @@ import { resolveWindowsProviderAuth, resolveSnapshot, run, - runStreaming, say, startHostServer, warn, @@ -30,15 +29,15 @@ import { type ProviderAuth, type SnapshotInfo, } from "./common.ts"; -import { WindowsGuest } from "./guest-transports.ts"; +import { runWindowsBackgroundPowerShell, WindowsGuest } from "./guest-transports.ts"; import { runSmokeLane, type SmokeLane, type SmokeLaneStatus } from "./lane-runner.ts"; import { waitForVmStatus } from "./parallels-vm.ts"; import { PhaseRunner } from "./phase-runner.ts"; import { - encodePowerShell, psSingleQuote, windowsAgentTurnConfigPatchScript, windowsOpenClawResolver, + windowsScopedEnvFunction, } from "./powershell.ts"; import { ensureGuestGit, prepareMinGitZip } from "./windows-git.ts"; @@ -641,166 +640,16 @@ if ($LASTEXITCODE -ne 0) { throw "openclaw onboard failed with exit code $LASTEX script: string, timeoutMs: number, ): Promise { - const safeLabel = label.replaceAll(/[^A-Za-z0-9_-]/g, "-"); - const nonce = `${safeLabel}-${Date.now()}-${Math.floor(Math.random() * 100000)}`; - const fileBase = `openclaw-parallels-${nonce}`; - const pathsScript = `$base = Join-Path $env:TEMP ${psSingleQuote(fileBase)} -$scriptPath = "$base.ps1" -$logPath = "$base.log" -$donePath = "$base.done" -$exitPath = "$base.exit"`; - const payload = Buffer.from( - `$ErrorActionPreference = 'Stop' -$PSNativeCommandUseErrorActionPreference = $false -${windowsOpenClawResolver} -${pathsScript} -try { - & { -${script} - } *>&1 | ForEach-Object { $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 } - Set-Content -Path $exitPath -Value '0' -Encoding UTF8 -} catch { - $_ | Out-String | Add-Content -Path $logPath -Encoding UTF8 - Set-Content -Path $exitPath -Value '1' -Encoding UTF8 -} finally { - Set-Content -Path $donePath -Value 'done' -Encoding UTF8 -}`, - "utf8", - ).toString("base64"); - this.guestPowerShell( - `$payload = ${psSingleQuote(payload)} -${pathsScript} -Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue -[System.IO.File]::WriteAllText($scriptPath, [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($payload)), [System.Text.UTF8Encoding]::new($false)) -if (!(Test-Path $scriptPath)) { throw "background script was not written" }`, - { timeoutMs: 30_000 }, - ); - let launched = false; - let lastLaunchStatus = 0; - for (let attempt = 1; attempt <= 5; attempt++) { - this.waitForGuestReady(120); - const launchLogPath = path.join(this.runDir, `${safeLabel}-launch-${attempt}.log`); - const launchStatus = await runStreaming( - "prlctl", - [ - "exec", - this.options.vmName, - "--current-user", - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -Start-Process -FilePath powershell.exe -WindowStyle Hidden -ArgumentList @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $scriptPath) -'started'`), - ], - { logPath: launchLogPath, quiet: true, timeoutMs: this.remainingPhaseTimeoutMs(30_000) }, - ); - const launchLog = await readFile(launchLogPath, "utf8").catch(() => ""); - this.log(launchLog); - if (launchStatus === 0 && launchLog.includes("started")) { - launched = true; - break; - } - if (launchStatus === 0 || launchStatus === 124) { - const materialized = this.waitForBackgroundMaterialized(pathsScript, 45_000); - if (!materialized) { - warn(`${label} launch retry ${attempt}: background log/done file did not materialize`); - lastLaunchStatus = launchStatus; - continue; - } - launched = true; - break; - } - lastLaunchStatus = launchStatus; - if (launchLog.includes("restoring")) { - warn(`${label} launch retry ${attempt}: VM is still restoring`); - this.waitForVmNotRestoring(120); - continue; - } - throw new Error(`${label} background launch failed with exit code ${launchStatus}`); - } - if (!launched) { - throw new Error(`${label} background launch failed with exit code ${lastLaunchStatus}`); - } - const deadline = Date.now() + timeoutMs; - let lastLogOffset = 0; - while (Date.now() < deadline) { - const result = this.guest.run( - [ - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -$offset = ${lastLogOffset} -if (Test-Path $logPath) { - $bytes = [System.IO.File]::ReadAllBytes($logPath) - if ($bytes.Length -gt $offset) { - "__OPENCLAW_LOG_OFFSET__:$($bytes.Length)" - [System.Text.Encoding]::UTF8.GetString($bytes, $offset, $bytes.Length - $offset) - } -} -if (Test-Path $donePath) { - $backgroundExit = if (Test-Path $exitPath) { (Get-Content -Path $exitPath -Raw).Trim() } else { '0' } - "__OPENCLAW_BACKGROUND_EXIT__:$backgroundExit" - '__OPENCLAW_BACKGROUND_DONE__' - if ($backgroundExit -ne '0') { exit 23 } - exit 0 -}`), - ], - { check: false, timeoutMs: this.remainingPhaseTimeoutMs(30_000) }, - ); - const offsetMatch = result.stdout.match(/__OPENCLAW_LOG_OFFSET__:(\d+)/); - if (offsetMatch) { - lastLogOffset = Number(offsetMatch[1]); - } - if (result.stdout.includes("__OPENCLAW_BACKGROUND_DONE__")) { - const exitMatch = result.stdout.match(/__OPENCLAW_BACKGROUND_EXIT__:(\S+)/); - const backgroundExit = exitMatch?.[1] ?? "0"; - if (backgroundExit !== "0" || (result.status !== 0 && result.status !== 124)) { - throw new Error(`${label} failed`); - } - this.guestPowerShell( - `${pathsScript} -Remove-Item -Path $scriptPath, $logPath, $donePath, $exitPath -Force -ErrorAction SilentlyContinue`, - { - check: false, - timeoutMs: 30_000, - }, - ); - return; - } - run("sleep", ["5"], { quiet: true }); - } - throw new Error(`${label} timed out`); - } - - private waitForBackgroundMaterialized(pathsScript: string, timeoutMs: number): boolean { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const result = this.guest.run( - [ - "powershell.exe", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - encodePowerShell(`${pathsScript} -if ((Test-Path $logPath) -or (Test-Path $donePath)) { - 'materialized' -}`), - ], - { check: false, timeoutMs: this.remainingPhaseTimeoutMs(15_000) }, - ); - if (result.stdout.includes("materialized")) { - return true; - } - run("sleep", ["2"], { quiet: true }); - } - return false; + await runWindowsBackgroundPowerShell({ + append: (chunk) => + this.log(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8")), + beforeLaunchAttempt: () => this.waitForGuestReady(120), + label, + onLaunchRetry: warn, + script: `${windowsOpenClawResolver}\n${script}`, + timeoutMs, + vmName: this.options.vmName, + }); } private runDevChannelUpdate(): void { @@ -814,10 +663,13 @@ if ($null -eq $config.update) { } $config.update | Add-Member -Force -MemberType NoteProperty -Name channel -Value 'dev' $config | ConvertTo-Json -Depth 100 | Set-Content -Path $configPath -Encoding utf8 -$env:OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1' -$env:OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1' -Invoke-OpenClaw update --channel dev --yes --json -if ($LASTEXITCODE -ne 0) { throw "openclaw update failed with exit code $LASTEXITCODE" } +${windowsScopedEnvFunction} +$script:OpenClawUpdateExit = 0 +Invoke-WithScopedEnv @{ OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS = '1'; OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1' } { + Invoke-OpenClaw update --channel dev --yes --json + $script:OpenClawUpdateExit = $LASTEXITCODE +} +if ($script:OpenClawUpdateExit -ne 0) { throw "openclaw update failed with exit code $script:OpenClawUpdateExit" } Invoke-OpenClaw --version Invoke-OpenClaw update status --json`, { timeoutMs: Number(process.env.OPENCLAW_PARALLELS_WINDOWS_UPDATE_TIMEOUT_S || 1200) * 1000 }, diff --git a/test/scripts/parallels-npm-update-smoke.test.ts b/test/scripts/parallels-npm-update-smoke.test.ts index 4507b61e71b..64285d7e9d5 100644 --- a/test/scripts/parallels-npm-update-smoke.test.ts +++ b/test/scripts/parallels-npm-update-smoke.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { windowsUpdateScript } from "../../scripts/e2e/parallels/npm-update-scripts.ts"; const SCRIPT_PATH = "scripts/e2e/parallels/npm-update-smoke.ts"; +const GUEST_TRANSPORTS_PATH = "scripts/e2e/parallels/guest-transports.ts"; const UPDATE_SCRIPTS_PATH = "scripts/e2e/parallels/npm-update-scripts.ts"; const TEST_AUTH = { authChoice: "openai", @@ -23,12 +24,13 @@ describe("parallels npm update smoke", () => { it("runs Windows updates through a detached done-file runner", () => { const script = readFileSync(SCRIPT_PATH, "utf8"); + const transports = readFileSync(GUEST_TRANSPORTS_PATH, "utf8"); - expect(script).toContain("openclaw-parallels-npm-update-windows"); - expect(script).toContain("runStreaming"); - expect(script).toContain("__OPENCLAW_BACKGROUND_EXIT__"); - expect(script).toContain("__OPENCLAW_BACKGROUND_DONE__"); - expect(script).toContain("Windows update timed out"); + expect(script).toContain("runWindowsBackgroundPowerShell"); + expect(transports).toContain("runWindowsBackgroundPowerShell"); + expect(transports).toContain("__OPENCLAW_BACKGROUND_EXIT__"); + expect(transports).toContain("__OPENCLAW_BACKGROUND_DONE__"); + expect(transports).toContain("${options.label} timed out"); }); it("keeps macOS sudo fallback update scripts readable by the desktop user", () => { @@ -47,7 +49,7 @@ describe("parallels npm update smoke", () => { expect(script).toContain("delete plugins.entries.whatsapp"); expect(script).toContain("Remove-FuturePluginEntries\nStop-OpenClawGatewayProcesses"); expect(script).toContain("scrub_future_plugin_entries\nstop_openclaw_gateway_processes"); - expect(script).toContain("$env:OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'"); + expect(script).toContain("Invoke-WithScopedEnv @{ OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'"); expect(script).toContain( "OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 /opt/homebrew/bin/openclaw update --tag", ); @@ -58,6 +60,28 @@ describe("parallels npm update smoke", () => { expect(script).toContain("OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 openclaw gateway stop"); }); + it("reenables bundled plugins before Windows post-update verification", () => { + const script = windowsUpdateScript({ + auth: TEST_AUTH, + expectedNeedle: "2026.5.3-beta.2", + updateTarget: "2026.5.3-beta.2", + }); + + const updateIndex = script.indexOf("Invoke-OpenClaw update --tag"); + const scopedIndex = script.indexOf("Invoke-WithScopedEnv @{ OPENCLAW_DISABLE_BUNDLED_PLUGINS"); + const versionIndex = script.indexOf("Invoke-OpenClaw --version", scopedIndex); + const restartIndex = script.indexOf("Invoke-OpenClaw gateway restart"); + const agentIndex = script.indexOf("Invoke-OpenClaw agent --local"); + + expect(updateIndex).toBeGreaterThanOrEqual(0); + expect(scopedIndex).toBeGreaterThanOrEqual(0); + expect(updateIndex).toBeGreaterThan(scopedIndex); + expect(versionIndex).toBeGreaterThan(updateIndex); + expect(restartIndex).toBeGreaterThan(updateIndex); + expect(agentIndex).toBeGreaterThan(updateIndex); + expect(script).not.toContain("$env:OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'"); + }); + it("generates a .NET-safe Windows stale import regex in the update-failure guard", () => { const script = windowsUpdateScript({ auth: TEST_AUTH, diff --git a/test/scripts/parallels-smoke-model.test.ts b/test/scripts/parallels-smoke-model.test.ts index 0049dfdf745..caad8296451 100644 --- a/test/scripts/parallels-smoke-model.test.ts +++ b/test/scripts/parallels-smoke-model.test.ts @@ -447,16 +447,18 @@ console.log(JSON.stringify(result)); it("runs Windows ref onboarding through a detached done-file runner", () => { const script = readFileSync(TS_PATHS.windows, "utf8"); + const transports = readFileSync(TS_PATHS.guestTransports, "utf8"); expect(script).toContain("guestPowerShellBackground"); - expect(script).toContain("Join-Path $env:TEMP"); - expect(script).toContain("__OPENCLAW_BACKGROUND_DONE__"); - expect(script).toContain("__OPENCLAW_BACKGROUND_EXIT__"); - expect(script).toContain("__OPENCLAW_LOG_OFFSET__"); - expect(script).toContain("result.status !== 0 && result.status !== 124"); - expect(script).toContain("Start-Process -FilePath powershell.exe"); - expect(script).toContain('launchLog.includes("started")'); - expect(script).toContain("waitForBackgroundMaterialized(pathsScript, 45_000)"); + expect(script).toContain("runWindowsBackgroundPowerShell"); + expect(transports).toContain("Join-Path $env:TEMP"); + expect(transports).toContain("__OPENCLAW_BACKGROUND_DONE__"); + expect(transports).toContain("__OPENCLAW_BACKGROUND_EXIT__"); + expect(transports).toContain("__OPENCLAW_LOG_OFFSET__"); + expect(transports).toContain("poll.status !== 0 && poll.status !== 124"); + expect(transports).toContain("Start-Process -FilePath powershell.exe"); + expect(transports).toContain('launch.stdout.includes("started")'); + expect(transports).toContain("waitForWindowsBackgroundMaterialized"); }); it("returns timed-out host command status when check is disabled", () => { @@ -520,10 +522,33 @@ console.log(JSON.stringify({ it("waits through transient Windows restoring state before VM operations", () => { const script = readFileSync(TS_PATHS.windows, "utf8"); + const transports = readFileSync(TS_PATHS.guestTransports, "utf8"); expect(script).toContain("waitForVmNotRestoring"); expect(script).toContain("snapshot-switch retry"); - expect(script).toContain("launch retry"); + expect(transports).toContain("launch retry"); + }); + + it("keeps Windows update-only env flags scoped before verification", () => { + const windows = readFileSync(TS_PATHS.windows, "utf8"); + const powershell = readFileSync(TS_PATHS.powershell, "utf8"); + + expect(powershell).toContain("windowsScopedEnvFunction"); + expect(windows).toContain( + "Invoke-WithScopedEnv @{ OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS", + ); + expect(windows).toContain("$script:OpenClawUpdateExit = $LASTEXITCODE"); + expect(windows).not.toContain("$env:OPENCLAW_DISABLE_BUNDLED_PLUGINS = '1'"); + }); + + it("writes Parallels phase timing artifacts", () => { + const phaseRunner = readFileSync(TS_PATHS.phaseRunner, "utf8"); + const npmUpdate = readFileSync(TS_PATHS.npmUpdate, "utf8"); + + expect(phaseRunner).toContain("phase-timings.json"); + expect(phaseRunner).toContain("slowest"); + expect(npmUpdate).toContain("timings: this.timings"); + expect(npmUpdate).toContain("recordTiming"); }); it("resolves Windows OpenClaw commands without assuming the npm shim path", () => {