From bae211f72a7bbb86edefce99368e9bb539eaae52 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 1 May 2026 14:20:49 +0100 Subject: [PATCH] test: require parallels agent responses --- scripts/e2e/parallels/linux-smoke.ts | 46 +++++++++++- scripts/e2e/parallels/macos-smoke.ts | 47 +++++++++++- scripts/e2e/parallels/npm-update-scripts.ts | 83 +++++++++++++++++++-- scripts/e2e/parallels/powershell.ts | 13 ++++ scripts/e2e/parallels/provider-auth.ts | 14 ++++ scripts/e2e/parallels/windows-smoke.ts | 8 +- test/scripts/parallels-smoke-model.test.ts | 10 +++ 7 files changed, 209 insertions(+), 12 deletions(-) diff --git a/scripts/e2e/parallels/linux-smoke.ts b/scripts/e2e/parallels/linux-smoke.ts index 2cf29c8cc8a..9dad0f402b1 100755 --- a/scripts/e2e/parallels/linux-smoke.ts +++ b/scripts/e2e/parallels/linux-smoke.ts @@ -12,10 +12,12 @@ import { parseBoolEnv, parseMode, parseProvider, + providerIdFromModelId, repoRoot, resolveHostIp, resolveHostPort, resolveLatestVersion, + resolveParallelsModelTimeoutSeconds, resolveProviderAuth, resolveSnapshot, run, @@ -687,6 +689,15 @@ rm -rf /root/.openclaw/test-bad-plugin`); private verifyLocalTurn(): void { this.guestExec(["openclaw", "models", "set", this.auth.modelId]); + const providerId = providerIdFromModelId(this.auth.modelId) || this.options.provider; + this.guestExec([ + "openclaw", + "config", + "set", + `models.providers.${providerId}.timeoutSeconds`, + String(resolveParallelsModelTimeoutSeconds("linux")), + "--strict-json", + ]); this.guestExec([ "openclaw", "config", @@ -698,9 +709,38 @@ rm -rf /root/.openclaw/test-bad-plugin`); this.guestExec(["openclaw", "config", "set", "tools.profile", "minimal"]); this.prepareAgentWorkspace(); this.guestBash( - `exec /usr/bin/env ${shellQuote(`${this.auth.apiKeyEnv}=${this.auth.apiKeyValue}`)} openclaw agent --local --agent main --session-id parallels-linux-smoke --message ${shellQuote( - "Reply with exact ASCII text OK only.", - )} --thinking minimal --json`, + `agent_ok=false +for attempt in 1 2; do + session_id="parallels-linux-smoke" + if [ "$attempt" -gt 1 ]; then session_id="parallels-linux-smoke-retry-$attempt"; fi + rm -f "$HOME/.openclaw/agents/main/sessions/$session_id.jsonl" + output_file="$(mktemp)" + set +e + /usr/bin/env ${shellQuote(`${this.auth.apiKeyEnv}=${this.auth.apiKeyValue}`)} openclaw agent --local --agent main --session-id "$session_id" --message ${shellQuote( + "Reply with exact ASCII text OK only.", + )} --thinking minimal --json >"$output_file" 2>&1 + rc=$? + set -e + cat "$output_file" + if [ "$rc" -ne 0 ]; then + rm -f "$output_file" + exit "$rc" + fi + if grep -Eq '"finalAssistant(Raw|Visible)Text"[[:space:]]*:[[:space:]]*"OK"' "$output_file"; then + agent_ok=true + rm -f "$output_file" + break + fi + rm -f "$output_file" + if [ "$attempt" -lt 2 ]; then + echo "agent turn attempt $attempt finished without OK response; retrying" + sleep 3 + fi +done +if [ "$agent_ok" != true ]; then + echo "openclaw agent finished without OK response" >&2 + exit 1 +fi`, ); } diff --git a/scripts/e2e/parallels/macos-smoke.ts b/scripts/e2e/parallels/macos-smoke.ts index bc7d0589a0f..cadc880dcf4 100755 --- a/scripts/e2e/parallels/macos-smoke.ts +++ b/scripts/e2e/parallels/macos-smoke.ts @@ -11,9 +11,11 @@ import { packOpenClaw, parseMode, parseProvider, + providerIdFromModelId, resolveHostIp, resolveHostPort, resolveLatestVersion, + resolveParallelsModelTimeoutSeconds, resolveProviderAuth, resolveSnapshot, run, @@ -971,6 +973,16 @@ exit 1`); private verifyTurn(): void { this.guestExec([guestNode, guestOpenClawEntry, "models", "set", this.auth.modelId]); + const providerId = providerIdFromModelId(this.auth.modelId) || this.options.provider; + this.guestExec([ + guestNode, + guestOpenClawEntry, + "config", + "set", + `models.providers.${providerId}.timeoutSeconds`, + String(resolveParallelsModelTimeoutSeconds("macos")), + "--strict-json", + ]); this.guestExec([ guestNode, guestOpenClawEntry, @@ -983,9 +995,38 @@ exit 1`); this.guestExec([guestNode, guestOpenClawEntry, "config", "set", "tools.profile", "minimal"]); this.guestSh( `${posixAgentWorkspaceScript("Parallels macOS smoke test assistant.")} -exec /usr/bin/env ${shellQuote(`${this.auth.apiKeyEnv}=${this.auth.apiKeyValue}`)} ${guestNode} ${guestOpenClawEntry} agent --local --agent main --session-id parallels-macos-smoke --message ${shellQuote( - "Reply with exact ASCII text OK only.", - )} --thinking minimal --json`, +agent_ok=false +for attempt in 1 2; do + session_id="parallels-macos-smoke" + if [ "$attempt" -gt 1 ]; then session_id="parallels-macos-smoke-retry-$attempt"; fi + rm -f "$HOME/.openclaw/agents/main/sessions/$session_id.jsonl" + output_file="$(mktemp)" + set +e + /usr/bin/env ${shellQuote(`${this.auth.apiKeyEnv}=${this.auth.apiKeyValue}`)} ${guestNode} ${guestOpenClawEntry} agent --local --agent main --session-id "$session_id" --message ${shellQuote( + "Reply with exact ASCII text OK only.", + )} --thinking minimal --json >"$output_file" 2>&1 + rc=$? + set -e + cat "$output_file" + if [ "$rc" -ne 0 ]; then + rm -f "$output_file" + exit "$rc" + fi + if grep -Eq '"finalAssistant(Raw|Visible)Text"[[:space:]]*:[[:space:]]*"OK"' "$output_file"; then + agent_ok=true + rm -f "$output_file" + break + fi + rm -f "$output_file" + if [ "$attempt" -lt 2 ]; then + echo "agent turn attempt $attempt finished without OK response; retrying" + sleep 3 + fi +done +if [ "$agent_ok" != true ]; then + echo "openclaw agent finished without OK response" >&2 + exit 1 +fi`, ); } diff --git a/scripts/e2e/parallels/npm-update-scripts.ts b/scripts/e2e/parallels/npm-update-scripts.ts index 8d2dd64b969..666630056d3 100644 --- a/scripts/e2e/parallels/npm-update-scripts.ts +++ b/scripts/e2e/parallels/npm-update-scripts.ts @@ -1,7 +1,12 @@ import { posixAgentWorkspaceScript, windowsAgentWorkspaceScript } from "./agent-workspace.ts"; import { shellQuote } from "./host-command.ts"; -import { psSingleQuote, windowsOpenClawResolver } from "./powershell.ts"; -import type { ProviderAuth } from "./types.ts"; +import { + psSingleQuote, + windowsModelProviderTimeoutScript, + windowsOpenClawResolver, +} from "./powershell.ts"; +import { providerIdFromModelId, resolveParallelsModelTimeoutSeconds } from "./provider-auth.ts"; +import type { Platform, ProviderAuth } from "./types.ts"; export interface NpmUpdateScriptInput { auth: ProviderAuth; @@ -9,6 +14,53 @@ export interface NpmUpdateScriptInput { updateTarget: string; } +function posixModelProviderTimeoutCommand( + command: string, + modelId: string, + platform: Platform, +): string { + const providerId = providerIdFromModelId(modelId); + if (!providerId) { + return ""; + } + return `${command} config set ${shellQuote( + `models.providers.${providerId}.timeoutSeconds`, + )} ${resolveParallelsModelTimeoutSeconds(platform)} --strict-json`; +} + +function posixAssertAgentOkScript(command: string, input: NpmUpdateScriptInput, sessionId: string) { + return `agent_ok=false +for attempt in 1 2; do + session_id=${shellQuote(sessionId)} + if [ "$attempt" -gt 1 ]; then session_id=${shellQuote(`${sessionId}-retry`)}"-$attempt"; fi + rm -f "$HOME/.openclaw/agents/main/sessions/$session_id.jsonl" + output_file="$(mktemp)" + set +e + ${input.auth.apiKeyEnv}=${shellQuote(input.auth.apiKeyValue)} ${command} agent --local --agent main --session-id "$session_id" --message 'Reply with exact ASCII text OK only.' --thinking minimal --json >"$output_file" 2>&1 + rc=$? + set -e + cat "$output_file" + if [ "$rc" -ne 0 ]; then + rm -f "$output_file" + exit "$rc" + fi + if grep -Eq '"finalAssistant(Raw|Visible)Text"[[:space:]]*:[[:space:]]*"OK"' "$output_file"; then + agent_ok=true + rm -f "$output_file" + break + fi + rm -f "$output_file" + if [ "$attempt" -lt 2 ]; then + echo "agent turn attempt $attempt finished without OK response; retrying" + sleep 3 + fi +done +if [ "$agent_ok" != true ]; then + echo "openclaw agent finished without OK response" >&2 + exit 1 +fi`; +} + export function macosUpdateScript(input: NpmUpdateScriptInput): string { return String.raw`set -euo pipefail export PATH=/opt/homebrew/bin:/opt/homebrew/opt/node/bin:/opt/homebrew/sbin:/usr/bin:/bin:/usr/sbin:/sbin @@ -70,10 +122,11 @@ ${posixVersionCheck("/opt/homebrew/bin/openclaw", input.expectedNeedle)} start_openclaw_gateway wait_for_gateway /opt/homebrew/bin/openclaw models set ${shellQuote(input.auth.modelId)} +${posixModelProviderTimeoutCommand("/opt/homebrew/bin/openclaw", input.auth.modelId, "macos")} /opt/homebrew/bin/openclaw config set agents.defaults.skipBootstrap true --strict-json /opt/homebrew/bin/openclaw config set tools.profile minimal ${posixAgentWorkspaceScript("Parallels npm update smoke test assistant.")} -${input.auth.apiKeyEnv}=${shellQuote(input.auth.apiKeyValue)} /opt/homebrew/bin/openclaw agent --local --agent main --session-id parallels-npm-update-macos --message 'Reply with exact ASCII text OK only.' --thinking minimal --json`; +${posixAssertAgentOkScript("/opt/homebrew/bin/openclaw", input, "parallels-npm-update-macos")}`; } export function windowsUpdateScript(input: NpmUpdateScriptInput): string { @@ -142,13 +195,32 @@ if ($LASTEXITCODE -ne 0) { } Wait-OpenClawGateway Invoke-OpenClaw models set ${psSingleQuote(input.auth.modelId)} +${windowsModelProviderTimeoutScript(input.auth.modelId)} Invoke-OpenClaw config set agents.defaults.skipBootstrap true --strict-json Invoke-OpenClaw config set tools.profile minimal $sessionPath = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions\\parallels-npm-update-windows.jsonl' Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue ${windowsAgentWorkspaceScript("Parallels npm update smoke test assistant.")} Set-Item -Path ('Env:' + ${psSingleQuote(input.auth.apiKeyEnv)}) -Value ${psSingleQuote(input.auth.apiKeyValue)} -Invoke-OpenClaw agent --local --agent main --session-id parallels-npm-update-windows --message 'Reply with exact ASCII text OK only.' --thinking minimal --json`; +$agentOk = $false +for ($attempt = 1; $attempt -le 2; $attempt++) { + $sessionId = if ($attempt -eq 1) { 'parallels-npm-update-windows' } else { "parallels-npm-update-windows-retry-$attempt" } + $sessionsDir = Join-Path $env:USERPROFILE '.openclaw\\agents\\main\\sessions' + $sessionPath = Join-Path $sessionsDir "$sessionId.jsonl" + Remove-Item $sessionPath -Force -ErrorAction SilentlyContinue + $output = Invoke-OpenClaw agent --local --agent main --session-id $sessionId --message 'Reply with exact ASCII text OK only.' --thinking minimal --json 2>&1 + if ($null -ne $output) { $output | ForEach-Object { $_ } } + if ($LASTEXITCODE -ne 0) { throw "agent failed with exit code $LASTEXITCODE" } + if (($output | Out-String) -match '"finalAssistant(Raw|Visible)Text":\\s*"OK"') { + $agentOk = $true + break + } + if ($attempt -lt 2) { + Write-Host "agent turn attempt $attempt finished without OK response; retrying" + Start-Sleep -Seconds 3 + } +} +if (-not $agentOk) { throw 'openclaw agent finished without OK response' }`; } export function linuxUpdateScript(input: NpmUpdateScriptInput): string { @@ -207,10 +279,11 @@ ${posixVersionCheck("openclaw", input.expectedNeedle)} start_openclaw_gateway wait_for_gateway openclaw models set ${shellQuote(input.auth.modelId)} +${posixModelProviderTimeoutCommand("openclaw", input.auth.modelId, "linux")} openclaw config set agents.defaults.skipBootstrap true --strict-json openclaw config set tools.profile minimal ${posixAgentWorkspaceScript("Parallels npm update smoke test assistant.")} -${input.auth.apiKeyEnv}=${shellQuote(input.auth.apiKeyValue)} openclaw agent --local --agent main --session-id parallels-npm-update-linux --message 'Reply with exact ASCII text OK only.' --thinking minimal --json`; +${posixAssertAgentOkScript("openclaw", input, "parallels-npm-update-linux")}`; } function posixVersionCheck(command: string, expectedNeedle: string): string { diff --git a/scripts/e2e/parallels/powershell.ts b/scripts/e2e/parallels/powershell.ts index 66f5eaffe95..0c1f8dfb960 100644 --- a/scripts/e2e/parallels/powershell.ts +++ b/scripts/e2e/parallels/powershell.ts @@ -1,3 +1,5 @@ +import { providerIdFromModelId, resolveParallelsModelTimeoutSeconds } from "./provider-auth.ts"; + export function psSingleQuote(value: string): string { return `'${value.replaceAll("'", "''")}'`; } @@ -12,6 +14,17 @@ export function encodePowerShell(script: string): string { ); } +export function windowsModelProviderTimeoutScript(modelId: string): string { + const providerId = providerIdFromModelId(modelId); + if (!providerId) { + return ""; + } + return `Invoke-OpenClaw config set ${psSingleQuote( + `models.providers.${providerId}.timeoutSeconds`, + )} ${resolveParallelsModelTimeoutSeconds("windows")} --strict-json +if ($LASTEXITCODE -ne 0) { throw "model provider timeout config set failed" }`; +} + export const windowsOpenClawResolver = String.raw`function Resolve-OpenClawCommand { if ($script:OpenClawResolvedCommand) { return $script:OpenClawResolvedCommand } $shimCandidates = @() diff --git a/scripts/e2e/parallels/provider-auth.ts b/scripts/e2e/parallels/provider-auth.ts index b1a3019f443..0166e87fb9f 100644 --- a/scripts/e2e/parallels/provider-auth.ts +++ b/scripts/e2e/parallels/provider-auth.ts @@ -72,6 +72,20 @@ export function resolveWindowsProviderAuth(input: { return { ...auth, modelId: "openai/gpt-4.1-mini" }; } +export function providerIdFromModelId(modelId: string): string { + const providerId = modelId.split("/", 1)[0]?.trim() ?? ""; + return /^[A-Za-z0-9_-]+$/u.test(providerId) ? providerId : ""; +} + +export function resolveParallelsModelTimeoutSeconds(platform?: Platform): number { + const platformEnv = + platform === undefined + ? undefined + : process.env[`OPENCLAW_PARALLELS_${platform.toUpperCase()}_MODEL_TIMEOUT_S`]; + const raw = Number(platformEnv || process.env.OPENCLAW_PARALLELS_MODEL_TIMEOUT_S || 600); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 600; +} + export function parseProvider(value: string): Provider { if (value === "openai" || value === "anthropic" || value === "minimax") { return value; diff --git a/scripts/e2e/parallels/windows-smoke.ts b/scripts/e2e/parallels/windows-smoke.ts index 9d92ccf1efa..5ea74481734 100755 --- a/scripts/e2e/parallels/windows-smoke.ts +++ b/scripts/e2e/parallels/windows-smoke.ts @@ -33,7 +33,12 @@ import { WindowsGuest } from "./guest-transports.ts"; import { runSmokeLane, type SmokeLane, type SmokeLaneStatus } from "./lane-runner.ts"; import { waitForVmStatus } from "./parallels-vm.ts"; import { PhaseRunner } from "./phase-runner.ts"; -import { encodePowerShell, psSingleQuote, windowsOpenClawResolver } from "./powershell.ts"; +import { + encodePowerShell, + psSingleQuote, + windowsModelProviderTimeoutScript, + windowsOpenClawResolver, +} from "./powershell.ts"; import { ensureGuestGit, prepareMinGitZip } from "./windows-git.ts"; interface WindowsOptions { @@ -887,6 +892,7 @@ if ($LASTEXITCODE -ne 0) { throw "gateway ${action} failed with exit code $LASTE $PSNativeCommandUseErrorActionPreference = $false Invoke-OpenClaw models set ${psSingleQuote(this.auth.modelId)} if ($LASTEXITCODE -ne 0) { throw "models set failed" } +${windowsModelProviderTimeoutScript(this.auth.modelId)} Invoke-OpenClaw config set agents.defaults.skipBootstrap true --strict-json if ($LASTEXITCODE -ne 0) { throw "config set failed" } Invoke-OpenClaw config set tools.profile minimal diff --git a/test/scripts/parallels-smoke-model.test.ts b/test/scripts/parallels-smoke-model.test.ts index 4b1d9d9b500..d8039347327 100644 --- a/test/scripts/parallels-smoke-model.test.ts +++ b/test/scripts/parallels-smoke-model.test.ts @@ -319,13 +319,20 @@ console.log(JSON.stringify(result)); expect(script, scriptPath).toContain("tools.profile"); expect(script, scriptPath).toContain("--thinking"); expect(script, scriptPath).toContain("minimal"); + expect(script, scriptPath).toContain("finalAssistant(Raw|Visible)Text"); } + expect(readFileSync(TS_PATHS.macos, "utf8")).toContain("resolveParallelsModelTimeoutSeconds"); + expect(readFileSync(TS_PATHS.linux, "utf8")).toContain("resolveParallelsModelTimeoutSeconds"); + expect(readFileSync(TS_PATHS.windows, "utf8")).toContain("windowsModelProviderTimeoutScript"); const npmUpdateScripts = readFileSync(TS_PATHS.npmUpdateScripts, "utf8"); expect(npmUpdateScripts).toContain("posixAgentWorkspaceScript"); expect(npmUpdateScripts).toContain("windowsAgentWorkspaceScript"); expect(npmUpdateScripts).toContain("tools.profile"); expect(npmUpdateScripts).toContain("--thinking minimal"); + expect(npmUpdateScripts).toContain("finalAssistant(Raw|Visible)Text"); + expect(npmUpdateScripts).toContain("posixAssertAgentOkScript"); + expect(npmUpdateScripts).toContain("windowsModelProviderTimeoutScript"); }); it("clears phase timers and applies phase deadlines to guest commands", () => { @@ -451,6 +458,7 @@ console.log(JSON.stringify(result)); expect(script).toContain('guestPowerShellBackground(\n "agent-turn"'); expect(script).toContain("OPENCLAW_PARALLELS_WINDOWS_AGENT_TIMEOUT_S"); + expect(script).toContain("windowsModelProviderTimeoutScript(this.auth.modelId)"); expect(script).toContain("finalAssistant(Raw|Visible)Text"); expect(script).toContain("parallels-windows-smoke-retry-$attempt"); expect(script).not.toContain("$config.models.providers"); @@ -471,6 +479,8 @@ console.log(JSON.stringify(result)); const windows = readFileSync(TS_PATHS.windows, "utf8"); expect(powershell).toContain("windowsOpenClawResolver"); + expect(powershell).toContain('resolveParallelsModelTimeoutSeconds("windows")'); + expect(powershell).toContain("models.providers.${providerId}.timeoutSeconds"); expect(powershell).toContain("Resolve-OpenClawCommand"); expect(powershell).toContain("npm\\node_modules\\openclaw\\openclaw.mjs"); expect(powershell).toContain("$ErrorActionPreference = 'Continue'");