diff --git a/extensions/google/google.live.test.ts b/extensions/google/google.live.test.ts index e95d79530bf..5f63845dd59 100644 --- a/extensions/google/google.live.test.ts +++ b/extensions/google/google.live.test.ts @@ -13,6 +13,17 @@ const GOOGLE_API_KEY = const LIVE = isLiveTestEnabled() && GOOGLE_API_KEY.length > 0; const describeLive = LIVE ? describe : describe.skip; +function isTransientGeminiSearchError(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + if (error.name === "AbortError") { + return true; + } + const message = error.message.toLowerCase(); + return message.includes("timeout") || message.includes("aborted"); +} + const registerGooglePlugin = () => registerProviderPlugin({ plugin, @@ -87,10 +98,26 @@ describeLive("google plugin live", () => { const provider = createGeminiWebSearchProvider(); const tool = provider.createTool?.({ config: {}, - searchConfig: { gemini: { apiKey: GOOGLE_API_KEY }, cacheTtlMinutes: 0 }, + searchConfig: { gemini: { apiKey: GOOGLE_API_KEY }, cacheTtlMinutes: 0, timeoutSeconds: 90 }, } as never); - const result = await tool?.execute({ query: "OpenClaw GitHub", count: 1 }); + let result: { provider?: string; content?: unknown; citations?: unknown } | undefined; + let lastError: unknown; + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + result = await tool?.execute({ query: "OpenClaw GitHub", count: 1 }); + lastError = undefined; + break; + } catch (error) { + lastError = error; + if (!isTransientGeminiSearchError(error) || attempt === 1) { + throw error; + } + } + } + if (lastError) { + throw lastError; + } expect(result?.provider).toBe("gemini"); expect(typeof result?.content).toBe("string"); diff --git a/scripts/lib/docker-build.sh b/scripts/lib/docker-build.sh index c12bbf923ab..910db2a7337 100644 --- a/scripts/lib/docker-build.sh +++ b/scripts/lib/docker-build.sh @@ -19,7 +19,7 @@ docker_build_on_missing_enabled() { [ "${OPENCLAW_TESTBOX:-0}" = "1" ] } -docker_build_exec() { +docker_build_command() { local build_cmd=(docker build) if [ "${OPENCLAW_DOCKER_BUILD_USE_BUILDX:-0}" = "1" ]; then build_cmd=(docker buildx build --load) @@ -31,23 +31,66 @@ docker_build_exec() { fi fi - env DOCKER_BUILDKIT=1 "${build_cmd[@]}" "$@" + printf '%s\0' env DOCKER_BUILDKIT=1 "${build_cmd[@]}" "$@" +} + +docker_build_transient_failure() { + local log_file="$1" + grep -Eqi \ + 'frontend grpc server closed unexpectedly|failed to dial gRPC|no active session|buildkit.*connection.*closed|rpc error: code = Unavailable' \ + "$log_file" +} + +docker_build_retry_count() { + local configured="${OPENCLAW_DOCKER_BUILD_RETRIES:-2}" + if [[ "$configured" =~ ^[0-9]+$ ]]; then + echo "$configured" + return 0 + fi + echo 2 +} + +docker_build_with_retries() { + local label="$1" + shift + local retries + retries="$(docker_build_retry_count)" + local attempt=1 + local max_attempts=$((retries + 1)) + local log_file + local command=() + while IFS= read -r -d '' part; do + command+=("$part") + done < <(docker_build_command "$@") + + while true; do + log_file="$(docker_e2e_run_log "$label")" + if "${command[@]}" >"$log_file" 2>&1; then + rm -f "$log_file" + return 0 + fi + + if [ "$attempt" -ge "$max_attempts" ] || ! docker_build_transient_failure "$log_file"; then + docker_e2e_print_log "$log_file" + rm -f "$log_file" + return 1 + fi + + echo "Docker build failed with a transient BuildKit transport error; retrying ($attempt/$retries)..." >&2 + docker_e2e_print_log "$log_file" + rm -f "$log_file" + attempt=$((attempt + 1)) + sleep "$attempt" + done +} + +docker_build_exec() { + docker_build_with_retries docker-build "$@" } docker_build_run() { local label="$1" shift - local build_cmd=(docker build) - if [ "${OPENCLAW_DOCKER_BUILD_USE_BUILDX:-0}" = "1" ]; then - build_cmd=(docker buildx build --load) - if [ -n "${OPENCLAW_DOCKER_BUILD_CACHE_FROM:-}" ]; then - build_cmd+=(--cache-from "${OPENCLAW_DOCKER_BUILD_CACHE_FROM}") - fi - if [ -n "${OPENCLAW_DOCKER_BUILD_CACHE_TO:-}" ]; then - build_cmd+=(--cache-to "${OPENCLAW_DOCKER_BUILD_CACHE_TO}") - fi - fi - - run_logged "$label" env DOCKER_BUILDKIT=1 "${build_cmd[@]}" "$@" + docker_build_with_retries "$label" "$@" } diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index b0518aea18c..85195f1b502 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -37,6 +37,9 @@ describe("docker build helper", () => { expect(helper).toContain("docker_build_exec()"); expect(helper).toContain("docker_build_run()"); expect(helper).toContain("docker buildx build --load"); + expect(helper).toContain("docker_build_transient_failure()"); + expect(helper).toContain("OPENCLAW_DOCKER_BUILD_RETRIES"); + expect(helper).toContain("frontend grpc server closed unexpectedly"); }); it("keeps shell-script Docker builds behind the helper", () => {