diff --git a/scripts/ci-docker-pull-retry.sh b/scripts/ci-docker-pull-retry.sh index 65d23435e42..0b15071f28f 100644 --- a/scripts/ci-docker-pull-retry.sh +++ b/scripts/ci-docker-pull-retry.sh @@ -27,12 +27,25 @@ if ! [[ "$retry_delay_seconds" =~ ^[0-9]+$ ]]; then fi last_status=1 -for attempt in $(seq 1 "$attempts"); do - echo "==> Pull Docker image attempt ${attempt}/${attempts}: ${image}" - if timeout --kill-after=30s "${timeout_seconds}s" docker pull "$image"; then - exit 0 +run_docker_pull() { + if ! command -v timeout >/dev/null 2>&1; then + echo "timeout command not found; cannot bound Docker pull after ${timeout_seconds}s" >&2 + return 127 + fi + if timeout --kill-after=1s 1s true >/dev/null 2>&1; then + timeout --kill-after=30s "${timeout_seconds}s" docker pull "$image" + else + timeout "${timeout_seconds}s" docker pull "$image" + fi +} + +for ((attempt = 1; attempt <= attempts; attempt++)); do + echo "==> Pull Docker image attempt ${attempt}/${attempts}: ${image}" + if run_docker_pull; then + exit 0 + else + last_status="$?" fi - last_status="$?" echo "Docker pull failed or timed out after ${timeout_seconds}s: status=${last_status}" >&2 if [[ "$attempt" -lt "$attempts" && "$retry_delay_seconds" -gt 0 ]]; then sleep "$retry_delay_seconds" diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 62d09b3e002..3c649d062c2 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -355,6 +355,7 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([ ["scripts/changed-lanes.mjs", ["test/scripts/changed-lanes.test.ts"]], ["scripts/check-changed.mjs", ["test/scripts/changed-lanes.test.ts"]], ["scripts/check-deadcode-unused-files.mjs", ["test/scripts/check-deadcode-unused-files.test.ts"]], + ["scripts/ci-docker-pull-retry.sh", ["test/scripts/ci-docker-pull-retry.test.ts"]], [ "scripts/deadcode-unused-files.allowlist.mjs", ["test/scripts/check-deadcode-unused-files.test.ts"], @@ -418,6 +419,7 @@ const TOOLING_TEST_TARGETS = new Map([ "test/scripts/check-deadcode-unused-files.test.ts", ["test/scripts/check-deadcode-unused-files.test.ts"], ], + ["test/scripts/ci-docker-pull-retry.test.ts", ["test/scripts/ci-docker-pull-retry.test.ts"]], ["test/scripts/live-docker-stage.test.ts", ["test/scripts/live-docker-stage.test.ts"]], ["test/scripts/openclaw-test-state.test.ts", ["test/scripts/openclaw-test-state.test.ts"]], [ @@ -488,6 +490,7 @@ const SOURCE_TEST_TARGETS = new Map([ CHANNEL_CONTRACT_REGISTRY_BACKED_TARGETS, ], ["test/helpers/normalize-text.ts", TEST_HELPER_NORMALIZE_TEXT_TARGETS], + ["ui/config/control-ui-chunking.ts", ["ui/src/ui/control-ui-chunking.test.ts"]], [ "src/plugin-sdk/test-helpers/directory-ids.ts", [ diff --git a/test/scripts/ci-docker-pull-retry.test.ts b/test/scripts/ci-docker-pull-retry.test.ts new file mode 100644 index 00000000000..e56fa393df7 --- /dev/null +++ b/test/scripts/ci-docker-pull-retry.test.ts @@ -0,0 +1,167 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const SCRIPT_PATH = path.resolve("scripts/ci-docker-pull-retry.sh"); +const tempDirs: string[] = []; + +function makeTempBin(prefix: string) { + const dir = mkdtempSync(path.join(tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +function writeExecutable(filePath: string, contents: string) { + writeFileSync(filePath, contents, "utf8"); + chmodSync(filePath, 0o755); +} + +function runPullHelper(binDir: string) { + return runPullHelperWithEnv(binDir, {}); +} + +function runPullHelperWithEnv(binDir: string, env: Record) { + return spawnSync("/bin/bash", [SCRIPT_PATH, "registry.example/openclaw:test"], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + OPENCLAW_DOCKER_PULL_ATTEMPTS: "1", + OPENCLAW_DOCKER_PULL_RETRY_DELAY_SECONDS: "0", + OPENCLAW_DOCKER_PULL_TIMEOUT_SECONDS: "42", + ...env, + PATH: binDir, + }, + }); +} + +afterEach(() => { + while (tempDirs.length > 0) { + rmSync(tempDirs.pop()!, { force: true, recursive: true }); + } +}); + +describe("scripts/ci-docker-pull-retry.sh", () => { + it("uses a kill-after grace period when timeout supports it", () => { + const binDir = makeTempBin("openclaw-ci-docker-pull-gnu-"); + const timeoutArgsPath = path.join(binDir, "timeout-args.txt"); + const dockerArgsPath = path.join(binDir, "docker-args.txt"); + + writeExecutable( + path.join(binDir, "timeout"), + [ + "#!/bin/bash", + "set -euo pipefail", + 'if [ "${1:-}" = "--kill-after=1s" ]; then exit 0; fi', + `printf "%s\\n" "$*" >${JSON.stringify(timeoutArgsPath)}`, + 'while [ "$#" -gt 0 ] && [ "$1" != "docker" ]; do shift; done', + 'exec "$@"', + "", + ].join("\n"), + ); + writeExecutable( + path.join(binDir, "docker"), + ["#!/bin/sh", "set -eu", `printf "%s\\n" "$*" >${JSON.stringify(dockerArgsPath)}`, ""].join( + "\n", + ), + ); + + const result = runPullHelper(binDir); + + expect(result.status).toBe(0); + expect(execFileSync("cat", [timeoutArgsPath], { encoding: "utf8" }).trim()).toBe( + "--kill-after=30s 42s docker pull registry.example/openclaw:test", + ); + expect(execFileSync("cat", [dockerArgsPath], { encoding: "utf8" }).trim()).toBe( + "pull registry.example/openclaw:test", + ); + }); + + it("falls back to plain timeout when kill-after is unavailable", () => { + const binDir = makeTempBin("openclaw-ci-docker-pull-plain-"); + const timeoutArgsPath = path.join(binDir, "timeout-args.txt"); + const dockerArgsPath = path.join(binDir, "docker-args.txt"); + + writeExecutable( + path.join(binDir, "timeout"), + [ + "#!/bin/bash", + "set -euo pipefail", + 'if [ "${1:-}" = "--kill-after=1s" ]; then exit 1; fi', + `printf "%s\\n" "$*" >${JSON.stringify(timeoutArgsPath)}`, + 'while [ "$#" -gt 0 ] && [ "$1" != "docker" ]; do shift; done', + 'exec "$@"', + "", + ].join("\n"), + ); + writeExecutable( + path.join(binDir, "docker"), + ["#!/bin/sh", "set -eu", `printf "%s\\n" "$*" >${JSON.stringify(dockerArgsPath)}`, ""].join( + "\n", + ), + ); + + const result = runPullHelper(binDir); + + expect(result.status).toBe(0); + expect(execFileSync("cat", [timeoutArgsPath], { encoding: "utf8" }).trim()).toBe( + "42s docker pull registry.example/openclaw:test", + ); + expect(execFileSync("cat", [dockerArgsPath], { encoding: "utf8" }).trim()).toBe( + "pull registry.example/openclaw:test", + ); + }); + + it("fails fast when timeout is unavailable", () => { + const binDir = makeTempBin("openclaw-ci-docker-pull-no-timeout-"); + const dockerArgsPath = path.join(binDir, "docker-args.txt"); + + writeExecutable( + path.join(binDir, "docker"), + ["#!/bin/sh", "set -eu", `printf "%s\\n" "$*" >${JSON.stringify(dockerArgsPath)}`, ""].join( + "\n", + ), + ); + + const result = runPullHelper(binDir); + + expect(result.status).toBe(127); + expect(result.stderr).toContain("timeout command not found; cannot bound Docker pull after 42s"); + expect(existsSync(dockerArgsPath)).toBe(false); + }); + + it("returns the last pull failure status after retries are exhausted", () => { + const binDir = makeTempBin("openclaw-ci-docker-pull-fail-"); + const dockerArgsPath = path.join(binDir, "docker-args.txt"); + + writeExecutable( + path.join(binDir, "timeout"), + [ + "#!/bin/bash", + "set -euo pipefail", + 'if [ "${1:-}" = "--kill-after=1s" ]; then exit 0; fi', + 'while [ "$#" -gt 0 ] && [ "$1" != "docker" ]; do shift; done', + 'exec "$@"', + "", + ].join("\n"), + ); + writeExecutable( + path.join(binDir, "docker"), + [ + "#!/bin/sh", + "set -eu", + `printf "%s\\n" "$*" >>${JSON.stringify(dockerArgsPath)}`, + "exit 42", + "", + ].join("\n"), + ); + + const result = runPullHelperWithEnv(binDir, { OPENCLAW_DOCKER_PULL_ATTEMPTS: "2" }); + + expect(result.status).toBe(42); + expect(result.stderr).toContain("Docker pull failed or timed out after 42s: status=42"); + expect(execFileSync("wc", ["-l", dockerArgsPath], { encoding: "utf8" }).trim()).toMatch(/^2\b/u); + }); +}); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 03230d6e95b..a1473cbe1ad 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -428,6 +428,11 @@ describe("package artifact reuse", () => { expect(pullHelper).toContain( 'timeout --kill-after=30s "${timeout_seconds}s" docker pull "$image"', ); + expect(pullHelper).toContain("timeout --kill-after=1s 1s true >/dev/null 2>&1"); + expect(pullHelper).toContain('timeout "${timeout_seconds}s" docker pull "$image"'); + expect(pullHelper).toContain( + 'timeout command not found; cannot bound Docker pull after ${timeout_seconds}s', + ); expect(dockerE2ePlanAction.match(/bash scripts\/ci-docker-pull-retry\.sh/g)?.length).toBe(2); expect(dockerE2ePlanAction).not.toContain('docker pull "${OPENCLAW_DOCKER_E2E_'); }); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index eb61062f24f..16b3fe6513b 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1,3 +1,4 @@ +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import fg from "fast-glob"; @@ -126,6 +127,15 @@ function listNormalFullSuiteTestFiles(): string[] { .toSorted((left, right) => left.localeCompare(right)); } +function hasGitGatewayFileListing(cwd: string): boolean { + const result = spawnSync("git", ["ls-files", "--", "src/gateway"], { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + return result.status === 0 && result.stdout.trim().length > 0; +} + describe("scripts/test-projects changed-target routing", () => { it("maps changed source files into scoped lane targets", () => { expect( @@ -170,6 +180,13 @@ describe("scripts/test-projects changed-target routing", () => { }); }); + it("routes Docker pull retry helper changes through its regression test", () => { + expect(resolveChangedTestTargetPlan(["scripts/ci-docker-pull-retry.sh"])).toEqual({ + mode: "targets", + targets: ["test/scripts/ci-docker-pull-retry.test.ts"], + }); + }); + it("routes group visible reply config changes through channel delivery regressions", () => { expect( resolveChangedTestTargetPlan([ @@ -1067,6 +1084,7 @@ describe("scripts/test-projects full-suite sharding", () => { let normalFullSuiteTestFiles: string[]; let leafShardPlans: ReturnType; let leafShardGatewayTreeReads: unknown[][]; + let leafShardHasGitGatewayListing: boolean; beforeAll(async () => { [fullSuiteMatches, normalFullSuiteTestFiles] = await Promise.all([ @@ -1078,6 +1096,7 @@ describe("scripts/test-projects full-suite sharding", () => { const gatewayServerConfig = "test/vitest/vitest.gateway-server.config.ts"; process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS = "1"; try { + leafShardHasGitGatewayListing = hasGitGatewayFileListing(process.cwd()); const captured = captureReaddirSyncCallsDuring(() => buildFullSuiteVitestRunPlans([], process.cwd()), ); @@ -1310,7 +1329,9 @@ describe("scripts/test-projects full-suite sharding", () => { const gatewayServerConfig = "test/vitest/vitest.gateway-server.config.ts"; const plans = leafShardPlans; - expect(leafShardGatewayTreeReads).toEqual([]); + if (leafShardHasGitGatewayListing) { + expect(leafShardGatewayTreeReads).toEqual([]); + } expect(leafShardPlans.map((plan) => plan.config)).toEqual([ "test/vitest/vitest.unit-fast.config.ts", "test/vitest/vitest.unit-src.config.ts",