workflows: retry transient Bazel download failures in setup-bazel (#4291) `bazel-contrib/setup-bazel` configures Bazelisk and restores caches, leaving the Bazel binary download to the first `bazel` invocation. Bazelisk retries initial HTTP GET headers, but does not retry mid-stream `io.Copy` failures (such as HTTP/2 stream resets from `releases.bazel.build`). A failed download also leaves an empty `~/.cache/bazelisk` directory that `setup-bazel`'s post step caches. To prevent transient network flakes, pin Bazelisk to 1.29.0 (for `windows-arm64` support) in the shared `setup-bazel` composite action, eagerly run `bazel --version` with exponential-backoff retries, delete partial Bazelisk cache directories if all retries fail, and reuse the shared action in the release artifact build workflow.
diff --git a/.github/actions/setup-bazel/BUILD.bazel b/.github/actions/setup-bazel/BUILD.bazel new file mode 100644 index 0000000..09c0ea2 --- /dev/null +++ b/.github/actions/setup-bazel/BUILD.bazel
@@ -0,0 +1,12 @@ +package(default_visibility = ["//tests:__subpackages__"]) + +exports_files( + glob(["*"]), + visibility = ["//tests:__subpackages__"], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//tests:__subpackages__"], +)
diff --git a/.github/actions/setup-bazel/action.yaml b/.github/actions/setup-bazel/action.yaml index 623cac8..60aa1d1 100644 --- a/.github/actions/setup-bazel/action.yaml +++ b/.github/actions/setup-bazel/action.yaml
@@ -5,8 +5,11 @@ using: "composite" steps: - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.19.0 + uses: bazel-contrib/setup-bazel@0.19.1 with: + # Windows images have a fixed-version `bazel` on PATH, and Bazelisk + # < 1.29.0 has no windows-arm64 build. + bazelisk-version: 1.29.0 bazelisk-cache: true disk-cache: true # Files hashed to compute the repository cache key. If only `true` is @@ -23,3 +26,11 @@ # pyrefly toolchain or type-check targets before running them. build --output_groups=-pyrefly build --//python/config_settings:py_pbs_distribution=install_only_stripped + + # setup-bazel only configures Bazelisk and restores caches; the Bazel + # binary itself is downloaded by Bazelisk on first invocation. Because + # Bazelisk does not retry mid-stream download failures, trigger and retry + # the download upfront via `bazel --version`. + - name: Download Bazel + shell: bash + run: "${GITHUB_ACTION_PATH}/download_bazel.sh"
diff --git a/.github/actions/setup-bazel/download_bazel.sh b/.github/actions/setup-bazel/download_bazel.sh new file mode 100755 index 0000000..088b470 --- /dev/null +++ b/.github/actions/setup-bazel/download_bazel.sh
@@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Downloads the Bazel binary via Bazelisk with retries on transient network +# errors. +# +# `bazel-contrib/setup-bazel` only configures Bazelisk and restores caches; the +# Bazel binary itself is downloaded by Bazelisk on first invocation. Bazelisk +# retries initial HTTP GET headers (`client.Do`), but does not retry mid-stream +# transfer errors during `io.Copy` (such as HTTP/2 `INTERNAL_ERROR` stream +# resets from `releases.bazel.build`). Running `bazel --version` here triggers +# the download upfront (without starting the Bazel server JVM) and retries on +# transient failures. +set -euo pipefail + +max_attempts="${BAZEL_DOWNLOAD_MAX_ATTEMPTS:-5}" +delay="${BAZEL_DOWNLOAD_INITIAL_DELAY:-2}" + +for ((attempt = 1; attempt <= max_attempts; attempt++)); do + if bazel --version; then + exit 0 + fi + if ((attempt < max_attempts)); then + echo "::warning::Failed to download Bazel (attempt $attempt/$max_attempts); retrying in ${delay}s..." + sleep "$delay" + delay=$((delay * 2)) + fi +done + +echo "::error::Failed to download Bazel after $max_attempts attempts" +# Remove any empty/partial Bazelisk cache directory created before the failed +# download so `setup-bazel`'s post step does not cache an empty directory. +if [[ -n "${BAZELISK_HOME:-}" ]]; then + rm -rf "$BAZELISK_HOME" +fi +rm -rf \ + "${XDG_CACHE_HOME:-$HOME/.cache}/bazelisk" \ + "$HOME/Library/Caches/bazelisk" \ + "$HOME/AppData/Local/bazelisk" +exit 1
diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml index ee8dfe8..0c254ef 100644 --- a/.github/workflows/release_build_artifacts.yaml +++ b/.github/workflows/release_build_artifacts.yaml
@@ -81,11 +81,7 @@ persist-credentials: false - name: Setup Bazel - uses: bazel-contrib/setup-bazel@0.19.1 - with: - # Windows images also have a fixed-version `bazel` on PATH. 1.20.0, - # used by the other workflows, has no windows-arm64 build. - bazelisk-version: 1.29.0 + uses: ./.github/actions/setup-bazel # Release commands run main's workflows, even for patch releases of older # release branches. Skip refs without dev/platforms, since their scripts
diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index f1e5625..b3f46d0 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel
@@ -75,3 +75,16 @@ "//python/runfiles", ], ) + +pytest_test( + name = "setup_bazel_test", + srcs = ["setup_bazel_test.py"], + data = [ + "//.github/actions/setup-bazel:all_files", + ], + target_compatible_with = NOT_WINDOWS, + deps = [ + "//python/runfiles", + "@pypi//pyyaml", + ], +)
diff --git a/tests/workflows/setup_bazel_test.py b/tests/workflows/setup_bazel_test.py new file mode 100644 index 0000000..951b9a8 --- /dev/null +++ b/tests/workflows/setup_bazel_test.py
@@ -0,0 +1,145 @@ +"""Tests for the .github/actions/setup-bazel composite action.""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import yaml + +from python.runfiles import runfiles + + +def _action_dir() -> Path: + return runfiles.CreateOrRaise().root() / "rules_python/.github/actions/setup-bazel" + + +def _write_fake_bazel(bin_dir: Path, body: str) -> Path: + bin_dir.mkdir(parents=True, exist_ok=True) + fake_bazel = bin_dir / "bazel" + fake_bazel.write_text( + f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n", encoding="utf-8" + ) + fake_bazel.chmod(0o755) + return fake_bazel + + +def test_action_yaml_invokes_executable_download_script(): + """Verifies setup-bazel action.yaml runs download_bazel.sh and the script is +x.""" + action_dir = _action_dir() + script = action_dir / "download_bazel.sh" + assert script.is_file() + assert bool(script.stat().st_mode & stat.S_IXUSR), ( + "download_bazel.sh must be executable" + ) + + action = yaml.safe_load((action_dir / "action.yaml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + assert any("download_bazel.sh" in step.get("run", "") for step in steps) + + +def test_download_bazel_succeeds_first_try(tmp_path: Path): + """Verifies download_bazel.sh succeeds without warnings when bazel --version succeeds.""" + script = _action_dir() / "download_bazel.sh" + calls_file = tmp_path / "calls.txt" + _write_fake_bazel( + tmp_path / "bin", + f'echo "$@" >> "{calls_file}"\necho "bazel 9.1.1"\n', + ) + + env = os.environ.copy() + env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}" + env["HOME"] = str(tmp_path / "home") + + result = subprocess.run( + [str(script)], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert calls_file.read_text(encoding="utf-8").splitlines() == ["--version"] + assert "::warning::" not in result.stdout + assert "::error::" not in result.stdout + + +def test_download_bazel_retries_transient_failure(tmp_path: Path): + """Verifies download_bazel.sh retries transient failures and succeeds.""" + script = _action_dir() / "download_bazel.sh" + calls_file = tmp_path / "calls.txt" + _write_fake_bazel( + tmp_path / "bin", + f""" +echo "$@" >> "{calls_file}" +count=$(wc -l < "{calls_file}") +if (( count < 3 )); then + echo "could not download Bazel: stream error: INTERNAL_ERROR" >&2 + exit 1 +fi +mkdir -p "$HOME/.cache/bazelisk/downloads" +echo "bazel 9.1.1" +""", + ) + + home = tmp_path / "home" + env = os.environ.copy() + env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}" + env["HOME"] = str(home) + env["BAZEL_DOWNLOAD_INITIAL_DELAY"] = "0" + + result = subprocess.run( + [str(script)], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert calls_file.read_text(encoding="utf-8").splitlines() == [ + "--version", + "--version", + "--version", + ] + assert "::warning::Failed to download Bazel (attempt 1/5)" in result.stdout + assert "::warning::Failed to download Bazel (attempt 2/5)" in result.stdout + assert (home / ".cache/bazelisk/downloads").is_dir() + + +def test_download_bazel_exhausts_retries_and_cleans_cache(tmp_path: Path): + """Verifies download_bazel.sh cleans up partial cache dirs when all retries fail.""" + script = _action_dir() / "download_bazel.sh" + calls_file = tmp_path / "calls.txt" + _write_fake_bazel( + tmp_path / "bin", + f""" +echo "$@" >> "{calls_file}" +mkdir -p "$HOME/.cache/bazelisk/downloads/_tmp" +mkdir -p "$BAZELISK_HOME/downloads/_tmp" +exit 1 +""", + ) + + home = tmp_path / "home" + bazelisk_home = tmp_path / "bazelisk_home" + env = os.environ.copy() + env["PATH"] = f"{tmp_path / 'bin'}:{env.get('PATH', '')}" + env["HOME"] = str(home) + env["BAZELISK_HOME"] = str(bazelisk_home) + env["BAZEL_DOWNLOAD_MAX_ATTEMPTS"] = "3" + env["BAZEL_DOWNLOAD_INITIAL_DELAY"] = "0" + + result = subprocess.run( + [str(script)], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert len(calls_file.read_text(encoding="utf-8").splitlines()) == 3 + assert "::error::Failed to download Bazel after 3 attempts" in result.stdout + assert not (home / ".cache/bazelisk").exists() + assert not bazelisk_home.exists()