docs: speed up Read the Docs builds with RBE and Bazelisk (#4452) Read the Docs builds previously installed Node.js and npm to fetch Bazelisk, ran all build actions locally without remote caching, and passed container-specific environment variables and unconditional stamping that busted action cache keys. Download Bazelisk directly via an async Python script, enable BuildBuddy remote cache and remote execution (using CAS-only presubmit keys for PRs and read/write postsubmit keys for branches and tags), disable lint aspects and intermediate artifact downloads for RTD builds, and restrict forwarded environment variables and stamping to avoid cache busting.
diff --git a/.bazelrc b/.bazelrc index 0b7e605..ee88287 100644 --- a/.bazelrc +++ b/.bazelrc
@@ -76,12 +76,24 @@ common:ci --//dev/release_artifacts:require_msvc_arm64=true # Additional config to use for readthedocs builds. -# See .readthedocs.yml for additional flags that can only be determined from -# the runtime environment. -build:rtd --stamp +# See .readthedocs.yml and docs/readthedocs_build.sh for additional flags that +# can only be determined from the runtime environment. # Some bzl files contain repos only available under bzlmod build:rtd --enable_bzlmod +# Disable typechecking and Rust lint aspects/output groups so doc builds do not +# fetch @rules_pyrefly/@rules_rust or run clippy/rustfmt checks. +build:rtd --aspects= +build:rtd --output_groups=-pyrefly,-clippy_checks,-rustfmt_checks +# Use the stripped Python runtime archive (without debug symbols) to reduce +# download size and share remote cache keys with GitHub Actions workflows. build:rtd --//python/config_settings:py_pbs_distribution=install_only_stripped +# Avoid downloading intermediate build outputs (such as protoc, shrubbery, and +# Stardoc protos) from remote cache/RBE; only materialize the top-level outputs +# needed by `bazel run //docs:readthedocs_install`. +build:rtd --remote_download_toplevel +# Fetch external repository archives through BuildBuddy's Remote Asset API +# cache so RTD builds do not hit GitHub Releases 504 timeouts. +build:rtd --experimental_remote_downloader=grpcs://rules-python.buildbuddy.io common --incompatible_python_disallow_native_rules common --incompatible_no_implicit_file_export @@ -105,7 +117,9 @@ # BuildBuddy's managed cloud pool only has Linux executors; disable remote # execution on macOS and Windows while keeping BES and remote caching enabled. common:macos --remote_executor= +build:macos --extra_execution_platforms= common:windows --remote_executor= +build:windows --extra_execution_platforms= import %workspace%/specialized_configs.bazelrc import %workspace%/buildbuddy.bazelrc
diff --git a/.github/actions/setup-bazel/action.yaml b/.github/actions/setup-bazel/action.yaml index 64f9b81..0f2c689 100644 --- a/.github/actions/setup-bazel/action.yaml +++ b/.github/actions/setup-bazel/action.yaml
@@ -58,7 +58,7 @@ # retries) or leaving the Bazel download to the first build step. - name: Download Bazel shell: bash - run: "${GITHUB_ACTION_PATH}/download_bazel.py" + run: "${GITHUB_ACTION_PATH}/../../../dev/download_bazel.py" # Use github-script with step-scoped env vars to write the key directly to # user.bazelrc without passing it via CLI args, job-wide environment
diff --git a/.readthedocs.yml b/.readthedocs.yml index 6e28fa3..efdfe1b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml
@@ -2,15 +2,21 @@ version: 2 build: - os: "ubuntu-22.04" - tools: - nodejs: "19" + os: "ubuntu-24.04" commands: # Log RTD env vars to help debug builds without dumping noisy or sensitive # unrelated env vars. - env | grep '^READTHEDOC*' - - npm install -g @bazel/bazelisk - - bazel version + - dev/download_bazel.py + # download_bazel.py installs into ~/.cache/bazelisk/bin, which is not on + # RTD's default PATH. Symlink into $READTHEDOCS_VIRTUALENV_PATH/bin, which + # is on PATH across steps. + - >- + mkdir -p "$READTHEDOCS_VIRTUALENV_PATH/bin" && + ln -s ~/.cache/bazelisk/bin/bazel "$READTHEDOCS_VIRTUALENV_PATH/bin/bazel" + # Verify bazel is on PATH and log the version in its own build step without + # starting the Bazel server JVM. + - bazel --version # Put the actual build behind a shell script because its easier to modify than # the yaml config. - docs/readthedocs_build.sh
diff --git a/MODULE.bazel b/MODULE.bazel index 8348c1d..30fbc28 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -191,6 +191,14 @@ bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True) bazel_dep(name = "rules_pkg", version = "1.2.0", dev_dependency = True) +bazel_dep(name = "toolchains_buildbuddy", version = "0.0.4", dev_dependency = True) + +buildbuddy = use_extension( + "@toolchains_buildbuddy//:extensions.bzl", + "buildbuddy", + dev_dependency = True, +) +buildbuddy.platform(buildbuddy_container_image = "UBUNTU24_04_IMAGE") rust = use_extension( "@rules_rust//rust:extensions.bzl",
diff --git a/buildbuddy.bazelrc b/buildbuddy.bazelrc index ae3c888..65606a4 100644 --- a/buildbuddy.bazelrc +++ b/buildbuddy.bazelrc
@@ -9,6 +9,10 @@ common:_buildbuddy_base --remote_cache=grpcs://rules-python.buildbuddy.io common:_buildbuddy_base --remote_timeout=10m common:_buildbuddy_base --remote_executor=grpcs://rules-python.buildbuddy.io +# Use BuildBuddy's Ubuntu 24.04 RBE container and hermetic GCC 13 toolchain +# instead of its default Ubuntu 16.04 executor image (GCC 5.4.0). +build:_buildbuddy_base --extra_execution_platforms=@toolchains_buildbuddy//platforms:linux_x86_64 +build:_buildbuddy_base --extra_toolchains=@toolchains_buildbuddy//toolchains/cc:ubuntu_gcc_x86_64 # Compress remote cache transfers with zstd to improve network throughput. common:_buildbuddy_base --remote_cache_compression # Chunk large blobs using FastCDC (SplitBlob/SpliceBlob) for efficient @@ -26,10 +30,10 @@ common:_buildbuddy_base --experimental_profile_include_target_label common:_buildbuddy_base --experimental_profile_include_primary_output # Re-apply `--enable_platform_specific_config` after setting `--remote_executor` -# so `common:macos` and `common:windows` in `.bazelrc` inherit the priority of -# `--config=buildbuddy_*` (e.g. when set in `~/.bazelrc` by `setup-bazel`, -# which has higher priority than `.bazelrc`'s top-level -# `--enable_platform_specific_config`). +# and `--extra_execution_platforms` so `macos` and `windows` in `.bazelrc` +# inherit the priority of `--config=buildbuddy_*` (e.g. when set in +# `~/.bazelrc` by `setup-bazel`, which has higher priority than `.bazelrc`'s +# top-level `--enable_platform_specific_config`). common:_buildbuddy_base --enable_platform_specific_config # For PR presubmit runs.
diff --git a/dev/BUILD.bazel b/dev/BUILD.bazel index 86a2298..056dfa4 100644 --- a/dev/BUILD.bazel +++ b/dev/BUILD.bazel
@@ -3,6 +3,11 @@ licenses(["notice"]) +exports_files( + ["download_bazel.py"], + visibility = ["//:__subpackages__"], +) + # Run bazel run //dev:uv_lock.update lock( name = "uv_lock",
diff --git a/.github/actions/setup-bazel/download_bazel.py b/dev/download_bazel.py similarity index 88% rename from .github/actions/setup-bazel/download_bazel.py rename to dev/download_bazel.py index 29c96bc..f64ce1d 100755 --- a/.github/actions/setup-bazel/download_bazel.py +++ b/dev/download_bazel.py
@@ -21,16 +21,15 @@ from __future__ import annotations +import asyncio import hashlib import io import os import platform import shutil import stat -import subprocess import sys import tarfile -import time import urllib.request from pathlib import Path @@ -143,10 +142,16 @@ ) -def _fetch_bazelisk(url: str, os_name: str, arch_name: str, ext: str) -> bytes | None: +def _read_url(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECONDS) as resp: + return resp.read() + + +async def _fetch_bazelisk( + url: str, os_name: str, arch_name: str, ext: str +) -> bytes | None: try: - with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response: - data = response.read() + data = await asyncio.to_thread(_read_url, url) if url.endswith(".tgz"): member_name = f"package/bazelisk-{os_name}_{arch_name}{ext}" with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: @@ -172,7 +177,7 @@ _add_to_path(bin_dir) -def _setup_bazelisk( +async def _setup_bazelisk( max_attempts: int = MAX_ATTEMPTS, initial_delay: float = INITIAL_DELAY_SECONDS, ) -> None: @@ -202,17 +207,18 @@ delay = initial_delay for attempt in range(1, max_attempts + 1): for url, expected_sha in sources: - data = _fetch_bazelisk(url, os_name, arch_name, ext) + data = await _fetch_bazelisk(url, os_name, arch_name, ext) if data is not None and _verify_sha256_bytes(data, expected_sha): _install_bazelisk_binary(data, bin_dir, bazel_bin, bazelisk_bin) return if attempt < max_attempts: print( - f"::warning::Failed to download Bazelisk (attempt {attempt}/{max_attempts}); retrying in {delay:g}s...", + f"::warning::Failed to download Bazelisk " + f"(attempt {attempt}/{max_attempts}); retrying in {delay:g}s...", flush=True, ) - time.sleep(delay) + await asyncio.sleep(delay) delay *= 2 print( @@ -223,27 +229,29 @@ sys.exit(1) -def _run_bazel_version() -> bool: +async def _run_bazel_version() -> bool: bazel_exe = shutil.which("bazel", path=os.environ.get("PATH")) or "bazel" - result = subprocess.run([bazel_exe, "--version"], check=False) - return result.returncode == 0 + proc = await asyncio.create_subprocess_exec(bazel_exe, "--version") + returncode = await proc.wait() + return returncode == 0 -def _download_bazel( +async def _download_bazel( max_attempts: int = MAX_ATTEMPTS, initial_delay: float = INITIAL_DELAY_SECONDS, ) -> None: delay = initial_delay for attempt in range(1, max_attempts + 1): - if _run_bazel_version(): + if await _run_bazel_version(): return if attempt < max_attempts: print( - f"::warning::Failed to download Bazel (attempt {attempt}/{max_attempts}); retrying in {delay:g}s...", + f"::warning::Failed to download Bazel " + f"(attempt {attempt}/{max_attempts}); retrying in {delay:g}s...", flush=True, ) - time.sleep(delay) + await asyncio.sleep(delay) delay *= 2 print( @@ -254,10 +262,10 @@ sys.exit(1) -def main() -> None: - _setup_bazelisk() - _download_bazel() +async def main() -> None: + await _setup_bazelisk() + await _download_bazel() if __name__ == "__main__": - main() + asyncio.run(main())
diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 4eb0f92..3c44bd6 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel
@@ -25,6 +25,11 @@ licenses(["notice"]) # Apache 2.0 +exports_files( + ["readthedocs_build.sh"], + visibility = ["//:__subpackages__"], +) + # We only build for Linux and Mac because: # 1. The actual doc process only runs on Linux # 2. Mac is a common development platform, and is close enough to Linux
diff --git a/docs/readthedocs_build.sh b/docs/readthedocs_build.sh index cd60792..4a69061 100755 --- a/docs/readthedocs_build.sh +++ b/docs/readthedocs_build.sh
@@ -2,19 +2,66 @@ set -eou pipefail -declare -a extra_env -while IFS='=' read -r -d '' name value; do - if [[ "$name" == READTHEDOCS* ]]; then - extra_env+=("--@sphinxdocs//sphinxdocs:extra_env=$name=$value") +# Only forward environment variables that docs/conf.py actually reads so +# volatile per-container variables (like HOSTNAME or READTHEDOCS_OUTPUT) do not +# bust the SphinxBuildDocs action cache. +declare -a build_flags=( + "--@sphinxdocs//sphinxdocs:extra_defines=version=${READTHEDOCS_VERSION:-0.0.0}" + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS=${READTHEDOCS:-}" +) + +if [[ "${READTHEDOCS_VERSION_TYPE:-}" != "external" ]]; then + build_flags+=( + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_GIT_COMMIT_HASH=${READTHEDOCS_GIT_COMMIT_HASH:-}" + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_GIT_IDENTIFIER=${READTHEDOCS_GIT_IDENTIFIER:-}" + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_VERSION=${READTHEDOCS_VERSION:-}" + ) +fi + +if [[ "${READTHEDOCS_VERSION_TYPE:-}" == "tag" || \ + "${READTHEDOCS_VERSION:-}" == "stable" ]]; then + build_flags+=("--stamp") +fi + +buildbuddy_config="" +buildbuddy_key="" +if [[ "${READTHEDOCS_VERSION_TYPE:-}" == "external" && \ + -n "${BUILDBUDDY_CAS_ONLY_KEY:-}" ]]; then + buildbuddy_config="buildbuddy_presubmit" + buildbuddy_key="${BUILDBUDDY_CAS_ONLY_KEY}" +elif [[ "${READTHEDOCS_VERSION_TYPE:-}" != "external" && \ + -n "${BUILDBUDDY_READ_WRITE_KEY:-}" ]]; then + buildbuddy_config="buildbuddy_postsubmit" + buildbuddy_key="${BUILDBUDDY_READ_WRITE_KEY}" +fi + +if [[ -n "${buildbuddy_config}" && -n "${buildbuddy_key}" ]]; then + printf 'common:%s --remote_header=x-buildbuddy-api-key=%s\n' \ + "${buildbuddy_config}" "${buildbuddy_key}" >> user.bazelrc + build_flags+=( + "--config=${buildbuddy_config}" + "--build_metadata=ROLE=CI" + "--build_metadata=VISIBILITY=PUBLIC" + ) + if [[ -n "${READTHEDOCS_GIT_COMMIT_HASH:-}" ]]; then + build_flags+=( + "--build_metadata=COMMIT_SHA=${READTHEDOCS_GIT_COMMIT_HASH}" + ) fi -done < <(env -0) + if [[ "${READTHEDOCS_VERSION_TYPE:-}" == "external" && \ + -n "${READTHEDOCS_VERSION:-}" ]]; then + build_flags+=("--build_metadata=PR_NUMBER=${READTHEDOCS_VERSION}") + elif [[ -n "${READTHEDOCS_GIT_IDENTIFIER:-}" ]]; then + # ${var//[^...]/} strips characters outside [a-zA-Z0-9._/-]. + safe_branch="${READTHEDOCS_GIT_IDENTIFIER//[^a-zA-Z0-9._\/-]/}" + build_flags+=("--build_metadata=BRANCH_NAME=${safe_branch}") + fi +fi -# In order to get the build number, we extract it from the host name -extra_env+=("--@sphinxdocs//sphinxdocs:extra_env=HOSTNAME=$HOSTNAME") - -set -x -bazel run \ - --config=rtd \ - "--@sphinxdocs//sphinxdocs:extra_defines=version=$READTHEDOCS_VERSION" \ - "${extra_env[@]}" \ - //docs:readthedocs_install +( + set -x + bazel run \ + --config=rtd \ + "${build_flags[@]}" \ + //docs:readthedocs_install +)
diff --git a/downloader_config.cfg b/downloader_config.cfg index c673c3b..fb6061f 100644 --- a/downloader_config.cfg +++ b/downloader_config.cfg
@@ -1,23 +1,49 @@ # Try GitHub first (primary) +rewrite ^github\.com/abseil/abseil-cpp/(.*) github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) github.com/protocolbuffers/protobuf/$1 # Fall back to mirror (secondary) # Tracking upstream BCR mirror addition: https://github.com/bazelbuild/platforms/issues/139 +rewrite ^github\.com/abseil/abseil-cpp/(.*) mirror.bazel.build/github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) mirror.bazel.build/github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) mirror.bazel.build/github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) mirror.bazel.build/github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) mirror.bazel.build/github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) mirror.bazel.build/github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) mirror.bazel.build/github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) mirror.bazel.build/github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) mirror.bazel.build/github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) mirror.bazel.build/github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) mirror.bazel.build/github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) mirror.bazel.build/github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) mirror.bazel.build/github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) mirror.bazel.build/github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) mirror.bazel.build/github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) mirror.bazel.build/github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) mirror.bazel.build/github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) mirror.bazel.build/github.com/protocolbuffers/protobuf/$1
diff --git a/gazelle/downloader_config.cfg b/gazelle/downloader_config.cfg index c673c3b..fb6061f 100644 --- a/gazelle/downloader_config.cfg +++ b/gazelle/downloader_config.cfg
@@ -1,23 +1,49 @@ # Try GitHub first (primary) +rewrite ^github\.com/abseil/abseil-cpp/(.*) github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) github.com/protocolbuffers/protobuf/$1 # Fall back to mirror (secondary) # Tracking upstream BCR mirror addition: https://github.com/bazelbuild/platforms/issues/139 +rewrite ^github\.com/abseil/abseil-cpp/(.*) mirror.bazel.build/github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) mirror.bazel.build/github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) mirror.bazel.build/github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) mirror.bazel.build/github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) mirror.bazel.build/github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) mirror.bazel.build/github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) mirror.bazel.build/github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) mirror.bazel.build/github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) mirror.bazel.build/github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) mirror.bazel.build/github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) mirror.bazel.build/github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) mirror.bazel.build/github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) mirror.bazel.build/github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) mirror.bazel.build/github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) mirror.bazel.build/github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) mirror.bazel.build/github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) mirror.bazel.build/github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) mirror.bazel.build/github.com/protocolbuffers/protobuf/$1
diff --git a/sphinxdocs/downloader_config.cfg b/sphinxdocs/downloader_config.cfg index c673c3b..fb6061f 100644 --- a/sphinxdocs/downloader_config.cfg +++ b/sphinxdocs/downloader_config.cfg
@@ -1,23 +1,49 @@ # Try GitHub first (primary) +rewrite ^github\.com/abseil/abseil-cpp/(.*) github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) github.com/protocolbuffers/protobuf/$1 # Fall back to mirror (secondary) # Tracking upstream BCR mirror addition: https://github.com/bazelbuild/platforms/issues/139 +rewrite ^github\.com/abseil/abseil-cpp/(.*) mirror.bazel.build/github.com/abseil/abseil-cpp/$1 rewrite ^github\.com/bazel-contrib/bazel-gazelle/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel-gazelle/$1 rewrite ^github\.com/bazel-contrib/bazel_features/(.*) mirror.bazel.build/github.com/bazel-contrib/bazel_features/$1 rewrite ^github\.com/bazel-contrib/rules_go/(.*) mirror.bazel.build/github.com/bazel-contrib/rules_go/$1 +rewrite ^github\.com/bazel-contrib/supply-chain/(.*) mirror.bazel.build/github.com/bazel-contrib/supply-chain/$1 +rewrite ^github\.com/bazelbuild/apple_support/(.*) mirror.bazel.build/github.com/bazelbuild/apple_support/$1 rewrite ^github\.com/bazelbuild/bazel-skylib/(.*) mirror.bazel.build/github.com/bazelbuild/bazel-skylib/$1 rewrite ^github\.com/bazelbuild/platforms/(.*) mirror.bazel.build/github.com/bazelbuild/platforms/$1 -rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 -rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_android/(.*) mirror.bazel.build/github.com/bazelbuild/rules_android/$1 +rewrite ^github\.com/bazelbuild/rules_cc/(.*) mirror.bazel.build/github.com/bazelbuild/rules_cc/$1 rewrite ^github\.com/bazelbuild/rules_java/(.*) mirror.bazel.build/github.com/bazelbuild/rules_java/$1 +rewrite ^github\.com/bazelbuild/rules_jvm_external/(.*) mirror.bazel.build/github.com/bazelbuild/rules_jvm_external/$1 +rewrite ^github\.com/bazelbuild/rules_kotlin/(.*) mirror.bazel.build/github.com/bazelbuild/rules_kotlin/$1 +rewrite ^github\.com/bazelbuild/rules_license/(.*) mirror.bazel.build/github.com/bazelbuild/rules_license/$1 +rewrite ^github\.com/bazelbuild/rules_pkg/(.*) mirror.bazel.build/github.com/bazelbuild/rules_pkg/$1 +rewrite ^github\.com/bazelbuild/rules_shell/(.*) mirror.bazel.build/github.com/bazelbuild/rules_shell/$1 +rewrite ^github\.com/bazelbuild/rules_swift/(.*) mirror.bazel.build/github.com/bazelbuild/rules_swift/$1 +rewrite ^github\.com/bazelbuild/rules_testing/(.*) mirror.bazel.build/github.com/bazelbuild/rules_testing/$1 rewrite ^github\.com/bazelbuild/stardoc/(.*) mirror.bazel.build/github.com/bazelbuild/stardoc/$1 +rewrite ^github\.com/google/googletest/(.*) mirror.bazel.build/github.com/google/googletest/$1 +rewrite ^github\.com/jvolkman/toml\.bzl/(.*) mirror.bazel.build/github.com/jvolkman/toml.bzl/$1 +rewrite ^github\.com/protocolbuffers/protobuf/(.*) mirror.bazel.build/github.com/protocolbuffers/protobuf/$1
diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index 1f99e5a..0b1a539 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel
@@ -83,8 +83,8 @@ py_library( name = "download_bazel", testonly = True, - srcs = ["//.github/actions/setup-bazel:download_bazel.py"], - imports = ["../../.github/actions/setup-bazel"], + srcs = ["//dev:download_bazel.py"], + imports = ["../../dev"], target_compatible_with = NOT_WINDOWS, ) @@ -95,6 +95,7 @@ "//:.bazelrc", "//:buildbuddy.bazelrc", "//.github/actions/setup-bazel:all_files", + "//dev:download_bazel.py", ], target_compatible_with = NOT_WINDOWS, deps = [ @@ -103,3 +104,15 @@ "@pypi//pyyaml", ], ) + +pytest_test( + name = "readthedocs_test", + srcs = ["readthedocs_test.py"], + data = [ + "//docs:readthedocs_build.sh", + ], + target_compatible_with = NOT_WINDOWS, + deps = [ + "//python/runfiles", + ], +)
diff --git a/tests/workflows/readthedocs_test.py b/tests/workflows/readthedocs_test.py new file mode 100644 index 0000000..0bee6f7 --- /dev/null +++ b/tests/workflows/readthedocs_test.py
@@ -0,0 +1,163 @@ +"""Tests for Read the Docs build script.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from python.runfiles import runfiles + + +def _write_fake_bazel(bin_dir: Path, body: str) -> Path: + bin_dir.mkdir(parents=True, exist_ok=True) + fake_bazel = bin_dir / "bazel" + fake_bazel.write_text( + f"#!/usr/bin/env bash\nset -euo pipefail\n{body}\n", encoding="utf-8" + ) + fake_bazel.chmod(0o755) + return fake_bazel + + +def _rtd_script() -> Path: + rf = runfiles.CreateOrRaise() + return rf.root() / "rules_python/docs/readthedocs_build.sh" + + +def test_pr_uses_cas_only_and_presubmit(tmp_path: Path): + """Verifies PR builds use BUILDBUDDY_CAS_ONLY_KEY and presubmit config.""" + calls_file = tmp_path / "bazel_calls.txt" + _write_fake_bazel( + tmp_path / "bin", + f'printf "%s\\n" "$*" >> "{calls_file}"\n', + ) + env = { + "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ.get('PATH', '')}", + "HOME": str(tmp_path / "home"), + "HOSTNAME": "build-999-project-1-rules-python", + "READTHEDOCS": "True", + "READTHEDOCS_VERSION": "4242", + "READTHEDOCS_VERSION_TYPE": "external", + "READTHEDOCS_GIT_IDENTIFIER": "4242", + "READTHEDOCS_GIT_COMMIT_HASH": "deadbeef1234", + "READTHEDOCS_OUTPUT": str(tmp_path / "out"), + "BUILDBUDDY_CAS_ONLY_KEY": "rtd_cas_only_val", + "BUILDBUDDY_READ_WRITE_KEY": "rtd_rw_val", + } + + result = subprocess.run( + [str(_rtd_script())], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=True, + ) + + combined_output = result.stdout + result.stderr + assert "rtd_cas_only_val" not in combined_output + assert "rtd_rw_val" not in combined_output + expected_rc = ( + "common:buildbuddy_presubmit " + "--remote_header=x-buildbuddy-api-key=rtd_cas_only_val\n" + ) + assert (tmp_path / "user.bazelrc").read_text(encoding="utf-8") == (expected_rc) + + calls = calls_file.read_text(encoding="utf-8").splitlines() + assert len(calls) == 1 + run_call = calls[0] + assert "--config=rtd" in run_call + assert "--config=buildbuddy_presubmit" in run_call + assert "--build_metadata=PR_NUMBER=4242" in run_call + assert "--build_metadata=COMMIT_SHA=deadbeef1234" in run_call + assert "--stamp" not in run_call + assert "HOSTNAME" not in run_call + assert "READTHEDOCS_OUTPUT" not in run_call + assert "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS=True" in run_call + assert "READTHEDOCS_GIT_COMMIT_HASH" not in run_call + assert "READTHEDOCS_GIT_IDENTIFIER" not in run_call + assert "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_VERSION=" not in run_call + + +def test_non_pr_uses_read_write_and_stamps_tags( + tmp_path: Path, +): + """Verifies non-PR builds use BUILDBUDDY_READ_WRITE_KEY and stamp tags.""" + calls_file = tmp_path / "bazel_calls.txt" + _write_fake_bazel( + tmp_path / "bin", + f'printf "%s\\n" "$*" >> "{calls_file}"\n', + ) + base_env = { + "PATH": f"{tmp_path / 'bin'}{os.pathsep}{os.environ.get('PATH', '')}", + "HOME": str(tmp_path / "home"), + "READTHEDOCS": "True", + "READTHEDOCS_GIT_COMMIT_HASH": "cafebabe5678", + "BUILDBUDDY_CAS_ONLY_KEY": "rtd_cas_only_val", + "BUILDBUDDY_READ_WRITE_KEY": "rtd_rw_val", + } + + # 1. Branch build (latest / main): no --stamp + branch_dir = tmp_path / "branch_run" + branch_dir.mkdir() + subprocess.run( + [str(_rtd_script())], + cwd=branch_dir, + env={ + **base_env, + "READTHEDOCS_VERSION": "latest", + "READTHEDOCS_VERSION_TYPE": "branch", + "READTHEDOCS_GIT_IDENTIFIER": "main", + }, + capture_output=True, + text=True, + check=True, + ) + expected_branch_rc = ( + "common:buildbuddy_postsubmit --remote_header=x-buildbuddy-api-key=rtd_rw_val\n" + ) + assert (branch_dir / "user.bazelrc").read_text(encoding="utf-8") == ( + expected_branch_rc + ) + + # 2. Tag build (release): passes --stamp + tag_dir = tmp_path / "tag_run" + tag_dir.mkdir() + subprocess.run( + [str(_rtd_script())], + cwd=tag_dir, + env={ + **base_env, + "READTHEDOCS_VERSION": "1.2.3", + "READTHEDOCS_VERSION_TYPE": "tag", + "READTHEDOCS_GIT_IDENTIFIER": "1.2.3", + }, + capture_output=True, + text=True, + check=True, + ) + + calls = calls_file.read_text(encoding="utf-8").splitlines() + assert len(calls) == 2 + branch_run_call = calls[0] + tag_run_call = calls[1] + + assert "--config=buildbuddy_postsubmit" in branch_run_call + assert "--build_metadata=BRANCH_NAME=main" in branch_run_call + assert "--stamp" not in branch_run_call + assert ( + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_GIT_COMMIT_HASH=cafebabe5678" + in branch_run_call + ) + assert ( + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_GIT_IDENTIFIER=main" + in branch_run_call + ) + assert ( + "--@sphinxdocs//sphinxdocs:extra_env=READTHEDOCS_VERSION=latest" + in branch_run_call + ) + + assert "--config=buildbuddy_postsubmit" in tag_run_call + assert "--build_metadata=BRANCH_NAME=1.2.3" in tag_run_call + assert "--stamp" in tag_run_call
diff --git a/tests/workflows/setup_bazel_test.py b/tests/workflows/setup_bazel_test.py index 5d7fcaa..4b1ff6b 100644 --- a/tests/workflows/setup_bazel_test.py +++ b/tests/workflows/setup_bazel_test.py
@@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import hashlib import os import stat @@ -35,8 +36,9 @@ def test_action_yaml_invokes_executable_download_script(): """Verifies setup-bazel action.yaml runs download_bazel.py and omits bazelisk-version.""" + rf = runfiles.CreateOrRaise() action_dir = _action_dir() - script = action_dir / "download_bazel.py" + script = rf.root() / "rules_python/dev/download_bazel.py" assert script.is_file() assert bool(script.stat().st_mode & stat.S_IXUSR), ( "download_bazel.py must be executable" @@ -44,7 +46,7 @@ action = yaml.safe_load((action_dir / "action.yaml").read_text(encoding="utf-8")) steps = action["runs"]["steps"] - assert any("download_bazel.py" in step.get("run", "") for step in steps) + assert any("dev/download_bazel.py" in step.get("run", "") for step in steps) setup_step = next(s for s in steps if s.get("name") == "Setup Bazel") assert "bazelisk-version" not in setup_step["with"] @@ -115,12 +117,25 @@ executor_line = ( "common:_buildbuddy_base --remote_executor=grpcs://rules-python.buildbuddy.io" ) + rbe_platform_line = ( + "build:_buildbuddy_base " + "--extra_execution_platforms=@toolchains_buildbuddy//platforms:linux_x86_64" + ) + rbe_toolchain_line = ( + "build:_buildbuddy_base " + "--extra_toolchains=@toolchains_buildbuddy//toolchains/cc:ubuntu_gcc_x86_64" + ) platform_config_line = "common:_buildbuddy_base --enable_platform_specific_config" assert executor_line in buildbuddy_lines + assert rbe_platform_line in buildbuddy_lines + assert rbe_toolchain_line in buildbuddy_lines assert platform_config_line in buildbuddy_lines assert buildbuddy_lines.index(executor_line) < buildbuddy_lines.index( platform_config_line ) + assert buildbuddy_lines.index(rbe_platform_line) < buildbuddy_lines.index( + platform_config_line + ) bazelrc_path = rf.root() / "rules_python/.bazelrc" bazelrc_lines = [ @@ -129,7 +144,9 @@ if line.strip() and not line.strip().startswith("#") ] assert "common:macos --remote_executor=" in bazelrc_lines + assert "build:macos --extra_execution_platforms=" in bazelrc_lines assert "common:windows --remote_executor=" in bazelrc_lines + assert "build:windows --extra_execution_platforms=" in bazelrc_lines def test_download_bazelisk_from_primary_url( @@ -160,7 +177,7 @@ [(asset.as_uri(), _sha256_file(asset))], ) - download_bazel.main() + asyncio.run(download_bazel.main()) out = capsys.readouterr().out bin_dir = home / ".cache/bazelisk/bin" @@ -204,7 +221,7 @@ [(missing_url, sha), (tgz_path.as_uri(), sha)], ) - download_bazel.main() + asyncio.run(download_bazel.main()) out = capsys.readouterr().out assert f"::warning::Failed to fetch {missing_url}" in out @@ -237,7 +254,7 @@ [((tmp_path / "missing").as_uri(), _sha256_file(cached_bazel))], ) - download_bazel.main() + asyncio.run(download_bazel.main()) out = capsys.readouterr().out assert (home / ".cache/bazelisk/bin/bazelisk").is_file() @@ -269,7 +286,7 @@ ) with pytest.raises(SystemExit) as exc_info: - download_bazel._setup_bazelisk(max_attempts=2, initial_delay=0) + asyncio.run(download_bazel._setup_bazelisk(max_attempts=2, initial_delay=0)) assert exc_info.value.code == 1 out = capsys.readouterr().out @@ -294,7 +311,7 @@ ) monkeypatch.setenv("HOME", str(tmp_path / "home")) - download_bazel._download_bazel(max_attempts=5, initial_delay=0) + asyncio.run(download_bazel._download_bazel(max_attempts=5, initial_delay=0)) out = capsys.readouterr().out assert calls_file.read_text(encoding="utf-8").splitlines() == ["--version"] @@ -329,7 +346,7 @@ ) monkeypatch.setenv("HOME", str(home)) - download_bazel._download_bazel(max_attempts=5, initial_delay=0) + asyncio.run(download_bazel._download_bazel(max_attempts=5, initial_delay=0)) out = capsys.readouterr().out assert calls_file.read_text(encoding="utf-8").splitlines() == [ @@ -369,7 +386,7 @@ monkeypatch.setenv("BAZELISK_HOME", str(bazelisk_home)) with pytest.raises(SystemExit) as exc_info: - download_bazel._download_bazel(max_attempts=3, initial_delay=0) + asyncio.run(download_bazel._download_bazel(max_attempts=3, initial_delay=0)) assert exc_info.value.code == 1 out = capsys.readouterr().out