build(release): build tools for more python-build-standalone platforms (#4248) Prebuilt tools need to run wherever rules_python runs Python, i.e. on every platform that python-build-standalone supports. Linux x86_64 runners now also cross-compile static musl binaries for armv7, i686, ppc64le and riscv64, and a Windows x86_64 runner cross-compiles for 32-bit Windows with MSVC's x86 tools. Static Linux binaries don't depend on the system's libc, so one binary per CPU covers both glibc and musl, and arm64e Macs run the aarch64 binary. No job builds s390x yet: Rust has no musl std for it, and a static glibc build needs a C cross toolchain. The release transition now passes +crt-static for musl too, since Rust links musl dynamically by default for some CPUs, e.g. riscv64. crate_universe now resolves the crates' conditional deps for triples that match the release platforms by CPU and OS, and no longer marks the crates incompatible with other platforms, since rules_rust has no ppc64le triple. With the default triples, the crates were incompatible with e.g. x86_64 macOS and arm64 Windows, so those existing jobs would only have warned. The Linux Rust toolchain set is renamed to rust_release_linux to shorten its labels. Its name, and the new Windows set's, must sort after rules_rust's default sets' so that the defaults' x86_64 toolchains take precedence. Work towards #4216
diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml index 998c203..8971c6c 100644 --- a/.github/workflows/release_build_tools.yaml +++ b/.github/workflows/release_build_tools.yaml
@@ -1,7 +1,8 @@ -# Builds `//dev/release_artifacts:artifacts_for_release` for each platform; see -# dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the files -# for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so -# they're built natively. +# Builds `//dev/release_artifacts:artifacts_for_release` for the platforms in +# dev/platforms, which cover python-build-standalone's platforms except s390x; +# see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the +# files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so +# they're built on those OSes. # # For now, this only verifies that the artifacts build; they aren't attached to # releases yet. @@ -42,15 +43,25 @@ runner: macos-15 - triple: aarch64-pc-windows-msvc runner: windows-11-arm - # Cross-compiled; see dev/platforms/BUILD.bazel. - - triple: aarch64-unknown-linux-musl - runner: ubuntu-24.04 - triple: x86_64-apple-darwin runner: macos-15-intel - triple: x86_64-pc-windows-msvc runner: windows-2022 - triple: x86_64-unknown-linux-gnu runner: ubuntu-24.04 + # Cross-compiled; see dev/platforms/BUILD.bazel. + - triple: aarch64-unknown-linux-musl + runner: ubuntu-24.04 + - triple: armv7-unknown-linux-musleabihf + runner: ubuntu-24.04 + - triple: i686-pc-windows-msvc + runner: windows-2022 + - triple: i686-unknown-linux-musl + runner: ubuntu-24.04 + - triple: powerpc64le-unknown-linux-musl + runner: ubuntu-24.04 + - triple: riscv64gc-unknown-linux-musl + runner: ubuntu-24.04 env: REF: ${{ inputs.ref || github.ref_name }} TRIPLE: ${{ matrix.triple }} @@ -69,8 +80,9 @@ bazelisk-version: 1.29.0 # Release commands run main's workflows, even for patch releases of older - # release branches. Skip refs without the platforms that build.sh takes, - # since their scripts don't build for the job's platform, or don't exist. + # release branches. Skip refs without dev/platforms, since their scripts + # don't build for the job's platform, or don't exist. Refs whose + # dev/platforms lacks the job's platform fail the build, which only warns. - name: Build artifacts id: build if: hashFiles('dev/platforms/BUILD.bazel') != ''
diff --git a/MODULE.bazel b/MODULE.bazel index f0abc99..60975fc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -109,11 +109,37 @@ package = "sha2", version = "0.10.8", ) -rust_crates.from_specs() + +# The crates must build for every platform in dev/platforms. Conditional +# dependencies are resolved for these triples; rules_rust matches them by CPU +# and OS, e.g. i686-unknown-linux-gnu also matches the i686-unknown-linux-musl +# platform. +rust_crates.from_specs( + supported_platform_triples = [ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "armv7-unknown-linux-gnueabi", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + ], +) + +# rules_rust has no @rules_rust//rust/platform config_setting for Linux ppc64le, +# so don't mark the crates as incompatible with platforms that don't match one +# of the triples above. Such platforms get only the unconditional dependencies, +# which suffice for sha2. +rust_crates.render_config( + generate_target_compatible_with = False, +) use_repo(rust_crates, "crates") -# A Rust toolchain to cross-compile the release artifacts for Linux aarch64 -# from Linux x86_64; see dev/release_artifacts. It targets musl, which Rust can +# Rust toolchains to cross-compile the release artifacts for other Linux CPUs +# from Linux x86_64; see dev/release_artifacts. They target musl, which Rust can # link statically with its bundled rust-lld, so no C cross toolchain is needed. rust = use_extension( "@rules_rust//rust:extensions.bzl", @@ -124,7 +150,7 @@ # rules_rust also registers an x86_64 to x86_64 toolchain from this set, in # name order with its default ones, so the name must sort after # rust_linux_x86_64 for the default x86_64 toolchain to take precedence. - name = "rust_release_linux_x86_64", + name = "rust_release_linux", # The default allocator library needs a C toolchain for the target platform, # but there isn't one. This one is what rules_rust uses for e.g. Windows. allocator_library = "@rules_rust//rust/private/cc:empty", @@ -141,15 +167,85 @@ # built by the same version. versions = ["1.97.1"], ) -use_repo(rust, "rust_release_linux_x86_64__aarch64-unknown-linux-musl__stable") -# Registered here so that it takes precedence over rules_rust's default -# toolchains, whose Linux x86_64 to aarch64 one also matches the musl platform. +# Other targets for the same set; the first tag sets the shared attributes. +rust.repository_set( + name = "rust_release_linux", + target_compatible_with = [ + "@platforms//cpu:armv7", + "@platforms//os:linux", + "//dev/platforms:musl", + ], + target_triple = "armv7-unknown-linux-musleabihf", +) +rust.repository_set( + name = "rust_release_linux", + target_compatible_with = [ + "@platforms//cpu:x86_32", + "@platforms//os:linux", + "//dev/platforms:musl", + ], + target_triple = "i686-unknown-linux-musl", +) +rust.repository_set( + name = "rust_release_linux", + target_compatible_with = [ + "@platforms//cpu:ppc64le", + "@platforms//os:linux", + "//dev/platforms:musl", + ], + target_triple = "powerpc64le-unknown-linux-musl", +) +rust.repository_set( + name = "rust_release_linux", + target_compatible_with = [ + "@platforms//cpu:riscv64", + "@platforms//os:linux", + "//dev/platforms:musl", + ], + target_triple = "riscv64gc-unknown-linux-musl", +) +use_repo( + rust, + "rust_release_linux__aarch64-unknown-linux-musl__stable", + "rust_release_linux__armv7-unknown-linux-musleabihf__stable", + "rust_release_linux__i686-unknown-linux-musl__stable", + "rust_release_linux__powerpc64le-unknown-linux-musl__stable", + "rust_release_linux__riscv64gc-unknown-linux-musl__stable", +) + +# Registered here so that they take precedence over rules_rust's default +# toolchains, e.g. its Linux x86_64 to aarch64 one also matches the aarch64 +# musl platform. register_toolchains( - "@rust_release_linux_x86_64__aarch64-unknown-linux-musl__stable//:toolchain", + "@rust_release_linux__aarch64-unknown-linux-musl__stable//:toolchain", + "@rust_release_linux__armv7-unknown-linux-musleabihf__stable//:toolchain", + "@rust_release_linux__i686-unknown-linux-musl__stable//:toolchain", + "@rust_release_linux__powerpc64le-unknown-linux-musl__stable//:toolchain", + "@rust_release_linux__riscv64gc-unknown-linux-musl__stable//:toolchain", dev_dependency = True, ) +# A Rust toolchain to cross-compile the release artifacts for 32-bit Windows +# from Windows x86_64, with MSVC's x86 tools. Unlike the Linux ones, it isn't +# registered here: no default toolchain targets 32-bit Windows, and rules_rust +# registers it. +rust.repository_set( + # As above, its x86_64 toolchain's name must sort after the default one's. + # On Windows, rules_rust names toolchains after a hash of the full name: + # rw-1886791612 for this one, after rust_windows_x86_64's rw-1704990954. + name = "rust_release_windows", + edition = "2021", + exec_triple = "x86_64-pc-windows-msvc", + target_compatible_with = [ + "@platforms//cpu:x86_32", + "@platforms//os:windows", + ], + target_triple = "i686-pc-windows-msvc", + # See above. + versions = ["1.97.1"], +) + # Use the Rust-based build tools when developing rules_python itself. register_toolchains( "//dev/dev_only_toolchains:exe_zip_maker_toolchain",
diff --git a/dev/platforms/BUILD.bazel b/dev/platforms/BUILD.bazel index d381862..52880c4 100644 --- a/dev/platforms/BUILD.bazel +++ b/dev/platforms/BUILD.bazel
@@ -1,7 +1,13 @@ # Platforms that rules_python's release artifacts are built for, named after -# their Rust target triples; see dev/release_artifacts/build.sh. Linux x86_64 -# hosts cross-compile for the musl one with the musl Rust toolchain in -# MODULE.bazel. The others are built on hosts that match them. +# their Rust target triples; see dev/release_artifacts/build.sh. They cover the +# platforms that python-build-standalone supports: the Linux files are static, +# so they also run on Linux platforms with other libcs, and arm64e Macs run the +# aarch64 macOS file. s390x isn't covered yet: Rust ships no musl std for it, +# and a static glibc build needs a C cross toolchain. +# +# Linux x86_64 hosts cross-compile for the musl ones, and Windows x86_64 hosts +# for i686-pc-windows-msvc, with the Rust toolchains in MODULE.bazel. The +# others are built on hosts that match them. package(default_visibility = ["//:__subpackages__"]) @@ -31,6 +37,50 @@ ) platform( + name = "armv7-unknown-linux-musleabihf", + constraint_values = [ + "@platforms//cpu:armv7", + "@platforms//os:linux", + ":musl", + ], +) + +platform( + name = "i686-pc-windows-msvc", + constraint_values = [ + "@platforms//cpu:x86_32", + "@platforms//os:windows", + ], +) + +platform( + name = "i686-unknown-linux-musl", + constraint_values = [ + "@platforms//cpu:x86_32", + "@platforms//os:linux", + ":musl", + ], +) + +platform( + name = "powerpc64le-unknown-linux-musl", + constraint_values = [ + "@platforms//cpu:ppc64le", + "@platforms//os:linux", + ":musl", + ], +) + +platform( + name = "riscv64gc-unknown-linux-musl", + constraint_values = [ + "@platforms//cpu:riscv64", + "@platforms//os:linux", + ":musl", + ], +) + +platform( name = "x86_64-apple-darwin", constraint_values = [ "@platforms//cpu:x86_64", @@ -55,13 +105,13 @@ ) # The libc of Linux platforms. Only musl is a constraint value: platforms -# without it, like the host, use glibc. It also keeps the musl Rust toolchain +# without it, like the host, use glibc. It also keeps the musl Rust toolchains # in MODULE.bazel from being used for glibc platforms. constraint_setting(name = "libc") constraint_value( name = "musl", constraint_setting = ":libc", - # The musl Rust toolchain's repo, generated by rules_rust, refers to it. + # The musl Rust toolchains' repos, generated by rules_rust, refer to it. visibility = ["//visibility:public"], )
diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index 857f049..b5bde5f 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl
@@ -8,15 +8,15 @@ def _release_transition_impl(settings, attr): features = [] rustc_flags = [] - if attr.target_os == "linux" and attr.target_libc != "musl": + if attr.target_os == "linux": # Rust links glibc dynamically, so the files would require at least # the build machine's glibc version. Link it statically so they run on - # older distros too. The gold linker can't link static glibc, so use - # bfd. Rust already links musl statically. - rustc_flags = [ - "-Ctarget-feature=+crt-static", - "-Clink-arg=-fuse-ld=bfd", - ] + # older distros too. Rust links musl statically for most CPUs, but not + # all, e.g. riscv64. + rustc_flags = ["-Ctarget-feature=+crt-static"] + if attr.target_libc != "musl": + # The gold linker can't link static glibc, so use bfd. + rustc_flags.append("-Clink-arg=-fuse-ld=bfd") elif attr.target_os == "windows": # Statically link the C runtime so that the VC++ redistributable isn't # required.
diff --git a/dev/release_artifacts/verify.sh b/dev/release_artifacts/verify.sh index 8e86d10..8b93107 100755 --- a/dev/release_artifacts/verify.sh +++ b/dev/release_artifacts/verify.sh
@@ -21,17 +21,24 @@ echo "::error::$bin isn't statically linked: $info" exit 1 fi - # Some Linux files are cross-compiled, so check that they're for the - # right CPU. + # Most Linux files are cross-compiled, so check that they're for the + # right CPU, as `file` describes it. `cpu` is a glob because file 5.46 + # renamed "Intel 80386" to "Intel i386". case "$triple" in aarch64-*) cpu="ARM aarch64" ;; + armv7-*) cpu="ARM" ;; + i686-*) cpu="Intel *386" ;; + powerpc64le-*) cpu="64-bit PowerPC or cisco 7500" ;; + riscv64gc-*) cpu="UCB RISC-V" ;; x86_64-*) cpu="x86-64" ;; *) echo "::error::$bin: no CPU check for $triple" exit 1 ;; esac - if [[ "$info" != *", $cpu,"* ]]; then + pattern="*, $cpu,*" + # shellcheck disable=SC2053 # The pattern is a glob. + if [[ "$info" != $pattern ]]; then echo "::error::$bin isn't for $cpu: $info" exit 1 fi