build(release): build prebuilt tools in the release workflow (#4217) This makes the release workflow build the exe zip maker tool as part of releases. Because more plumbing is needed to integrate prebuilt binaries, they are just built, but not actually included into the final release. For that same reason, failures to build are ignored for now. This provides some basic verification of the overall flow and will give an idea of how expensive it is to build them. The binaries are built to be fully statically linked to minimize system dependencies. Work towards https://github.com/bazel-contrib/rules_python/issues/4216
diff --git a/.github/workflows/release_build_tools.yaml b/.github/workflows/release_build_tools.yaml new file mode 100644 index 0000000..0801df0 --- /dev/null +++ b/.github/workflows/release_build_tools.yaml
@@ -0,0 +1,89 @@ +# Builds `//dev/release_artifacts:artifacts_for_release` natively for each +# platform; see dev/release_artifacts/build.sh. +# +# For now, this only verifies that the artifacts build; they aren't attached to +# releases yet. +name: "Release: Build Tools" + +on: + workflow_call: + inputs: + ref: + description: "The git ref (e.g. release tag) to build" + required: true + type: string + workflow_dispatch: + inputs: + ref: + description: "The git ref (e.g. release tag) to build. Defaults to the selected ref." + required: false + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + build: + name: Build ${{ matrix.triple }} + runs-on: ${{ matrix.runner }} + strategy: + # Report the result of every platform instead of stopping at the first + # failure. + fail-fast: false + matrix: + include: + - triple: aarch64-apple-darwin + runner: macos-15 + - triple: aarch64-pc-windows-msvc + runner: windows-11-arm + - triple: aarch64-unknown-linux-gnu + runner: ubuntu-24.04-arm + - triple: x86_64-apple-darwin + runner: macos-15-intel + - triple: x86_64-pc-windows-msvc + runner: windows-2022 + - triple: x86_64-unknown-linux-gnu + runner: ubuntu-24.04 + env: + REF: ${{ inputs.ref || github.ref_name }} + TRIPLE: ${{ matrix.triple }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.19.0 + with: + # Windows images also have a fixed-version `bazel` on PATH. 1.20.0, + # used by the other workflows, has no windows-arm64 build. + bazelisk-version: 1.29.0 + + # Release commands run main's workflows, even for patch releases of older + # release branches, which don't have the script; skip those. + - name: Build artifacts + id: build + if: hashFiles('dev/release_artifacts/build.sh') != '' + run: dev/release_artifacts/build.sh "$REF" + + # build.sh only sets `files` if the build succeeded. Otherwise, it adds a + # warning instead of failing, since releases don't use the files yet. + - name: Verify artifacts + if: steps.build.outputs.files != '' + run: dev/release_artifacts/verify.sh "$TRIPLE" + + # Stores the files with this workflow run so they can be downloaded from + # the run's page. Nothing is added to the GitHub release. + - name: Upload artifacts + if: steps.build.outputs.files != '' + uses: actions/upload-artifact@v7 + with: + name: release-tools-${{ matrix.triple }} + path: ${{ steps.build.outputs.files }} + if-no-files-found: error
diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index eedf653..cfa34cc 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml
@@ -46,6 +46,15 @@ default: true jobs: + build_tools: + # For now, this only verifies that the tools build. They aren't attached + # to the release, so no other job waits on this one. A failed build only + # adds a warning to this run; a failed check of the built files fails it. + name: Build tools + uses: ./.github/workflows/release_build_tools.yaml + with: + ref: ${{ inputs.tag_name || github.ref_name }} + release: name: Release runs-on: ubuntu-latest
diff --git a/RELEASING.md b/RELEASING.md index 2f94fde..29251e0 100644 --- a/RELEASING.md +++ b/RELEASING.md
@@ -67,6 +67,12 @@ gh workflow run release_publish.yaml --ref <TAG> -f publish_to_pypi=false ``` +### Prebuilt tools + +The release workflow also builds `//dev/release_artifacts:artifacts_for_release` +for each platform to verify it builds. For now, the files aren't attached to +the release. + ### Manually publishing to PyPI If PyPI publishing failed or was skipped during the main release, the PyPI
diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel new file mode 100644 index 0000000..d44628d --- /dev/null +++ b/dev/release_artifacts/BUILD.bazel
@@ -0,0 +1,20 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load(":release_files.bzl", "release_files") + +package(default_visibility = ["//:__subpackages__"]) + +# Files that the release workflow builds for each platform; see build.sh. +release_files( + name = "artifacts_for_release", + srcs = ["//crates/exe_zip_maker"], + # Under WORKSPACE, rules_rust is a stub without the rustc flags setting + # that release_files sets, so building this would fail. + tags = [] if BZLMOD_ENABLED else ["manual"], +) + +bzl_library( + name = "release_files", + srcs = ["release_files.bzl"], + deps = ["//python/private:common_labels"], +)
diff --git a/dev/release_artifacts/build.sh b/dev/release_artifacts/build.sh new file mode 100755 index 0000000..1460a34 --- /dev/null +++ b/dev/release_artifacts/build.sh
@@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Builds //dev/release_artifacts:artifacts_for_release for the host platform. +# The target applies the release settings; see release_files.bzl. In GitHub +# Actions, the paths of the built files, relative to the workspace root, are +# set as the step's `files` output, one per line. +# +# Usage: dev/release_artifacts/build.sh EMBED_LABEL +set -euo pipefail + +embed_label="$1" + +# The paths below are relative to the workspace root. +cd "$(dirname "$0")/../.." + +# Keep Git Bash on Windows from rewriting `//foo` labels into paths. +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" + +if ! bazel build --verbose_failures --compilation_mode=opt --stamp \ + --embed_label="$embed_label" //dev/release_artifacts:artifacts_for_release; then + # Releases don't use the files yet, so don't fail the release over them. + echo "::warning::Building the release artifacts failed. Releases don't use them yet." + exit 0 +fi + +# The target lists the paths of its files in a manifest; see release_files.bzl. +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "files<<EOF" + cat bazel-bin/dev/release_artifacts/artifacts_for_release.txt + echo "EOF" + } >> "$GITHUB_OUTPUT" +fi
diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl new file mode 100644 index 0000000..7bd3662 --- /dev/null +++ b/dev/release_artifacts/release_files.bzl
@@ -0,0 +1,96 @@ +"""Macro to build the files for a release and list where they are.""" + +load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl-visibility + +_FEATURES = "//command_line_option:features" +_RUSTC_FLAGS = "@rules_rust//rust/settings:extra_rustc_flags" + +def _release_transition_impl(settings, attr): + features = [] + rustc_flags = [] + if attr.target_os == "linux": + # Rust links glibc dynamically, so the files would require at least + # the build machine's glibc version. Link it statically so they run on + # older distros too. The gold linker can't link static glibc, so use + # bfd. + rustc_flags = [ + "-Ctarget-feature=+crt-static", + "-Clink-arg=-fuse-ld=bfd", + ] + elif attr.target_os == "windows": + # Statically link the C runtime so that the VC++ redistributable isn't + # required. + features = ["static_link_msvcrt"] + rustc_flags = ["-Ctarget-feature=+crt-static"] + return { + "//command_line_option:compilation_mode": "opt", + # Only affects macOS. Without it, the minimum OS version is the build + # machine's SDK version. + "//command_line_option:macos_minimum_os": "11.0", + _FEATURES: settings[_FEATURES] + features, + _RUSTC_FLAGS: settings[_RUSTC_FLAGS] + rustc_flags, + } + +_release_transition = transition( + implementation = _release_transition_impl, + inputs = [_FEATURES, _RUSTC_FLAGS], + outputs = [ + "//command_line_option:compilation_mode", + "//command_line_option:macos_minimum_os", + _FEATURES, + _RUSTC_FLAGS, + ], +) + +def _release_files_impl(ctx): + manifest = ctx.actions.declare_file(ctx.label.name + ".txt") + ctx.actions.write( + output = manifest, + # End every line with "\n", including the last: `while read` loops skip + # an unterminated last line, and build.sh adds a line after the paths. + content = "".join([file.path + "\n" for file in ctx.files.srcs]), + ) + return [DefaultInfo(files = depset([manifest] + ctx.files.srcs))] + +_release_files = rule( + implementation = _release_files_impl, + attrs = { + "srcs": attr.label_list( + allow_files = True, + cfg = _release_transition, + doc = "The files to build and list.", + ), + "target_os": attr.string( + doc = "The OS that the files are built for. Set by the macro.", + ), + }, +) + +def release_files(name, srcs, **kwargs): + """Builds files for a release and writes their paths to `<name>.txt`. + + The files are built with release settings: optimized, and linked so that + they run on older OS versions than the build machine's, without extra + runtime libraries. + + The paths, one per line, are relative to the execroot, e.g. + `bazel-out/k8-opt-ST-1234/bin/foo/foo`. Paths of generated files also + resolve from the workspace root through the `bazel-out` convenience + symlink. This lets scripts find the files without running `bazel cquery`, + which re-analyzes the build. + + Args: + name: The target name. + srcs: The files to build and list. + **kwargs: Additional attributes for the rule. + """ + _release_files( + name = name, + srcs = srcs, + target_os = select({ + Label("@platforms//os:linux"): "linux", + labels.PLATFORMS_OS_WINDOWS: "windows", + "//conditions:default": "", + }), + **kwargs + )
diff --git a/dev/release_artifacts/verify.sh b/dev/release_artifacts/verify.sh new file mode 100755 index 0000000..8982e6f --- /dev/null +++ b/dev/release_artifacts/verify.sh
@@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Checks that the files built by build.sh don't require a newer OS than users +# may have. Only Linux and macOS files are checked. +# +# Usage: dev/release_artifacts/verify.sh TRIPLE +set -euo pipefail + +triple="$1" + +# The paths below are relative to the workspace root. +cd "$(dirname "$0")/../.." + +# The target lists the paths of its files in a manifest; see release_files.bzl. +while IFS= read -r bin; do + case "$triple" in + *-linux-gnu) + linkage="$(file "$bin")" + if [[ "$linkage" != *"statically linked"* && + "$linkage" != *"static-pie linked"* ]]; then + echo "::error::$bin isn't statically linked: $linkage" + exit 1 + fi + ;; + *-apple-darwin) + minos="$(otool -l "$bin" | awk ' + $2 == "LC_BUILD_VERSION" || $2 == "LC_VERSION_MIN_MACOSX" { found = 1 } + found && minos == "" && ($1 == "minos" || $1 == "version") { minos = $2 } + END { print minos } + ')" + if [[ -z "$minos" || "${minos%%.*}" -gt 11 ]]; then + echo "::error::$bin requires macOS ${minos:-<unknown>}; expected 11 or lower" + exit 1 + fi + ;; + esac +done < bazel-bin/dev/release_artifacts/artifacts_for_release.txt