build(release): cross-build all Linux release artifacts in one RBE job (#4304) Spinning up a separate GitHub Actions runner for each of the six Linux platforms is unnecessary now that Linux release builds run on BuildBuddy RBE. Replace the six Linux matrix entries with a single `linux` job that builds `//dev/release_artifacts:cross_compiled_artifacts`, and include `x86_64-unknown-linux-gnu` in that target along with a manifest file so one runner builds and verifies all Linux binaries in a single Bazel invocation. Work towards #4216
diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml index ca2cdcd..073c863 100644 --- a/.github/workflows/release_build_artifacts.yaml +++ b/.github/workflows/release_build_artifacts.yaml
@@ -55,20 +55,11 @@ runner: macos-15-intel - triple: x86_64-pc-windows-msvc runner: windows-2022 - - triple: x86_64-unknown-linux-gnu - runner: ubuntu-24.04 - # Cross-compiled; see dev/platforms/BUILD.bazel. - - triple: aarch64-unknown-linux-musl - runner: ubuntu-24.04 - - triple: armv7-unknown-linux-musleabihf - runner: ubuntu-24.04 + # Cross-compiled; see dev/platforms/BUILD.bazel and + # dev/release_artifacts/BUILD.bazel. - triple: i686-pc-windows-msvc runner: windows-2022 - - triple: i686-unknown-linux-musl - runner: ubuntu-24.04 - - triple: powerpc64le-unknown-linux-musl - runner: ubuntu-24.04 - - triple: riscv64gc-unknown-linux-musl + - triple: linux runner: ubuntu-24.04 env: REF: ${{ inputs.ref || github.sha }} @@ -198,7 +189,7 @@ if: hashFiles('artifacts_manifest.json') != '' uses: actions/upload-artifact@v7 with: - name: release-artifacts-manifest + name: release-manifest path: artifacts_manifest.json if-no-files-found: error
diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 0e17bbb..49bb174 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml
@@ -72,9 +72,7 @@ - name: Download built artifacts uses: actions/download-artifact@v8 with: - # Match platform triples (<cpu>-<vendor>-<os>), excluding - # release-artifacts-manifest. - pattern: release-artifacts-*-* + pattern: release-artifacts-* path: release-artifacts merge-multiple: true
diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index 8f5f0c6..26895e1 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel
@@ -40,6 +40,7 @@ "//dev/platforms:i686-unknown-linux-musl", "//dev/platforms:powerpc64le-unknown-linux-musl", "//dev/platforms:riscv64gc-unknown-linux-musl", + "//dev/platforms:x86_64-unknown-linux-gnu", ], ":is_macos_aarch64": ["//dev/platforms:x86_64-apple-darwin"], ":is_windows_x86_64": ["//dev/platforms:i686-pc-windows-msvc"],
diff --git a/dev/release_artifacts/build.sh b/dev/release_artifacts/build.sh index 3ceb0c8..b84d128 100755 --- a/dev/release_artifacts/build.sh +++ b/dev/release_artifacts/build.sh
@@ -14,29 +14,40 @@ # The paths below are relative to the workspace root. cd "$(dirname "$0")/../.." -# Fail for a triple without a platform: it's the caller's mistake, unlike a -# failed build, which only warns. -if ! grep -qF "name = \"$triple\"," dev/platforms/BUILD.bazel; then +if [[ "$triple" == "linux" ]]; then + build_args=(//dev/release_artifacts:cross_compiled_artifacts) + manifest="bazel-bin/dev/release_artifacts/cross_compiled_artifacts.txt" +elif ! grep -qF "name = \"$triple\"," dev/platforms/BUILD.bazel; then + # Fail for a triple without a platform: it's the caller's mistake, unlike a + # failed build, which only warns. echo "::error::dev/platforms has no $triple platform" exit 1 +else + build_args=( + --platforms="//dev/platforms:$triple" + //dev/release_artifacts:artifacts_for_release + ) + manifest="bazel-bin/dev/release_artifacts/artifacts_for_release.txt" fi # Keep Git Bash on Windows from rewriting `//foo` labels into paths. export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" if ! bazel build --verbose_failures --compilation_mode=opt --stamp \ - --embed_label="$embed_label" --platforms="//dev/platforms:$triple" \ - //dev/release_artifacts:artifacts_for_release; then + --embed_label="$embed_label" "${build_args[@]}"; then # Releases don't use the files yet, so don't fail the release over them. echo "::warning::Building the release artifacts failed. Releases don't use them yet." exit 0 fi # The target lists the paths of its files in a manifest; see release_files.bzl. +# Copy them into a single directory so upload-artifact strips the directory +# prefix instead of preserving per-platform bazel-out/<cfg> subdirectories. if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - { - echo "files<<EOF" - cat bazel-bin/dev/release_artifacts/artifacts_for_release.txt - echo "EOF" - } >> "$GITHUB_OUTPUT" + rm -rf release-artifacts + mkdir -p release-artifacts + while IFS= read -r file; do + cp "$file" release-artifacts/ + done < "$manifest" + echo "files=release-artifacts/*" >> "$GITHUB_OUTPUT" fi
diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index 32a84b3..e9604cd 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl
@@ -5,6 +5,11 @@ _FEATURES = "//command_line_option:features" _RUSTC_FLAGS = "@rules_rust//rust/settings:extra_rustc_flags" +_ReleaseFilesInfo = provider( + doc = "Files built by _release_files, excluding the .txt manifest.", + fields = ["files"], +) + def _release_transition_impl(settings, attr): features = [] rustc_flags = [] @@ -82,7 +87,10 @@ # an unterminated last line, and build.sh adds a line after the paths. content = "".join([file.path + "\n" for file in files]), ) - return [DefaultInfo(files = depset([manifest] + files))] + return [ + DefaultInfo(files = depset([manifest] + files)), + _ReleaseFilesInfo(files = files), + ] _release_files = rule( implementation = _release_files_impl, @@ -168,8 +176,17 @@ return [DefaultInfo()] # With a transition, ctx.attr.src is a list: one target per platform. - files = [src[DefaultInfo].files for src in ctx.attr.src] - return [DefaultInfo(files = depset(transitive = files))] + files = [] + for src in ctx.attr.src: + files.extend(src[_ReleaseFilesInfo].files) + manifest = ctx.actions.declare_file(ctx.label.name + ".txt") + ctx.actions.write( + output = manifest, + # End every line with "\n", including the last: `while read` loops skip + # an unterminated last line, and build.sh adds a line after the paths. + content = "".join([file.path + "\n" for file in files]), + ) + return [DefaultInfo(files = depset([manifest] + files))] cross_compiled_files = rule( implementation = _cross_compiled_files_impl, @@ -188,6 +205,7 @@ cfg = _platforms_transition, doc = "The target to build.", mandatory = True, + providers = [_ReleaseFilesInfo], ), }, )
diff --git a/dev/release_artifacts/verify.sh b/dev/release_artifacts/verify.sh index 0c55db7..bce09cf 100755 --- a/dev/release_artifacts/verify.sh +++ b/dev/release_artifacts/verify.sh
@@ -7,13 +7,25 @@ # Usage: dev/release_artifacts/verify.sh TRIPLE set -euo pipefail -triple="$1" +triple_arg="$1" # The paths below are relative to the workspace root. cd "$(dirname "$0")/../.." +if [[ "$triple_arg" == "linux" ]]; then + manifest="bazel-bin/dev/release_artifacts/cross_compiled_artifacts.txt" +else + manifest="bazel-bin/dev/release_artifacts/artifacts_for_release.txt" +fi + # The target lists the paths of its files in a manifest; see release_files.bzl. while IFS= read -r bin; do + if [[ "$triple_arg" == "linux" ]]; then + name="${bin##*/}" + triple="${name#*-}" + else + triple="$triple_arg" + fi case "$triple" in *-linux-*) info="$(file -L -b "$bin")" @@ -84,4 +96,4 @@ exit 1 ;; esac -done < bazel-bin/dev/release_artifacts/artifacts_for_release.txt +done < "$manifest"
diff --git a/tests/workflows/release_build_artifacts_test.py b/tests/workflows/release_build_artifacts_test.py index 93bdd7c..11232dc 100644 --- a/tests/workflows/release_build_artifacts_test.py +++ b/tests/workflows/release_build_artifacts_test.py
@@ -39,8 +39,11 @@ # e.g. because of a comment. calls = re.findall(r"^platform\(", build_file, re.MULTILINE) assert len(platforms) == len(calls), "Couldn't parse every platform's name" + expected = { + "linux" if "-linux-" in platform else platform for platform in platforms + } # Sets, so that failures list the triples that differ. - assert set(triples) == set(platforms), "Matrix (left) vs. dev/platforms (right)" + assert set(triples) == expected, "Matrix (left) vs. dev/platforms (right)" duplicates = sorted({triple for triple in triples if triples.count(triple) > 1}) assert not duplicates, f"Triples with more than one job: {duplicates}"