build(ci): optimize BuildBuddy RBE caching and inject CI build metadata (#4313) Optimize remote execution, remote caching, and diagnostic telemetry for BuildBuddy invocations across local and CI runs. Invocation analysis (0a836a44-4d93-4312-b6d1-1e2fffd7a4ff) highlighted opportunities to enhance CAS throughput with compression and chunking, prevent build failures from CAS eviction through lost-input rewinding, and collect rich diagnostic and profiling traces for the BuildBuddy UI. Additionally, GitHub Actions runs previously lacked build metadata to correlate invocations with workflows and PRs, and developers needed an automated, secret-safe method to link workspace credentials. To address this, enable remote cache compression, experimental chunking, lost-input rewinding, diagnostic logging (bazel-remote-grpc.log and bazel-execution-log.binpb.zst), and detailed execution profiling in _buildbuddy_base. In setup-bazel, inject non-secret CI build metadata (ROLE, VISIBILITY, commit SHA, PR number, branch name, run ID, actor, and GHA workflow run links) unconditionally on build. Replace the legacy shell script with a standalone Python client setup script to configure Git triangle tracking and symlink user.bazelrc cleanly. Work towards #4289, #4300, #4304.
diff --git a/.agents/rules/workspace.md b/.agents/rules/workspace.md index 20d8027..7f87eba 100644 --- a/.agents/rules/workspace.md +++ b/.agents/rules/workspace.md
@@ -14,7 +14,7 @@ * Run the workspace helper script to configure upstream tracking and safe pushing: ```bash - .agents/scripts/setup_triangle_branch.sh <branch> + .agents/scripts/setup_client.py <branch> ``` * Never use `git push -u` or `--set-upstream` (it breaks upstream tracking). * When pushing to a checked-out PR branch from an external fork, push
diff --git a/.agents/scripts/setup_client.py b/.agents/scripts/setup_client.py new file mode 100755 index 0000000..dc031c5 --- /dev/null +++ b/.agents/scripts/setup_client.py
@@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Configures git tracking and user environment for rules_python worktrees.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +def _get_current_branch() -> str: + result = subprocess.run( + ["git", "symbolic-ref", "--short", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def _get_workspace_root() -> Path: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ) + return Path(result.stdout.strip()) + + +def setup_triangle_branch(branch: str) -> None: + """Configures local git tracking to pull from upstream and push to origin.""" + print(f"Setting up upstream tracking for branch: {branch}") + subprocess.check_call( + ["git", "config", "--local", f"branch.{branch}.remote", "upstream"] + ) + subprocess.check_call( + ["git", "config", "--local", f"branch.{branch}.merge", "refs/heads/main"] + ) + subprocess.check_call( + ["git", "config", "--local", f"branch.{branch}.pushRemote", "origin"] + ) + subprocess.check_call(["git", "config", "--local", "push.default", "current"]) + print( + "Successfully configured upstream/main tracking and origin pushRemote " + f"for {branch}!" + ) + + +def setup_buildbuddy_user_bazelrc(workspace_root: Path) -> None: + """Links user BuildBuddy config to workspace user.bazelrc if not present.""" + source_rc = ( + Path.home() / ".config" / "rules_python" / "buildbuddy.bazelrc" + ).resolve() + target_rc = workspace_root / "user.bazelrc" + + if not source_rc.exists(): + print( + f"Note: {source_rc} not found. Skipping user.bazelrc link " + "(BuildBuddy RBE may not work for local invocations)." + ) + return + + # Only create the symlink if user.bazelrc does not already exist + if target_rc.exists() or target_rc.is_symlink(): + print(f"user.bazelrc already exists at {target_rc}. Skipping link.") + return + + target_rc.symlink_to(source_rc) + print(f"Linked {source_rc} -> {target_rc}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Setup git tracking and environment for rules_python workspace." + ) + parser.add_argument( + "branch", + nargs="?", + default=None, + help="Branch name to configure (defaults to current branch)", + ) + args = parser.parse_args() + + branch = args.branch or _get_current_branch() + if not branch: + print( + "Error: Could not detect current branch name and none was provided.", + file=sys.stderr, + ) + return 1 + + workspace_root = _get_workspace_root() + setup_triangle_branch(branch) + setup_buildbuddy_user_bazelrc(workspace_root) + return 0 + + +if __name__ == "__main__": + sys.exit(main())
diff --git a/.agents/scripts/setup_triangle_branch.sh b/.agents/scripts/setup_triangle_branch.sh deleted file mode 100755 index b6bdc2b..0000000 --- a/.agents/scripts/setup_triangle_branch.sh +++ /dev/null
@@ -1,25 +0,0 @@ -#!/bin/bash -# Helper script to set up upstream tracking for fork-and-PR workflow. -set -e - -# Detect current branch if not provided as argument -BRANCH_NAME="${1:-$(git symbolic-ref --short HEAD)}" - -if [ -z "$BRANCH_NAME" ]; then - echo "Error: Could not detect current branch name." >&2 - exit 1 -fi - -echo "Setting up upstream tracking for branch: $BRANCH_NAME" - -# 1. Set the pull/fetch behavior to track the canonical repo's main branch -git config --local "branch.${BRANCH_NAME}.remote" upstream -git config --local "branch.${BRANCH_NAME}.merge" refs/heads/main - -# 2. Override the destination remote for pushing -git config --local "branch.${BRANCH_NAME}.pushRemote" origin - -# 3. Defend against global push.default settings (ensure it pushes to current branch name) -git config --local push.default current - -echo "Successfully configured upstream/main tracking and origin pushRemote for $BRANCH_NAME!"
diff --git a/.github/actions/setup-bazel/action.yaml b/.github/actions/setup-bazel/action.yaml index aac5fd6..c1ae945 100644 --- a/.github/actions/setup-bazel/action.yaml +++ b/.github/actions/setup-bazel/action.yaml
@@ -37,6 +37,14 @@ build --output_groups=-pyrefly build --//python/config_settings:py_pbs_distribution=install_only_stripped ${{ inputs.buildbuddy_api_key && format('common --config={0}', inputs.buildbuddy_config) || '' }} + build --build_metadata=ROLE=CI + build --build_metadata=VISIBILITY=PUBLIC + build --build_metadata=RUN_ID=${{ github.run_id }} + build --build_metadata=ACTOR=${{ github.actor }} + build --build_metadata=COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }} + ${{ github.event.pull_request.number && format('build --build_metadata=PR_NUMBER={0}', github.event.pull_request.number) || '' }} + ${{ (github.head_ref || github.ref_name) && format('build --build_metadata=BRANCH_NAME={0}', github.head_ref || github.ref_name) || '' }} + build --build_metadata="BUILDBUDDY_LINKS=[GHA Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" # setup-bazel only configures Bazelisk and restores caches; the Bazel # binary itself is downloaded by Bazelisk on first invocation. Because
diff --git a/buildbuddy.bazelrc b/buildbuddy.bazelrc index 861e70e..ae3c888 100644 --- a/buildbuddy.bazelrc +++ b/buildbuddy.bazelrc
@@ -5,9 +5,26 @@ # `user.bazelrc` or passed by CI. build:_buildbuddy_base --bes_results_url=https://rules-python.buildbuddy.io/invocation/ build:_buildbuddy_base --bes_backend=grpcs://rules-python.buildbuddy.io +build:_buildbuddy_base --build_metadata=REPO_URL=https://github.com/bazel-contrib/rules_python common:_buildbuddy_base --remote_cache=grpcs://rules-python.buildbuddy.io common:_buildbuddy_base --remote_timeout=10m common:_buildbuddy_base --remote_executor=grpcs://rules-python.buildbuddy.io +# Compress remote cache transfers with zstd to improve network throughput. +common:_buildbuddy_base --remote_cache_compression +# Chunk large blobs using FastCDC (SplitBlob/SpliceBlob) for efficient +# deduplicated transfer. +common:_buildbuddy_base --experimental_remote_cache_chunking +# Automatically rewind and regenerate lost inputs when evicted from remote cache. +common:_buildbuddy_base --rewind_lost_inputs + +# Diagnostic logs for troubleshooting cache misses and gRPC errors +common:_buildbuddy_base --remote_grpc_log=bazel-remote-grpc.log +common:_buildbuddy_base --execution_log_compact_file=bazel-execution-log.binpb.zst + +# Detailed timing profiles for the BuildBuddy Timing tab +common:_buildbuddy_base --noslim_profile +common:_buildbuddy_base --experimental_profile_include_target_label +common:_buildbuddy_base --experimental_profile_include_primary_output # Re-apply `--enable_platform_specific_config` after setting `--remote_executor` # so `common:macos` and `common:windows` in `.bazelrc` inherit the priority of # `--config=buildbuddy_*` (e.g. when set in `~/.bazelrc` by `setup-bazel`,
diff --git a/tests/workflows/setup_bazel_test.py b/tests/workflows/setup_bazel_test.py index ba04a96..aebfb63 100644 --- a/tests/workflows/setup_bazel_test.py +++ b/tests/workflows/setup_bazel_test.py
@@ -45,14 +45,55 @@ action = yaml.safe_load((_action_dir() / "action.yaml").read_text(encoding="utf-8")) steps = action["runs"]["steps"] setup_step = next(s for s in steps if s.get("name") == "Setup Bazel") - assert "inputs.buildbuddy_api_key" in setup_step["with"]["bazelrc"] - assert "runner.os" not in setup_step["with"]["bazelrc"] + bazelrc = setup_step["with"]["bazelrc"] + assert "inputs.buildbuddy_api_key" in bazelrc + assert "runner.os" not in bazelrc + assert "ROLE=CI" in bazelrc + assert "VISIBILITY=PUBLIC" in bazelrc + assert "RUN_ID=" in bazelrc + assert "ACTOR=" in bazelrc + assert "COMMIT_SHA=" in bazelrc + assert "PR_NUMBER=" in bazelrc + assert "BRANCH_NAME=" in bazelrc + assert '--build_metadata="BUILDBUDDY_LINKS=' in bazelrc secret_step = next( s for s in steps if s.get("name") == "Write secret to user.bazelrc" ) assert secret_step["if"] == "${{ inputs.buildbuddy_api_key != '' }}" +def test_buildbuddy_bazelrc_cache_and_diagnostics(): + """Verifies buildbuddy.bazelrc includes recommended cache, diagnostic, and + profiling flags. + """ + rf = runfiles.CreateOrRaise() + buildbuddy_path = rf.root() / "rules_python/buildbuddy.bazelrc" + buildbuddy_lines = [ + line.strip() + for line in buildbuddy_path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#") + ] + expected_flags = [ + ( + "build:_buildbuddy_base " + "--build_metadata=REPO_URL=https://github.com/bazel-contrib/rules_python" + ), + "common:_buildbuddy_base --remote_cache_compression", + "common:_buildbuddy_base --experimental_remote_cache_chunking", + "common:_buildbuddy_base --rewind_lost_inputs", + "common:_buildbuddy_base --remote_grpc_log=bazel-remote-grpc.log", + ( + "common:_buildbuddy_base " + "--execution_log_compact_file=bazel-execution-log.binpb.zst" + ), + "common:_buildbuddy_base --noslim_profile", + "common:_buildbuddy_base --experimental_profile_include_target_label", + "common:_buildbuddy_base --experimental_profile_include_primary_output", + ] + for flag in expected_flags: + assert flag in buildbuddy_lines, f"Expected '{flag}' in buildbuddy.bazelrc" + + def test_buildbuddy_bazelrc_disables_remote_executor_on_macos_and_windows(): """Verifies .bazelrc and buildbuddy.bazelrc disable remote_executor on macOS and Windows.""" rf = runfiles.CreateOrRaise()