build(release): upload artifacts_manifest.json to GitHub releases (#4390) Switching the release manifest generator to emit `artifacts_manifest.bzl` (#4370) dropped the standalone `artifacts_manifest.json` output, and the publish workflow never downloaded `release-manifest` to attach it to the GitHub release. Generate both `artifacts_manifest.bzl` (bundled into the source archive) and `artifacts_manifest.json` via explicit `--bzl-output` and `--json-output` flags, upload `artifacts_manifest.json` in the `release-manifest` workflow artifact, and download and attach it to the GitHub release during publishing. Work towards #4250
diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml index aa73b07..32a9680 100644 --- a/.github/workflows/release_build_artifacts.yaml +++ b/.github/workflows/release_build_artifacts.yaml
@@ -4,7 +4,8 @@ # see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the # files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, # so they're built on those OSes. -# - `SHA256SUMS` and `artifacts_manifest.bzl` for the built platform binaries. +# - `SHA256SUMS`, `artifacts_manifest.json`, and `artifacts_manifest.bzl` for +# the built platform binaries. # - The release source archive (`rules_python-<ref>.tar.gz`) and # `release_notes.txt`. # - The `bazel_runfiles` wheel (`//python/runfiles:wheel.dist`). @@ -187,14 +188,16 @@ hashFiles('release-artifacts/*') != '' run: >- dev/release_artifacts/generate_artifacts_manifest.py - "$REF" release-artifacts artifacts_manifest.bzl + --bzl-output=artifacts_manifest.bzl + --json-output=artifacts_manifest.json + -- "$REF" release-artifacts - name: Upload artifacts manifest - if: hashFiles('artifacts_manifest.bzl') != '' + if: hashFiles('artifacts_manifest.json') != '' uses: actions/upload-artifact@v7 with: name: release-manifest - path: artifacts_manifest.bzl + path: artifacts_manifest.json if-no-files-found: error - name: Create release archive and notes
diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 49bb174..4b64bc5 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml
@@ -69,6 +69,11 @@ with: name: release-archive + - name: Download artifacts manifest + uses: actions/download-artifact@v8 + with: + name: release-manifest + - name: Download built artifacts uses: actions/download-artifact@v8 with: @@ -88,6 +93,7 @@ files: | rules_python-*.tar.gz release-artifacts/* + artifacts_manifest.json SHA256SUMS tag_name: ${{ inputs.tag_name || github.ref_name }}
diff --git a/RELEASING.md b/RELEASING.md index a52bf47..4b687ba 100644 --- a/RELEASING.md +++ b/RELEASING.md
@@ -70,9 +70,9 @@ ### Release artifacts The release workflow also builds `//dev/release_artifacts:artifacts_for_release` -for each platform and uploads the built artifacts and `SHA256SUMS` to the GitHub -release. For now, build failures only emit a warning rather than failing the -release. +for each platform and uploads the built artifacts, `artifacts_manifest.json`, +and `SHA256SUMS` to the GitHub release. For now, build failures only emit a +warning rather than failing the release. Presubmit builds it for every platform too: natively on each CI host, and through `//dev/release_artifacts:cross_compiled_artifacts` for the others.
diff --git a/dev/release_artifacts/generate_artifacts_manifest.py b/dev/release_artifacts/generate_artifacts_manifest.py index 30b5b4b..ae8ad4a 100755 --- a/dev/release_artifacts/generate_artifacts_manifest.py +++ b/dev/release_artifacts/generate_artifacts_manifest.py
@@ -1,5 +1,16 @@ #!/usr/bin/env python3 -"""Generates an artifacts_manifest.bzl file for built release artifacts.""" +"""Generates artifacts_manifest.json and artifacts_manifest.bzl files. + +Reads built release binaries named ``<artifact>-<triple>[.exe]`` from +``artifacts_dir`` and writes their SHA-256 digests, download URLs, and platform +constraints to both JSON and Starlark manifest files. + +Usage: + dev/release_artifacts/generate_artifacts_manifest.py \\ + --bzl-output=artifacts_manifest.bzl \\ + --json-output=artifacts_manifest.json \\ + -- 2.4.0 release-artifacts [bazel-contrib/rules_python] +""" from __future__ import annotations @@ -86,8 +97,12 @@ return {"artifacts": dict(sorted(artifacts.items()))} +def render_json(manifest: dict[str, dict[str, list[ArtifactEntry]]]) -> str: + return json.dumps(manifest, indent=2, sort_keys=True) + "\n" + + def render_bzl(manifest: dict[str, dict[str, list[ArtifactEntry]]]) -> str: - manifest_json = json.dumps(manifest, indent=1, sort_keys=True) + manifest_json = render_json(manifest) # Embedded in a Starlark string literal, so it must not end the literal # early or rely on escape sequences. if '"""' in manifest_json or "\\" in manifest_json: @@ -95,12 +110,24 @@ return ( '"""Manifest of prebuilt tool binaries for this release."""\n' "\n" - f'ARTIFACTS_MANIFEST_JSON = """\n{manifest_json}\n"""\n' + f'ARTIFACTS_MANIFEST_JSON = """\n{manifest_json}"""\n' ) def _main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--bzl-output", + type=Path, + required=True, + help="Path to write the Starlark (.bzl) manifest file.", + ) + parser.add_argument( + "--json-output", + type=Path, + required=True, + help="Path to write the JSON (.json) manifest file.", + ) parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).") parser.add_argument( "artifacts_dir", @@ -108,11 +135,6 @@ help="Directory containing built <artifact>-<triple>[.exe] files.", ) parser.add_argument( - "output_file", - type=Path, - help="Path to the artifacts_manifest.bzl file to write.", - ) - parser.add_argument( "repo", nargs="?", default=os.environ.get("GITHUB_REPOSITORY") or "bazel-contrib/rules_python", @@ -121,12 +143,16 @@ args = parser.parse_args(argv) try: manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo) - manifest_bzl = render_bzl(manifest) + outputs = ( + (args.json_output, render_json(manifest)), + (args.bzl_output, render_bzl(manifest)), + ) except ValueError as err: print(f"::error::{err}") return 1 - args.output_file.parent.mkdir(parents=True, exist_ok=True) - args.output_file.write_text(manifest_bzl, encoding="utf-8") + for out_path, content in outputs: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(content, encoding="utf-8") return 0
diff --git a/tests/workflows/BUILD.bazel b/tests/workflows/BUILD.bazel index de839eb..3fdcb98 100644 --- a/tests/workflows/BUILD.bazel +++ b/tests/workflows/BUILD.bazel
@@ -42,8 +42,7 @@ name = "release_build_artifacts_test", srcs = ["release_build_artifacts_test.py"], data = [ - "//.github/workflows:create_archive_and_notes.sh", - "//.github/workflows:release_build_artifacts.yaml", + "//.github/workflows:all_files", "//dev/platforms:BUILD.bazel", ], target_compatible_with = ["@platforms//os:linux"],
diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py index c503d7a..b66b38d 100644 --- a/tests/workflows/release_artifacts_test.py +++ b/tests/workflows/release_artifacts_test.py
@@ -20,7 +20,8 @@ artifacts_dir = tmp_path / "release-artifacts" artifacts_dir.mkdir(parents=True) - manifest_file = tmp_path / "artifacts_manifest.bzl" + manifest_json_file = tmp_path / "artifacts_manifest.json" + manifest_bzl_file = tmp_path / "artifacts_manifest.bzl" linux_bin = artifacts_dir / "exe_zip_maker-x86_64-unknown-linux-gnu" linux_bin.write_bytes(b"linux-binary-contents") @@ -37,20 +38,23 @@ subprocess.run( [ str(generate_script), + f"--bzl-output={manifest_bzl_file}", + f"--json-output={manifest_json_file}", + "--", "2.4.0", str(artifacts_dir), - str(manifest_file), "bazel-contrib/rules_python", ], check=True, ) - manifest_bzl = manifest_file.read_text(encoding="utf-8") + json_manifest = json.loads(manifest_json_file.read_text(encoding="utf-8")) + manifest_bzl = manifest_bzl_file.read_text(encoding="utf-8") _, marker, manifest_json = manifest_bzl.partition('ARTIFACTS_MANIFEST_JSON = """') assert marker - manifest = json.loads(manifest_json.removesuffix('"""\n')) + bzl_manifest = json.loads(manifest_json.removesuffix('"""\n')) base_url = "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0" - assert manifest == { + expected_manifest = { "artifacts": { "exe_zip_maker": [ { @@ -83,6 +87,8 @@ ] } } + assert json_manifest == expected_manifest + assert bzl_manifest == expected_manifest def test_create_archive_and_notes_with_manifest(tmp_path: Path):
diff --git a/tests/workflows/release_build_artifacts_test.py b/tests/workflows/release_build_artifacts_test.py index 9262d49..ad933a1 100644 --- a/tests/workflows/release_build_artifacts_test.py +++ b/tests/workflows/release_build_artifacts_test.py
@@ -13,9 +13,9 @@ from python.runfiles import runfiles -def _load_workflow() -> dict: +def _load_workflow(filename: str = "release_build_artifacts.yaml") -> dict: root = runfiles.CreateOrRaise().root() / "rules_python" - workflow_path = root / ".github/workflows/release_build_artifacts.yaml" + workflow_path = root / ".github/workflows" / filename return yaml.safe_load(workflow_path.read_text(encoding="utf-8")) @@ -91,6 +91,50 @@ assert "secrets.BUILDBUDDY_READ_WRITE_KEY" in api_key +def test_artifacts_manifest_json_built_and_published(): + """Verifies artifacts_manifest.json is generated, uploaded, and published.""" + build_workflow = _load_workflow() + package_steps = build_workflow["jobs"]["package"]["steps"] + + generate_steps = [ + s for s in package_steps if s.get("name") == "Generate artifacts manifest" + ] + assert len(generate_steps) == 1 + generate_cmd = generate_steps[0]["run"] + assert "--bzl-output=artifacts_manifest.bzl" in generate_cmd + assert "--json-output=artifacts_manifest.json" in generate_cmd + + upload_steps = [ + s for s in package_steps if s.get("name") == "Upload artifacts manifest" + ] + assert len(upload_steps) == 1 + assert upload_steps[0]["with"]["name"] == "release-manifest" + assert "artifacts_manifest.json" in upload_steps[0]["with"]["path"] + + publish_workflow = _load_workflow("release_publish.yaml") + release_steps = publish_workflow["jobs"]["release"]["steps"] + download_manifest_steps = [ + s + for s in release_steps + if s.get("uses", "").startswith("actions/download-artifact") + and s.get("with", {}).get("name") == "release-manifest" + ] + assert len(download_manifest_steps) == 1 + + gh_release_steps = [ + s + for s in release_steps + if s.get("uses", "").startswith("softprops/action-gh-release") + ] + assert len(gh_release_steps) == 1 + release_files = [ + line.strip() + for line in gh_release_steps[0]["with"]["files"].splitlines() + if line.strip() + ] + assert "artifacts_manifest.json" in release_files + + def test_create_archive_and_notes_fetches_and_expands_tag(tmp_path: Path): """Verifies create_archive_and_notes.sh fetches missing tags and expands export-subst.""" script = (