refactor(toolchain): use bazel to extract `zstd` archives (#2412)
Here we remove the dependence on the `zstd` archive downloaded from GH
for extracting `zstd` toolchains because bazel now supports `zstd`
extraction for a long time.
This mainly cleans up the code which is again used a lot more due to
#2386.
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dd1c2ff..5eaa3fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -52,7 +52,7 @@
{#v0-0-0-changed}
### Changed
-* Nothing changed.
+* (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
{#v0-0-0-fixed}
### Fixed
@@ -89,6 +89,11 @@
### Removed
* (publish) Remove deprecated `requirements.txt` for the `twine` dependencies.
Please use `requirements_linux.txt` instead.
+* (python_repository) Use bazel's built in `zstd` support and remove attributes
+ for customizing the `zstd` binary to be used for `zstd` archives in the
+ {bzl:obj}`python_repository` repository_rule. This affects the
+ {bzl:obj}`python_register_toolchains` and
+ {bzl:obj}`python_register_multi_toolchains` callers in the `WORKSPACE`.
{#v0-39-0}
## [0.39.0] - 2024-11-13
diff --git a/python/private/python.bzl b/python/private/python.bzl
index 8632554..ec6f73e 100644
--- a/python/private/python.bzl
+++ b/python/private/python.bzl
@@ -462,7 +462,7 @@
"strip_prefix": {
platform: item["strip_prefix"]
for platform in item["sha256"]
- },
+ } if type(item["strip_prefix"]) == type("") else item["strip_prefix"],
"url": {
platform: [item["url"]]
for platform in item["sha256"]
diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl
index 9ffa196..c7407c8 100644
--- a/python/private/python_repository.bzl
+++ b/python/private/python_repository.bzl
@@ -15,7 +15,7 @@
"""This file contains repository rules and macros to support toolchain registration.
"""
-load("//python:versions.bzl", "FREETHREADED", "PLATFORMS")
+load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS")
load(":auth.bzl", "get_auth")
load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load(":text_util.bzl", "render")
@@ -72,51 +72,13 @@
urls = rctx.attr.urls or [rctx.attr.url]
auth = get_auth(rctx, urls)
- if release_filename.endswith(".zst"):
- rctx.download(
+ if INSTALL_ONLY in release_filename:
+ rctx.download_and_extract(
url = urls,
sha256 = rctx.attr.sha256,
- output = release_filename,
+ stripPrefix = rctx.attr.strip_prefix,
auth = auth,
)
- unzstd = rctx.which("unzstd")
- if not unzstd:
- url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version)
- rctx.download_and_extract(
- url = url,
- sha256 = rctx.attr.zstd_sha256,
- auth = auth,
- )
- working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version)
-
- repo_utils.execute_checked(
- rctx,
- op = "python_repository.MakeZstd",
- arguments = [
- repo_utils.which_checked(rctx, "make"),
- "--jobs=4",
- ],
- timeout = 600,
- quiet = True,
- working_directory = working_directory,
- logger = logger,
- )
- zstd = "{working_directory}/zstd".format(working_directory = working_directory)
- unzstd = "./unzstd"
- rctx.symlink(zstd, unzstd)
-
- repo_utils.execute_checked(
- rctx,
- op = "python_repository.ExtractRuntime",
- arguments = [
- repo_utils.which_checked(rctx, "tar"),
- "--extract",
- "--strip-components=2",
- "--use-compress-program={unzstd}".format(unzstd = unzstd),
- "--file={}".format(release_filename),
- ],
- logger = logger,
- )
else:
rctx.download_and_extract(
url = urls,
@@ -125,6 +87,12 @@
auth = auth,
)
+ # Strip the things that are not present in the INSTALL_ONLY builds
+ # NOTE: if the dirs are not present, we will not fail here
+ rctx.delete("python/build")
+ rctx.delete("python/licenses")
+ rctx.delete("python/PYTHON.json")
+
patches = rctx.attr.patches
if patches:
for patch in patches:
@@ -378,15 +346,6 @@
"urls": attr.string_list(
doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
),
- "zstd_sha256": attr.string(
- default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
- ),
- "zstd_url": attr.string(
- default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz",
- ),
- "zstd_version": attr.string(
- default = "1.5.2",
- ),
"_rule_name": attr.string(default = "python_repository"),
},
environ = [REPO_DEBUG_ENV_VAR],
diff --git a/python/versions.bzl b/python/versions.bzl
index 774c24d..688c4e2 100644
--- a/python/versions.bzl
+++ b/python/versions.bzl
@@ -20,6 +20,7 @@
LINUX_NAME = "linux"
WINDOWS_NAME = "windows"
FREETHREADED = "freethreaded"
+INSTALL_ONLY = "install_only"
DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download"
@@ -52,7 +53,7 @@
"x86_64-apple-darwin": "8d06bec08db8cdd0f64f4f05ee892cf2fcbc58cfb1dd69da2caab78fac420238",
"x86_64-unknown-linux-gnu": "aec8c4c53373b90be7e2131093caa26063be6d9d826f599c935c0e1042af3355",
},
- "strip_prefix": "python",
+ "strip_prefix": "python/install",
},
"3.8.12": {
"url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz",
@@ -579,7 +580,22 @@
"x86_64-pc-windows-msvc-freethreaded": "bfd89f9acf866463bc4baf01733da5e767d13f5d0112175a4f57ba91f1541310",
"x86_64-unknown-linux-gnu-freethreaded": "a73adeda301ad843cce05f31a2d3e76222b656984535a7b87696a24a098b216c",
},
- "strip_prefix": "python",
+ "strip_prefix": {
+ "aarch64-apple-darwin": "python",
+ "aarch64-unknown-linux-gnu": "python",
+ "ppc64le-unknown-linux-gnu": "python",
+ "s390x-unknown-linux-gnu": "python",
+ "x86_64-apple-darwin": "python",
+ "x86_64-pc-windows-msvc": "python",
+ "x86_64-unknown-linux-gnu": "python",
+ "aarch64-apple-darwin-freethreaded": "python/install",
+ "aarch64-unknown-linux-gnu-freethreaded": "python/install",
+ "ppc64le-unknown-linux-gnu-freethreaded": "python/install",
+ "s390x-unknown-linux-gnu-freethreaded": "python/install",
+ "x86_64-apple-darwin-freethreaded": "python/install",
+ "x86_64-pc-windows-msvc-freethreaded": "python/install",
+ "x86_64-unknown-linux-gnu-freethreaded": "python/install",
+ },
},
}
@@ -777,7 +793,7 @@
}[p],
)
else:
- build = "install_only"
+ build = INSTALL_ONLY
if WINDOWS_NAME in platform:
build = "shared-" + build
