feat(twine): support 'bzlmod' users out of the box (#1572)
Implements a test that starts a [`pypiserver`] and checks
that the publishing with the new machinery still works.
Fixes #1369
[pypiserver]: https://github.com/pypiserver/pypiserver
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff61b2b..2beb8bd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,9 @@
* (wheel) Add support for `data_files` attributes in py_wheel rule
([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
+* (py_wheel) `bzlmod` installations now provide a `twine` setup for the default
+ Python toolchain in `rules_python` for version 3.11.
+
[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
[python_default_visibility]: gazelle/README.md#directive-python_default_visibility
@@ -269,7 +272,6 @@
attribute for every target in the package. This is enabled through a separate
directive `python_generation_mode_per_file_include_init`.
-
## [0.27.0] - 2023-11-16
[0.27.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.27.0
diff --git a/MODULE.bazel b/MODULE.bazel
index 15a33c8..a165a94 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -48,11 +48,24 @@
is_default = True,
python_version = "3.11",
)
-use_repo(python, "pythons_hub")
+use_repo(python, "python_versions", "pythons_hub")
# This call registers the Python toolchains.
register_toolchains("@pythons_hub//:all")
+#####################
+# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
+
+pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip.parse(
+ hub_name = "rules_python_publish_deps",
+ python_version = "3.11",
+ requirements_darwin = "//tools/publish:requirements_darwin.txt",
+ requirements_lock = "//tools/publish:requirements.txt",
+ requirements_windows = "//tools/publish:requirements_windows.txt",
+)
+use_repo(pip, "rules_python_publish_deps")
+
# ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
@@ -84,24 +97,12 @@
python_version = "3.11",
requirements_lock = "//docs/sphinx:requirements.txt",
)
-
-#####################
-# Install twine for our own runfiles wheel publishing.
-# Eventually we might want to install twine automatically for users too, see:
-# https://github.com/bazelbuild/rules_python/issues/1016.
-
dev_pip.parse(
- hub_name = "publish_deps",
+ hub_name = "pypiserver",
python_version = "3.11",
- requirements_darwin = "//tools/publish:requirements_darwin.txt",
- requirements_lock = "//tools/publish:requirements.txt",
- requirements_windows = "//tools/publish:requirements_windows.txt",
+ requirements_lock = "//examples/wheel:requirements_server.txt",
)
-use_repo(
- dev_pip,
- "dev_pip",
- publish_deps_twine = "publish_deps_311_twine",
-)
+use_repo(dev_pip, "dev_pip", "pypiserver")
# Bazel integration test setup below
diff --git a/WORKSPACE b/WORKSPACE
index 75c8e56..86a80bd 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -94,17 +94,27 @@
load("@rules_python//python:pip.bzl", "pip_parse")
pip_parse(
- name = "publish_deps",
+ name = "rules_python_publish_deps",
python_interpreter_target = interpreter,
requirements_darwin = "//tools/publish:requirements_darwin.txt",
requirements_lock = "//tools/publish:requirements.txt",
requirements_windows = "//tools/publish:requirements_windows.txt",
)
-load("@publish_deps//:requirements.bzl", "install_deps")
+load("@rules_python_publish_deps//:requirements.bzl", "install_deps")
install_deps()
+pip_parse(
+ name = "pypiserver",
+ python_interpreter_target = interpreter,
+ requirements_lock = "//examples/wheel:requirements_server.txt",
+)
+
+load("@pypiserver//:requirements.bzl", install_pypiserver = "install_deps")
+
+install_pypiserver()
+
#####################
# Install sphinx for doc generation.
diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel
index 2e45d7d..aa063ce 100644
--- a/examples/wheel/BUILD.bazel
+++ b/examples/wheel/BUILD.bazel
@@ -17,7 +17,10 @@
load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags")
load("//python:defs.bzl", "py_library", "py_test")
load("//python:packaging.bzl", "py_package", "py_wheel")
+load("//python:pip.bzl", "compile_pip_requirements")
load("//python:versions.bzl", "gen_python_config_settings")
+load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
package(default_visibility = ["//visibility:public"])
@@ -56,6 +59,10 @@
# Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
distribution = "example_minimal_library",
python_tag = "py3",
+ # NOTE: twine_binary = "//tools/publish:twine" does not work on non-bzlmod
+ # setups because the `//tools/publish:twine` produces multiple files and is
+ # unsuitable as the `src` to the underlying native_binary rule.
+ twine = None if BZLMOD_ENABLED else "@rules_python_publish_deps_twine//:pkg",
version = "0.0.1",
deps = [
"//examples/wheel/lib:module_with_data",
@@ -348,3 +355,39 @@
"//python/runfiles",
],
)
+
+# Test wheel publishing
+
+compile_pip_requirements(
+ name = "requirements_server",
+ src = "requirements_server.in",
+)
+
+py_test(
+ name = "test_publish",
+ srcs = ["test_publish.py"],
+ data = [
+ ":minimal_with_py_library",
+ ":minimal_with_py_library.publish",
+ ":pypiserver",
+ ],
+ env = {
+ "PUBLISH_PATH": "$(location :minimal_with_py_library.publish)",
+ "SERVER_PATH": "$(location :pypiserver)",
+ "WHEEL_PATH": "$(rootpath :minimal_with_py_library)",
+ },
+ target_compatible_with = select({
+ "@platforms//os:linux": [],
+ "@platforms//os:macos": [],
+ "//conditions:default": ["@platforms//:incompatible"],
+ }),
+ deps = [
+ "@pypiserver//pypiserver",
+ ],
+)
+
+py_console_script_binary(
+ name = "pypiserver",
+ pkg = "@pypiserver//pypiserver",
+ script = "pypi-server",
+)
diff --git a/examples/wheel/requirements_server.in b/examples/wheel/requirements_server.in
new file mode 100644
index 0000000..d5d483d
--- /dev/null
+++ b/examples/wheel/requirements_server.in
@@ -0,0 +1,2 @@
+# This is for running publishing tests
+pypiserver
diff --git a/examples/wheel/requirements_server.txt b/examples/wheel/requirements_server.txt
new file mode 100644
index 0000000..eccab12
--- /dev/null
+++ b/examples/wheel/requirements_server.txt
@@ -0,0 +1,16 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+# bazel run //examples/wheel:requirements_server.update
+#
+pypiserver==2.0.1 \
+ --hash=sha256:1dd98fb99d2da4199fb44c7284e57d69a9f7fda2c6c8dc01975c151c592677bf \
+ --hash=sha256:7b58fbd54468235f79e4de07c4f7a9ff829e7ac6869bef47ec11e0710138e162
+ # via -r examples/wheel/requirements_server.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==24.0 \
+ --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
+ --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
+ # via pypiserver
diff --git a/examples/wheel/test_publish.py b/examples/wheel/test_publish.py
new file mode 100644
index 0000000..496642a
--- /dev/null
+++ b/examples/wheel/test_publish.py
@@ -0,0 +1,117 @@
+import os
+import socket
+import subprocess
+import textwrap
+import time
+import unittest
+from contextlib import closing
+from pathlib import Path
+from urllib.request import urlopen
+
+
+def find_free_port():
+ with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+ s.bind(("", 0))
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ return s.getsockname()[1]
+
+
+class TestTwineUpload(unittest.TestCase):
+ def setUp(self):
+ self.maxDiff = 1000
+ self.port = find_free_port()
+ self.url = f"http://localhost:{self.port}"
+ self.dir = Path(os.environ["TEST_TMPDIR"])
+
+ self.log_file = self.dir / "pypiserver-log.txt"
+ self.log_file.touch()
+ _storage_dir = self.dir / "data"
+ for d in [_storage_dir]:
+ d.mkdir(exist_ok=True)
+
+ print("Starting PyPI server...")
+ self._server = subprocess.Popen(
+ [
+ str(Path(os.environ["SERVER_PATH"])),
+ "run",
+ "--verbose",
+ "--log-file",
+ str(self.log_file),
+ "--host",
+ "localhost",
+ "--port",
+ str(self.port),
+ # Allow unauthenticated access
+ "--authenticate",
+ ".",
+ "--passwords",
+ ".",
+ str(_storage_dir),
+ ],
+ )
+
+ line = "Hit Ctrl-C to quit"
+ interval = 0.1
+ wait_seconds = 40
+ for _ in range(int(wait_seconds / interval)): # 40 second timeout
+ current_logs = self.log_file.read_text()
+ if line in current_logs:
+ print(current_logs.strip())
+ print("...")
+ break
+
+ time.sleep(0.1)
+ else:
+ raise RuntimeError(
+ f"Could not get the server running fast enough, waited for {wait_seconds}s"
+ )
+
+ def tearDown(self):
+ self._server.terminate()
+ print(f"Stopped PyPI server, all logs:\n{self.log_file.read_text()}")
+
+ def test_upload_and_query_simple_api(self):
+ # Given
+ script_path = Path(os.environ["PUBLISH_PATH"])
+ whl = Path(os.environ["WHEEL_PATH"])
+
+ # When I publish a whl to a package registry
+ subprocess.check_output(
+ [
+ str(script_path),
+ "--no-color",
+ "upload",
+ str(whl),
+ "--verbose",
+ "--non-interactive",
+ "--disable-progress-bar",
+ ],
+ env={
+ "TWINE_REPOSITORY_URL": self.url,
+ "TWINE_USERNAME": "dummy",
+ "TWINE_PASSWORD": "dummy",
+ },
+ )
+
+ # Then I should be able to get its contents
+ with urlopen(self.url + "/example-minimal-library/") as response:
+ got_content = response.read().decode("utf-8")
+ want_content = """
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Links for example-minimal-library</title>
+ </head>
+ <body>
+ <h1>Links for example-minimal-library</h1>
+ <a href="/packages/example_minimal_library-0.0.1-py3-none-any.whl#sha256=79a4e9c1838c0631d5d8fa49a26efd6e9a364f6b38d9597c0f6df112271a0e28">example_minimal_library-0.0.1-py3-none-any.whl</a><br>
+ </body>
+</html>"""
+ self.assertEqual(
+ textwrap.dedent(want_content).strip(),
+ textwrap.dedent(got_content).strip(),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
index cc6348a..d586347 100644
--- a/python/BUILD.bazel
+++ b/python/BUILD.bazel
@@ -76,11 +76,13 @@
srcs = ["packaging.bzl"],
deps = [
":py_binary_bzl",
+ "//python/private:bzlmod_enabled_bzl",
"//python/private:py_package.bzl",
"//python/private:py_wheel_bzl",
"//python/private:py_wheel_normalize_pep440.bzl",
"//python/private:stamp_bzl",
"//python/private:util_bzl",
+ "@bazel_skylib//rules:native_binary",
],
)
diff --git a/python/packaging.bzl b/python/packaging.bzl
index f811965..a5ac25b 100644
--- a/python/packaging.bzl
+++ b/python/packaging.bzl
@@ -14,7 +14,9 @@
"""Public API for for building wheels."""
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
load("//python:py_binary.bzl", "py_binary")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
load("//python/private:py_package.bzl", "py_package_lib")
load("//python/private:py_wheel.bzl", _PyWheelInfo = "PyWheelInfo", _py_wheel = "py_wheel")
load("//python/private:util.bzl", "copy_propagating_kwargs")
@@ -70,7 +72,7 @@
},
)
-def py_wheel(name, twine = None, publish_args = [], **kwargs):
+def py_wheel(name, twine = None, twine_binary = Label("//tools/publish:twine") if BZLMOD_ENABLED else None, publish_args = [], **kwargs):
"""Builds a Python Wheel.
Wheels are Python distribution format defined in https://www.python.org/dev/peps/pep-0427/.
@@ -115,19 +117,21 @@
)
```
- To publish the wheel to PyPI, the twine package is required.
- rules_python doesn't provide twine itself, see [https://github.com/bazelbuild/rules_python/issues/1016].
- However you can install it with [pip_parse](#pip_parse), just like we do in the WORKSPACE file in rules_python.
+ To publish the wheel to PyPI, the twine package is required and it is installed
+ by default on `bzlmod` setups. On legacy `WORKSPACE`, `rules_python`
+ doesn't provide `twine` itself
+ (see https://github.com/bazelbuild/rules_python/issues/1016), but
+ you can install it with `pip_parse`, just like we do any other dependencies.
- Once you've installed twine, you can pass its label to the `twine` attribute of this macro,
- to get a "[name].publish" target.
+ Once you've installed twine, you can pass its label to the `twine`
+ attribute of this macro, to get a "[name].publish" target.
Example:
```python
py_wheel(
name = "my_wheel",
- twine = "@publish_deps_twine//:pkg",
+ twine = "@publish_deps//twine",
...
)
```
@@ -143,6 +147,7 @@
Args:
name: A unique name for this target.
twine: A label of the external location of the py_library target for twine
+ twine_binary: A label of the external location of a binary target for twine.
publish_args: arguments passed to twine, e.g. ["--repository-url", "https://pypi.my.org/simple/"].
These are subject to make var expansion, as with the `args` attribute.
Note that you can also pass additional args to the bazel run command as in the example above.
@@ -158,16 +163,32 @@
_py_wheel(name = name, **kwargs)
- if twine:
- if not twine.endswith(":pkg"):
- fail("twine label should look like @my_twine_repo//:pkg")
- twine_main = twine.replace(":pkg", ":rules_python_wheel_entry_point_twine.py")
+ twine_args = []
+ if twine or twine_binary:
twine_args = ["upload"]
twine_args.extend(publish_args)
twine_args.append("$(rootpath :{})/*".format(_dist_target))
- # TODO: use py_binary from //python:defs.bzl after our stardoc setup is less brittle
- # buildifier: disable=native-py
+ if twine_binary:
+ twine_kwargs = {"tags": ["manual"]}
+ native_binary(
+ name = "{}.publish".format(name),
+ src = twine_binary,
+ out = select({
+ "@platforms//os:windows": "{}.publish_script.exe".format(name),
+ "//conditions:default": "{}.publish_script".format(name),
+ }),
+ args = twine_args,
+ data = [_dist_target],
+ visibility = kwargs.get("visibility"),
+ **copy_propagating_kwargs(kwargs, twine_kwargs)
+ )
+ elif twine:
+ if not twine.endswith(":pkg"):
+ fail("twine label should look like @my_twine_repo//:pkg")
+
+ twine_main = twine.replace(":pkg", ":rules_python_wheel_entry_point_twine.py")
+
py_binary(
name = "{}.publish".format(name),
srcs = [twine_main],
diff --git a/python/private/py_console_script_binary.bzl b/python/private/py_console_script_binary.bzl
index d0c58bf..7347ebe 100644
--- a/python/private/py_console_script_binary.bzl
+++ b/python/private/py_console_script_binary.bzl
@@ -31,10 +31,19 @@
* @pypi//pylint
* @pypi//pylint:pkg
* Label("@pypi//pylint:pkg")
+ * Label("@pypi//pylint")
"""
- # str() is called to convert Label objects
- return str(pkg).replace(":pkg", "") + ":dist_info"
+ if type(pkg) == type(""):
+ label = native.package_relative_label(pkg)
+ else:
+ label = pkg
+
+ if hasattr(label, "same_package_label"):
+ return label.same_package_label("dist_info")
+ else:
+ # NOTE @aignas 2024-03-25: this is deprecated but kept for compatibility
+ return label.relative("dist_info")
def py_console_script_binary(
*,
diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel
index dde5b45..c1fc027 100644
--- a/python/runfiles/BUILD.bazel
+++ b/python/runfiles/BUILD.bazel
@@ -14,6 +14,7 @@
load("//python:defs.bzl", "py_library")
load("//python:packaging.bzl", "py_wheel")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
filegroup(
name = "distribution",
@@ -52,7 +53,7 @@
homepage = "https://github.com/bazelbuild/rules_python",
python_requires = ">=3.7",
strip_path_prefixes = ["python"],
- twine = "@publish_deps_twine//:pkg",
+ twine = None if BZLMOD_ENABLED else "@rules_python_publish_deps_twine//:pkg",
# this can be replaced by building with --stamp --embed_label=1.2.3
version = "{BUILD_EMBED_LABEL}",
visibility = ["//visibility:public"],
diff --git a/tests/entry_points/BUILD.bazel b/tests/entry_points/BUILD.bazel
index 7a22d3c..c877462 100644
--- a/tests/entry_points/BUILD.bazel
+++ b/tests/entry_points/BUILD.bazel
@@ -28,12 +28,19 @@
py_console_script_binary_in_a_macro(
name = "twine",
- pkg = "@publish_deps_twine//:pkg",
+ pkg = "@rules_python_publish_deps//twine",
+)
+
+py_console_script_binary_in_a_macro(
+ name = "twine_pkg",
+ pkg = "@rules_python_publish_deps//twine:pkg",
+ script = "twine",
)
build_test(
name = "build_entry_point",
targets = [
":twine",
+ ":twine_pkg",
],
)
diff --git a/tests/entry_points/simple_macro.bzl b/tests/entry_points/simple_macro.bzl
index 4764a3f..c56f2e1 100644
--- a/tests/entry_points/simple_macro.bzl
+++ b/tests/entry_points/simple_macro.bzl
@@ -18,14 +18,16 @@
load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
-def py_console_script_binary_in_a_macro(name, pkg):
+def py_console_script_binary_in_a_macro(name, pkg, **kwargs):
"""A simple macro to see that we can use our macro in a macro.
Args:
name, str: the name of the target
pkg, str: the pkg target
+ **kwargs, Any: extra kwargs passed through.
"""
py_console_script_binary(
name = name,
pkg = Label(pkg),
+ **kwargs
)
diff --git a/tests/runfiles/BUILD.bazel b/tests/runfiles/BUILD.bazel
index 6193ee9..5c92026 100644
--- a/tests/runfiles/BUILD.bazel
+++ b/tests/runfiles/BUILD.bazel
@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
load("@rules_python//python:py_test.bzl", "py_test")
load("@rules_python//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility
@@ -9,3 +10,10 @@
},
deps = ["//python/runfiles"],
)
+
+build_test(
+ name = "publishing",
+ targets = [
+ "//python/runfiles:wheel.publish",
+ ],
+)
diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel
index 51bd56d..b2aca5c 100644
--- a/tools/BUILD.bazel
+++ b/tools/BUILD.bazel
@@ -29,6 +29,7 @@
srcs = [
"BUILD.bazel",
"wheelmaker.py",
+ "//tools/publish:distribution",
],
visibility = ["//:__pkg__"],
)
diff --git a/tools/publish/BUILD.bazel b/tools/publish/BUILD.bazel
index 4759a31..a51693b 100644
--- a/tools/publish/BUILD.bazel
+++ b/tools/publish/BUILD.bazel
@@ -1,4 +1,6 @@
load("//python:pip.bzl", "compile_pip_requirements")
+load("//python/config_settings:transition.bzl", "py_binary")
+load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
compile_pip_requirements(
name = "requirements",
@@ -6,3 +8,27 @@
requirements_darwin = "requirements_darwin.txt",
requirements_windows = "requirements_windows.txt",
)
+
+py_console_script_binary(
+ name = "twine",
+ # We use a py_binary rule with version transitions to ensure that we do not
+ # rely on the default version of the registered python toolchain. What is more
+ # we are using this instead of `@python_versions//3.11:defs.bzl` because loading
+ # that file relies on bzlmod being enabled.
+ binary_rule = py_binary,
+ pkg = "@rules_python_publish_deps//twine",
+ python_version = "3.11",
+ script = "twine",
+ visibility = ["//visibility:public"],
+)
+
+filegroup(
+ name = "distribution",
+ srcs = [
+ "BUILD.bazel",
+ "requirements.txt",
+ "requirements_darwin.txt",
+ "requirements_windows.txt",
+ ],
+ visibility = ["//tools:__pkg__"],
+)
