blob: 612ca2cfdfdbb84cce790f29c91e9d7b511e5021 [file] [log] [blame]
# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""
load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
load("//python/private:envsubst.bzl", "envsubst")
load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter")
load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load(":attrs.bzl", "ATTRS", "use_isolated")
load(":deps.bzl", "all_repo_names")
load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
load(":parse_whl_name.bzl", "parse_whl_name")
load(":patch_whl.bzl", "patch_whl")
load(":pypi_repo_utils.bzl", "pypi_repo_utils")
load(":whl_target_platforms.bzl", "whl_target_platforms")
_CPPFLAGS = "CPPFLAGS"
_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point"
def _get_xcode_location_cflags(rctx):
"""Query the xcode sdk location to update cflags
Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so.
Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
"""
# Only run on MacOS hosts
if not rctx.os.name.lower().startswith("mac os"):
return []
xcode_sdk_location = repo_utils.execute_unchecked(
rctx,
op = "GetXcodeLocation",
arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"],
)
if xcode_sdk_location.return_code != 0:
return []
xcode_root = xcode_sdk_location.stdout.strip()
if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower():
# This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer
# so we need to change the path to to the macos specific tools which are in a different relative
# path than xcode installed command line tools.
xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root)
return [
"-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root),
]
def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None):
"""Gather cflags from a standalone toolchain for unix systems.
Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg
otherwise. See https://github.com/indygreg/python-build-standalone/issues/103
"""
# Only run on Unix systems
if not rctx.os.name.lower().startswith(("mac os", "linux")):
return []
# Only update the location when using a standalone toolchain.
if not is_standalone_interpreter(rctx, python_interpreter, logger = logger):
return []
stdout = repo_utils.execute_checked_stdout(
rctx,
op = "GetPythonVersionForUnixCflags",
arguments = [
python_interpreter,
"-c",
"import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')",
],
)
_python_version = stdout
include_path = "{}/include/python{}".format(
python_interpreter.dirname,
_python_version,
)
return ["-isystem {}".format(include_path)]
def _parse_optional_attrs(rctx, args, extra_pip_args = None):
"""Helper function to parse common attributes of pip_repository and whl_library repository rules.
This function also serializes the structured arguments as JSON
so they can be passed on the command line to subprocesses.
Args:
rctx: Handle to the rule repository context.
args: A list of parsed args for the rule.
extra_pip_args: The pip args to pass.
Returns: Augmented args list.
"""
if use_isolated(rctx, rctx.attr):
args.append("--isolated")
# Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3)
# support rctx.getenv(name, default): When building incrementally, any change to the value of
# the variable named by name will cause this repository to be re-fetched.
if "getenv" in dir(rctx):
getenv = rctx.getenv
else:
getenv = rctx.os.environ.get
# Check for None so we use empty default types from our attrs.
# Some args want to be list, and some want to be dict.
if extra_pip_args != None:
args += [
"--extra_pip_args",
json.encode(struct(arg = [
envsubst(pip_arg, rctx.attr.envsubst, getenv)
for pip_arg in extra_pip_args
])),
]
if rctx.attr.download_only:
args.append("--download_only")
if rctx.attr.pip_data_exclude != None:
args += [
"--pip_data_exclude",
json.encode(struct(arg = rctx.attr.pip_data_exclude)),
]
if rctx.attr.enable_implicit_namespace_pkgs:
args.append("--enable_implicit_namespace_pkgs")
if rctx.attr.environment != None:
args += [
"--environment",
json.encode(struct(arg = rctx.attr.environment)),
]
return args
def _create_repository_execution_environment(rctx, python_interpreter, logger = None):
"""Create a environment dictionary for processes we spawn with rctx.execute.
Args:
rctx (repository_ctx): The repository context.
python_interpreter (path): The resolved python interpreter.
logger: Optional logger to use for operations.
Returns:
Dictionary of environment variable suitable to pass to rctx.execute.
"""
# Gather any available CPPFLAGS values
cppflags = []
cppflags.extend(_get_xcode_location_cflags(rctx))
cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter, logger = logger))
env = {
"PYTHONPATH": pypi_repo_utils.construct_pythonpath(
rctx,
entries = rctx.attr._python_path_entries,
),
_CPPFLAGS: " ".join(cppflags),
}
return env
def _whl_library_impl(rctx):
logger = repo_utils.logger(rctx)
python_interpreter = pypi_repo_utils.resolve_python_interpreter(
rctx,
python_interpreter = rctx.attr.python_interpreter,
python_interpreter_target = rctx.attr.python_interpreter_target,
)
args = [
python_interpreter,
"-m",
"python.private.pypi.whl_installer.wheel_installer",
"--requirement",
rctx.attr.requirement,
]
extra_pip_args = []
extra_pip_args.extend(rctx.attr.extra_pip_args)
# Manually construct the PYTHONPATH since we cannot use the toolchain here
environment = _create_repository_execution_environment(rctx, python_interpreter, logger = logger)
whl_path = None
if rctx.attr.whl_file:
whl_path = rctx.path(rctx.attr.whl_file)
# Simulate the behaviour where the whl is present in the current directory.
rctx.symlink(whl_path, whl_path.basename)
whl_path = rctx.path(whl_path.basename)
elif rctx.attr.urls:
filename = rctx.attr.filename
urls = rctx.attr.urls
if not filename:
_, _, filename = urls[0].rpartition("/")
if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")):
if rctx.attr.filename:
msg = "got '{}'".format(filename)
else:
msg = "detected '{}' from url:\n{}".format(filename, urls[0])
fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg))
result = rctx.download(
url = urls,
output = filename,
sha256 = rctx.attr.sha256,
auth = get_auth(rctx, urls),
)
if not result.success:
fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
if filename.endswith(".whl"):
whl_path = rctx.path(rctx.attr.filename)
else:
# It is an sdist and we need to tell PyPI to use a file in this directory
# and, allow getting build dependencies from PYTHONPATH, which we
# setup in this repository rule, but still download any necessary
# build deps from PyPI (e.g. `flit_core`) if they are missing.
extra_pip_args.extend(["--find-links", "."])
args = _parse_optional_attrs(rctx, args, extra_pip_args)
if not whl_path:
if rctx.attr.urls:
op_tmpl = "whl_library.BuildWheelFromSource({name}, {requirement})"
elif rctx.attr.download_only:
op_tmpl = "whl_library.DownloadWheel({name}, {requirement})"
else:
op_tmpl = "whl_library.ResolveRequirement({name}, {requirement})"
repo_utils.execute_checked(
rctx,
# truncate the requirement value when logging it / reporting
# progress since it may contain several ' --hash=sha256:...
# --hash=sha256:...' substrings that fill up the console
op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]),
arguments = args,
environment = environment,
quiet = rctx.attr.quiet,
timeout = rctx.attr.timeout,
logger = logger,
)
whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"])
if not rctx.delete("whl_file.json"):
fail("failed to delete the whl_file.json file")
if rctx.attr.whl_patches:
patches = {}
for patch_file, json_args in rctx.attr.whl_patches.items():
patch_dst = struct(**json.decode(json_args))
if whl_path.basename in patch_dst.whls:
patches[patch_file] = patch_dst.patch_strip
if patches:
whl_path = patch_whl(
rctx,
op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement),
python_interpreter = python_interpreter,
whl_path = whl_path,
patches = patches,
quiet = rctx.attr.quiet,
timeout = rctx.attr.timeout,
)
target_platforms = rctx.attr.experimental_target_platforms
if target_platforms:
parsed_whl = parse_whl_name(whl_path.basename)
if parsed_whl.platform_tag != "any":
# NOTE @aignas 2023-12-04: if the wheel is a platform specific
# wheel, we only include deps for that target platform
target_platforms = [
p.target_platform
for p in whl_target_platforms(
platform_tag = parsed_whl.platform_tag,
abi_tag = parsed_whl.abi_tag,
)
]
repo_utils.execute_checked(
rctx,
op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
arguments = args + [
"--whl-file",
whl_path,
] + ["--platform={}".format(p) for p in target_platforms],
environment = environment,
quiet = rctx.attr.quiet,
timeout = rctx.attr.timeout,
logger = logger,
)
metadata = json.decode(rctx.read("metadata.json"))
rctx.delete("metadata.json")
# NOTE @aignas 2024-06-22: this has to live on until we stop supporting
# passing `twine` as a `:pkg` library via the `WORKSPACE` builds.
#
# See ../../packaging.bzl line 190
entry_points = {}
for item in metadata["entry_points"]:
name = item["name"]
module = item["module"]
attribute = item["attribute"]
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name
entry_point_target_name = (
_WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py
)
entry_point_script_name = entry_point_target_name + ".py"
rctx.file(
entry_point_script_name,
_generate_entry_point_contents(module, attribute),
)
entry_points[entry_point_without_py] = entry_point_script_name
build_file_contents = generate_whl_library_build_bazel(
name = whl_path.basename,
dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
dependencies = metadata["deps"],
dependencies_by_platform = metadata["deps_by_platform"],
group_name = rctx.attr.group_name,
group_deps = rctx.attr.group_deps,
data_exclude = rctx.attr.pip_data_exclude,
tags = [
"pypi_name=" + metadata["name"],
"pypi_version=" + metadata["version"],
],
entry_points = entry_points,
annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
)
rctx.file("BUILD.bazel", build_file_contents)
return
def _generate_entry_point_contents(
module,
attribute,
shebang = "#!/usr/bin/env python3"):
"""Generate the contents of an entry point script.
Args:
module (str): The name of the module to use.
attribute (str): The name of the attribute to call.
shebang (str, optional): The shebang to use for the entry point python
file.
Returns:
str: A string of python code.
"""
contents = """\
{shebang}
import sys
from {module} import {attribute}
if __name__ == "__main__":
sys.exit({attribute}())
""".format(
shebang = shebang,
module = module,
attribute = attribute,
)
return contents
# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique
whl_library_attrs = dict({
"annotation": attr.label(
doc = (
"Optional json encoded file containing annotation to apply to the extracted wheel. " +
"See `package_annotation`"
),
allow_files = True,
),
"dep_template": attr.string(
doc = """
The dep template to use for referencing the dependencies. It should have `{name}`
and `{target}` tokens that will be replaced with the normalized distribution name
and the target that we need respectively.
""",
),
"filename": attr.string(
doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
),
"group_deps": attr.string_list(
doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
default = [],
),
"group_name": attr.string(
doc = "Name of the group, if any.",
),
"repo": attr.string(
mandatory = True,
doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
),
"repo_prefix": attr.string(
doc = """
Prefix for the generated packages will be of the form `@<prefix><sanitized-package-name>//...`
DEPRECATED. Only left for people who vendor requirements.bzl.
""",
),
"requirement": attr.string(
mandatory = True,
doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.",
),
"sha256": attr.string(
doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.",
),
"urls": attr.string_list(
doc = """\
The list of urls of the whl to be downloaded using bazel downloader. Using this
attr makes `extra_pip_args` and `download_only` ignored.""",
),
"whl_file": attr.label(
doc = "The whl file that should be used instead of downloading or building the whl.",
),
"whl_patches": attr.label_keyed_string_dict(
doc = """a label-keyed-string dict that has
json.encode(struct([whl_file], patch_strip]) as values. This
is to maintain flexibility and correct bzlmod extension interface
until we have a better way to define whl_library and move whl
patching to a separate place. INTERNAL USE ONLY.""",
),
"_python_path_entries": attr.label_list(
# Get the root directory of these rules and keep them as a default attribute
# in order to avoid unnecessary repository fetching restarts.
#
# This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478
default = [
Label("//:BUILD.bazel"),
] + [
# Includes all the external dependencies from repositories.bzl
Label("@" + repo + "//:BUILD.bazel")
for repo in all_repo_names
],
),
"_rule_name": attr.string(default = "whl_library"),
}, **ATTRS)
whl_library_attrs.update(AUTH_ATTRS)
whl_library = repository_rule(
attrs = whl_library_attrs,
doc = """
Download and extracts a single wheel based into a bazel repo based on the requirement string passed in.
Instantiated from pip_repository and inherits config options from there.""",
implementation = _whl_library_impl,
environ = [
"RULES_PYTHON_PIP_ISOLATED",
REPO_DEBUG_ENV_VAR,
],
)