| # Copyright 2024 The Bazel Authors. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Requirements parsing for whl_library creation. | |
| Use cases that the code needs to cover: | |
| * A single requirements_lock file that is used for the host platform. | |
| * Per-OS requirements_lock files that are used for the host platform. | |
| * A target platform specific requirements_lock that is used with extra | |
| pip arguments with --platform, etc and download_only = True. | |
| In the last case only a single `requirements_lock` file is allowed, in all | |
| other cases we assume that there may be a desire to resolve the requirements | |
| file for the host platform to be backwards compatible with the legacy | |
| behavior. | |
| """ | |
| load("//python/private:normalize_name.bzl", "normalize_name") | |
| load(":index_sources.bzl", "index_sources") | |
| load(":parse_requirements_txt.bzl", "parse_requirements_txt") | |
| load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms") | |
| # This includes the vendored _translate_cpu and _translate_os from | |
| # @platforms//host:extension.bzl at version 0.0.9 so that we don't | |
| # force the users to depend on it. | |
| def _translate_cpu(arch): | |
| if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: | |
| return "x86_32" | |
| if arch in ["amd64", "x86_64", "x64"]: | |
| return "x86_64" | |
| if arch in ["ppc", "ppc64", "ppc64le"]: | |
| return "ppc" | |
| if arch in ["arm", "armv7l"]: | |
| return "arm" | |
| if arch in ["aarch64"]: | |
| return "aarch64" | |
| if arch in ["s390x", "s390"]: | |
| return "s390x" | |
| if arch in ["mips64el", "mips64"]: | |
| return "mips64" | |
| if arch in ["riscv64"]: | |
| return "riscv64" | |
| return arch | |
| def _translate_os(os): | |
| if os.startswith("mac os"): | |
| return "osx" | |
| if os.startswith("freebsd"): | |
| return "freebsd" | |
| if os.startswith("openbsd"): | |
| return "openbsd" | |
| if os.startswith("linux"): | |
| return "linux" | |
| if os.startswith("windows"): | |
| return "windows" | |
| return os | |
| # TODO @aignas 2024-05-13: consider using the same platform tags as are used in | |
| # the //python:versions.bzl | |
| DEFAULT_PLATFORMS = [ | |
| "linux_aarch64", | |
| "linux_arm", | |
| "linux_ppc", | |
| "linux_s390x", | |
| "linux_x86_64", | |
| "osx_aarch64", | |
| "osx_x86_64", | |
| "windows_x86_64", | |
| ] | |
| def _default_platforms(*, filter): | |
| if not filter: | |
| fail("Must specific a filter string, got: {}".format(filter)) | |
| if filter.startswith("cp3"): | |
| # TODO @aignas 2024-05-23: properly handle python versions in the filter. | |
| # For now we are just dropping it to ensure that we don't fail. | |
| _, _, filter = filter.partition("_") | |
| sanitized = filter.replace("*", "").replace("_", "") | |
| if sanitized and not sanitized.isalnum(): | |
| fail("The platform filter can only contain '*', '_' and alphanumerics") | |
| if "*" in filter: | |
| prefix = filter.rstrip("*") | |
| if "*" in prefix: | |
| fail("The filter can only contain '*' at the end of it") | |
| if not prefix: | |
| return DEFAULT_PLATFORMS | |
| return [p for p in DEFAULT_PLATFORMS if p.startswith(prefix)] | |
| else: | |
| return [p for p in DEFAULT_PLATFORMS if filter in p] | |
| def _platforms_from_args(extra_pip_args): | |
| platform_values = [] | |
| for arg in extra_pip_args: | |
| if platform_values and platform_values[-1] == "": | |
| platform_values[-1] = arg | |
| continue | |
| if arg == "--platform": | |
| platform_values.append("") | |
| continue | |
| if not arg.startswith("--platform"): | |
| continue | |
| _, _, plat = arg.partition("=") | |
| if not plat: | |
| _, _, plat = arg.partition(" ") | |
| if plat: | |
| platform_values.append(plat) | |
| else: | |
| platform_values.append("") | |
| if not platform_values: | |
| return [] | |
| platforms = { | |
| p.target_platform: None | |
| for arg in platform_values | |
| for p in whl_target_platforms(arg) | |
| } | |
| return list(platforms.keys()) | |
| def parse_requirements( | |
| ctx, | |
| *, | |
| requirements_by_platform = {}, | |
| requirements_osx = None, | |
| requirements_linux = None, | |
| requirements_lock = None, | |
| requirements_windows = None, | |
| extra_pip_args = [], | |
| get_index_urls = None, | |
| python_version = None, | |
| logger = None, | |
| fail_fn = fail): | |
| """Get the requirements with platforms that the requirements apply to. | |
| Args: | |
| ctx: A context that has .read function that would read contents from a label. | |
| requirements_by_platform (label_keyed_string_dict): a way to have | |
| different package versions (or different packages) for different | |
| os, arch combinations. | |
| requirements_osx (label): The requirements file for the osx OS. | |
| requirements_linux (label): The requirements file for the linux OS. | |
| requirements_lock (label): The requirements file for all OSes, or used as a fallback. | |
| requirements_windows (label): The requirements file for windows OS. | |
| extra_pip_args (string list): Extra pip arguments to perform extra validations and to | |
| be joined with args fined in files. | |
| get_index_urls: Callable[[ctx, list[str]], dict], a callable to get all | |
| of the distribution URLs from a PyPI index. Accepts ctx and | |
| distribution names to query. | |
| python_version: str or None. This is needed when the get_index_urls is | |
| specified. It should be of the form "3.x.x", | |
| logger: repo_utils.logger or None, a simple struct to log diagnostic messages. | |
| fail_fn (Callable[[str], None]): A failure function used in testing failure cases. | |
| Returns: | |
| A tuple where the first element a dict of dicts where the first key is | |
| the normalized distribution name (with underscores) and the second key | |
| is the requirement_line, then value and the keys are structs with the | |
| following attributes: | |
| * distribution: The non-normalized distribution name. | |
| * srcs: The Simple API downloadable source list. | |
| * requirement_line: The original requirement line. | |
| * target_platforms: The list of target platforms that this package is for. | |
| * is_exposed: A boolean if the package should be exposed via the hub | |
| repository. | |
| The second element is extra_pip_args should be passed to `whl_library`. | |
| """ | |
| if not ( | |
| requirements_lock or | |
| requirements_linux or | |
| requirements_osx or | |
| requirements_windows or | |
| requirements_by_platform | |
| ): | |
| fail_fn( | |
| "A 'requirements_lock' attribute must be specified, a platform-specific lockfiles " + | |
| "via 'requirements_by_platform' or an os-specific lockfiles must be specified " + | |
| "via 'requirements_*' attributes", | |
| ) | |
| return None | |
| platforms = _platforms_from_args(extra_pip_args) | |
| if platforms: | |
| lock_files = [ | |
| f | |
| for f in [ | |
| requirements_lock, | |
| requirements_linux, | |
| requirements_osx, | |
| requirements_windows, | |
| ] + list(requirements_by_platform.keys()) | |
| if f | |
| ] | |
| if len(lock_files) > 1: | |
| # If the --platform argument is used, check that we are using | |
| # a single `requirements_lock` file instead of the OS specific ones as that is | |
| # the only correct way to use the API. | |
| fail_fn("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute") | |
| return None | |
| files_by_platform = [ | |
| (lock_files[0], platforms), | |
| ] | |
| else: | |
| files_by_platform = { | |
| file: [ | |
| platform | |
| for filter_or_platform in specifier.split(",") | |
| for platform in (_default_platforms(filter = filter_or_platform) if filter_or_platform.endswith("*") else [filter_or_platform]) | |
| ] | |
| for file, specifier in requirements_by_platform.items() | |
| }.items() | |
| for f in [ | |
| # If the users need a greater span of the platforms, they should consider | |
| # using the 'requirements_by_platform' attribute. | |
| (requirements_linux, _default_platforms(filter = "linux_*")), | |
| (requirements_osx, _default_platforms(filter = "osx_*")), | |
| (requirements_windows, _default_platforms(filter = "windows_*")), | |
| (requirements_lock, None), | |
| ]: | |
| if f[0]: | |
| files_by_platform.append(f) | |
| configured_platforms = {} | |
| options = {} | |
| requirements = {} | |
| for file, plats in files_by_platform: | |
| if plats: | |
| for p in plats: | |
| if p in configured_platforms: | |
| fail_fn( | |
| "Expected the platform '{}' to be map only to a single requirements file, but got multiple: '{}', '{}'".format( | |
| p, | |
| configured_platforms[p], | |
| file, | |
| ), | |
| ) | |
| return None | |
| configured_platforms[p] = file | |
| else: | |
| plats = [ | |
| p | |
| for p in DEFAULT_PLATFORMS | |
| if p not in configured_platforms | |
| ] | |
| for p in plats: | |
| configured_platforms[p] = file | |
| contents = ctx.read(file) | |
| # Parse the requirements file directly in starlark to get the information | |
| # needed for the whl_library declarations later. | |
| parse_result = parse_requirements_txt(contents) | |
| # Replicate a surprising behavior that WORKSPACE builds allowed: | |
| # Defining a repo with the same name multiple times, but only the last | |
| # definition is respected. | |
| # The requirement lines might have duplicate names because lines for extras | |
| # are returned as just the base package name. e.g., `foo[bar]` results | |
| # in an entry like `("foo", "foo[bar] == 1.0 ...")`. | |
| requirements_dict = { | |
| normalize_name(entry[0]): entry | |
| for entry in sorted( | |
| parse_result.requirements, | |
| # Get the longest match and fallback to original WORKSPACE sorting, | |
| # which should get us the entry with most extras. | |
| # | |
| # FIXME @aignas 2024-05-13: The correct behaviour might be to get an | |
| # entry with all aggregated extras, but it is unclear if we | |
| # should do this now. | |
| key = lambda x: (len(x[1].partition("==")[0]), x), | |
| ) | |
| }.values() | |
| tokenized_options = [] | |
| for opt in parse_result.options: | |
| for p in opt.split(" "): | |
| tokenized_options.append(p) | |
| pip_args = tokenized_options + extra_pip_args | |
| for p in plats: | |
| requirements[p] = requirements_dict | |
| options[p] = pip_args | |
| requirements_by_platform = {} | |
| for target_platform, reqs_ in requirements.items(): | |
| extra_pip_args = options[target_platform] | |
| for distribution, requirement_line in reqs_: | |
| for_whl = requirements_by_platform.setdefault( | |
| normalize_name(distribution), | |
| {}, | |
| ) | |
| for_req = for_whl.setdefault( | |
| (requirement_line, ",".join(extra_pip_args)), | |
| struct( | |
| distribution = distribution, | |
| srcs = index_sources(requirement_line), | |
| requirement_line = requirement_line, | |
| target_platforms = [], | |
| extra_pip_args = extra_pip_args, | |
| download = len(platforms) > 0, | |
| ), | |
| ) | |
| for_req.target_platforms.append(target_platform) | |
| index_urls = {} | |
| if get_index_urls: | |
| if not python_version: | |
| fail_fn("'python_version' must be provided") | |
| return None | |
| index_urls = get_index_urls( | |
| ctx, | |
| # Use list({}) as a way to have a set | |
| list({ | |
| req.distribution: None | |
| for reqs in requirements_by_platform.values() | |
| for req in reqs.values() | |
| }), | |
| ) | |
| ret = {} | |
| for whl_name, reqs in requirements_by_platform.items(): | |
| requirement_target_platforms = {} | |
| for r in reqs.values(): | |
| for p in r.target_platforms: | |
| requirement_target_platforms[p] = None | |
| is_exposed = len(requirement_target_platforms) == len(configured_platforms) | |
| if not is_exposed and logger: | |
| logger.debug(lambda: "Package {} will not be exposed because it is only present on a subset of platforms: {} out of {}".format( | |
| whl_name, | |
| sorted(requirement_target_platforms), | |
| sorted(configured_platforms), | |
| )) | |
| for r in sorted(reqs.values(), key = lambda r: r.requirement_line): | |
| whls, sdist = _add_dists( | |
| r, | |
| index_urls.get(whl_name), | |
| python_version = python_version, | |
| logger = logger, | |
| ) | |
| ret.setdefault(whl_name, []).append( | |
| struct( | |
| distribution = r.distribution, | |
| srcs = r.srcs, | |
| requirement_line = r.requirement_line, | |
| target_platforms = sorted(r.target_platforms), | |
| extra_pip_args = r.extra_pip_args, | |
| download = r.download, | |
| whls = whls, | |
| sdist = sdist, | |
| is_exposed = is_exposed, | |
| ), | |
| ) | |
| return ret | |
| def select_requirement(requirements, *, platform): | |
| """A simple function to get a requirement for a particular platform. | |
| Args: | |
| requirements (list[struct]): The list of requirements as returned by | |
| the `parse_requirements` function above. | |
| platform (str): The host platform. Usually an output of the | |
| `host_platform` function. | |
| Returns: | |
| None if not found or a struct returned as one of the values in the | |
| parse_requirements function. The requirement that should be downloaded | |
| by the host platform will be returned. | |
| """ | |
| maybe_requirement = [ | |
| req | |
| for req in requirements | |
| if platform in req.target_platforms or req.download | |
| ] | |
| if not maybe_requirement: | |
| # Sometimes the package is not present for host platform if there | |
| # are whls specified only in particular requirements files, in that | |
| # case just continue, however, if the download_only flag is set up, | |
| # then the user can also specify the target platform of the wheel | |
| # packages they want to download, in that case there will be always | |
| # a requirement here, so we will not be in this code branch. | |
| return None | |
| return maybe_requirement[0] | |
| def host_platform(repository_os): | |
| """Return a string representation of the repository OS. | |
| Args: | |
| repository_os (struct): The `module_ctx.os` or `repository_ctx.os` attribute. | |
| See https://bazel.build/rules/lib/builtins/repository_os.html | |
| Returns: | |
| The string representation of the platform that we can later used in the `pip` | |
| machinery. | |
| """ | |
| return "{}_{}".format( | |
| _translate_os(repository_os.name.lower()), | |
| _translate_cpu(repository_os.arch.lower()), | |
| ) | |
| def _add_dists(requirement, index_urls, python_version, logger = None): | |
| """Populate dists based on the information from the PyPI index. | |
| This function will modify the given requirements_by_platform data structure. | |
| Args: | |
| requirement: The result of parse_requirements function. | |
| index_urls: The result of simpleapi_download. | |
| python_version: The version of the python interpreter. | |
| logger: A logger for printing diagnostic info. | |
| """ | |
| if not index_urls: | |
| return [], None | |
| whls = [] | |
| sdist = None | |
| # TODO @aignas 2024-05-22: it is in theory possible to add all | |
| # requirements by version instead of by sha256. This may be useful | |
| # for some projects. | |
| for sha256 in requirement.srcs.shas: | |
| # For now if the artifact is marked as yanked we just ignore it. | |
| # | |
| # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api | |
| maybe_whl = index_urls.whls.get(sha256) | |
| if maybe_whl and not maybe_whl.yanked: | |
| whls.append(maybe_whl) | |
| continue | |
| maybe_sdist = index_urls.sdists.get(sha256) | |
| if maybe_sdist and not maybe_sdist.yanked: | |
| sdist = maybe_sdist | |
| continue | |
| if logger: | |
| logger.warn(lambda: "Could not find a whl or an sdist with sha256={}".format(sha256)) | |
| yanked = {} | |
| for dist in whls + [sdist]: | |
| if dist and dist.yanked: | |
| yanked.setdefault(dist.yanked, []).append(dist.filename) | |
| if yanked: | |
| logger.warn(lambda: "\n".join([ | |
| "the following distributions got yanked:", | |
| ] + [ | |
| "reason: {}\n {}".format(reason, "\n".join(sorted(dists))) | |
| for reason, dists in yanked.items() | |
| ])) | |
| # Filter out the wheels that are incompatible with the target_platforms. | |
| whls = select_whls( | |
| whls = whls, | |
| want_abis = [ | |
| "none", | |
| "abi3", | |
| "cp" + python_version.replace(".", ""), | |
| # Older python versions have wheels for the `*m` ABI. | |
| "cp" + python_version.replace(".", "") + "m", | |
| ], | |
| want_platforms = requirement.target_platforms, | |
| want_python_version = python_version, | |
| logger = logger, | |
| ) | |
| return whls, sdist |