python/private/pypi/pip_compile.bzl

# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Rules to verify and update pip-compile locked requirements.txt.

NOTE @aignas 2024-06-23: We are using the implementation specific name here to
make it possible to have multiple tools inside the `pypi` directory
"""

load("//python:py_binary.bzl", _py_binary = "py_binary")
load("//python:py_test.bzl", _py_test = "py_test")

# The `data` attribute does not allow duplicate labels, but user-provided `data`
# can overlap with labels added from attributes like `src`.
def _dedupe_data(data):
    res = []
    for d in data:
        if d not in res:
            res.append(d)
    return res

def pip_compile(
        name,
        srcs = None,
        src = None,
        extra_args = [],
        extra_deps = [],
        generate_hashes = True,
        py_binary = _py_binary,
        py_test = _py_test,
        requirements_in = None,
        requirements_txt = None,
        requirements_darwin = None,
        requirements_linux = None,
        requirements_windows = None,
        visibility = ["//visibility:private"],
        tags = None,
        constraints = [],
        data = [],
        **kwargs):
    """Generates targets for managing pip dependencies with pip-compile (piptools).

    By default this rules generates a filegroup named "[name]" which can be included in the data
    of some other compile_pip_requirements rule that references these requirements
    (e.g. with `-r ../other/requirements.txt`).
    It also generates two targets for running pip-compile:

    - validate with `bazel test [name].test`
    - update with   `bazel run [name].update`

    If you are using a version control system, the requirements.txt generated by this rule should
    be checked into it to ensure that all developers/users have the same dependency versions.

    Args:
        name: base name for generated targets, typically "requirements".
        srcs: a list of files containing inputs to dependency resolution. If not specified,
            defaults to `["pyproject.toml"]`. Supported formats are:
            * a requirements text file, usually named `requirements.in`
            * A `.toml` file, where the `project.dependencies` list is used as per
              {pep}`PEP 621`.
        src: file containing inputs to dependency resolution. If not specified,
            defaults to `pyproject.toml`. Supported formats are:
            * a requirements text file, usually named `requirements.in`
            * A `.toml` file, where the `project.dependencies` list is used as per
              {pep}`PEP 621`.
        extra_args: passed to pip-compile (aka `piptools`). See the
            [pip-compile docs](https://pip-tools.readthedocs.io/en/latest/cli/pip-compile)
            for args and meaning (passing `-h` and/or `--version` can help
            inform what args are available)
        extra_deps: extra dependencies passed to pip-compile.
        generate_hashes: whether to put hashes in the requirements_txt file.
        py_binary: the py_binary rule to be used.
        py_test: the py_test rule to be used.
        requirements_in: file expressing desired dependencies. Deprecated, use src or srcs instead.
        requirements_txt: result of "compiling" the requirements.in file.
        requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes.
        requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes.
        requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes.
        tags: tagging attribute common to all build rules, passed to both the _test and .update rules.
        visibility: passed to both the _test and .update rules.
        constraints: a list of files containing constraints to pass to pip-compile with `--constraint`.
        data: A list of labels to include as part of the `data` attribute in the generated `py_binary`.
        **kwargs: other bazel attributes passed to the "_test" rule.
    """
    if len([x for x in [srcs, src, requirements_in] if x != None]) > 1:
        fail("At most one of 'srcs', 'src', and 'requirements_in' attributes may be provided")

    if requirements_in:
        srcs = [requirements_in]
    elif src:
        srcs = [src]
    else:
        srcs = srcs or ["pyproject.toml"]

    requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt

    data = data or []

    # "Default" target produced by this macro
    # Allow a compile_pip_requirements rule to include another one in the data
    # for a requirements file that does `-r ../other/requirements.txt`
    native.filegroup(
        name = name,
        srcs = data + [requirements_txt],
        visibility = visibility,
    )

    data = _dedupe_data(data + [name, requirements_txt] + srcs + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + constraints)

    # Use the Label constructor so this is expanded in the context of the file
    # where it appears, which is to say, in @rules_python
    pip_compile = Label("//python/private/pypi/dependency_resolver:dependency_resolver.py")

    loc = "$(rlocationpath {})"

    args = ["--src=%s" % loc.format(src) for src in srcs] + [
        loc.format(requirements_txt),
        "//%s:%s" % (native.package_name(), name),
        "--resolver=backtracking",
        "--allow-unsafe",
    ]
    if generate_hashes:
        args.append("--generate-hashes")
    if requirements_linux:
        args.append("--requirements-linux={}".format(loc.format(requirements_linux)))
    if requirements_darwin:
        args.append("--requirements-darwin={}".format(loc.format(requirements_darwin)))
    if requirements_windows:
        args.append("--requirements-windows={}".format(loc.format(requirements_windows)))
    for constraint in constraints:
        args.append("--constraint=$(location {})".format(constraint))
    args.extend(extra_args)

    deps = [
        Label("@pypi__build//:lib"),
        Label("@pypi__click//:lib"),
        Label("@pypi__colorama//:lib"),
        Label("@pypi__importlib_metadata//:lib"),
        Label("@pypi__more_itertools//:lib"),
        Label("@pypi__packaging//:lib"),
        Label("@pypi__pep517//:lib"),
        Label("@pypi__pip//:lib"),
        Label("@pypi__pip_tools//:lib"),
        Label("@pypi__pyproject_hooks//:lib"),
        Label("@pypi__setuptools//:lib"),
        Label("@pypi__tomli//:lib"),
        Label("@pypi__zipp//:lib"),
        Label("//python/runfiles:runfiles"),
    ] + extra_deps

    tags = tags or []
    tags.append("requires-network")
    tags.append("no-remote-exec")
    tags.append("no-sandbox")
    attrs = {
        "args": args,
        "data": data,
        "deps": deps,
        "main": pip_compile,
        "srcs": [pip_compile],
        "tags": tags,
        "visibility": visibility,
    }

    env = kwargs.pop("env", {})
    env_inherit = kwargs.pop("env_inherit", [])
    proxy_variables = ["https_proxy", "http_proxy", "no_proxy", "HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY"]

    for var in proxy_variables:
        if var not in env_inherit:
            env_inherit.append(var)

    py_binary(
        name = name + ".update",
        env = env,
        python_version = kwargs.get("python_version", None),
        target_compatible_with = kwargs.get("target_compatible_with", []),
        **attrs
    )

    timeout = kwargs.pop("timeout", "short")

    py_test(
        name = name + ".test",
        timeout = timeout,
        # setuptools (the default python build tool) attempts to find user
        # configuration in the user's home direcotory. This seems to work fine on
        # linux and macOS, but fails on Windows, so we conditionally provide a fake
        # USERPROFILE env variable to allow setuptools to proceed without finding
        # user-provided configuration.
        env = select({
            "@@platforms//os:windows": {"USERPROFILE": "Z:\\FakeSetuptoolsHomeDirectoryHack"},
            "//conditions:default": {},
        }) | env,
        env_inherit = env_inherit,
        # kwargs could contain test-specific attributes like size
        **dict(attrs, **kwargs)
    )

    native.alias(
        name = "{}_test".format(name),
        actual = ":{}.test".format(name),
        deprecation = "Use '{}.test' instead. The '*_test' target will be removed in the next major release.".format(name),
        tags = ["manual"],
    )