This adds support for "extras". "Extras" are additional dependencies of a given library, which are consumed by passing the "extra" name in brackets after the distribution name, for example: ``` mock[docs]==1.0.1 ``` We see this in the dependencies of several Google Cloud libraries, which depend on: `googleapis_common_protos[grpc]` I've added a simple test that the dependency structure we synthesize for this kind of thing is correct via an "extras" test that has a `requirements.txt` of: ``` google-cloud-language==0.27.0 ``` Fixes: https://github.com/bazelbuild/rules_python/issues/12
diff --git a/README.md b/README.md index a6a8710..aea05d5 100644 --- a/README.md +++ b/README.md
@@ -106,6 +106,10 @@ `requirement` pattern in case the need arises for us to make changes to this format in the future. +["Extras"]( +https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras) +will have a target of the extra name (in place of `pkg` above). + ## Updating `docs/` All of the content (except `BUILD`) under `docs/` is generated. To update the
diff --git a/WORKSPACE b/WORKSPACE index 0f205be..430dfe7 100644 --- a/WORKSPACE +++ b/WORKSPACE
@@ -26,8 +26,8 @@ git_repository( name = "io_bazel_skydoc", - remote = "https://github.com/bazelbuild/skydoc.git", commit = "e9be81cf5be41e4200749f5d8aa2db7955f8aacc", + remote = "https://github.com/bazelbuild/skydoc.git", ) load("@io_bazel_skydoc//skylark:skylark.bzl", "skydoc_repositories") @@ -92,6 +92,15 @@ "mock-2.0.0-py2.py3-none-any.whl"), ) +http_file( + name = "google_cloud_language_whl", + sha256 = "a2dd34f0a0ebf5705dcbe34bd41199b1d0a55c4597d38ed045bd183361a561e9", + # From https://pypi.python.org/pypi/google-cloud-language + url = ("https://pypi.python.org/packages/6e/86/" + + "cae57e4802e72d9e626ee5828ed5a646cf4016b473a4a022f1038dba3460/" + + "google_cloud_language-0.29.0-py2.py3-none-any.whl"), +) + # Imports for examples pip_import( name = "examples_helloworld", @@ -128,3 +137,15 @@ ) _boto_install() + +pip_import( + name = "examples_extras", + requirements = "//examples/extras:requirements.txt", +) + +load( + "@examples_extras//:requirements.bzl", + _extras_install = "pip_install", +) + +_extras_install()
diff --git a/examples/extras/BUILD b/examples/extras/BUILD new file mode 100644 index 0000000..94880ce --- /dev/null +++ b/examples/extras/BUILD
@@ -0,0 +1,29 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +load("@examples_extras//:requirements.bzl", "requirement") +load("//python:python.bzl", "py_test") + +py_test( + name = "extras_test", + srcs = ["extras_test.py"], + deps = [ + requirement("google-cloud-language"), + # Make sure that we can resolve the "extra" dependency + requirement("googleapis-common-protos[grpc]"), + ], +)
diff --git a/examples/extras/extras_test.py b/examples/extras/extras_test.py new file mode 100644 index 0000000..3ad249d --- /dev/null +++ b/examples/extras/extras_test.py
@@ -0,0 +1,25 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +# The test is the build itself, which should not work if extras are missing. +class ExtrasTest(unittest.TestCase): + def test_nothing(self): + pass + + +if __name__ == '__main__': + unittest.main()
diff --git a/examples/extras/requirements.txt b/examples/extras/requirements.txt new file mode 100644 index 0000000..743bbe7 --- /dev/null +++ b/examples/extras/requirements.txt
@@ -0,0 +1 @@ +google-cloud-language==0.27.0
diff --git a/examples/extras/version_test.py b/examples/extras/version_test.py new file mode 100644 index 0000000..9b469b7 --- /dev/null +++ b/examples/extras/version_test.py
@@ -0,0 +1,26 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pip +import unittest + + +class VersionTest(unittest.TestCase): + + def test_version(self): + self.assertEqual(pip.__version__, '9.0.1') + + +if __name__ == '__main__': + unittest.main()
diff --git a/python/whl.bzl b/python/whl.bzl index f7827ed..7a19947 100644 --- a/python/whl.bzl +++ b/python/whl.bzl
@@ -16,12 +16,17 @@ def _whl_impl(repository_ctx): """Core implementation of whl_library.""" - result = repository_ctx.execute([ + args = [ "python", repository_ctx.path(repository_ctx.attr._script), "--whl", repository_ctx.path(repository_ctx.attr.whl), "--requirements", repository_ctx.attr.requirements, - ]) + ] + + if repository_ctx.attr.extras: + args += ["--extras", ",".join(repository_ctx.attr.extras)] + + result = repository_ctx.execute(args) if result.return_code: fail("whl_library failed: %s (%s)" % (result.stdout, result.stderr)) @@ -33,6 +38,7 @@ single_file = True, ), "requirements": attr.string(), + "extras": attr.string_list(), "_script": attr.label( executable = True, default = Label("//rules_python:whl.py"), @@ -64,4 +70,7 @@ requirements: The name of the pip_import repository rule from which to load this .whl's dependencies. + + extras: A subset of the "extras" available from this <code>.whl</code> for which + <code>requirements</code> has the dependencies. """
diff --git a/rules_python/BUILD b/rules_python/BUILD index 08aca7e..e0bf796 100644 --- a/rules_python/BUILD +++ b/rules_python/BUILD
@@ -28,6 +28,7 @@ data = [ "@futures_3_1_1_whl//file", "@futures_2_2_0_whl//file", + "@google_cloud_language_whl//file", "@grpc_whl//file", "@mock_whl//file", ],
diff --git a/rules_python/piptool.py b/rules_python/piptool.py index 98dfce9..f5d504a 100644 --- a/rules_python/piptool.py +++ b/rules_python/piptool.py
@@ -18,6 +18,7 @@ import json import os import pkgutil +import pkg_resources import re import shutil import sys @@ -78,36 +79,7 @@ argv = ["--disable-pip-version-check", "--cert", cert_path] + argv return pip.main(argv) - -# TODO(mattmoor): We can't easily depend on other libraries when -# being invoked as a raw .py file. Once bundled, we should be able -# to remove this fallback on a stub implementation of Wheel. -try: - from rules_python.whl import Wheel -except: - class Wheel(object): - - def __init__(self, path): - self._path = path - - def basename(self): - return os.path.basename(self._path) - - def distribution(self): - # See https://www.python.org/dev/peps/pep-0427/#file-name-convention - parts = self.basename().split('-') - return parts[0] - - def version(self): - # See https://www.python.org/dev/peps/pep-0427/#file-name-convention - parts = self.basename().split('-') - return parts[1] - - def repository_name(self): - # Returns the canonical name of the Bazel repository for this package. - canonical = 'pypi__{}_{}'.format(self.distribution(), self.version()) - # Escape any illegal characters with underscore. - return re.sub('[-.]', '_', canonical) +from rules_python.whl import Wheel parser = argparse.ArgumentParser( description='Import Python dependencies into Bazel.') @@ -124,6 +96,59 @@ parser.add_argument('--directory', action='store', help=('The directory into which to put .whl files.')) +def determine_possible_extras(whls): + """Determines the list of possible "extras" for each .whl + + The possibility of an extra is determined by looking at its + additional requirements, and determinine whether they are + satisfied by the complete list of available wheels. + + Args: + whls: a list of Wheel objects + + Returns: + a dict that is keyed by the Wheel objects in whls, and whose + values are lists of possible extras. + """ + whl_map = { + whl.distribution(): whl + for whl in whls + } + + # TODO(mattmoor): Consider memoizing if this recursion ever becomes + # expensive enough to warrant it. + def is_possible(distro, extra): + distro = distro.replace("-", "_") + # If we don't have the .whl at all, then this isn't possible. + if distro not in whl_map: + return False + whl = whl_map[distro] + # If we have the .whl, and we don't need anything extra then + # we can satisfy this dependency. + if not extra: + return True + # If we do need something extra, then check the extra's + # dependencies to make sure they are fully satisfied. + for extra_dep in whl.dependencies(extra=extra): + req = pkg_resources.Requirement.parse(extra_dep) + # Check that the dep and any extras are all possible. + if not is_possible(req.project_name, None): + return False + for e in req.extras: + if not is_possible(req.project_name, e): + return False + # If all of the dependencies of the extra are satisfiable then + # it is possible to construct this dependency. + return True + + return { + whl: [ + extra + for extra in whl.extras() + if is_possible(whl.distribution(), extra) + ] + for whl in whls + } def main(): args = parser.parse_args() @@ -140,6 +165,9 @@ if fname.endswith('.whl'): yield os.path.join(root, fname) + whls = [Wheel(path) for path in list_whls()] + possible_extras = determine_possible_extras(whls) + def whl_library(wheel): # Indentation here matters. whl_library must be within the scope # of the function below. We also avoid reimporting an existing WHL. @@ -149,10 +177,25 @@ name = "{repo_name}", whl = "@{name}//:{path}", requirements = "@{name}//:requirements.bzl", + extras = [{extras}] )""".format(name=args.name, repo_name=wheel.repository_name(), - path=wheel.basename()) + path=wheel.basename(), + extras=','.join([ + '"%s"' % extra + for extra in possible_extras.get(wheel, []) + ])) - whls = [Wheel(path) for path in list_whls()] + whl_targets = ','.join([ + ','.join([ + '"%s": "@%s//:pkg"' % (whl.distribution().lower(), whl.repository_name()) + ] + [ + # For every extra that is possible from this requirements.txt + '"%s[%s]": "@%s//:%s"' % (whl.distribution().lower(), extra.lower(), + whl.repository_name(), extra) + for extra in possible_extras.get(whl, []) + ]) + for whl in whls + ]) with open(args.output, 'w') as f: f.write("""\ @@ -178,10 +221,7 @@ return _requirements[name_key] """.format(input=args.input, whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass", - mappings=','.join([ - '"%s": "@%s//:pkg"' % (wheel.distribution().lower(), wheel.repository_name()) - for wheel in whls - ]))) + mappings=whl_targets)) if __name__ == '__main__': main()
diff --git a/rules_python/whl.py b/rules_python/whl.py index 14d775a..e3829bd 100644 --- a/rules_python/whl.py +++ b/rules_python/whl.py
@@ -70,13 +70,21 @@ def name(self): return self.metadata().get('name') - def dependencies(self): + def dependencies(self, extra=None): + """Access the dependencies of this Wheel. + + Args: + extra: if specified, include the additional dependencies + of the named "extra". + + Yields: + the names of requirements from the metadata.json + """ # TODO(mattmoor): Is there a schema to follow for this? run_requires = self.metadata().get('run_requires', []) for requirement in run_requires: - if 'extra' in requirement: - # TODO(mattmoor): What's the best way to support "extras"? - # https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras + if requirement.get('extra') != extra: + # Match the requirements for the extra we're looking for. continue if 'environment' in requirement: # TODO(mattmoor): What's the best way to support "environment"? @@ -89,6 +97,9 @@ parts = re.split('[ ><=()]', entry) yield parts[0] + def extras(self): + return self.metadata().get('extras', []) + def expand(self, directory): with zipfile.ZipFile(self.path(), 'r') as whl: whl.extractall(directory) @@ -112,6 +123,9 @@ parser.add_argument('--directory', action='store', default='.', help='The directory into which to expand things.') +parser.add_argument('--extras', action='append', + help='The set of extras for which to generate library targets.') + def main(): args = parser.parse_args() whl = Wheel(args.whl) @@ -126,19 +140,33 @@ load("{requirements}", "requirement") py_library( - name = "pkg", - srcs = glob(["**/*.py"]), - data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]), - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["."], - deps = [{dependencies}], - )""".format( - requirements=args.requirements, - dependencies=','.join([ - 'requirement("%s")' % d - for d in whl.dependencies() - ]))) + name = "pkg", + srcs = glob(["**/*.py"]), + data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = [{dependencies}], +) +{extras}""".format( + requirements=args.requirements, + dependencies=','.join([ + 'requirement("%s")' % d + for d in whl.dependencies() + ]), + extras='\n\n'.join([ + """py_library( + name = "{extra}", + deps = [ + ":pkg",{deps} + ], +)""".format(extra=extra, + deps=','.join([ + 'requirement("%s")' % dep + for dep in whl.dependencies(extra) + ])) + for extra in args.extras or [] + ]))) if __name__ == '__main__': main()
diff --git a/rules_python/whl_test.py b/rules_python/whl_test.py index e148328..c56a4e9 100644 --- a/rules_python/whl_test.py +++ b/rules_python/whl_test.py
@@ -33,6 +33,7 @@ self.assertEqual(set(wheel.dependencies()), set(['enum34', 'futures', 'protobuf', 'six'])) self.assertEqual('pypi__grpcio_1_6_0', wheel.repository_name()) + self.assertEqual([], wheel.extras()) def test_futures_whl(self): td = TestData('futures_3_1_1_whl/file/futures-3.1.1-py2-none-any.whl') @@ -42,6 +43,7 @@ self.assertEqual(wheel.version(), '3.1.1') self.assertEqual(set(wheel.dependencies()), set()) self.assertEqual('pypi__futures_3_1_1', wheel.repository_name()) + self.assertEqual([], wheel.extras()) def test_whl_with_METADATA_file(self): td = TestData('futures_2_2_0_whl/file/futures-2.2.0-py2.py3-none-any.whl') @@ -61,6 +63,23 @@ self.assertEqual(set(wheel.dependencies()), set(['pbr', 'six'])) self.assertEqual('pypi__mock_2_0_0', wheel.repository_name()) + self.assertEqual(['docs', 'test'], wheel.extras()) + self.assertEqual(set(wheel.dependencies(extra='docs')), set()) + self.assertEqual(set(wheel.dependencies(extra='test')), set(['unittest2'])) + + def test_google_cloud_language_whl(self): + td = TestData('google_cloud_language_whl/file/' + + 'google_cloud_language-0.29.0-py2.py3-none-any.whl') + wheel = whl.Wheel(td) + self.assertEqual(wheel.name(), 'google-cloud-language') + self.assertEqual(wheel.distribution(), 'google_cloud_language') + self.assertEqual(wheel.version(), '0.29.0') + self.assertEqual(set(wheel.dependencies()), + set(['google-gax', 'google-cloud-core', + 'googleapis-common-protos[grpc]'])) + self.assertEqual('pypi__google_cloud_language_0_29_0', + wheel.repository_name()) + self.assertEqual([], wheel.extras()) if __name__ == '__main__': unittest.main()
diff --git a/tools/piptool.par b/tools/piptool.par index 29b1289..4101923 100755 --- a/tools/piptool.par +++ b/tools/piptool.par Binary files differ