Merge rules python external (#348)
Co-authored-by: Dillon Giacoppo <dillon@canva.com>
Co-authored-by: Dillon Giacoppo <dillon.giacoppo@gmail.com>
Co-authored-by: Jonathon Belotti <jonathon@canva.com>
Co-authored-by: Greg Roodt <groodt@gmail.com>
Co-authored-by: Kevin Gessner <kevin@kevingessner.com>
Co-authored-by: benjamin-fleischmann <56046658+benjamin-fleischmann@users.noreply.github.com>
Co-authored-by: Gergely Fábián <gergo.fb@gmail.com>
Co-authored-by: JoshuaCrestone <57449606+JoshuaCrestone@users.noreply.github.com>
Co-authored-by: Allan Clark <allanc@chickenandporn.com>
Co-authored-by: Sebastian Pietras <seba00767@gmail.com>
diff --git a/.bazelignore b/.bazelignore
new file mode 100644
index 0000000..a4647e1
--- /dev/null
+++ b/.bazelignore
@@ -0,0 +1 @@
+experimental/rules_python_external
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
new file mode 100644
index 0000000..3084b21
--- /dev/null
+++ b/.git-blame-ignore-revs
@@ -0,0 +1 @@
+1b1ee606b7d19978094c6198c2173a8b5ebbc9e7
diff --git a/experimental/rules_python_external/.bazelignore b/experimental/rules_python_external/.bazelignore
new file mode 100644
index 0000000..90c978b
--- /dev/null
+++ b/experimental/rules_python_external/.bazelignore
@@ -0,0 +1 @@
+example/
diff --git a/experimental/rules_python_external/.bazelrc b/experimental/rules_python_external/.bazelrc
new file mode 100644
index 0000000..fa4bb2e
--- /dev/null
+++ b/experimental/rules_python_external/.bazelrc
@@ -0,0 +1,2 @@
+build --aspects @mypy_integration//:mypy.bzl%mypy_aspect
+build --output_groups=+mypy
diff --git a/experimental/rules_python_external/.gitattributes b/experimental/rules_python_external/.gitattributes
new file mode 100644
index 0000000..0bad51c
--- /dev/null
+++ b/experimental/rules_python_external/.gitattributes
@@ -0,0 +1 @@
+example/* linguist-vendored
diff --git a/experimental/rules_python_external/.github/workflows/continuous-integration.yml b/experimental/rules_python_external/.github/workflows/continuous-integration.yml
new file mode 100644
index 0000000..01c4cdb
--- /dev/null
+++ b/experimental/rules_python_external/.github/workflows/continuous-integration.yml
@@ -0,0 +1,24 @@
+name: CI
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+
+ steps:
+ # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it
+ - uses: actions/checkout@v2
+
+ - name: Setup Bazel
+ uses: abhinavsingh/setup-bazel@v3
+ with:
+ # Bazel version to install e.g. 1.2.1, 2.0.0, ...
+ version: 2.0.0 # optional, default is 2.0.0
+
+ - name: Run tests
+ run: bazel test //...
diff --git a/experimental/rules_python_external/.gitignore b/experimental/rules_python_external/.gitignore
new file mode 100644
index 0000000..74570cb
--- /dev/null
+++ b/experimental/rules_python_external/.gitignore
@@ -0,0 +1,135 @@
+# Intellij
+.ijwb/
+.idea/
+
+# Bazel
+bazel-*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
diff --git a/experimental/rules_python_external/BUILD b/experimental/rules_python_external/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/experimental/rules_python_external/BUILD
diff --git a/experimental/rules_python_external/LICENSE b/experimental/rules_python_external/LICENSE
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ b/experimental/rules_python_external/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/experimental/rules_python_external/README.md b/experimental/rules_python_external/README.md
new file mode 100644
index 0000000..c21dda0
--- /dev/null
+++ b/experimental/rules_python_external/README.md
@@ -0,0 +1,125 @@
+# rules_python_external 
+
+Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file.
+
+## Features
+
+The rules address most of the top packaging issues in [`bazelbuild/rules_python`](https://github.com/bazelbuild/rules_python). This means the rules support common packages such
+as [`tensorflow`](https://pypi.org/project/tensorflow/) and [`google.cloud`](https://github.com/googleapis/google-cloud-python) natively.
+
+* Transitive dependency resolution:
+ [#35](https://github.com/bazelbuild/rules_python/issues/35),
+ [#102](https://github.com/bazelbuild/rules_python/issues/102)
+* Minimal runtime dependencies:
+ [#184](https://github.com/bazelbuild/rules_python/issues/184)
+* Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl):
+ [#71](https://github.com/bazelbuild/rules_python/issues/71)
+* Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/):
+ [#14](https://github.com/bazelbuild/rules_python/issues/14),
+ [#55](https://github.com/bazelbuild/rules_python/issues/55),
+ [#65](https://github.com/bazelbuild/rules_python/issues/65),
+ [#93](https://github.com/bazelbuild/rules_python/issues/93),
+ [#189](https://github.com/bazelbuild/rules_python/issues/189)
+* Fetches pip packages only for building Python targets:
+ [#96](https://github.com/bazelbuild/rules_python/issues/96)
+* Reproducible builds:
+ [#154](https://github.com/bazelbuild/rules_python/issues/154),
+ [#176](https://github.com/bazelbuild/rules_python/issues/176)
+
+## Usage
+
+#### Prerequisites
+
+The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)).
+
+#### Setup `WORKSPACE`
+
+```python
+rules_python_external_version = "{COMMIT_SHA}"
+
+http_archive(
+ name = "rules_python_external",
+ sha256 = "", # Fill in with correct sha256 of your COMMIT_SHA version
+ strip_prefix = "rules_python_external-{version}".format(version = rules_python_external_version),
+ url = "https://github.com/dillon-giacoppo/rules_python_external/archive/v{version}.zip".format(version = rules_python_external_version),
+)
+
+# Install the rule dependencies
+load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies")
+rules_python_external_dependencies()
+
+load("@rules_python_external//:defs.bzl", "pip_install")
+pip_install(
+ name = "py_deps",
+ requirements = "//:requirements.txt",
+ # (Optional) You can provide a python interpreter (by path):
+ python_interpreter = "/usr/bin/python3.8",
+ # (Optional) Alternatively you can provide an in-build python interpreter, that is available as a Bazel target.
+ # This overrides `python_interpreter`.
+ # Note: You need to set up the interpreter target beforehand (not shown here). Please see the `example` folder for further details.
+ #python_interpreter_target = "@python_interpreter//:python_bin",
+)
+```
+
+#### Example `BUILD` file.
+
+```python
+load("@py_deps//:requirements.bzl", "requirement")
+
+py_binary(
+ name = "main",
+ srcs = ["main.py"],
+ deps = [
+ requirement("boto3"),
+ ],
+)
+```
+
+Note that above you do not need to add transitively required packages to `deps = [ ... ]`
+
+#### Setup `requirements.txt`
+
+While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended.
+But if you want to just have top-level packages listed, that also will work.
+
+Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we
+recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile).
+
+For example, `pip-compile` takes a `requirements.in` like this:
+
+```
+boto3~=1.9.227
+botocore~=1.12.247
+click~=7.0
+```
+
+`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to
+`pip_install` below:
+
+```
+boto3==1.9.253
+botocore==1.12.253
+click==7.0
+docutils==0.15.2 # via botocore
+jmespath==0.9.4 # via boto3, botocore
+python-dateutil==2.8.1 # via botocore
+s3transfer==0.2.1 # via boto3
+six==1.14.0 # via python-dateutil
+urllib3==1.25.8 # via botocore
+```
+
+### Demo
+
+You can find a demo in the [example/](./example) directory.
+
+## Development
+
+### Testing
+
+`bazel test //...`
+
+## Adopters
+
+Here's a (non-exhaustive) list of companies that use `rules_python_external` in production. Don't see yours? [You can add it in a PR](https://github.com/dillon-giacoppo/rules_python_external/edit/master/README.md)!
+
+* [Canva](https://www.canva.com/)
diff --git a/experimental/rules_python_external/WORKSPACE b/experimental/rules_python_external/WORKSPACE
new file mode 100644
index 0000000..ad38285
--- /dev/null
+++ b/experimental/rules_python_external/WORKSPACE
@@ -0,0 +1,43 @@
+workspace(name = "rules_python_external")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+ name = "rules_python",
+ sha256 = "d2865e2ce23ee217aaa408ddaa024ca472114a6f250b46159d27de05530c75e3",
+ strip_prefix = "rules_python-7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c",
+ url = "https://github.com/bazelbuild/rules_python/archive/7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+py_repositories()
+
+load("//:repositories.bzl", "rules_python_external_dependencies")
+rules_python_external_dependencies()
+
+mypy_integration_version = "0.0.7" # latest @ Feb 10th 2020
+
+http_archive(
+ name = "mypy_integration",
+ sha256 = "bf7ecd386740328f96c343dca095a63b93df7f86f8d3e1e2e6ff46e400880077", # for 0.0.7
+ strip_prefix = "bazel-mypy-integration-{version}".format(version = mypy_integration_version),
+ url = "https://github.com/thundergolfer/bazel-mypy-integration/archive/{version}.zip".format(
+ version = mypy_integration_version
+ ),
+)
+
+load(
+ "@mypy_integration//repositories:repositories.bzl",
+ mypy_integration_repositories = "repositories",
+)
+
+mypy_integration_repositories()
+
+load("@mypy_integration//:config.bzl", "mypy_configuration")
+mypy_configuration("//tools/typing:mypy.ini")
+
+load("@mypy_integration//repositories:deps.bzl", mypy_integration_deps = "deps")
+mypy_integration_deps("//tools/typing:mypy_version.txt")
+
+load("@mypy_integration//repositories:pip_repositories.bzl", "pip_deps")
+pip_deps()
diff --git a/experimental/rules_python_external/defs.bzl b/experimental/rules_python_external/defs.bzl
new file mode 100644
index 0000000..a11bff1
--- /dev/null
+++ b/experimental/rules_python_external/defs.bzl
@@ -0,0 +1,108 @@
+""
+
+load("//:repositories.bzl", "all_requirements")
+
+DEFAULT_REPOSITORY_NAME = "pip"
+
+def _pip_repository_impl(rctx):
+ python_interpreter = rctx.attr.python_interpreter
+ if rctx.attr.python_interpreter_target != None:
+ target = rctx.attr.python_interpreter_target
+ python_interpreter = rctx.path(target)
+ else:
+ if "/" not in python_interpreter:
+ python_interpreter = rctx.which(python_interpreter)
+ if not python_interpreter:
+ fail("python interpreter not found")
+
+ rctx.file("BUILD", "")
+
+ # Get the root directory of these rules
+ rules_root = rctx.path(Label("//:BUILD")).dirname
+ thirdparty_roots = [
+ # Includes all the external dependencies from repositories.bzl
+ rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname
+ for repo in all_requirements
+ ]
+ separator = ":" if not "windows" in rctx.os.name.lower() else ";"
+ pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots])
+
+ args = [
+ python_interpreter,
+ "-m",
+ "extract_wheels",
+ "--requirements",
+ rctx.path(rctx.attr.requirements),
+ "--repo",
+ "@%s" % rctx.attr.name,
+ ]
+
+ if rctx.attr.extra_pip_args:
+ args += [
+ "--extra_pip_args",
+ struct(args = rctx.attr.extra_pip_args).to_json(),
+ ]
+
+ if rctx.attr.pip_data_exclude:
+ args += [
+ "--pip_data_exclude",
+ struct(exclude = rctx.attr.pip_data_exclude).to_json(),
+ ]
+
+ if rctx.attr.enable_implicit_namespace_pkgs:
+ args.append("--enable_implicit_namespace_pkgs")
+
+ result = rctx.execute(
+ args,
+ environment = {
+ # Manually construct the PYTHONPATH since we cannot use the toolchain here
+ "PYTHONPATH": pypath,
+ },
+ timeout = rctx.attr.timeout,
+ quiet = rctx.attr.quiet,
+ )
+ if result.return_code:
+ fail("rules_python_external failed: %s (%s)" % (result.stdout, result.stderr))
+
+ return
+
+pip_repository = repository_rule(
+ attrs = {
+ "enable_implicit_namespace_pkgs": attr.bool(
+ default = False,
+ doc = """
+If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary
+and py_test targets must specify either `legacy_create_init=False` or the global Bazel option
+`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory.
+
+This option is required to support some packages which cannot handle the conversion to pkg-util style.
+ """,
+ ),
+ "extra_pip_args": attr.string_list(
+ doc = "Extra arguments to pass on to pip. Must not contain spaces.",
+ ),
+ "pip_data_exclude": attr.string_list(
+ doc = "Additional data exclusion parameters to add to the pip packages BUILD file.",
+ ),
+ "python_interpreter": attr.string(default = "python3"),
+ "python_interpreter_target": attr.label(allow_single_file = True, doc = """
+If you are using a custom python interpreter built by another repository rule,
+use this attribute to specify its BUILD target. This allows pip_repository to invoke
+pip using the same interpreter as your toolchain. If set, takes precedence over
+python_interpreter.
+"""),
+ "quiet": attr.bool(default = True),
+ "requirements": attr.label(allow_single_file = True, mandatory = True),
+ # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
+ "timeout": attr.int(default = 600),
+ "wheel_env": attr.string_dict(),
+ },
+ implementation = _pip_repository_impl,
+)
+
+def pip_install(requirements, name = DEFAULT_REPOSITORY_NAME, **kwargs):
+ pip_repository(
+ name = name,
+ requirements = requirements,
+ **kwargs
+ )
diff --git a/experimental/rules_python_external/example/BUILD b/experimental/rules_python_external/example/BUILD
new file mode 100644
index 0000000..6c77b7b
--- /dev/null
+++ b/experimental/rules_python_external/example/BUILD
@@ -0,0 +1,36 @@
+load("@pip//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_binary")
+
+# Toolchain setup, this is optional.
+# Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE).
+#
+#load("@rules_python//python:defs.bzl", "py_runtime_pair")
+#
+#py_runtime(
+# name = "python3_runtime",
+# files = ["@python_interpreter//:files"],
+# interpreter = "@python_interpreter//:python_bin",
+# python_version = "PY3",
+# visibility = ["//visibility:public"],
+#)
+#
+#py_runtime_pair(
+# name = "my_py_runtime_pair",
+# py2_runtime = None,
+# py3_runtime = ":python3_runtime",
+#)
+#
+#toolchain(
+# name = "my_py_toolchain",
+# toolchain = ":my_py_runtime_pair",
+# toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+#)
+# End of toolchain setup.
+
+py_binary(
+ name = "main",
+ srcs = ["main.py"],
+ deps = [
+ requirement("boto3"),
+ ],
+)
diff --git a/experimental/rules_python_external/example/WORKSPACE b/experimental/rules_python_external/example/WORKSPACE
new file mode 100644
index 0000000..639308a
--- /dev/null
+++ b/experimental/rules_python_external/example/WORKSPACE
@@ -0,0 +1,90 @@
+workspace(name = "example_repo")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+ name = "rules_python",
+ url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
+ strip_prefix = "rules_python-0.0.2",
+ sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+local_repository(
+ name = "rules_python_external",
+ path = "../",
+)
+
+load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies")
+
+rules_python_external_dependencies()
+
+load("@rules_python_external//:defs.bzl", "pip_install")
+
+pip_install(
+ # (Optional) You can provide extra parameters to pip.
+ # Here, make pip output verbose (this is usable with `quiet = False`).
+ #extra_pip_args = ["-v"],
+
+ # (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages.
+ # Exclude directories with spaces in their names in this example (avoids build errors if there are such directories).
+ #pip_data_exclude = ["**/* */**"],
+
+ # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that
+ # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.:
+ # 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
+ # 2. Pre-compiled python interpreter included with http_archive
+ # 3. Wrapper script, like in the autodetecting python toolchain.
+ #python_interpreter_target = "@python_interpreter//:python_bin",
+
+ # (Optional) You can set quiet to False if you want to see pip output.
+ #quiet = False,
+
+ # Uses the default repository name "pip"
+ requirements = "//:requirements.txt",
+)
+
+# You could optionally use an in-build, compiled python interpreter as a toolchain,
+# and also use it to execute pip.
+#
+# Special logic for building python interpreter with OpenSSL from homebrew.
+# See https://devguide.python.org/setup/#macos-and-os-x
+#_py_configure = """
+#if [[ "$OSTYPE" == "darwin"* ]]; then
+# ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl)
+#else
+# ./configure --prefix=$(pwd)/bazel_install
+#fi
+#"""
+#
+# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS).
+# E.g. on Ubuntu: `sudo apt install libssl-dev`
+#http_archive(
+# name = "python_interpreter",
+# build_file_content = """
+#exports_files(["python_bin"])
+#filegroup(
+# name = "files",
+# srcs = glob(["bazel_install/**"], exclude = ["**/* *"]),
+# visibility = ["//visibility:public"],
+#)
+#""",
+# patch_cmds = [
+# "mkdir $(pwd)/bazel_install",
+# _py_configure,
+# "make",
+# "make install",
+# "ln -s bazel_install/bin/python3 python_bin",
+# ],
+# sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864",
+# strip_prefix = "Python-3.8.3",
+# urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"],
+#)
+
+# Optional:
+# Register the toolchain with the same python interpreter we used for pip in pip_install().
+#register_toolchains("//:my_py_toolchain")
+# End of in-build Python interpreter setup.
diff --git a/experimental/rules_python_external/example/main.py b/experimental/rules_python_external/example/main.py
new file mode 100644
index 0000000..e5c690b
--- /dev/null
+++ b/experimental/rules_python_external/example/main.py
@@ -0,0 +1,4 @@
+import boto3
+
+if __name__ == "__main__":
+ pass
diff --git a/experimental/rules_python_external/example/requirements.txt b/experimental/rules_python_external/example/requirements.txt
new file mode 100644
index 0000000..30ddf82
--- /dev/null
+++ b/experimental/rules_python_external/example/requirements.txt
@@ -0,0 +1 @@
+boto3
diff --git a/experimental/rules_python_external/extract_wheels/BUILD b/experimental/rules_python_external/extract_wheels/BUILD
new file mode 100644
index 0000000..27434db
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/BUILD
@@ -0,0 +1,8 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+py_binary(
+ name = "extract_wheels",
+ srcs = ["__init__.py", "__main__.py"],
+ main = "__main__.py",
+ deps = ["//extract_wheels/lib"],
+)
diff --git a/experimental/rules_python_external/extract_wheels/__init__.py b/experimental/rules_python_external/extract_wheels/__init__.py
new file mode 100644
index 0000000..8184dac
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/__init__.py
@@ -0,0 +1,110 @@
+"""extract_wheels
+
+extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a
+requirements.txt. It generates the required BUILD files to consume these packages as Python libraries.
+
+Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels.
+"""
+import argparse
+import glob
+import os
+import subprocess
+import sys
+import json
+
+from extract_wheels.lib import bazel, requirements
+
+
+def configure_reproducible_wheels() -> None:
+ """Modifies the environment to make wheel building reproducible.
+
+ Wheels created from sdists are not reproducible by default. We can however workaround this by
+ patching in some configuration with environment variables.
+ """
+
+ # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file
+ # We can override this behavior by disabling debug symbols entirely.
+ # https://github.com/pypa/pip/issues/6505
+ if "CFLAGS" in os.environ:
+ os.environ["CFLAGS"] += " -g0"
+ else:
+ os.environ["CFLAGS"] = "-g0"
+
+ # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels
+ # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
+ if "SOURCE_DATE_EPOCH" not in os.environ:
+ os.environ["SOURCE_DATE_EPOCH"] = "315532800"
+
+ # Python wheel metadata files can be unstable.
+ # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff
+ if "PYTHONHASHSEED" not in os.environ:
+ os.environ["PYTHONHASHSEED"] = "0"
+
+
+def main() -> None:
+ """Main program.
+
+ Exits zero on successful program termination, non-zero otherwise.
+ """
+
+ configure_reproducible_wheels()
+
+ parser = argparse.ArgumentParser(
+ description="Resolve and fetch artifacts transitively from PyPI"
+ )
+ parser.add_argument(
+ "--requirements",
+ action="store",
+ required=True,
+ help="Path to requirements.txt from where to install dependencies",
+ )
+ parser.add_argument(
+ "--repo",
+ action="store",
+ required=True,
+ help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
+ )
+ parser.add_argument(
+ "--extra_pip_args", action="store", help="Extra arguments to pass down to pip.",
+ )
+ parser.add_argument(
+ "--pip_data_exclude",
+ action="store",
+ help="Additional data exclusion parameters to add to the pip packages BUILD file.",
+ )
+ parser.add_argument(
+ "--enable_implicit_namespace_pkgs",
+ action="store_true",
+ help="Disables conversion of implicit namespace packages into pkg-util style packages.",
+ )
+ args = parser.parse_args()
+
+ pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
+ if args.extra_pip_args:
+ pip_args += json.loads(args.extra_pip_args)["args"]
+
+ # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
+ subprocess.run(pip_args, check=True)
+
+ extras = requirements.parse_extras(args.requirements)
+
+ if args.pip_data_exclude:
+ pip_data_exclude = json.loads(args.pip_data_exclude)["exclude"]
+ else:
+ pip_data_exclude = []
+
+ targets = [
+ '"%s%s"'
+ % (
+ args.repo,
+ bazel.extract_wheel(
+ whl, extras, pip_data_exclude, args.enable_implicit_namespace_pkgs
+ ),
+ )
+ for whl in glob.glob("*.whl")
+ ]
+
+ with open("requirements.bzl", "w") as requirement_file:
+ requirement_file.write(
+ bazel.generate_requirements_file_contents(args.repo, targets)
+ )
diff --git a/experimental/rules_python_external/extract_wheels/__main__.py b/experimental/rules_python_external/extract_wheels/__main__.py
new file mode 100644
index 0000000..939e8b9
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/__main__.py
@@ -0,0 +1,5 @@
+"""Main entry point."""
+import extract_wheels
+
+if __name__ == "__main__":
+ extract_wheels.main()
diff --git a/experimental/rules_python_external/extract_wheels/lib/BUILD b/experimental/rules_python_external/extract_wheels/lib/BUILD
new file mode 100644
index 0000000..415bb1b
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/BUILD
@@ -0,0 +1,42 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("//:repositories.bzl", "requirement")
+
+py_library(
+ name = "lib",
+ visibility = ["//extract_wheels:__subpackages__"],
+ srcs = [
+ "bazel.py",
+ "namespace_pkgs.py",
+ "purelib.py",
+ "requirements.py",
+ "wheel.py",
+ ],
+ deps = [
+ requirement("pkginfo"),
+ requirement("setuptools"),
+ ],
+)
+
+py_test(
+ name = "namespace_pkgs_test",
+ size = "small",
+ srcs = [
+ "namespace_pkgs_test.py",
+ ],
+ tags = ["unit"],
+ deps = [
+ ":lib",
+ ],
+)
+
+py_test(
+ name = "requirements_test",
+ size = "small",
+ srcs = [
+ "requirements_test.py",
+ ],
+ tags = ["unit"],
+ deps = [
+ ":lib",
+ ],
+)
diff --git a/experimental/rules_python_external/extract_wheels/lib/__init__.py b/experimental/rules_python_external/extract_wheels/lib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/__init__.py
diff --git a/experimental/rules_python_external/extract_wheels/lib/bazel.py b/experimental/rules_python_external/extract_wheels/lib/bazel.py
new file mode 100644
index 0000000..acda4c2
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/bazel.py
@@ -0,0 +1,166 @@
+"""Utility functions to manipulate Bazel files"""
+import os
+import textwrap
+import json
+from typing import Iterable, List, Dict, Set
+
+from extract_wheels.lib import namespace_pkgs, wheel, purelib
+
+
+def generate_build_file_contents(
+ name: str, dependencies: List[str], pip_data_exclude: List[str]
+) -> str:
+ """Generate a BUILD file for an unzipped Wheel
+
+ Args:
+ name: the target name of the py_library
+ dependencies: a list of Bazel labels pointing to dependencies of the library
+
+ Returns:
+ A complete BUILD file as a string
+
+ We allow for empty Python sources as for Wheels containing only compiled C code
+ there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`).
+ """
+
+ data_exclude = ["**/*.py", "**/* *", "BUILD", "WORKSPACE"] + pip_data_exclude
+
+ return textwrap.dedent(
+ """\
+ package(default_visibility = ["//visibility:public"])
+
+ load("@rules_python//python:defs.bzl", "py_library")
+
+ py_library(
+ name = "{name}",
+ srcs = glob(["**/*.py"], allow_empty = True),
+ data = glob(["**/*"], exclude={data_exclude}),
+ # This makes this directory a top-level in the python import
+ # search path for anything that depends on this.
+ imports = ["."],
+ deps = [{dependencies}],
+ )
+ """.format(
+ name=name,
+ dependencies=",".join(dependencies),
+ data_exclude=json.dumps(data_exclude),
+ )
+ )
+
+
+def generate_requirements_file_contents(repo_name: str, targets: Iterable[str]) -> str:
+ """Generate a requirements.bzl file for a given pip repository
+
+ The file allows converting the PyPI name to a bazel label. Additionally, it adds a function which can glob all the
+ installed dependencies. This is provided for legacy reasons and can be considered deprecated.
+
+ Args:
+ repo_name: the name of the pip repository
+ targets: a list of Bazel labels pointing to all the generated targets
+
+ Returns:
+ A complete requirements.bzl file as a string
+ """
+
+ return textwrap.dedent(
+ """\
+ # Deprecated. This will be removed in a future release
+ all_requirements = [{requirement_labels}]
+
+ def requirement(name):
+ name_key = name.replace("-", "_").replace(".", "_").lower()
+ return "{repo}//pypi__" + name_key
+ """.format(
+ repo=repo_name, requirement_labels=",".join(sorted(targets))
+ )
+ )
+
+
+def sanitise_name(name: str) -> str:
+ """Sanitises the name to be compatible with Bazel labels.
+
+ There are certain requirements around Bazel labels that we need to consider. From the Bazel docs:
+
+ Package names must be composed entirely of characters drawn from the set A-Z, a–z, 0–9, '/', '-', '.', and '_',
+ and cannot start with a slash.
+
+ Due to restrictions on Bazel labels we also cannot allow hyphens. See
+ https://github.com/bazelbuild/bazel/issues/6841
+
+ Further, rules-python automatically adds the repository root to the PYTHONPATH, meaning a package that has the same
+ name as a module is picked up. We workaround this by prefixing with `pypi__`. Alternatively we could require
+ `--noexperimental_python_import_all_repositories` be set, however this breaks rules_docker.
+ See: https://github.com/bazelbuild/bazel/issues/2636
+ """
+
+ return "pypi__" + name.replace("-", "_").replace(".", "_").lower()
+
+
+def setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
+ """Converts native namespace packages to pkgutil-style packages
+
+ Namespace packages can be created in one of three ways. They are detailed here:
+ https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
+
+ 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
+ 'native namespace packages' (1) do not.
+
+ We ensure compatibility with Bazel of method 1 by converting them into method 2.
+
+ Args:
+ wheel_dir: the directory of the wheel to convert
+ """
+
+ namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
+ wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,],
+ )
+
+ for ns_pkg_dir in namespace_pkg_dirs:
+ namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
+
+
+def extract_wheel(
+ wheel_file: str,
+ extras: Dict[str, Set[str]],
+ pip_data_exclude: List[str],
+ enable_implicit_namespace_pkgs: bool,
+) -> str:
+ """Extracts wheel into given directory and creates a py_library target.
+
+ Args:
+ wheel_file: the filepath of the .whl
+ extras: a list of extras to add as dependencies for the installed wheel
+ pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library
+ enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
+
+ Returns:
+ The Bazel label for the extracted wheel, in the form '//path/to/wheel'.
+ """
+
+ whl = wheel.Wheel(wheel_file)
+ directory = sanitise_name(whl.name)
+
+ os.mkdir(directory)
+ whl.unzip(directory)
+
+ # Note: Order of operations matters here
+ purelib.spread_purelib_into_root(directory)
+
+ if not enable_implicit_namespace_pkgs:
+ setup_namespace_pkg_compatibility(directory)
+
+ extras_requested = extras[whl.name] if whl.name in extras else set()
+
+ sanitised_dependencies = [
+ '"//%s"' % sanitise_name(d) for d in sorted(whl.dependencies(extras_requested))
+ ]
+
+ with open(os.path.join(directory, "BUILD"), "w") as build_file:
+ contents = generate_build_file_contents(
+ sanitise_name(whl.name), sanitised_dependencies, pip_data_exclude,
+ )
+ build_file.write(contents)
+
+ os.remove(whl.path)
+
+ return "//%s" % directory
diff --git a/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py
new file mode 100644
index 0000000..cb9e164
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs.py
@@ -0,0 +1,72 @@
+"""Utility functions to discover python package types"""
+import os
+import textwrap
+from typing import Set, List, Optional
+
+from extract_wheels.lib import wheel
+
+
+def implicit_namespace_packages(
+ directory: str, ignored_dirnames: Optional[List[str]] = None
+) -> Set[str]:
+ """Discovers namespace packages implemented using the 'native namespace packages' method.
+
+ AKA 'implicit namespace packages', which has been supported since Python 3.3.
+ See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages
+
+ Args:
+ directory: The root directory to recursively find packages in.
+ ignored_dirnames: A list of directories to exclude from the search
+
+ Returns:
+ The set of directories found under root to be packages using the native namespace method.
+ """
+ namespace_pkg_dirs = set()
+ for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
+ # We are only interested in dirs with no __init__.py file
+ if "__init__.py" in filenames:
+ dirnames[:] = [] # Remove dirnames from search
+ continue
+
+ for ignored_dir in ignored_dirnames or []:
+ if ignored_dir in dirnames:
+ dirnames.remove(ignored_dir)
+
+ non_empty_directory = dirnames or filenames
+ if (
+ non_empty_directory
+ and
+ # The root of the directory should never be an implicit namespace
+ dirpath != directory
+ ):
+ namespace_pkg_dirs.add(dirpath)
+
+ return namespace_pkg_dirs
+
+
+def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None:
+ """Adds 'pkgutil-style namespace packages' init file to the given directory
+
+ See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+
+ Args:
+ dir_path: The directory to create an __init__.py for.
+
+ Raises:
+ ValueError: If the directory already contains an __init__.py file
+ """
+ ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py")
+
+ if os.path.isfile(ns_pkg_init_filepath):
+ raise ValueError("%s already contains an __init__.py file." % dir_path)
+
+ with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f:
+ # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
+ ns_pkg_init_f.write(
+ textwrap.dedent(
+ """\
+ # __path__ manipulation added by rules_python_external to support namespace pkgs.
+ __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+ """
+ )
+ )
diff --git a/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py
new file mode 100644
index 0000000..899f7bc
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/namespace_pkgs_test.py
@@ -0,0 +1,73 @@
+import pathlib
+import shutil
+import tempfile
+from typing import Optional
+import unittest
+
+from extract_wheels.lib import namespace_pkgs
+
+
+class TempDir:
+ def __init__(self) -> None:
+ self.dir = tempfile.mkdtemp()
+
+ def root(self) -> str:
+ return self.dir
+
+ def add_dir(self, rel_path: str) -> None:
+ d = pathlib.Path(self.dir, rel_path)
+ d.mkdir(parents=True)
+
+ def add_file(self, rel_path: str, contents: Optional[str] = None) -> None:
+ f = pathlib.Path(self.dir, rel_path)
+ f.parent.mkdir(parents=True, exist_ok=True)
+ if contents:
+ with open(str(f), "w") as writeable_f:
+ writeable_f.write(contents)
+ else:
+ f.touch()
+
+ def remove(self) -> None:
+ shutil.rmtree(self.dir)
+
+
+class TestImplicitNamespacePackages(unittest.TestCase):
+ def test_finds_correct_namespace_packages(self) -> None:
+ directory = TempDir()
+ directory.add_file("foo/bar/biz.py")
+ directory.add_file("foo/bee/boo.py")
+ directory.add_file("foo/buu/__init__.py")
+ directory.add_file("foo/buu/bii.py")
+
+ expected = {
+ directory.root() + "/foo",
+ directory.root() + "/foo/bar",
+ directory.root() + "/foo/bee",
+ }
+ actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+ self.assertEqual(actual, expected)
+
+ def test_ignores_empty_directories(self) -> None:
+ directory = TempDir()
+ directory.add_file("foo/bar/biz.py")
+ directory.add_dir("foo/cat")
+
+ expected = {
+ directory.root() + "/foo",
+ directory.root() + "/foo/bar",
+ }
+ actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+ self.assertEqual(actual, expected)
+
+ def test_empty_case(self) -> None:
+ directory = TempDir()
+ directory.add_file("foo/__init__.py")
+ directory.add_file("foo/bar/__init__.py")
+ directory.add_file("foo/bar/biz.py")
+
+ actual = namespace_pkgs.implicit_namespace_packages(directory.root())
+ self.assertEqual(actual, set())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/rules_python_external/extract_wheels/lib/purelib.py b/experimental/rules_python_external/extract_wheels/lib/purelib.py
new file mode 100644
index 0000000..ffcda8f
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/purelib.py
@@ -0,0 +1,56 @@
+"""Functions to make purelibs Bazel compatible"""
+import pathlib
+import shutil
+
+from extract_wheels.lib import wheel
+
+
+def spread_purelib_into_root(wheel_dir: str) -> None:
+ """Unpacks purelib directories into the root.
+
+ Args:
+ wheel_dir: The root of the extracted wheel directory.
+ """
+ dist_info = wheel.get_dist_info(wheel_dir)
+ wheel_metadata_file_path = pathlib.Path(dist_info, "WHEEL")
+ wheel_metadata_dict = wheel.parse_wheel_meta_file(str(wheel_metadata_file_path))
+
+ if "Root-Is-Purelib" not in wheel_metadata_dict:
+ raise ValueError(
+ "Invalid WHEEL file '%s'. Expected key 'Root-Is-Purelib'."
+ % wheel_metadata_file_path
+ )
+ root_is_purelib = wheel_metadata_dict["Root-Is-Purelib"]
+
+ if root_is_purelib.lower() == "true":
+ # The Python package code is in the root of the Wheel, so no need to 'spread' anything.
+ return
+
+ dot_data_dir = wheel.get_dot_data_directory(wheel_dir)
+ # 'Root-Is-Purelib: false' is no guarantee a .date directory exists with
+ # package code in it. eg. the 'markupsafe' package.
+ if not dot_data_dir:
+ return
+
+ for child in pathlib.Path(dot_data_dir).iterdir():
+ # TODO(Jonathon): Should all other potential folders get ignored? eg. 'platlib'
+ if str(child).endswith("purelib"):
+ _spread_purelib(child, wheel_dir)
+
+
+def _spread_purelib(purelib_dir: pathlib.Path, root_dir: str) -> None:
+ """Recursively moves all sibling directories of the purelib to the root.
+
+ Args:
+ purelib_dir: The directory of the purelib.
+ root_dir: The directory to move files into.
+ """
+ for grandchild in purelib_dir.iterdir():
+ # Some purelib Wheels, like Tensorflow 2.0.0, have directories
+ # split between the root and the purelib directory. In this case
+ # we should leave the purelib 'sibling' alone.
+ # See: https://github.com/dillon-giacoppo/rules_python_external/issues/8
+ if not pathlib.Path(root_dir, grandchild.name).exists():
+ shutil.move(
+ src=str(grandchild), dst=root_dir,
+ )
diff --git a/experimental/rules_python_external/extract_wheels/lib/requirements.py b/experimental/rules_python_external/extract_wheels/lib/requirements.py
new file mode 100644
index 0000000..e246379
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/requirements.py
@@ -0,0 +1,45 @@
+import re
+from typing import Dict, Set, Tuple, Optional
+
+
+def parse_extras(requirements_path: str) -> Dict[str, Set[str]]:
+ """Parse over the requirements.txt file to find extras requested.
+
+ Args:
+ requirements_path: The filepath for the requirements.txt file to parse.
+
+ Returns:
+ A dictionary mapping the requirement name to a set of extras requested.
+ """
+
+ extras_requested = {}
+ with open(requirements_path, "r") as requirements:
+ # Merge all backslash line continuations so we parse each requirement as a single line.
+ for line in requirements.read().replace("\\\n", "").split("\n"):
+ requirement, extras = _parse_requirement_for_extra(line)
+ if requirement and extras:
+ extras_requested[requirement] = extras
+
+ return extras_requested
+
+
+def _parse_requirement_for_extra(
+ requirement: str,
+) -> Tuple[Optional[str], Optional[Set[str]]]:
+ """Given a requirement string, returns the requirement name and set of extras, if extras specified.
+ Else, returns (None, None)
+ """
+
+ # https://www.python.org/dev/peps/pep-0508/#grammar
+ extras_pattern = re.compile(
+ r"^\s*([0-9A-Za-z][0-9A-Za-z_.\-]*)\s*\[\s*([0-9A-Za-z][0-9A-Za-z_.\-]*(?:\s*,\s*[0-9A-Za-z][0-9A-Za-z_.\-]*)*)\s*\]"
+ )
+
+ matches = extras_pattern.match(requirement)
+ if matches:
+ return (
+ matches.group(1),
+ {extra.strip() for extra in matches.group(2).split(",")},
+ )
+
+ return None, None
diff --git a/experimental/rules_python_external/extract_wheels/lib/requirements_test.py b/experimental/rules_python_external/extract_wheels/lib/requirements_test.py
new file mode 100644
index 0000000..2b96a75
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/requirements_test.py
@@ -0,0 +1,32 @@
+import unittest
+
+from extract_wheels.lib import requirements
+
+
+class TestRequirementExtrasParsing(unittest.TestCase):
+ def test_parses_requirement_for_extra(self) -> None:
+ cases = [
+ ("name[foo]", ("name", frozenset(["foo"]))),
+ ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))),
+ (" name1[ foo ] ", ("name1", frozenset(["foo"]))),
+ (
+ "name [fred,bar] @ http://foo.com ; python_version=='2.7'",
+ ("name", frozenset(["fred", "bar"])),
+ ),
+ (
+ "name[quux, strange];python_version<'2.7' and platform_version=='2'",
+ ("name", frozenset(["quux", "strange"])),
+ ),
+ ("name; (os_name=='a' or os_name=='b') and os_name=='c'", (None, None),),
+ ("name@http://foo.com", (None, None),),
+ ]
+
+ for case, expected in cases:
+ with self.subTest():
+ self.assertTupleEqual(
+ requirements._parse_requirement_for_extra(case), expected
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/rules_python_external/extract_wheels/lib/wheel.py b/experimental/rules_python_external/extract_wheels/lib/wheel.py
new file mode 100644
index 0000000..c13f4e8
--- /dev/null
+++ b/experimental/rules_python_external/extract_wheels/lib/wheel.py
@@ -0,0 +1,147 @@
+"""Utility class to inspect an extracted wheel directory"""
+import glob
+import os
+import stat
+import zipfile
+from typing import Dict, Optional, Set
+
+import pkg_resources
+import pkginfo
+
+
+def current_umask() -> int:
+ """Get the current umask which involves having to set it temporarily."""
+ mask = os.umask(0)
+ os.umask(mask)
+ return mask
+
+
+def set_extracted_file_to_default_mode_plus_executable(path: str) -> None:
+ """
+ Make file present at path have execute for user/group/world
+ (chmod +x) is no-op on windows per python docs
+ """
+ os.chmod(path, (0o777 & ~current_umask() | 0o111))
+
+
+class Wheel:
+ """Representation of the compressed .whl file"""
+
+ def __init__(self, path: str):
+ self._path = path
+
+ @property
+ def path(self) -> str:
+ return self._path
+
+ @property
+ def name(self) -> str:
+ return str(self.metadata.name)
+
+ @property
+ def metadata(self) -> pkginfo.Wheel:
+ return pkginfo.get_metadata(self.path)
+
+ def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]:
+ dependency_set = set()
+
+ for wheel_req in self.metadata.requires_dist:
+ req = pkg_resources.Requirement(wheel_req) # type: ignore
+
+ if req.marker is None or any(
+ req.marker.evaluate({"extra": extra})
+ for extra in extras_requested or [""]
+ ):
+ dependency_set.add(req.name) # type: ignore
+
+ return dependency_set
+
+ def unzip(self, directory: str) -> None:
+ with zipfile.ZipFile(self.path, "r") as whl:
+ whl.extractall(directory)
+ # The following logic is borrowed from Pip:
+ # https://github.com/pypa/pip/blob/cc48c07b64f338ac5e347d90f6cb4efc22ed0d0b/src/pip/_internal/utils/unpacking.py#L240
+ for info in whl.infolist():
+ name = info.filename
+ # Do not attempt to modify directories.
+ if name.endswith("/") or name.endswith("\\"):
+ continue
+ mode = info.external_attr >> 16
+ # if mode and regular file and any execute permissions for
+ # user/group/world?
+ if mode and stat.S_ISREG(mode) and mode & 0o111:
+ name = os.path.join(directory, name)
+ set_extracted_file_to_default_mode_plus_executable(name)
+
+
+def get_dist_info(wheel_dir: str) -> str:
+ """"Returns the relative path to the dist-info directory if it exists.
+
+ Args:
+ wheel_dir: The root of the extracted wheel directory.
+
+ Returns:
+ Relative path to the dist-info directory if it exists, else, None.
+ """
+ dist_info_dirs = glob.glob(os.path.join(wheel_dir, "*.dist-info"))
+ if not dist_info_dirs:
+ raise ValueError(
+ "No *.dist-info directory found. %s is not a valid Wheel." % wheel_dir
+ )
+
+ if len(dist_info_dirs) > 1:
+ raise ValueError(
+ "Found more than 1 *.dist-info directory. %s is not a valid Wheel."
+ % wheel_dir
+ )
+
+ return dist_info_dirs[0]
+
+
+def get_dot_data_directory(wheel_dir: str) -> Optional[str]:
+ """Returns the relative path to the data directory if it exists.
+
+ See: https://www.python.org/dev/peps/pep-0491/#the-data-directory
+
+ Args:
+ wheel_dir: The root of the extracted wheel directory.
+
+ Returns:
+ Relative path to the data directory if it exists, else, None.
+ """
+
+ dot_data_dirs = glob.glob(os.path.join(wheel_dir, "*.data"))
+ if not dot_data_dirs:
+ return None
+
+ if len(dot_data_dirs) > 1:
+ raise ValueError(
+ "Found more than 1 *.data directory. %s is not a valid Wheel." % wheel_dir
+ )
+
+ return dot_data_dirs[0]
+
+
+def parse_wheel_meta_file(wheel_dir: str) -> Dict[str, str]:
+ """Parses the given WHEEL file into a dictionary.
+
+ Args:
+ wheel_dir: The file path of the WHEEL metadata file in dist-info.
+
+ Returns:
+ The WHEEL file mapped into a dictionary.
+ """
+ contents = {}
+ with open(wheel_dir, "r") as wheel_file:
+ for line in wheel_file:
+ cleaned = line.strip()
+ if not cleaned:
+ continue
+ try:
+ key, value = cleaned.split(":", maxsplit=1)
+ contents[key] = value.strip()
+ except ValueError:
+ raise RuntimeError(
+ "Encounted invalid line in WHEEL file: '%s'" % cleaned
+ )
+ return contents
diff --git a/experimental/rules_python_external/repositories.bzl b/experimental/rules_python_external/repositories.bzl
new file mode 100644
index 0000000..60c6e5d
--- /dev/null
+++ b/experimental/rules_python_external/repositories.bzl
@@ -0,0 +1,59 @@
+""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+_RULE_DEPS = [
+ (
+ "pypi__pip",
+ "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl",
+ "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7",
+ ),
+ (
+ "pypi__pkginfo",
+ "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl",
+ "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32",
+ ),
+ (
+ "pypi__setuptools",
+ "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl",
+ "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6",
+ ),
+ (
+ "pypi__wheel",
+ "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl",
+ "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28",
+ ),
+]
+
+_GENERIC_WHEEL = """\
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_python//python:defs.bzl", "py_library")
+
+py_library(
+ name = "lib",
+ srcs = glob(["**/*.py"]),
+ data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]),
+ # This makes this directory a top-level in the python import
+ # search path for anything that depends on this.
+ imports = ["."],
+)
+"""
+
+# Collate all the repository names so they can be easily consumed
+all_requirements = [name for (name, _, _) in _RULE_DEPS]
+
+def requirement(pkg):
+ return "@pypi__"+ pkg + "//:lib"
+
+def rules_python_external_dependencies():
+ for (name, url, sha256) in _RULE_DEPS:
+ maybe(
+ http_archive,
+ name,
+ url=url,
+ sha256=sha256,
+ type="zip",
+ build_file_content=_GENERIC_WHEEL,
+ )
diff --git a/experimental/rules_python_external/tools/typing/BUILD b/experimental/rules_python_external/tools/typing/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/experimental/rules_python_external/tools/typing/BUILD
diff --git a/experimental/rules_python_external/tools/typing/mypy.ini b/experimental/rules_python_external/tools/typing/mypy.ini
new file mode 100644
index 0000000..ed392f2
--- /dev/null
+++ b/experimental/rules_python_external/tools/typing/mypy.ini
@@ -0,0 +1,24 @@
+[mypy]
+# This should be the oldest supported release of Python
+# https://devguide.python.org/#status-of-python-branches
+python_version = 3.5
+
+# Third-Party packages without Stub files
+# https://mypy.readthedocs.io/en/latest/stubs.html
+[mypy-pkginfo.*]
+ignore_missing_imports = True
+
+[mypy-extract_wheels.*]
+check_untyped_defs = True
+disallow_incomplete_defs = True
+disallow_untyped_calls = True
+disallow_untyped_decorators = True
+disallow_untyped_defs = True
+no_implicit_optional = True
+strict_equality = True
+strict_optional = True
+warn_no_return = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_ignores = True
diff --git a/experimental/rules_python_external/tools/typing/mypy_version.txt b/experimental/rules_python_external/tools/typing/mypy_version.txt
new file mode 100644
index 0000000..9e7ef88
--- /dev/null
+++ b/experimental/rules_python_external/tools/typing/mypy_version.txt
@@ -0,0 +1 @@
+mypy==0.780
