Fix unnecessary namespace directory creation (#54)
diff --git a/example/BUILD b/example/BUILD index 699c10d..64a3718 100644 --- a/example/BUILD +++ b/example/BUILD
@@ -1,27 +1,29 @@ load("@pip//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_runtime_pair") # Toolchain setup, this is optional. # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE). -py_runtime( - name = "python3_runtime", - files = ["@python_interpreter//:files"], - interpreter = "@python_interpreter//:python_bin", - python_version = "PY3", - visibility = ["//visibility:public"], -) - -py_runtime_pair( - name = "my_py_runtime_pair", - py2_runtime = None, - py3_runtime = ":python3_runtime", -) - -toolchain( - name = "my_py_toolchain", - toolchain = ":my_py_runtime_pair", - toolchain_type = "@bazel_tools//tools/python:toolchain_type", -) +# +#load("@rules_python//python:defs.bzl", "py_runtime_pair") +# +#py_runtime( +# name = "python3_runtime", +# files = ["@python_interpreter//:files"], +# interpreter = "@python_interpreter//:python_bin", +# python_version = "PY3", +# visibility = ["//visibility:public"], +#) +# +#py_runtime_pair( +# name = "my_py_runtime_pair", +# py2_runtime = None, +# py3_runtime = ":python3_runtime", +#) +# +#toolchain( +# name = "my_py_toolchain", +# toolchain = ":my_py_runtime_pair", +# toolchain_type = "@bazel_tools//tools/python:toolchain_type", +#) # End of toolchain setup. py_binary(
diff --git a/example/WORKSPACE b/example/WORKSPACE index f42f7c5..639308a 100644 --- a/example/WORKSPACE +++ b/example/WORKSPACE
@@ -4,52 +4,15 @@ http_archive( name = "rules_python", - sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz", + strip_prefix = "rules_python-0.0.2", + sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c", ) load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() -# You could optionally use an in-build, compiled python interpreter as a toolchain, -# and also use it to execute pip. - -# Special logic for building python interpreter with OpenSSL from homebrew. -# See https://devguide.python.org/setup/#macos-and-os-x -_py_configure = """ -if [[ "$OSTYPE" == "darwin"* ]]; then - ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl) -else - ./configure --prefix=$(pwd)/bazel_install -fi -""" - -# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS). -# E.g. on Ubuntu: `sudo apt install libssl-dev` -http_archive( - name = "python_interpreter", - build_file_content = """ -exports_files(["python_bin"]) -filegroup( - name = "files", - srcs = glob(["bazel_install/**"], exclude = ["**/* *"]), - visibility = ["//visibility:public"], -) -""", - patch_cmds = [ - "mkdir $(pwd)/bazel_install", - _py_configure, - "make", - "make install", - "ln -s bazel_install/bin/python3 python_bin", - ], - sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864", - strip_prefix = "Python-3.8.3", - urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"], -) -# End of in-build Python interpreter setup. - local_repository( name = "rules_python_external", path = "../", @@ -75,7 +38,7 @@ # 1. Python interpreter that you compile in the build file (as above in @python_interpreter). # 2. Pre-compiled python interpreter included with http_archive # 3. Wrapper script, like in the autodetecting python toolchain. - python_interpreter_target = "@python_interpreter//:python_bin", + #python_interpreter_target = "@python_interpreter//:python_bin", # (Optional) You can set quiet to False if you want to see pip output. #quiet = False, @@ -84,6 +47,44 @@ requirements = "//:requirements.txt", ) +# You could optionally use an in-build, compiled python interpreter as a toolchain, +# and also use it to execute pip. +# +# Special logic for building python interpreter with OpenSSL from homebrew. +# See https://devguide.python.org/setup/#macos-and-os-x +#_py_configure = """ +#if [[ "$OSTYPE" == "darwin"* ]]; then +# ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl) +#else +# ./configure --prefix=$(pwd)/bazel_install +#fi +#""" +# +# NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS). +# E.g. on Ubuntu: `sudo apt install libssl-dev` +#http_archive( +# name = "python_interpreter", +# build_file_content = """ +#exports_files(["python_bin"]) +#filegroup( +# name = "files", +# srcs = glob(["bazel_install/**"], exclude = ["**/* *"]), +# visibility = ["//visibility:public"], +#) +#""", +# patch_cmds = [ +# "mkdir $(pwd)/bazel_install", +# _py_configure, +# "make", +# "make install", +# "ln -s bazel_install/bin/python3 python_bin", +# ], +# sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864", +# strip_prefix = "Python-3.8.3", +# urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"], +#) + # Optional: # Register the toolchain with the same python interpreter we used for pip in pip_install(). -register_toolchains("//:my_py_toolchain") +#register_toolchains("//:my_py_toolchain") +# End of in-build Python interpreter setup.
diff --git a/example/main.py b/example/main.py index 24d1dea..e5c690b 100644 --- a/example/main.py +++ b/example/main.py
@@ -1,3 +1,4 @@ import boto3 -print("Hello World") +if __name__ == "__main__": + pass
diff --git a/extract_wheels/__init__.py b/extract_wheels/__init__.py index 6cd8e5f..fccf1b0 100644 --- a/extract_wheels/__init__.py +++ b/extract_wheels/__init__.py
@@ -64,18 +64,22 @@ required=True, help="The external repo name to install dependencies. In the format '@{REPO_NAME}'", ) - parser.add_argument('--extra_pip_args', action='store', - help=('Extra arguments to pass down to pip.')) + parser.add_argument( + "--extra_pip_args", + action="store", + help=("Extra arguments to pass down to pip."), + ) parser.add_argument( "--pip_data_exclude", - action='store', - help='Additional data exclusion parameters to add to the pip packages BUILD file.' + action="store", + help="Additional data exclusion parameters to add to the pip packages BUILD file.", ) args = parser.parse_args() pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements] if args.extra_pip_args: pip_args += json.loads(args.extra_pip_args)["args"] + # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails subprocess.run(pip_args, check=True)
diff --git a/extract_wheels/lib/bazel.py b/extract_wheels/lib/bazel.py index cc7d879..85ba965 100644 --- a/extract_wheels/lib/bazel.py +++ b/extract_wheels/lib/bazel.py
@@ -7,7 +7,9 @@ from extract_wheels.lib import namespace_pkgs, wheel, purelib -def generate_build_file_contents(name: str, dependencies: List[str], pip_data_exclude: List[str]) -> str: +def generate_build_file_contents( + name: str, dependencies: List[str], pip_data_exclude: List[str] +) -> str: """Generate a BUILD file for an unzipped Wheel Args: @@ -39,7 +41,9 @@ deps = [{dependencies}], ) """.format( - name=name, dependencies=",".join(dependencies), data_exclude=json.dumps(data_exclude) + name=name, + dependencies=",".join(dependencies), + data_exclude=json.dumps(data_exclude), ) ) @@ -93,33 +97,31 @@ def setup_namespace_pkg_compatibility(wheel_dir: str) -> None: - """Converts native namespace packages and pkg_resource-style packages to pkgutil-style packages + """Converts native namespace packages to pkgutil-style packages Namespace packages can be created in one of three ways. They are detailed here: https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package - 'pkgutil-style namespace packages' (2) works in Bazel, but 'native namespace packages' (1) and - 'pkg_resources-style namespace packages' (3) do not. + 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but + 'native namespace packages' (1) do not. - We ensure compatibility with Bazel of methods 1 and 3 by converting them into method 2. + We ensure compatibility with Bazel of method 1 by converting them into method 2. Args: wheel_dir: the directory of the wheel to convert """ - namespace_pkg_dirs = namespace_pkgs.pkg_resources_style_namespace_packages( - wheel_dir + namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( + wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,], ) - if not namespace_pkg_dirs and namespace_pkgs.native_namespace_packages_supported(): - namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( - wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,], - ) for ns_pkg_dir in namespace_pkg_dirs: namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) -def extract_wheel(wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str]) -> str: +def extract_wheel( + wheel_file: str, extras: Dict[str, Set[str]], pip_data_exclude: List[str] +) -> str: """Extracts wheel into given directory and creates a py_library target. Args:
diff --git a/extract_wheels/lib/namespace_pkgs.py b/extract_wheels/lib/namespace_pkgs.py index 3a1b33d..cb9e164 100644 --- a/extract_wheels/lib/namespace_pkgs.py +++ b/extract_wheels/lib/namespace_pkgs.py
@@ -1,36 +1,11 @@ """Utility functions to discover python package types""" import os -import sys import textwrap from typing import Set, List, Optional from extract_wheels.lib import wheel -def pkg_resources_style_namespace_packages(wheel_dir: str) -> Set[str]: - """Discovers namespace packages implemented using the 'pkg_resources-style namespace packages' method. - - "While this approach is no longer recommended, it is widely present in most existing namespace packages." - PyPA - See https://packaging.python.org/guides/packaging-namespace-packages/#pkg-resources-style-namespace-packages - """ - namespace_pkg_dirs = set() - - dist_info = wheel.get_dist_info(wheel_dir) - namespace_packages_record_file = os.path.join(dist_info, "namespace_packages.txt") - if os.path.exists(namespace_packages_record_file): - with open(namespace_packages_record_file) as nspkg: - for line in nspkg.readlines(): - namespace = line.strip().replace(".", os.sep) - if namespace: - namespace_pkg_dirs.add(os.path.join(wheel_dir, namespace)) - return namespace_pkg_dirs - - -def native_namespace_packages_supported() -> bool: - """Returns true if this version of Python supports native namespace packages.""" - return (sys.version_info.major, sys.version_info.minor) >= (3, 3) - - def implicit_namespace_packages( directory: str, ignored_dirnames: Optional[List[str]] = None ) -> Set[str]: @@ -84,8 +59,7 @@ if os.path.isfile(ns_pkg_init_filepath): raise ValueError("%s already contains an __init__.py file." % dir_path) - if not os.path.exists(dir_path): - os.makedirs(dir_path) + with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages ns_pkg_init_f.write(
diff --git a/extract_wheels/lib/namespace_pkgs_test.py b/extract_wheels/lib/namespace_pkgs_test.py index 195cdd9..899f7bc 100644 --- a/extract_wheels/lib/namespace_pkgs_test.py +++ b/extract_wheels/lib/namespace_pkgs_test.py
@@ -1,100 +1,38 @@ import pathlib import shutil import tempfile +from typing import Optional import unittest from extract_wheels.lib import namespace_pkgs class TempDir: - def __init__(self): + def __init__(self) -> None: self.dir = tempfile.mkdtemp() - def root(self): + def root(self) -> str: return self.dir - def add_dir(self, rel_path): + def add_dir(self, rel_path: str) -> None: d = pathlib.Path(self.dir, rel_path) d.mkdir(parents=True) - def add_file(self, rel_path, contents=None): + def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: f = pathlib.Path(self.dir, rel_path) f.parent.mkdir(parents=True, exist_ok=True) if contents: - with open(f, "w") as writeable_f: + with open(str(f), "w") as writeable_f: writeable_f.write(contents) else: f.touch() - def remove(self): + def remove(self) -> None: shutil.rmtree(self.dir) -class TestPkgResourcesStyleNamespacePackages(unittest.TestCase): - def test_finds_correct_namespace_packages(self): - directory = TempDir() - directory.add_file("google/auth/__init__.py") - directory.add_file("google/auth/foo.py") - directory.add_file( - "google_auth-1.8.2.dist-info/namespace_packages.txt", contents="google\n" - ) - - expected = { - directory.root() + "/google", - } - actual = namespace_pkgs.pkg_resources_style_namespace_packages(directory.root()) - self.assertEqual(actual, expected) - - def test_nested_namespace_packages(self): - directory = TempDir() - directory.add_file("google/auth/__init__.py") - directory.add_file("google/auth/foo.py") - directory.add_file("google/bar/biz/__init__.py") - directory.add_file("google/bar/biz/bee.py") - directory.add_file( - "google_auth-1.8.2.dist-info/namespace_packages.txt", - contents="google\ngoogle.bar\n", - ) - - expected = { - directory.root() + "/google", - directory.root() + "/google/bar", - } - actual = namespace_pkgs.pkg_resources_style_namespace_packages(directory.root()) - self.assertEqual(actual, expected) - - def test_empty_case(self): - # Even though this directory contains directories with no __init__.py - # it has an empty namespace_packages.txt file so no namespace packages - # should be returned. - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - directory.add_file("foo-1.0.0.dist-info/namespace_packages.txt") - - actual = namespace_pkgs.pkg_resources_style_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - def test_missing_namespace_pkgs_record_file(self): - # Even though this directory contains directories with no __init__.py - # it has no namespace_packages.txt file, so no namespace packages should - # be found and returned. - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - directory.add_file("foo-1.0.0.dist-info/METADATA") - directory.add_file("foo-1.0.0.dist-info/RECORD") - - actual = namespace_pkgs.pkg_resources_style_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - class TestImplicitNamespacePackages(unittest.TestCase): - def test_finds_correct_namespace_packages(self): + def test_finds_correct_namespace_packages(self) -> None: directory = TempDir() directory.add_file("foo/bar/biz.py") directory.add_file("foo/bee/boo.py") @@ -109,7 +47,7 @@ actual = namespace_pkgs.implicit_namespace_packages(directory.root()) self.assertEqual(actual, expected) - def test_ignores_empty_directories(self): + def test_ignores_empty_directories(self) -> None: directory = TempDir() directory.add_file("foo/bar/biz.py") directory.add_dir("foo/cat") @@ -121,7 +59,7 @@ actual = namespace_pkgs.implicit_namespace_packages(directory.root()) self.assertEqual(actual, expected) - def test_empty_case(self): + def test_empty_case(self) -> None: directory = TempDir() directory.add_file("foo/__init__.py") directory.add_file("foo/bar/__init__.py") @@ -131,12 +69,5 @@ self.assertEqual(actual, set()) -class TestaddPkgutilStyleNamespacePkgInit(unittest.TestCase): - def test_missing_directory_is_created(self): - missing_directory = pathlib.Path(TempDir().root(), "missing_directory") - namespace_pkgs.add_pkgutil_style_namespace_pkg_init(str(missing_directory)) - self.assertTrue(missing_directory.is_dir()) - - if __name__ == "__main__": unittest.main()
diff --git a/tools/typing/mypy.ini b/tools/typing/mypy.ini index 65254d7..ed392f2 100644 --- a/tools/typing/mypy.ini +++ b/tools/typing/mypy.ini
@@ -7,3 +7,18 @@ # https://mypy.readthedocs.io/en/latest/stubs.html [mypy-pkginfo.*] ignore_missing_imports = True + +[mypy-extract_wheels.*] +check_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_untyped_defs = True +no_implicit_optional = True +strict_equality = True +strict_optional = True +warn_no_return = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True +warn_unused_ignores = True
diff --git a/tools/typing/mypy_version.txt b/tools/typing/mypy_version.txt index 4436991..9e7ef88 100644 --- a/tools/typing/mypy_version.txt +++ b/tools/typing/mypy_version.txt
@@ -1 +1 @@ -mypy==0.750 +mypy==0.780