wip Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com>
diff --git a/examples/pip_parse/.bazelversion b/examples/pip_parse/.bazelversion new file mode 100644 index 0000000..af8c8ec --- /dev/null +++ b/examples/pip_parse/.bazelversion
@@ -0,0 +1 @@ +4.2.2
diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD index 92b59ae..4ae065a 100644 --- a/examples/pip_parse/BUILD +++ b/examples/pip_parse/BUILD
@@ -4,6 +4,7 @@ "dist_info_requirement", "entry_point", ) +load("@rules_python//python/pip_install:pip_repository.bzl", "wheel") load("@rules_python//python:defs.bzl", "py_binary", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -79,3 +80,56 @@ }, deps = ["@rules_python//python/runfiles"], ) + +wheel( + name = "numpy", + src = "@numpy_wheel_source_distribution//file", +) + +# wheel( +# name = "pybind11", +# src = "@pybind11_wheel_source_distribution//file", +# ) + +# wheel( +# name = "pythran", +# src = "@pythran_wheel_source_distribution//file", +# ) + +# wheel( +# name = "gast", +# src = "@gast_wheel_source_distribution//file", +# ) + +# wheel( +# name = "beniget", +# src = "@beniget_wheel_source_distribution//file", +# ) + +# wheel( +# name = "ply", +# src = "@ply_wheel_source_distribution//file", +# ) + +# wheel( +# name = "scipy", +# src = "@scipy_wheel_source_distribution//file", +# deps = [ +# ":numpy", +# ":pybind11", +# ":pythran", +# ":gast", +# ":beniget", +# ":ply", +# ], +# ) + +wheel( + name = "boto3", + src = "@boto3_wheel_source_distribution//file", +) + +wheel( + name = "django", + src = "@django_wheel_source_distribution//file", +)
diff --git a/examples/pip_parse/WORKSPACE b/examples/pip_parse/WORKSPACE index e96db9f..b706fd6 100644 --- a/examples/pip_parse/WORKSPACE +++ b/examples/pip_parse/WORKSPACE
@@ -1,18 +1,37 @@ workspace(name = "rules_python_pip_parse_example") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") + +http_archive( + name = "bazel_skylib", + sha256 = "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz", + ], +) + +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + +bazel_skylib_workspace() + local_repository( name = "rules_python", path = "../..", ) +load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") + +pip_install_dependencies() + load("@rules_python//python:repositories.bzl", "python_register_toolchains") python_register_toolchains( - name = "python39", - python_version = "3.9", + name = "python310", + python_version = "3.10", ) -load("@python39//:defs.bzl", "interpreter") +load("@python310//:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( @@ -48,3 +67,66 @@ # Initialize repositories for all packages in requirements_lock.txt. install_deps() + +http_file( + name = "numpy_wheel_source_distribution", + downloaded_file_path = "numpy-1.22.3.zip", + sha256 = "dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18", + urls = ["https://files.pythonhosted.org/packages/64/4a/b008d1f8a7b9f5206ecf70a53f84e654707e7616a771d84c05151a4713e9/numpy-1.22.3.zip"], +) + +http_file( + name = "pybind11_wheel_source_distribution", + downloaded_file_path = "pybind11-2.9.2.tar.gz", + sha256 = "e5541f8bccf9111d1a94f7897593b55c4cf1a28d5e8cfc8225a855651f011071", + urls = ["https://files.pythonhosted.org/packages/cf/6a/a7f2c40fdb43fcf59bc1eebb0a4c4206a99ccddee6391a1168fa6efebce9/pybind11-2.9.2.tar.gz"], +) + +http_file( + name = "pythran_wheel_source_distribution", + downloaded_file_path = "pythran-0.11.0.tar.gz", + sha256 = "0b2cba712e09f7630879dff69f268460bfe34a6d6000451b47d598558a92a875", + urls = ["https://files.pythonhosted.org/packages/88/9f/161f08131abf7f23920cee29b691de27f10fd97ac09fb2f3532b3a7f9b96/pythran-0.11.0.tar.gz"], +) + +http_file( + name = "gast_wheel_source_distribution", + downloaded_file_path = "gast-0.5.3.tar.gz", + sha256 = "cfbea25820e653af9c7d1807f659ce0a0a9c64f2439421a7bba4f0983f532dea", + urls = ["https://files.pythonhosted.org/packages/48/a3/0bd844c54ae8141642088b7ae09dd38fec2ec7faa9b7d25bb6a23c1f266f/gast-0.5.3.tar.gz"], +) + +http_file( + name = "beniget_wheel_source_distribution", + downloaded_file_path = "beniget-0.4.1.tar.gz", + sha256 = "75554b3b8ad0553ce2f607627dad3d95c60c441189875b98e097528f8e23ac0c", + urls = ["https://files.pythonhosted.org/packages/14/e7/50cbac38f77eca8efd39516be6651fdb9f3c4c0fab8cf2cf05f612578737/beniget-0.4.1.tar.gz"], +) + +http_file( + name = "ply_wheel_source_distribution", + downloaded_file_path = "ply-3.11.tar.gz", + sha256 = "00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", + urls = ["https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz"], +) + +http_file( + name = "scipy_wheel_source_distribution", + downloaded_file_path = "scipy-1.8.0.tar.gz", + sha256 = "31d4f2d6b724bc9a98e527b5849b8a7e589bf1ea630c33aa563eda912c9ff0bd", + urls = ["https://files.pythonhosted.org/packages/b4/a2/4faa34bf0cdbefd5c706625f1234987795f368eb4e97bde9d6f46860843e/scipy-1.8.0.tar.gz"], +) + +http_file( + name = "boto3_wheel_source_distribution", + downloaded_file_path = "boto3-1.21.38.tar.gz", + sha256 = "62dde36a57697b40b4693e0ad0d39013f1e187e5a3c52fdb50dbe710633061bb", + urls = ["https://files.pythonhosted.org/packages/df/89/6b20ad811c1502f9fbb29369556010ef49e830650ad4d6ac257d40134220/boto3-1.21.38.tar.gz"], +) + +http_file( + name = "django_wheel_source_distribution", + downloaded_file_path = "Django-4.0.4.tar.gz", + sha256 = "4e8177858524417563cc0430f29ea249946d831eacb0068a1455686587df40b5", + urls = ["https://files.pythonhosted.org/packages/d2/95/93d1f75da95624bf89e373d079fa72debf77f9b10acc31998cc52a5ff3f9/Django-4.0.4.tar.gz"], +)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index e587f34..d5e2a31 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl
@@ -1,5 +1,15 @@ "" +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load( + "@bazel_tools//tools/build_defs/cc:action_names.bzl", + "ASSEMBLE_ACTION_NAME", + "CPP_COMPILE_ACTION_NAME", + "CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME", + "CPP_LINK_STATIC_LIBRARY_ACTION_NAME", + "C_COMPILE_ACTION_NAME", +) load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") @@ -434,3 +444,209 @@ data_exclude_glob = data_exclude_glob, srcs_exclude_glob = srcs_exclude_glob, )) + +def _wheel_impl(ctx): + tools = [ + ("CC", C_COMPILE_ACTION_NAME), + ("CXX", CPP_COMPILE_ACTION_NAME), + ("AS", ASSEMBLE_ACTION_NAME), + ("LD", CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME), + ("AR", CPP_LINK_STATIC_LIBRARY_ACTION_NAME), + ] + flags = [ + ("LDFLAGS", CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, ctx.fragments.cpp.linkopts), + ("CFLAGS", C_COMPILE_ACTION_NAME, ctx.fragments.cpp.copts), + ("CXXFLAGS", CPP_COMPILE_ACTION_NAME, ctx.fragments.cpp.cxxopts), + ] + env = _environment_variables( + ctx, + tools, + flags, + ) + + python_toolchain = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"] + runtime = python_toolchain.py3_runtime + if not runtime.interpreter: + fail("py3_runtime must be set") + + build_deps = ctx.files._build_deps + deps = ctx.files.deps + + build_deps_pythonpath = _install_deps(ctx, runtime, env, [], "build_deps", build_deps) + deps_pythonpath = _install_deps(ctx, runtime, env, [build_deps_pythonpath], "deps", deps) + + output_name = paths.replace_extension(paths.basename(ctx.file.src.path), ".whl").replace(".tar", "") + whl = ctx.actions.declare_file(output_name) + progress_message = "Building {}".format(output_name) + + ctx.actions.run_shell( + command = """\ +set -o errexit -o nounset -o pipefail + +export PYTHONPATH="{pythonpath}" + +'{interpreter}' -m pip wheel \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-index \ + --no-deps \ + --no-build-isolation \ + --use-pep517 \ + --wheel-dir output/ \ + '{src}' + +mv output/*.whl '{whl}' +""".format( + interpreter = runtime.interpreter.path, + pythonpath = _construct_pythonpath([build_deps_pythonpath, deps_pythonpath]), + src = ctx.file.src.path, + whl = whl.path, + ), + env = env, + execution_requirements = { + "block-network": "1", + }, + inputs = build_deps + [ + ctx.file.src, + build_deps_pythonpath, + deps_pythonpath, + ], + mnemonic = "BuildWheel", + outputs = [whl], + progress_message = progress_message, + tools = runtime.files, + ) + + return [DefaultInfo( + files = depset([whl]), + runfiles = ctx.runfiles([whl]), + )] + +def _install_deps(ctx, runtime, env, extra_pythonpath, dirname, deps): + pythonpath = ctx.actions.declare_directory("{}_{}".format(dirname, ctx.attr.name)) + commands = [ + _install_dep_command(runtime.interpreter.path, dep.path, pythonpath.path) + for dep in deps + ] + ctx.actions.run_shell( + command = """\ +set -o errexit -o nounset -o pipefail + +export PYTHONPATH="{pythonpath}" + +{commands} +""".format( + pythonpath = _construct_pythonpath(extra_pythonpath + [pythonpath]), + commands = "\n".join(commands), + ), + env = env, + execution_requirements = { + "block-network": "1", + }, + inputs = deps + extra_pythonpath, + mnemonic = "InstallDeps", + outputs = [pythonpath], + progress_message = "Installing dependencies", + tools = runtime.files, + ) + return pythonpath + +def _construct_pythonpath(paths): + return ":".join([ + "$(pwd)/{}".format(p.path) + for p in paths + ]) + +def _install_dep_command(interpreter, dep, pythonpath): + return """\ +if [[ '{dep}' =~ \\.whl$ ]]; then + # '{interpreter}' -m wheel unpack \ + # --dest-dir '{pythonpath}' \ + # '{dep}' + '{interpreter}' <<'EOF' +import zipfile +with zipfile.ZipFile('{dep}', 'r') as zf: + zf.extractall('{pythonpath}') +EOF +else + '{interpreter}' -m pip install \ + --upgrade \ + --disable-pip-version-check \ + --no-cache-dir \ + --no-index \ + --no-deps \ + --no-build-isolation \ + --use-pep517 \ + --target '{pythonpath}' \ + '{dep}' +fi +""".format( + dep = dep, + interpreter = interpreter, + pythonpath = pythonpath, + ) + +def _environment_variables(ctx, tools, flags): + cc_toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + ) + env = {} + for (var, action) in tools: + env[var] = cc_common.get_tool_for_action( + feature_configuration = feature_configuration, + action_name = action, + ) + for (var, action, user_compile_flags) in flags: + compile_variables = cc_common.create_compile_variables( + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + user_compile_flags = user_compile_flags, + ) + env[var] = " ".join(cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = action, + variables = compile_variables, + )) + + # TODO(f0rmiga): some libraries will not respect the env vars above, so the + # only way to still get them to compile successfuly is to add a PATH. + # E.g. compiling numpy calls `as` hardcoded somewhere. Even setting -B in + # the CFLAGS, CXXFLAGS and LDFLAGS won't solve fully. + # An alternative is to get binutils hermetic and put in the PATH to be found + # first. + env["PATH"] = "/usr/bin" + return env + +wheel = rule( + _wheel_impl, + attrs = { + "src": attr.label( + allow_single_file = True, + doc = "The wheel source distribution file.", + mandatory = True, + ), + "deps": attr.label_list( + allow_files = True, + doc = "Wheel distributions to be installed before building src.", + mandatory = False, + ), + "_cc_toolchain": attr.label( + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), + "_build_deps": attr.label_list( + allow_files = True, + default = [ + Label("@pypi__setuptools_whl//file"), + Label("@pypi__wheel_whl//file"), + Label("@pypi__cython//file"), + ], + ), + }, + fragments = ["cpp"], + toolchains = [ + "@bazel_tools//tools/cpp:toolchain_type", + "@bazel_tools//tools/python:toolchain_type", + ], +)
diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 352f663..049f3f1 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl
@@ -1,6 +1,7 @@ "" -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") # Avoid a load from @bazel_skylib repository as users don't necessarily have it installed @@ -42,6 +43,11 @@ "https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl", "4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a", ), + ( + "pypi__cython", + "https://files.pythonhosted.org/packages/cb/da/54a5d7a7d9afc90036d21f4b58229058270cc14b4c81a86d9b2c77fd072e/Cython-0.29.28.tar.gz", + "d6fac2342802c30e51426828fe084ff4deb1b3387367cf98976bb2e64b6f8e45", + ), ] _GENERIC_WHEEL = """\ @@ -60,7 +66,7 @@ """ # Collate all the repository names so they can be easily consumed -all_requirements = [name for (name, _, _) in _RULE_DEPS] +all_requirements = [name for (name, url, _) in _RULE_DEPS if url.endswith(".whl")] def requirement(pkg): return "@pypi__" + pkg + "//:lib" @@ -79,11 +85,30 @@ versions.check("4.0.0") for (name, url, sha256) in _RULE_DEPS: - maybe( - http_archive, - name, - url = url, - sha256 = sha256, - type = "zip", - build_file_content = _GENERIC_WHEEL, - ) + filename = paths.basename(url) + + if filename.endswith(".whl"): + maybe( + http_file, + name + "_whl", + urls = [url], + sha256 = sha256, + downloaded_file_path = filename, + ) + + maybe( + http_archive, + name, + urls = [url], + sha256 = sha256, + type = "zip", + build_file_content = _GENERIC_WHEEL, + ) + else: + maybe( + http_file, + name, + urls = [url], + sha256 = sha256, + downloaded_file_path = filename, + )