pw_build/python.gni - pigweed/pigweed - Git at Google

 # Copyright 2020 The Pigweed Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
 # use this file except in compliance with the License. You may obtain a copy of
 # the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.

 import("//build_overrides/pigweed.gni")

 import("$dir_pw_build/input_group.gni")
 import("$dir_pw_build/python_action.gni")

 # Defines a Python package. GN Python packages contain several GN targets:
 #
 #   - $name - Provides the Python files in the build, but does not take any
 #         actions. All subtargets depend on this target.
 #   - $name.lint - Runs static analyis tools on the Python code. This is a group
 #     of two subtargets:
 #     - $name.lint.mypy - Runs mypy (if enabled).
 #     - $name.lint.pylint - Runs pylint (if enabled).
 #   - $name.tests - Runs all tests for this package.
 #   - $name.install - Installs the package in a venv.
 #   - $name.wheel - Builds a Python wheel for the package. (Not implemented.)
 #
 # TODO(pwbug/239): Implement wheel building.
 #
 # Args:
 #   setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
 #       which must all be in the same directory.
 #   sources: Python sources files in the package.
 #   tests: Test files for this Python package.
 #   python_deps: Dependencies on other pw_python_packages in the GN build.
 #   other_deps: Dependencies on GN targets that are not pw_python_packages.
 #   inputs: Other files to track, such as package_data.
 #   lint: If true (default), applies mypy and pylint to the package. If false,
 #       does not.
 #   pylintrc: Optional path to a pylintrc configuration file to use. If not
 #       provided, Pylint's default rcfile search is used. Pylint is executed
 #       from the package's setup directory, so pylintrc files in that directory
 #       will take precedence over others.
 #   mypy_ini: Optional path to a mypy configuration file to use. If not
 #       provided, mypy's default configuration file search is used. mypy is
 #       executed from the package's setup directory, so mypy.ini files in that
 #       directory will take precedence over others.
 template("pw_python_package") {
   if (defined(invoker.sources)) {
     _all_py_files = invoker.sources
   } else {
     _all_py_files = []
   }

   if (defined(invoker.tests)) {
     _test_sources = invoker.tests
   } else {
     _test_sources = []
   }

   _all_py_files += _test_sources

   # pw_python_script uses pw_python_package, but with a limited set of features.
   # _pw_standalone signals that this target is actually a pw_python_script.
   _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)

   # Some build targets generate Python packages, setting _pw_generated to
   # indicate this.
   _is_generated_package =
       defined(invoker._pw_generated) && invoker._pw_generated

   # Argument: invoker.lint = [true | false]; default = true.
   # Default to false for generated packages, but allow overrides.
   if (defined(invoker.lint)) {
     _should_lint = invoker.lint
   } else {
     _should_lint = !_is_generated_package
   }

   if (_is_package) {
     assert(defined(invoker.setup) && invoker.setup != [],
            "pw_python_package requires 'setup' to point to a setup.py file " +
                "or pyproject.toml and setup.cfg files")

     if (!_is_generated_package) {
       _all_py_files += invoker.setup
     }

     # Get the directories of the setup files. All files must be in the same dir.
     _setup_dirs = get_path_info(invoker.setup, "dir")
     _setup_dir = _setup_dirs[0]

     foreach(dir, _setup_dirs) {
       assert(dir == _setup_dir,
              "All files in 'setup' must be in the same directory")
     }

     # If sources are provided, make sure there is an __init__.py file.
     if (!_is_generated_package && defined(invoker.sources) &&
         invoker.sources != []) {
       assert(filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
              "Python packages must have at least one __init__.py file")
     }
   }

   _python_deps = []
   if (defined(invoker.python_deps)) {
     foreach(dep, invoker.python_deps) {
       # Use the fully qualified name so the subtarget can be appended as needed.
       _python_deps += [ get_label_info(dep, "label_no_toolchain") ]
     }
   }

   # Declare the main Python package group. This represents the Python files, but
   # does not take any actions. GN targets can depend on the package name to run
   # when any files in the package change.
   pw_input_group(target_name) {
     inputs = _all_py_files
     if (defined(invoker.inputs)) {
       inputs += invoker.inputs
     }

     deps = _python_deps

     if (defined(invoker.other_deps)) {
       deps += invoker.other_deps
     }
   }

   _package_target = ":$target_name"

   if (_is_package) {
     # Install this Python package and its dependencies in the current Python
     # environment.
     pw_python_action("$target_name.install") {
       module = "pip"
       args = [ "install" ]

       # Don't install generated packages with --editable, since the build
       # directory is ephemeral.
       if (!_is_generated_package) {
         args += [ "--editable" ]
       }
       args += [ rebase_path(_setup_dir) ]

       stamp = true

       # Parallel pip installations don't work, so serialize pip invocations.
       pool = "$dir_pw_build:pip_pool"

       deps = [ _package_target ]
       foreach(dep, _python_deps) {
         deps += [ "$dep.install" ]
       }
     }

     # TODO(pwbug/239): Add support for building groups of wheels. The code below
     #     is incomplete and untested.
     pw_python_action("$target_name.wheel") {
       script = "$dir_pw_build/py/pw_build/python_wheels.py"

       args = [
         "--out_dir",
         rebase_path(target_out_dir),
       ]
       args += rebase_path(_all_py_files)

       deps = [ _package_target ]
       stamp = true
     }
   } else {
     # If this is not a package, install or build wheels for its deps only.
     group("$target_name.install") {
       deps = []
       foreach(dep, _python_deps) {
         deps += [ "$dep.install" ]
       }
     }
     group("$target_name.wheel") {
       deps = []
       foreach(dep, _python_deps) {
         deps += [ "$dep.wheel" ]
       }
     }
   }

   # Define the static analysis targets for this package.
   group("$target_name.lint") {
     deps = [
       "$_package_target.lint.mypy",
       "$_package_target.lint.pylint",
     ]
   }

   # For packages that are not generated, create targets to run mypy and pylint.
   # Linting is not performed on generated packages.
   if (_should_lint) {
     # Run lint tools from the setup or target directory so that the tools detect
     # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
     # may be explicitly specified with the pylintrc or mypy_ini arguments.
     if (defined(_setup_dir)) {
       _lint_directory = rebase_path(_setup_dir)
     } else {
       _lint_directory = rebase_path(".")
     }

     pw_python_action("$target_name.lint.mypy") {
       module = "mypy"
       args = [
         "--pretty",
         "--show-error-codes",
       ]

       if (defined(invoker.mypy_ini)) {
         args += [ "--config-file=" + rebase_path(invoker.mypy_ini) ]
         inputs = [ invoker.mypy_ini ]
       }

       if (_is_package) {
         args += [ rebase_path(_setup_dir) ]
       } else {
         args += rebase_path(_all_py_files)
       }

       # Use this environment variable to force mypy to colorize output.
       # See https://github.com/python/mypy/issues/7771
       environment = [ "MYPY_FORCE_COLOR=1" ]

       directory = _lint_directory
       stamp = true

       deps = [ _package_target ]
       foreach(dep, _python_deps) {
         deps += [ "$dep.lint.mypy" ]
       }
     }

     # Create a target to run pylint on each of the Python files in this
     # package and its dependencies.
     pw_python_action_foreach("$target_name.lint.pylint") {
       module = "pylint"
       args = [
         rebase_path(".") + "/{{source_target_relative}}",
         "--jobs=1",
         "--output-format=colorized",
       ]

       if (defined(invoker.pylintrc)) {
         args += [ "--rcfile=" + rebase_path(invoker.pylintrc) ]
         inputs = [ invoker.pylintrc ]
       }

       if (host_os == "win") {
         # Allow CRLF on Windows, in case Git is set to switch line endings.
         args += [ "--disable=unexpected-line-ending-format" ]
       }

       sources = _all_py_files

       directory = _lint_directory
       stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"

       deps = [ _package_target ]
       foreach(dep, _python_deps) {
         deps += [ "$dep.lint.pylint" ]
       }
     }
   } else {
     pw_input_group("$target_name.lint.mypy") {
       if (defined(invoker.pylintrc)) {
         inputs = [ invoker.pylintrc ]
       }
     }
     pw_input_group("$target_name.lint.pylint") {
       if (defined(invoker.mypy_ini)) {
         inputs = [ invoker.mypy_ini ]
       }
     }
   }

   # Create a target for each test file.
   _test_targets = []

   foreach(test, _test_sources) {
     _test_name = string_replace(test, "/", "_")
     _test_target = "$target_name.tests.$_test_name"

     pw_python_action(_test_target) {
       script = test
       stamp = true

       deps = [ _package_target ]
       foreach(dep, _python_deps) {
         deps += [ "$dep.tests" ]
       }
     }

     _test_targets += [ ":$_test_target" ]
   }

   group("$target_name.tests") {
     deps = _test_targets
   }
 }

 # Declares a group of Python packages or other Python groups. pw_python_groups
 # expose the same set of subtargets as pw_python_package (e.g.
 # "$group_name.lint" and "$group_name.tests"), but these apply to all packages
 # in deps and their dependencies.
 template("pw_python_group") {
   if (defined(invoker.python_deps)) {
     _python_deps = invoker.python_deps
   } else {
     _python_deps = []
   }

   group(target_name) {
     deps = _python_deps
   }

   _subtargets = [
     "tests",
     "lint",
     "lint.mypy",
     "lint.pylint",
     "install",
     "wheel",
   ]

   foreach(subtarget, _subtargets) {
     group("$target_name.$subtarget") {
       deps = []
       foreach(dep, _python_deps) {
         # Split out the toolchain to support deps with a toolchain specified.
         _target = get_label_info(dep, "label_no_toolchain")
         _toolchain = get_label_info(dep, "toolchain")
         deps += [ "$_target.$subtarget($_toolchain)" ]
       }
     }
   }
 }

 # Declares Python scripts or tests that are not part of a Python package.
 # Similar to pw_python_package, but only supports a subset of its features.
 #
 # pw_python_script accepts the same arguments as pw_python_package, except
 # `setup` cannot be provided.
 #
 # pw_python_script provides the same subtargets as pw_python_package, but
 # $target_name.install and $target_name.wheel only affect the python_deps of
 # this GN target, not the target itself.
 template("pw_python_script") {
   _supported_variables = [
     "sources",
     "tests",
     "python_deps",
     "other_deps",
     "inputs",
     "pylintrc",
   ]

   pw_python_package(target_name) {
     _pw_standalone = true
     forward_variables_from(invoker, _supported_variables)
   }
 }
	# Copyright 2020 The Pigweed Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License"); you may not
	# use this file except in compliance with the License. You may obtain a copy of
	# the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	# License for the specific language governing permissions and limitations under
	# the License.

	import("//build_overrides/pigweed.gni")

	import("$dir_pw_build/input_group.gni")
	import("$dir_pw_build/python_action.gni")

	# Defines a Python package. GN Python packages contain several GN targets:
	#
	# - $name - Provides the Python files in the build, but does not take any
	# actions. All subtargets depend on this target.
	# - $name.lint - Runs static analyis tools on the Python code. This is a group
	# of two subtargets:
	# - $name.lint.mypy - Runs mypy (if enabled).
	# - $name.lint.pylint - Runs pylint (if enabled).
	# - $name.tests - Runs all tests for this package.
	# - $name.install - Installs the package in a venv.
	# - $name.wheel - Builds a Python wheel for the package. (Not implemented.)
	#
	# TODO(pwbug/239): Implement wheel building.
	#
	# Args:
	# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
	# which must all be in the same directory.
	# sources: Python sources files in the package.
	# tests: Test files for this Python package.
	# python_deps: Dependencies on other pw_python_packages in the GN build.
	# other_deps: Dependencies on GN targets that are not pw_python_packages.
	# inputs: Other files to track, such as package_data.
	# lint: If true (default), applies mypy and pylint to the package. If false,
	# does not.
	# pylintrc: Optional path to a pylintrc configuration file to use. If not
	# provided, Pylint's default rcfile search is used. Pylint is executed
	# from the package's setup directory, so pylintrc files in that directory
	# will take precedence over others.
	# mypy_ini: Optional path to a mypy configuration file to use. If not
	# provided, mypy's default configuration file search is used. mypy is
	# executed from the package's setup directory, so mypy.ini files in that
	# directory will take precedence over others.
	template("pw_python_package") {
	if (defined(invoker.sources)) {
	_all_py_files = invoker.sources
	} else {
	_all_py_files = []
	}

	if (defined(invoker.tests)) {
	_test_sources = invoker.tests
	} else {
	_test_sources = []
	}

	_all_py_files += _test_sources

	# pw_python_script uses pw_python_package, but with a limited set of features.
	# _pw_standalone signals that this target is actually a pw_python_script.
	_is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)

	# Some build targets generate Python packages, setting _pw_generated to
	# indicate this.
	_is_generated_package =
	defined(invoker._pw_generated) && invoker._pw_generated

	# Argument: invoker.lint = [true \| false]; default = true.
	# Default to false for generated packages, but allow overrides.
	if (defined(invoker.lint)) {
	_should_lint = invoker.lint
	} else {
	_should_lint = !_is_generated_package
	}

	if (_is_package) {
	assert(defined(invoker.setup) && invoker.setup != [],
	"pw_python_package requires 'setup' to point to a setup.py file " +
	"or pyproject.toml and setup.cfg files")

	if (!_is_generated_package) {
	_all_py_files += invoker.setup
	}

	# Get the directories of the setup files. All files must be in the same dir.
	_setup_dirs = get_path_info(invoker.setup, "dir")
	_setup_dir = _setup_dirs[0]

	foreach(dir, _setup_dirs) {
	assert(dir == _setup_dir,
	"All files in 'setup' must be in the same directory")
	}

	# If sources are provided, make sure there is an __init__.py file.
	if (!_is_generated_package && defined(invoker.sources) &&
	invoker.sources != []) {
	assert(filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
	"Python packages must have at least one __init__.py file")
	}
	}

	_python_deps = []
	if (defined(invoker.python_deps)) {
	foreach(dep, invoker.python_deps) {
	# Use the fully qualified name so the subtarget can be appended as needed.
	_python_deps += [ get_label_info(dep, "label_no_toolchain") ]
	}
	}

	# Declare the main Python package group. This represents the Python files, but
	# does not take any actions. GN targets can depend on the package name to run
	# when any files in the package change.
	pw_input_group(target_name) {
	inputs = _all_py_files
	if (defined(invoker.inputs)) {
	inputs += invoker.inputs
	}

	deps = _python_deps

	if (defined(invoker.other_deps)) {
	deps += invoker.other_deps
	}
	}

	_package_target = ":$target_name"

	if (_is_package) {
	# Install this Python package and its dependencies in the current Python
	# environment.
	pw_python_action("$target_name.install") {
	module = "pip"
	args = [ "install" ]

	# Don't install generated packages with --editable, since the build
	# directory is ephemeral.
	if (!_is_generated_package) {
	args += [ "--editable" ]
	}
	args += [ rebase_path(_setup_dir) ]

	stamp = true

	# Parallel pip installations don't work, so serialize pip invocations.
	pool = "$dir_pw_build:pip_pool"

	deps = [ _package_target ]
	foreach(dep, _python_deps) {
	deps += [ "$dep.install" ]
	}
	}

	# TODO(pwbug/239): Add support for building groups of wheels. The code below
	# is incomplete and untested.
	pw_python_action("$target_name.wheel") {
	script = "$dir_pw_build/py/pw_build/python_wheels.py"

	args = [
	"--out_dir",
	rebase_path(target_out_dir),
	]
	args += rebase_path(_all_py_files)

	deps = [ _package_target ]
	stamp = true
	}
	} else {
	# If this is not a package, install or build wheels for its deps only.
	group("$target_name.install") {
	deps = []
	foreach(dep, _python_deps) {
	deps += [ "$dep.install" ]
	}
	}
	group("$target_name.wheel") {
	deps = []
	foreach(dep, _python_deps) {
	deps += [ "$dep.wheel" ]
	}
	}
	}

	# Define the static analysis targets for this package.
	group("$target_name.lint") {
	deps = [
	"$_package_target.lint.mypy",
	"$_package_target.lint.pylint",
	]
	}

	# For packages that are not generated, create targets to run mypy and pylint.
	# Linting is not performed on generated packages.
	if (_should_lint) {
	# Run lint tools from the setup or target directory so that the tools detect
	# config files (e.g. pylintrc or mypy.ini) in that directory. Config files
	# may be explicitly specified with the pylintrc or mypy_ini arguments.
	if (defined(_setup_dir)) {
	_lint_directory = rebase_path(_setup_dir)
	} else {
	_lint_directory = rebase_path(".")
	}

	pw_python_action("$target_name.lint.mypy") {
	module = "mypy"
	args = [
	"--pretty",
	"--show-error-codes",
	]

	if (defined(invoker.mypy_ini)) {
	args += [ "--config-file=" + rebase_path(invoker.mypy_ini) ]
	inputs = [ invoker.mypy_ini ]
	}

	if (_is_package) {
	args += [ rebase_path(_setup_dir) ]
	} else {
	args += rebase_path(_all_py_files)
	}

	# Use this environment variable to force mypy to colorize output.
	# See https://github.com/python/mypy/issues/7771
	environment = [ "MYPY_FORCE_COLOR=1" ]

	directory = _lint_directory
	stamp = true

	deps = [ _package_target ]
	foreach(dep, _python_deps) {
	deps += [ "$dep.lint.mypy" ]
	}
	}

	# Create a target to run pylint on each of the Python files in this
	# package and its dependencies.
	pw_python_action_foreach("$target_name.lint.pylint") {
	module = "pylint"
	args = [
	rebase_path(".") + "/{{source_target_relative}}",
	"--jobs=1",
	"--output-format=colorized",
	]

	if (defined(invoker.pylintrc)) {
	args += [ "--rcfile=" + rebase_path(invoker.pylintrc) ]
	inputs = [ invoker.pylintrc ]
	}

	if (host_os == "win") {
	# Allow CRLF on Windows, in case Git is set to switch line endings.
	args += [ "--disable=unexpected-line-ending-format" ]
	}

	sources = _all_py_files

	directory = _lint_directory
	stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"

	deps = [ _package_target ]
	foreach(dep, _python_deps) {
	deps += [ "$dep.lint.pylint" ]
	}
	}
	} else {
	pw_input_group("$target_name.lint.mypy") {
	if (defined(invoker.pylintrc)) {
	inputs = [ invoker.pylintrc ]
	}
	}
	pw_input_group("$target_name.lint.pylint") {
	if (defined(invoker.mypy_ini)) {
	inputs = [ invoker.mypy_ini ]
	}
	}
	}

	# Create a target for each test file.
	_test_targets = []

	foreach(test, _test_sources) {
	_test_name = string_replace(test, "/", "_")
	_test_target = "$target_name.tests.$_test_name"

	pw_python_action(_test_target) {
	script = test
	stamp = true

	deps = [ _package_target ]
	foreach(dep, _python_deps) {
	deps += [ "$dep.tests" ]
	}
	}

	_test_targets += [ ":$_test_target" ]
	}

	group("$target_name.tests") {
	deps = _test_targets
	}
	}

	# Declares a group of Python packages or other Python groups. pw_python_groups
	# expose the same set of subtargets as pw_python_package (e.g.
	# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
	# in deps and their dependencies.
	template("pw_python_group") {
	if (defined(invoker.python_deps)) {
	_python_deps = invoker.python_deps
	} else {
	_python_deps = []
	}

	group(target_name) {
	deps = _python_deps
	}

	_subtargets = [
	"tests",
	"lint",
	"lint.mypy",
	"lint.pylint",
	"install",
	"wheel",
	]

	foreach(subtarget, _subtargets) {
	group("$target_name.$subtarget") {
	deps = []
	foreach(dep, _python_deps) {
	# Split out the toolchain to support deps with a toolchain specified.
	_target = get_label_info(dep, "label_no_toolchain")
	_toolchain = get_label_info(dep, "toolchain")
	deps += [ "$_target.$subtarget($_toolchain)" ]
	}
	}
	}
	}

	# Declares Python scripts or tests that are not part of a Python package.
	# Similar to pw_python_package, but only supports a subset of its features.
	#
	# pw_python_script accepts the same arguments as pw_python_package, except
	# `setup` cannot be provided.
	#
	# pw_python_script provides the same subtargets as pw_python_package, but
	# $target_name.install and $target_name.wheel only affect the python_deps of
	# this GN target, not the target itself.
	template("pw_python_script") {
	_supported_variables = [
	"sources",
	"tests",
	"python_deps",
	"other_deps",
	"inputs",
	"pylintrc",
	]

	pw_python_package(target_name) {
	_pw_standalone = true
	forward_variables_from(invoker, _supported_variables)
	}
	}