pw_package: Initial commit
Add pw_package module. This manages dependencies that aren't pulled in
through env setup. For now only nanopb is available through pw_package.
Change-Id: Ib8a20102baf27d5964bb275088c265f9334b6ff3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22020
Reviewed-by: Anthony DiGirolamo <tonymd@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/PW_PLUGINS b/PW_PLUGINS
index d789bf8..f588ded 100644
--- a/PW_PLUGINS
+++ b/PW_PLUGINS
@@ -14,3 +14,4 @@
presubmit pw_presubmit.pigweed_presubmit main
heap-viewer pw_allocator.heap_viewer main
rpc pw_hdlc_lite.rpc_console main
+package pw_package.pigweed_packages main
diff --git a/docs/BUILD.gn b/docs/BUILD.gn
index cc63f06..50abacc 100644
--- a/docs/BUILD.gn
+++ b/docs/BUILD.gn
@@ -79,6 +79,7 @@
"$dir_pw_metric:docs",
"$dir_pw_minimal_cpp_stdlib:docs",
"$dir_pw_module:docs",
+ "$dir_pw_package:docs",
"$dir_pw_polyfill:docs",
"$dir_pw_preprocessor:docs",
"$dir_pw_presubmit:docs",
diff --git a/modules.gni b/modules.gni
index e441a0f..974c2a4 100644
--- a/modules.gni
+++ b/modules.gni
@@ -51,6 +51,7 @@
dir_pw_minimal_cpp_stdlib = get_path_info("pw_minimal_cpp_stdlib", "abspath")
dir_pw_module = get_path_info("pw_module", "abspath")
dir_pw_fuzzer = get_path_info("pw_fuzzer", "abspath")
+ dir_pw_package = get_path_info("pw_package", "abspath")
dir_pw_polyfill = get_path_info("pw_polyfill", "abspath")
dir_pw_preprocessor = get_path_info("pw_preprocessor", "abspath")
dir_pw_presubmit = get_path_info("pw_presubmit", "abspath")
diff --git a/pw_package/BUILD.gn b/pw_package/BUILD.gn
new file mode 100644
index 0000000..dd021e8
--- /dev/null
+++ b/pw_package/BUILD.gn
@@ -0,0 +1,21 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("$dir_pw_docgen/docs.gni")
+
+pw_doc_group("docs") {
+ sources = [ "docs.rst" ]
+}
diff --git a/pw_package/docs.rst b/pw_package/docs.rst
new file mode 100644
index 0000000..b3055db
--- /dev/null
+++ b/pw_package/docs.rst
@@ -0,0 +1,110 @@
+.. _module-pw_package:
+
+==========
+pw_package
+==========
+The package module provides a mechanism to install additional tools used by
+Pigweed. Most Pigweed dependencies should be installed using
+:ref:`module-pw_env_setup`. Examples of reasons packages should be managed using
+this module instead are listed below.
+
+* The dependency is extremely large and not commonly used.
+* The dependency has a number of compatible versions and we want to allow
+ downstream projects to pick a version rather than being forced to use ours.
+* The dependency has license issues that make it complicated for Google to
+ include it directly as a submodule or distribute it as a CIPD package.
+* The dependency needs to be "installed" into the system in some manner beyond
+ just extraction and thus isn't a good match for distribution with CIPD.
+
+-----
+Usage
+-----
+The package module can be accessed through the ``pw package`` command. This
+has several subcommands.
+
+``pw package list``
+ Lists all the packages installed followed by all the packages available.
+
+``pw package install <package-name>``
+ Installs ``<package-name>``. Exactly how this works is package-dependent,
+ and packages can decide to do nothing because the package is current, do an
+ incremental update, or delete the current version and install anew. Use
+ ``--force`` to remove the package before installing.
+
+``pw package status <package-name>``
+ Indicates whether ``<packagxe-name>`` is installed.
+
+``pw package remove <package-name>``
+ Removes ``<package-name>``.
+
+-----------
+Configuring
+-----------
+
+Compatibility
+~~~~~~~~~~~~~
+Python 3
+
+Adding a New Package
+~~~~~~~~~~~~~~~~~~~~
+To add a new package create a class that subclasses ``Package`` from
+``pw_package/package_manager.py``.
+
+.. code-block:: python
+
+ class Package:
+ """Package to be installed.
+
+ Subclass this to implement installation of a specific package.
+ """
+ def __init__(self, name):
+ self._name = name
+
+ @property
+ def name(self):
+ return self._name
+
+ def install(self, path: pathlib.Path) -> None:
+ """Install the package at path.
+
+ Install the package in path. Cannot assume this directory is empty—it
+ may need to be deleted or updated.
+ """
+
+ def remove(self, path: pathlib.Path) -> None:
+ """Remove the package from path.
+
+ Removes the directory containing the package. For most packages this
+ should be sufficient to remove the package, and subclasses should not
+ need to override this package.
+ """
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def status(self, path: pathlib.Path) -> bool:
+ """Returns if package is installed at path and current.
+
+ This method will be skipped if the directory does not exist.
+ """
+
+There's also a helper class for retrieving specific revisions of Git
+repositories in ``pw_package/git_repo.py``.
+
+Then call ``pw_package.package_manager.register(PackageClass)`` to register
+the class with the package manager.
+
+Setting up a Project
+~~~~~~~~~~~~~~~~~~~~
+To set up the package manager for a new project create a file like below and
+add it to the ``PW_PLUGINS`` file (see :ref:`module-pw_cli` for details). This
+file is based off of ``pw_package/pigweed_packages.py``.
+
+.. code-block:: python
+
+ from pw_package import package_manager
+ # These modules register themselves so must be imported despite appearing
+ # unused.
+ from pw_package.packages import nanopb
+
+ def main(argv=None) -> int:
+ return package_manager.run(**vars(package_manager.parse_args(argv)))
diff --git a/pw_package/py/pw_package/git_repo.py b/pw_package/py/pw_package/git_repo.py
new file mode 100644
index 0000000..4b98366
--- /dev/null
+++ b/pw_package/py/pw_package/git_repo.py
@@ -0,0 +1,71 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and check status of Git repository-based packages."""
+
+import os
+import pathlib
+import shutil
+import subprocess
+from typing import Union
+
+import pw_package.package_manager
+
+PathOrStr = Union[pathlib.Path, str]
+
+
+def git_stdout(*args: PathOrStr,
+ show_stderr=False,
+ repo: PathOrStr = '.') -> str:
+ return subprocess.run(['git', '-C', repo, *args],
+ stdout=subprocess.PIPE,
+ stderr=None if show_stderr else subprocess.DEVNULL,
+ check=True).stdout.decode().strip()
+
+
+def git(*args: PathOrStr,
+ repo: PathOrStr = '.') -> subprocess.CompletedProcess:
+ return subprocess.run(['git', '-C', repo, *args], check=True)
+
+
+class GitRepo(pw_package.package_manager.Package):
+ """Install and check status of Git repository-based packages."""
+ def __init__(self, url, commit, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._url = url
+ self._commit = commit
+
+ def status(self, path: pathlib.Path) -> bool:
+ if not os.path.isdir(path / '.git'):
+ return False
+
+ remote = git_stdout('remote', 'get-url', 'origin', repo=path)
+ commit = git_stdout('rev-parse', 'HEAD', repo=path)
+ status = git_stdout('status', '--porcelain=v1', repo=path)
+ return remote == self._url and commit == self._commit and not status
+
+ def install(self, path: pathlib.Path) -> None:
+ # If already installed and at correct version exit now.
+ if self.status(path):
+ return
+
+ # Otherwise delete current version and clone again.
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+
+ # --filter=blob:none means we don't get history, just the current
+ # revision. If we later run commands that need history it will be
+ # retrieved on-demand. For small repositories the effect is negligible
+ # but for large repositories this should be a significant improvement.
+ git('clone', '--filter=blob:none', self._url, path)
+ git('reset', '--hard', self._commit, repo=path)
diff --git a/pw_package/py/pw_package/package_manager.py b/pw_package/py/pw_package/package_manager.py
new file mode 100644
index 0000000..dd20b44
--- /dev/null
+++ b/pw_package/py/pw_package/package_manager.py
@@ -0,0 +1,147 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and remove optional packages."""
+
+import argparse
+import logging
+import os
+import pathlib
+import shutil
+from typing import List
+
+_LOG: logging.Logger = logging.getLogger(__name__)
+
+
+class Package:
+ """Package to be installed.
+
+ Subclass this to implement installation of a specific package.
+ """
+ def __init__(self, name):
+ self._name = name
+
+ @property
+ def name(self):
+ return self._name
+
+ def install(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use
+ """Install the package at path.
+
+ Install the package in path. Cannot assume this directory is empty—it
+ may need to be deleted or updated.
+ """
+
+ def remove(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use
+ """Remove the package from path.
+
+ Removes the directory containing the package. For most packages this
+ should be sufficient to remove the package, and subclasses should not
+ need to override this package.
+ """
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+ def status(self, path: pathlib.Path) -> bool: # pylint: disable=no-self-use
+ """Returns if package is installed at path and current.
+
+ This method will be skipped if the directory does not exist.
+ """
+
+
+_PACKAGES = {}
+
+
+def register(package_class: type) -> None:
+ obj = package_class()
+ _PACKAGES[obj.name] = obj
+
+
+class PackageManager:
+ """Install and remove optional packages."""
+ def __init__(self):
+ self._pkg_root: pathlib.Path = None
+
+ def install(self, package: str, force=False):
+ pkg = _PACKAGES[package]
+ if force:
+ self.remove(package)
+ _LOG.info('Installing %s...', pkg.name)
+ pkg.install(self._pkg_root / pkg.name)
+ _LOG.info('Installing %s...done.', pkg.name)
+ return 0
+
+ def remove(self, package: str): # pylint: disable=no-self-use
+ pkg = _PACKAGES[package]
+ _LOG.info('Removing %s...', pkg.name)
+ pkg.remove(self._pkg_root / pkg.name)
+ _LOG.info('Removing %s...done.', pkg.name)
+ return 0
+
+ def status(self, package: str): # pylint: disable=no-self-use
+ pkg = _PACKAGES[package]
+ path = self._pkg_root / pkg.name
+ if os.path.isdir(path) and pkg.status(path):
+ _LOG.info('%s is installed.', pkg.name)
+ return 0
+
+ _LOG.info('%s is not installed.', pkg.name)
+ return -1
+
+ def list(self): # pylint: disable=no-self-use
+ _LOG.info('Installed packages:')
+ available = []
+ for package in sorted(_PACKAGES.keys()):
+ pkg = _PACKAGES[package]
+ if pkg.status(self._pkg_root / pkg.name):
+ _LOG.info(' %s', pkg.name)
+ else:
+ available.append(pkg.name)
+ _LOG.info('')
+
+ _LOG.info('Available packages:')
+ for pkg_name in available:
+ _LOG.info(' %s', pkg_name)
+ _LOG.info('')
+
+ return 0
+
+ def run(self, command: str, pkg_root: pathlib.Path, **kwargs):
+ os.makedirs(pkg_root, exist_ok=True)
+ self._pkg_root = pkg_root
+ return getattr(self, command)(**kwargs)
+
+
+def parse_args(argv: List[str] = None) -> argparse.Namespace:
+ parser = argparse.ArgumentParser("Manage packages.")
+ parser.add_argument(
+ '--package-root',
+ '-e',
+ dest='pkg_root',
+ type=pathlib.Path,
+ default=(pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT']) /
+ 'packages'),
+ )
+ subparsers = parser.add_subparsers(dest='command', required=True)
+ install = subparsers.add_parser('install')
+ install.add_argument('--force', '-f', action='store_true')
+ remove = subparsers.add_parser('remove')
+ status = subparsers.add_parser('status')
+ for cmd in (install, remove, status):
+ cmd.add_argument('package', choices=_PACKAGES.keys())
+ _ = subparsers.add_parser('list')
+ return parser.parse_args(argv)
+
+
+def run(**kwargs):
+ return PackageManager().run(**kwargs)
diff --git a/pw_package/py/pw_package/packages/__init__.py b/pw_package/py/pw_package/packages/__init__.py
new file mode 100644
index 0000000..2c8334f
--- /dev/null
+++ b/pw_package/py/pw_package/packages/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
diff --git a/pw_package/py/pw_package/packages/nanopb.py b/pw_package/py/pw_package/packages/nanopb.py
new file mode 100644
index 0000000..96955bd
--- /dev/null
+++ b/pw_package/py/pw_package/packages/nanopb.py
@@ -0,0 +1,30 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and check status of nanopb."""
+
+import pw_package.git_repo
+import pw_package.package_manager
+
+
+class NanoPB(pw_package.git_repo.GitRepo):
+ """Install and check status of nanopb."""
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args,
+ name='nanopb',
+ url='https://github.com/nanopb/nanopb.git',
+ commit='9f57cc871d8a025039019c2d2fde217591f4e30d',
+ **kwargs)
+
+
+pw_package.package_manager.register(NanoPB)
diff --git a/pw_package/py/pw_package/pigweed_packages.py b/pw_package/py/pw_package/pigweed_packages.py
new file mode 100644
index 0000000..734b6d7
--- /dev/null
+++ b/pw_package/py/pw_package/pigweed_packages.py
@@ -0,0 +1,29 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Install and remove optional packages for Pigweed."""
+
+import sys
+
+from pw_package import package_manager
+# These modules register themselves so must be imported despite appearing
+# unused.
+from pw_package.packages import nanopb # pylint: disable=unused-import
+
+
+def main(argv=None) -> int:
+ return package_manager.run(**vars(package_manager.parse_args(argv)))
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/pw_package/py/setup.py b/pw_package/py/setup.py
new file mode 100644
index 0000000..682e013
--- /dev/null
+++ b/pw_package/py/setup.py
@@ -0,0 +1,26 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""The pw_package package."""
+
+import setuptools
+
+setuptools.setup(
+ name='pw_package',
+ version='0.0.1',
+ author='Pigweed Authors',
+ author_email='pigweed-developers@googlegroups.com',
+ description='Tools for installing optional packages',
+ install_requires=[],
+ packages=setuptools.find_packages(),
+)
