pw_env_setup: Pin Python package versions
Add a file to pass to the '--constraint' option of 'pip'. This allows
setup.cfg files to only set actual requirements on dependencies but
ensures the actual version retrieved for a given commit is always the
same.
Added a new command 'pw python-packages'. 'pw python-packages list'
writes the list of versions of installed packages to a file.
'pw python-packages diff' compares the list of installed packages to the
constraint file. If there are new packages or updates to package
versions 'pw python-packages diff' will fail.
Added an option to pw_env_setup, '--unpin-pip-packages'. If this is set
the constraint file defined in the top-level '.gn' file is ignored. This
should periodically be used to update versions of packages in the
constraint file.
Change-Id: I49337e029b0c5c82ac1414448d97d952d9df4dae
Bug: 459
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/58280
Pigweed-Auto-Submit: Rob Mohr <mohrr@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>
Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/.gn b/.gn
index 60ce6bd..f898dae 100644
--- a/.gn
+++ b/.gn
@@ -13,3 +13,8 @@
# the License.
buildconfig = "//BUILDCONFIG.gn"
+
+default_args = {
+ pw_build_PIP_CONSTRAINTS =
+ [ "//pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list" ]
+}
diff --git a/pw_cli/py/pw_cli/pw_command_plugins.py b/pw_cli/py/pw_cli/pw_command_plugins.py
index a5c20df..845e007 100644
--- a/pw_cli/py/pw_cli/pw_command_plugins.py
+++ b/pw_cli/py/pw_cli/pw_command_plugins.py
@@ -30,6 +30,8 @@
# Register these by name to avoid circular dependencies.
registry.register_by_name('doctor', 'pw_doctor.doctor', 'main')
+ registry.register_by_name('python-packages',
+ 'pw_env_setup.python_packages', 'main')
registry.register_by_name('format', 'pw_presubmit.format_code', 'main')
registry.register_by_name('logdemo', 'pw_cli.log', 'main')
registry.register_by_name('module-check', 'pw_module.check', 'main')
diff --git a/pw_env_setup/docs.rst b/pw_env_setup/docs.rst
index cef79a6..9aca996 100644
--- a/pw_env_setup/docs.rst
+++ b/pw_env_setup/docs.rst
@@ -294,6 +294,24 @@
- ``PW_MYPROJECTNAME_CIPD_INSTALL_DIR``
- ``PW_PIGWEED_CIPD_INSTALL_DIR``
+Pinning Python Packages
+***********************
+Python modules usually express dependencies as ranges, which makes it easier to
+install many Python packages that might otherwise have conflicting dependencies.
+However, this means version of packages can often change underneath us and
+builds will not be hermetic.
+
+To ensure versions don't change without approval, run
+``pw python-packages list <path/to/constraints/file>`` and then add
+``pw_build_PIP_CONSTRAINTS = ["//path/to/constraints/file"]`` to your project's
+``.gn`` file (see `Pigweed's .gn file` for an example).
+
+.. _Pigweed's .gn file: https://cs.opensource.google/pigweed/pigweed/+/main:.gn
+
+To update packages, remove the ``pw_build_PIP_CONSTRAINTS`` line, delete the
+environment, and bootstrap again. Then run the ``list`` command from above
+again, and run ``pw presubmit``.
+
Environment Variables
*********************
The following environment variables affect env setup behavior. Most users will
diff --git a/pw_env_setup/py/BUILD.gn b/pw_env_setup/py/BUILD.gn
index 09055d2..1775ff9 100644
--- a/pw_env_setup/py/BUILD.gn
+++ b/pw_env_setup/py/BUILD.gn
@@ -33,6 +33,7 @@
"pw_env_setup/env_setup.py",
"pw_env_setup/environment.py",
"pw_env_setup/json_visitor.py",
+ "pw_env_setup/python_packages.py",
"pw_env_setup/shell_visitor.py",
"pw_env_setup/spinner.py",
"pw_env_setup/virtualenv_setup/__init__.py",
@@ -42,6 +43,7 @@
]
tests = [
"environment_test.py",
+ "python_packages_test.py",
"json_visitor_test.py",
]
pylintrc = "$dir_pigweed/.pylintrc"
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index dcab6c0..931a085 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -173,7 +173,8 @@
"""Run environment setup for Pigweed."""
def __init__(self, pw_root, cipd_cache_dir, shell_file, quiet, install_dir,
virtualenv_root, strict, virtualenv_gn_out_dir, json_file,
- project_root, config_file, use_existing_cipd):
+ project_root, config_file, use_existing_cipd,
+ use_pinned_pip_packages):
self._env = environment.Environment()
self._project_root = project_root
self._pw_root = pw_root
@@ -198,6 +199,7 @@
self._virtualenv_requirements = []
self._virtualenv_gn_targets = []
self._virtualenv_gn_args = []
+ self._use_pinned_pip_packages = use_pinned_pip_packages
self._optional_submodules = []
self._required_submodules = []
self._virtualenv_system_packages = False
@@ -534,6 +536,7 @@
python=new_python3,
env=self._env,
system_packages=self._virtualenv_system_packages,
+ use_pinned_pip_packages=self._use_pinned_pip_packages,
):
return result(_Result.Status.FAILED)
@@ -647,6 +650,13 @@
action='store_true',
)
+ parser.add_argument(
+ '--unpin-pip-packages',
+ dest='use_pinned_pip_packages',
+ help='Do not use pins of pip packages.',
+ action='store_false',
+ )
+
args = parser.parse_args(argv)
return args
diff --git a/pw_env_setup/py/pw_env_setup/python_packages.py b/pw_env_setup/py/pw_env_setup/python_packages.py
new file mode 100644
index 0000000..3f8d001
--- /dev/null
+++ b/pw_env_setup/py/pw_env_setup/python_packages.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Save list of installed packages and versions."""
+
+import argparse
+import subprocess
+import sys
+from typing import Dict, List, TextIO, Union
+
+
+def _installed_packages():
+ """Run pip python_packages and write to out."""
+ cmd = [
+ 'python',
+ '-m',
+ 'pip',
+ 'freeze',
+ '--exclude-editable',
+ '--local',
+ ]
+ proc = subprocess.run(cmd, capture_output=True)
+ for line in proc.stdout.decode().splitlines():
+ if ' @ ' not in line:
+ yield line
+
+
+def ls(out: TextIO) -> int: # pylint: disable=invalid-name
+ """Run pip python_packages and write to out."""
+ for package in _installed_packages():
+ print(package, file=out)
+
+ return 0
+
+
+class UpdateRequiredError(Exception):
+ pass
+
+
+def _stderr(*args, **kwargs):
+ return print(*args, file=sys.stderr, **kwargs)
+
+
+def diff(expected: TextIO) -> int:
+ """Report on differences between installed and expected versions."""
+ actual_lines = set(_installed_packages())
+ expected_lines = set(expected.read().splitlines())
+
+ if actual_lines == expected_lines:
+ _stderr('files are identical')
+ return 0
+
+ removed_entries: Dict[str, str] = dict(
+ x.split('==', 1) # type: ignore[misc]
+ for x in expected_lines - actual_lines)
+ added_entries: Dict[str, str] = dict(
+ x.split('==', 1) # type: ignore[misc]
+ for x in actual_lines - expected_lines)
+
+ new_packages = set(added_entries) - set(removed_entries)
+ removed_packages = set(removed_entries) - set(added_entries)
+ updated_packages = set(added_entries).intersection(set(removed_entries))
+
+ if removed_packages:
+ _stderr('Removed packages')
+ for package in removed_packages:
+ _stderr(f' {package}=={removed_entries[package]}')
+
+ if updated_packages:
+ _stderr('Updated packages')
+ for package in updated_packages:
+ _stderr(f' {package}=={added_entries[package]} (from '
+ f'{removed_entries[package]})')
+
+ if new_packages:
+ _stderr('New packages')
+ for package in new_packages:
+ _stderr(f' {package}=={added_entries[package]}')
+
+ if updated_packages or new_packages:
+ _stderr("Package versions don't match!")
+ _stderr(f"""
+Please do the following:
+
+* purge your environment directory
+ * Linux/Mac: 'rm -rf "$_PW_ACTUAL_ENVIRONMENT_ROOT"'
+ * Windows: 'rmdir /S %_PW_ACTUAL_ENVIRONMENT_ROOT%'
+* bootstrap
+ * Linux/Mac: '. ./bootstrap.sh'
+ * Windows: 'bootstrap.bat'
+* update the constraint file
+ * 'pw python-packages list {expected.name}'
+""")
+ return -1
+
+ return 0
+
+
+def parse(argv: Union[List[str], None] = None) -> argparse.Namespace:
+ """Parse command-line arguments."""
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest='cmd')
+
+ list_parser = subparsers.add_parser(
+ 'list', aliases=('ls', ), help='List installed package versions.')
+ list_parser.add_argument('out',
+ type=argparse.FileType('w'),
+ default=sys.stdout,
+ nargs='?')
+
+ diff_parser = subparsers.add_parser(
+ 'diff',
+ help='Show differences between expected and actual package versions.',
+ )
+ diff_parser.add_argument('expected', type=argparse.FileType('r'))
+
+ return parser.parse_args(argv)
+
+
+def main() -> int:
+ try:
+ args = vars(parse())
+ cmd = args.pop('cmd')
+ if cmd == 'diff':
+ return diff(**args)
+ if cmd == 'list':
+ return ls(**args)
+ return -1
+ except subprocess.CalledProcessError as err:
+ print(file=sys.stderr)
+ print(err.output, file=sys.stderr)
+ raise
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list
new file mode 100644
index 0000000..0815425
--- /dev/null
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list
@@ -0,0 +1,98 @@
+actdiag==2.0.0
+alabaster==0.7.12
+appdirs==1.4.4
+astroid==2.6.6
+Babel==2.9.1
+backcall==0.2.0
+beautifulsoup4==4.10.0
+blockdiag==2.0.1
+build==0.6.0.post1
+certifi==2021.5.30
+cffi==1.14.6
+charset-normalizer==2.0.4
+coloredlogs==15.0.1
+cryptography==3.4.8
+decorator==5.0.9
+docutils==0.16
+funcparserlib==0.3.5
+furo==2021.8.31
+future==0.18.2
+grpcio==1.39.0
+grpcio-tools==1.39.0
+httpwatcher==0.5.2
+humanfriendly==9.2
+idna==3.2
+imagesize==1.2.0
+ipython==7.27.0
+isort==5.9.3
+jedi==0.18.0
+Jinja2==3.0.1
+lazy-object-proxy==1.6.0
+MarkupSafe==2.0.1
+matplotlib-inline==0.1.3
+mccabe==0.6.1
+mypy==0.910
+mypy-extensions==0.4.3
+mypy-protobuf==2.9
+nwdiag==2.0.0
+packaging==21.0
+parameterized==0.8.1
+parso==0.8.2
+pep517==0.11.0
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==8.2.0
+prompt-toolkit==3.0.20
+protobuf==3.17.3
+psutil==5.8.0
+ptpython==3.0.19
+ptyprocess==0.7.0
+pycparser==2.20
+pyelftools==0.27
+Pygments==2.10.0
+pygments-style-dracula==1.2.5.1
+pygments-style-tomorrow==1.0.0.1
+pylint==2.9.3
+pyparsing==2.4.7
+pyperclip==1.8.2
+pyserial==3.5
+pytz==2021.1
+PyYAML==5.4.1
+requests==2.26.0
+robotframework==3.1
+scan-build==2.0.19
+seqdiag==2.0.0
+six==1.16.0
+snowballstemmer==2.1.0
+soupsieve==2.2.1
+Sphinx==4.1.2
+sphinx-design==0.0.12
+sphinx-rtd-theme==0.5.2
+sphinxcontrib-actdiag==2.0.0
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-blockdiag==2.0.0
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-mermaid==0.7.1
+sphinxcontrib-nwdiag==2.0.0
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-seqdiag==2.0.0
+sphinxcontrib-serializinghtml==1.1.5
+toml==0.10.2
+tomli==1.2.1
+tornado==4.5.3
+traitlets==5.1.0
+types-docutils==0.17.0
+types-futures==0.1.6
+types-protobuf==3.17.4
+types-Pygments==2.9.2
+types-setuptools==57.0.2
+types-six==1.16.1
+typing-extensions==3.10.0.2
+urllib3==1.26.6
+watchdog==2.1.5
+wcwidth==0.2.5
+webcolors==1.11.1
+wrapt==1.12.1
+yapf==0.31.0
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
index af34ab9..96136bd 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/install.py
@@ -128,7 +128,7 @@
shutil.rmtree(venv_path)
-def install(
+def install( # pylint: disable=too-many-arguments
project_root,
venv_path,
full_envsetup=True,
@@ -139,6 +139,7 @@
python=sys.executable,
env=None,
system_packages=False,
+ use_pinned_pip_packages=True,
):
"""Creates a venv and installs all packages in this Git repo."""
@@ -249,6 +250,9 @@
gn_cmd = ['gn', 'gen', build_dir]
args = list(gn_args)
+ if not use_pinned_pip_packages:
+ args.append('pw_build_PIP_CONSTRAINTS=[]')
+
args.append('dir_pigweed="{}"'.format(pw_root))
gn_cmd.append('--args={}'.format(' '.join(args)))
diff --git a/pw_env_setup/py/python_packages_test.py b/pw_env_setup/py/python_packages_test.py
new file mode 100755
index 0000000..27720c6
--- /dev/null
+++ b/pw_env_setup/py/python_packages_test.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests the python_packages module."""
+
+import collections
+import io
+import unittest
+from unittest import mock
+
+from pw_env_setup import python_packages
+
+
+def _subprocess_run_stdout(stdout=b'foo==1.0\nbar==2.0\npw-foo @ file:...\n'):
+ def subprocess_run(*unused_args, **unused_kwargs):
+ CompletedProcess = collections.namedtuple('CompletedProcess', 'stdout')
+ return CompletedProcess(stdout=stdout)
+
+ return subprocess_run
+
+
+class TestPythonPackages(unittest.TestCase):
+ """Tests the python_packages module."""
+ @mock.patch('pw_env_setup.python_packages.subprocess.run',
+ side_effect=_subprocess_run_stdout())
+ def test_list(self, unused_mock):
+ buf = io.StringIO()
+ python_packages.ls(buf)
+ self.assertIn('foo==1.0', buf.getvalue())
+ self.assertIn('bar==2.0', buf.getvalue())
+ self.assertNotIn('pw-foo', buf.getvalue())
+
+ @mock.patch('pw_env_setup.python_packages.subprocess.run',
+ side_effect=_subprocess_run_stdout())
+ @mock.patch('pw_env_setup.python_packages._stderr')
+ def test_diff_removed(self, stderr_mock, unused_mock):
+ expected = io.StringIO('foo==1.0\nbar==2.0\nbaz==3.0\n')
+ expected.name = 'test.name'
+ self.assertFalse(python_packages.diff(expected))
+
+ stderr_mock.assert_any_call('Removed packages')
+ stderr_mock.assert_any_call(' baz==3.0')
+
+ @mock.patch('pw_env_setup.python_packages.subprocess.run',
+ side_effect=_subprocess_run_stdout())
+ @mock.patch('pw_env_setup.python_packages._stderr')
+ def test_diff_updated(self, stderr_mock, unused_mock):
+ expected = io.StringIO('foo==1.0\nbar==1.9\n')
+ expected.name = 'test.name'
+ self.assertTrue(python_packages.diff(expected))
+
+ stderr_mock.assert_any_call('Updated packages')
+ stderr_mock.assert_any_call(' bar==2.0 (from 1.9)')
+ stderr_mock.assert_any_call("Package versions don't match!")
+
+ @mock.patch('pw_env_setup.python_packages.subprocess.run',
+ side_effect=_subprocess_run_stdout())
+ @mock.patch('pw_env_setup.python_packages._stderr')
+ def test_diff_new(self, stderr_mock, unused_mock):
+ expected = io.StringIO('foo==1.0\n')
+ expected.name = 'test.name'
+ self.assertTrue(python_packages.diff(expected))
+
+ stderr_mock.assert_any_call('New packages')
+ stderr_mock.assert_any_call(' bar==2.0')
+ stderr_mock.assert_any_call("Package versions don't match!")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index a20b8ab..3defaf6 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -526,6 +526,7 @@
# Configuration
r'^(?:.+/)?\..+$',
r'\bPW_PLUGINS$',
+ r'\bconstraint.list$',
# Metadata
r'^docker/tag$',
r'\bAUTHORS$',
