blob: e6f98b941de7a7ebc401b93fff9b165e063feef9 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2021 The Pigweed Authors
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Save list of installed packages and versions."""
import argparse
import itertools
import sys
from pathlib import Path
from typing import Dict, Iterator, List, Optional, Union
import pkg_resources
def _installed_packages() -> Iterator[str]:
"""Run pip python_packages and write to out."""
installed_packages = list(
pkg.as_requirement() for pkg in pkg_resources.working_set # pylint: disable=not-an-iterable
# Non-editable packages only
if isinstance(
pkg_resources.DistInfoDistribution # type: ignore
# This will skip packages with local versions.
# For example text after a plus sign: 1.2.3+dev456
and not pkg.parsed_version.local
# These are always installed by default in:
# pw_env_setup/py/pw_env_setup/virtualenv_setup/
and pkg.key not in ['pip', 'setuptools', 'wheel'])
for req in sorted(
key=lambda pkg: # type: ignore
yield str(req)
def ls(output_file: Optional[Path]) -> int: # pylint: disable=invalid-name
"""Run pip python_packages and write to output_file."""
actual_requirements = frozenset(
for line in _installed_packages())
missing_requirements = set()
# If updating an existing file, load the existing requirements to find lines
# that are missing in the active environment.
if output_file:
existing_lines = output_file.read_text().splitlines()
expected_requirements = set(
pkg_resources.Requirement.parse(line) for line in existing_lines)
missing_requirements = expected_requirements - actual_requirements
new_requirements: List[pkg_resources.Requirement] = list(
for requirement in missing_requirements:
# Preserve this requirement if it has a marker that doesn't apply to
# the current environment. For example a line with ;python_version <
# "3.9" if running on Python 3.9 or higher will be saved.
if requirement.marker and not requirement.marker.evaluate():
# If this package is in the active environment then it has a marker
# in the existing file that should be preserved.
found_package = pkg_resources.working_set.find(requirement)
# If the package version doesn't match, save the new version.
except pkg_resources.VersionConflict:
found_package = None
if found_package:
# Delete the old line with no marker.
# Add the existing requirement line that includes the marker.
out ='w') if output_file else sys.stdout
for package in sorted(
key=lambda pkg: # type: ignore
print(package, file=out)
if output_file:
return 0
class UpdateRequiredError(Exception):
def _stderr(*args, **kwargs):
return print(*args, file=sys.stderr, **kwargs)
def _load_requirements_lines(*req_files: Path) -> Iterator[str]:
for req_file in req_files:
for line in req_file.read_text().splitlines():
# Ignore comments and blank lines
if line.startswith('#') or line == '':
yield line
def diff(expected: Path,
ignore_requirements_file: Optional[List[Path]] = None) -> int:
"""Report on differences between installed and expected versions."""
actual_lines = set(_installed_packages())
expected_lines = set(_load_requirements_lines(expected))
ignored_lines = set()
if ignore_requirements_file:
ignored_lines = set(
if actual_lines == expected_lines:
_stderr('package versions are identical')
return 0
actual_requirements = frozenset(
pkg_resources.Requirement.parse(line) for line in actual_lines)
expected_requirements = frozenset(
pkg_resources.Requirement.parse(line) for line in expected_lines)
ignored_requirements = frozenset(
pkg_resources.Requirement.parse(line) for line in ignored_lines)
removed_requirements = expected_requirements - actual_requirements
added_requirements = actual_requirements - expected_requirements
removed_packages: Dict[pkg_resources.Requirement, str] = {}
updated_packages: Dict[pkg_resources.Requirement, str] = {}
new_packages: Dict[pkg_resources.Requirement, str] = {}
reformatted_packages: Dict[pkg_resources.Requirement, str] = {}
for line in expected_lines:
requirement = pkg_resources.Requirement.parse(line)
# Check for lines that need reformatting
# This will catch lines that use underscores instead of dashes in the
# name of missing spaces after specifiers.
# Match this requirement with the original one found in
# actual_requirements.
# Note the requirement variable may equal its counterpart in
# actual_requirements due to the .hashCpm on the Requirement class
# checking their normalized name. Since we are looking for formatting
# mismatches here we need to retrieve the requirement instance from the
# actual_requirements set.
matching_found_requirement = {req: req
for req in actual_requirements
# If the actual requirement line doesn't match.
if matching_found_requirement and str(
matching_found_requirement) != line:
reformatted_packages[matching_found_requirement] = line
# If this requirement isn't in the active enviroment and the line
# doesn't match the repr: flag for reformatting.
if not matching_found_requirement and str(requirement) != line:
reformatted_packages[requirement] = line
# If a requirement specifier is used and it doesn't apply, skip this
# line. See details for requirement specifiers at:
if requirement.marker and not requirement.marker.evaluate():
# Try to find this requirement in the current environment.
found_package = pkg_resources.working_set.find(requirement)
# If the package version doesn't match, save the new version.
except pkg_resources.VersionConflict as err:
found_package = None
if err.dist:
found_package = err.dist
# If this requirement isn't in the environment, it was removed.
if not found_package:
removed_packages[requirement] = line
# If found_package is set, the version doesn't match so it was updated.
if requirement.specs != found_package.as_requirement().specs:
updated_packages[found_package.as_requirement()] = line
ignored_distributions = list(
distribution for distribution in pkg_resources.working_set # pylint: disable=not-an-iterable
if distribution.as_requirement() in ignored_requirements)
expected_distributions = list(
distribution for distribution in pkg_resources.working_set # pylint: disable=not-an-iterable
if distribution.as_requirement() in expected_requirements)
def get_requirements(
dist_info: pkg_resources.Distribution
) -> Iterator[pkg_resources.Distribution]:
"""Return requirement that are not in expected_distributions."""
for req in dist_info.requires():
req_dist_info = pkg_resources.working_set.find(req)
if not req_dist_info:
if req_dist_info in expected_distributions:
yield req_dist_info
def expand_requirements(
reqs: List[pkg_resources.Distribution]
) -> Iterator[List[pkg_resources.Distribution]]:
"""Recursively expand requirements."""
for dist_info in reqs:
deps = list(get_requirements(dist_info))
if deps:
yield deps
yield from expand_requirements(deps)
ignored_transitive_deps = set(
# Check for new packages
for requirement in added_requirements - removed_requirements:
if requirement in updated_packages:
if requirement in ignored_requirements or ignored_transitive_deps:
new_packages[requirement] = str(requirement)
# Print status messages to stderr
if reformatted_packages:
_stderr('Requirements that need reformatting:')
for requirement, line in reformatted_packages.items():
_stderr(f' {line}')
_stderr(' should be:')
_stderr(f' {str(requirement)}')
if updated_packages:
_stderr('Updated packages')
for requirement, line in updated_packages.items():
_stderr(f' {str(requirement)} (from {line})')
if removed_packages:
_stderr('Removed packages')
for requirement in removed_packages:
_stderr(f' {requirement}')
if new_packages:
_stderr('New packages')
for requirement in new_packages:
_stderr(f' {requirement}')
if updated_packages or new_packages:
_stderr("Package versions don't match!")
Please do the following:
* purge your environment directory
* Linux/Mac: 'rm -rf "$_PW_ACTUAL_ENVIRONMENT_ROOT"'
* Windows: 'rmdir /S %_PW_ACTUAL_ENVIRONMENT_ROOT%'
* bootstrap
* Linux/Mac: '. ./'
* Windows: 'bootstrap.bat'
* update the constraint file
* 'pw python-packages list {}'
return -1
return 0
def parse(argv: Union[List[str], None] = None) -> argparse.Namespace:
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(
prog="python -m pw_env_setup.python_packages")
subparsers = parser.add_subparsers(dest='cmd')
list_parser = subparsers.add_parser(
'list', aliases=('ls', ), help='List installed package versions.')
list_parser.add_argument('output_file', type=Path, nargs='?')
diff_parser = subparsers.add_parser(
help='Show differences between expected and actual package versions.',
diff_parser.add_argument('expected', type=Path)
return parser.parse_args(argv)
def main() -> int:
args = vars(parse())
cmd = args.pop('cmd')
if cmd == 'diff':
return diff(**args)
if cmd == 'list':
return ls(**args)
return -1
if __name__ == '__main__':