blob: 49376fdcd6d406693a6cc766ef9c1e49134e83da [file] [log] [blame]
# Copyright 2021 The Pigweed Authors
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Dataclass for a Python package."""
import configparser
from contextlib import contextmanager
import copy
from dataclasses import dataclass, asdict
import io
import json
import os
from pathlib import Path
import pprint
import re
import shutil
from typing import Any, Dict, List, Optional, Iterable
_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat
# List of known environment markers supported by pip.
def change_working_dir(directory: Path):
original_dir = Path.cwd()
yield directory
class UnknownPythonPackageName(Exception):
"""Exception thrown when a Python package_name cannot be determined."""
class MissingSetupSources(Exception):
"""Exception thrown when a Python package is missing setup source files.
For example: setup.cfg and pyproject.toml.i
def _sanitize_install_requires(metadata_dict: dict) -> dict:
"""Convert install_requires lists into strings joined with line breaks."""
install_requires = metadata_dict['options']['install_requires']
if isinstance(install_requires, list):
metadata_dict['options']['install_requires'] = (
except KeyError:
return metadata_dict
class PythonPackage:
"""Class to hold a single Python package's metadata."""
sources: List[Path]
setup_sources: List[Path]
tests: List[Path]
inputs: List[Path]
gn_target_name: str = ''
generate_setup: Optional[Dict] = None
config: Optional[configparser.ConfigParser] = None
def from_dict(**kwargs) -> 'PythonPackage':
"""Build a PythonPackage instance from a dictionary."""
transformed_kwargs = copy.copy(kwargs)
# Transform string filenames to Paths
for attribute in ['sources', 'tests', 'inputs', 'setup_sources']:
transformed_kwargs[attribute] = [
Path(s) for s in kwargs[attribute]
return PythonPackage(**transformed_kwargs)
def __post_init__(self):
# Read the setup.cfg file if possible
if not self.config:
self.config = self._load_config()
def setup_dir(self) -> Optional[Path]:
if not self.setup_sources:
return None
# Assuming all setup_source files live in the same parent directory.
return self.setup_sources[0].parent
def setup_py(self) -> Path:
setup_py = [
setup_file for setup_file in self.setup_sources
if str(setup_file).endswith('')
# will not exist for GN generated Python packages
assert len(setup_py) == 1
return setup_py[0]
def setup_cfg(self) -> Optional[Path]:
setup_cfg = [
setup_file for setup_file in self.setup_sources
if str(setup_file).endswith('setup.cfg')
if len(setup_cfg) < 1:
return None
return setup_cfg[0]
def as_dict(self) -> Dict[Any, Any]:
"""Return a dict representation of this class."""
self_dict = asdict(self)
if self.config:
# Expand self.config into text.
setup_cfg_text = io.StringIO()
self_dict['config'] = setup_cfg_text.getvalue()
return self_dict
def package_name(self) -> str:
unknown_package_message = (
'Cannot determine the package_name for the Python '
f'library/package: {self.gn_target_name}\n\n'
'This could be due to a missing python dependency in GN for:\n'
if self.config:
name = self.config['metadata']['name']
except KeyError:
raise UnknownPythonPackageName(unknown_package_message +
return name
top_level_source_dir = self.top_level_source_dir
if top_level_source_dir:
actual_gn_target_name = self.gn_target_name.split(':')
if len(actual_gn_target_name) < 2:
raise UnknownPythonPackageName(unknown_package_message)
return actual_gn_target_name[-1]
def package_dir(self) -> Path:
if self.setup_cfg and self.setup_cfg.is_file():
return self.setup_cfg.parent / self.package_name
root_source_dir = self.top_level_source_dir
if root_source_dir:
return root_source_dir
return self.sources[0].parent
def top_level_source_dir(self) -> Optional[Path]:
source_dir_paths = sorted(set(
(len(, sfile.parent) for sfile in self.sources),
key=lambda s: s[1])
if not source_dir_paths:
return None
top_level_source_dir = source_dir_paths[0][1]
if not top_level_source_dir.is_dir():
return None
return top_level_source_dir
def _load_config(self) -> Optional[configparser.ConfigParser]:
config = configparser.ConfigParser()
# Check for a setup.cfg and load that config.
if self.setup_cfg:
if self.setup_cfg.is_file():
with as config_file:
return config
if self.setup_cfg.with_suffix('.json').is_file():
return self._load_setup_json_config()
# Fallback on the generate_setup scope from GN
if self.generate_setup:
return config
return None
def _load_setup_json_config(self) -> configparser.ConfigParser:
assert self.setup_cfg
setup_json = self.setup_cfg.with_suffix('.json')
config = configparser.ConfigParser()
with as json_fp:
json_dict = _sanitize_install_requires(json.load(json_fp))
return config
def copy_sources_to(self, destination: Path) -> None:
"""Copy this PythonPackage source files to another path."""
new_destination = destination /
new_destination.mkdir(parents=True, exist_ok=True)
shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True)
def install_requires_entries(self) -> List[str]:
"""Convert the install_requires entry into a list of strings."""
this_requires: List[str] = []
# If there's no setup.cfg, do nothing.
if not self.config:
return this_requires
# Requires are delimited by newlines or semicolons.
# Split existing list on either one.
for req in re.split(r' *[\n;] *',
# Skip empty lines.
if not req:
# Get the name part part of the dep, ignoring any spaces or
# other characters.
req_name_match = re.match(r'^(?P<name_part>[A-Za-z0-9_-]+)', req)
if not req_name_match:
req_name = req_name_match.groupdict().get('name_part', '')
# Check if this is an environment marker.
# Append this req as an environment marker for the previous
# requirement.
this_requires[-1] += f';{req}'
# Normal pip requirement, save to this_requires.
return this_requires
def load_packages(input_list_files: Iterable[Path],
ignore_missing=False) -> List[PythonPackage]:
"""Load Python package metadata and configs."""
packages = []
for input_path in input_list_files:
if ignore_missing and not input_path.is_file():
with as input_file:
# Each line contains the path to a json file.
for json_file in input_file.readlines():
# Load the json as a dict.
json_file_path = Path(json_file.strip()).resolve()
with as json_fp:
json_dict = json.load(json_fp)
return packages