pw_build/py/pw_build/generate_python_package.py

# Copyright 2021 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""Generates a setup.cfg file for a Python package."""

import argparse
from collections import defaultdict
import configparser
from dataclasses import dataclass
import json
from pathlib import Path
import sys
import textwrap
from typing import (
    Any,
    Dict,
    Iterable,
    Iterator,
    List,
    Optional,
    Sequence,
    Set,
    TextIO,
)

try:
    from pw_build.mirror_tree import mirror_paths
except ImportError:
    # Append this path to the module search path to allow running this module
    # before the pw_build package is installed.
    sys.path.append(str(Path(__file__).resolve().parent.parent))
    from pw_build.mirror_tree import mirror_paths


def _parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument('--label', help='Label for this Python package')
    parser.add_argument('--proto-library',
                        dest='proto_libraries',
                        type=argparse.FileType('r'),
                        default=[],
                        action='append',
                        help='Paths')
    parser.add_argument('--generated-root',
                        required=True,
                        type=Path,
                        help='The base directory for the Python package')
    parser.add_argument('--setup-json',
                        required=True,
                        type=argparse.FileType('r'),
                        help='setup.py keywords as JSON')
    parser.add_argument('--module-as-package',
                        action='store_true',
                        help='Generate an __init__.py that imports everything')
    parser.add_argument('files',
                        type=Path,
                        nargs='+',
                        help='Relative paths to the files in the package')
    return parser.parse_args()


def _check_nested_protos(label: str, proto_info: Dict[str, Any]) -> None:
    """Checks that the proto library refers to this package."""
    python_package = proto_info['nested_in_python_package']

    if python_package != label:
        raise ValueError(
            f"{label}'s 'proto_library' is set to {proto_info['label']}, but "
            f"that target's 'python_package' is {python_package or 'not set'}. "
            f"Set {proto_info['label']}'s 'python_package' to {label}.")


@dataclass(frozen=True)
class _ProtoInfo:
    root: Path
    sources: Sequence[Path]
    deps: Sequence[str]


def _collect_all_files(
        root: Path, files: List[Path],
        paths_to_collect: Iterable[_ProtoInfo]) -> Dict[str, Set[str]]:
    """Collects files in output dir, adds to files; returns package_data."""
    root.mkdir(exist_ok=True)

    for proto_info in paths_to_collect:
        # Mirror the proto files to this package.
        files += mirror_paths(proto_info.root, proto_info.sources, root)

    # Find all subpackages, including empty ones.
    subpackages: Set[Path] = set()
    for file in (f.relative_to(root) for f in files):
        subpackages.update(root / path for path in file.parents)
    subpackages.remove(root)

    # Make sure there are __init__.py and py.typed files for each subpackage.
    for pkg in subpackages:
        pytyped = pkg / 'py.typed'
        if not pytyped.exists():
            pytyped.touch()
        files.append(pytyped)

        # Create an __init__.py file if it doesn't already exist.
        initpy = pkg / '__init__.py'
        if not initpy.exists():
            # Use pkgutil.extend_path to treat this as a namespaced package.
            # This allows imports with the same name to live in multiple
            # separate PYTHONPATH locations.
            initpy.write_text(
                'from pkgutil import extend_path  # type: ignore\n'
                '__path__ = extend_path(__path__, __name__)  # type: ignore\n')
        files.append(initpy)

    pkg_data: Dict[str, Set[str]] = defaultdict(set)

    # Add all non-source files to package data.
    for file in (f for f in files if f.suffix != '.py'):
        pkg = file.parent

        package_name = pkg.relative_to(root).as_posix().replace('/', '.')
        pkg_data[package_name].add(file.name)

    return pkg_data


PYPROJECT_FILE = '''\
# Generated file. Do not modify.
[build-system]
requires = ['setuptools', 'wheel']
build-backend = 'setuptools.build_meta'
'''


def _get_setup_keywords(pkg_data: dict, keywords: dict) -> Dict:
    """Gather all setuptools.setup() keyword args."""
    options_keywords = dict(
        packages=list(pkg_data),
        package_data={pkg: list(files)
                      for pkg, files in pkg_data.items()},
    )

    keywords['options'].update(options_keywords)
    return keywords


def _write_to_config(config: configparser.ConfigParser,
                     data: Dict,
                     section: Optional[str] = None):
    """Populate a ConfigParser instance with the contents of a dict."""
    # Add a specified section if missing.
    if section is not None and not config.has_section(section):
        config.add_section(section)

    for key, value in data.items():
        # Value is a dict so create a new subsection
        if isinstance(value, dict):
            _write_to_config(
                config,
                value,
                f'{section}.{key}' if section else key,
            )
        elif isinstance(value, list):
            if value:
                assert section is not None
                # Convert the list to an allowed str format.
                config[section][key] = '\n' + '\n'.join(value)
        else:
            assert section is not None
            # Add the value as a string. See expected types here:
            # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html#specifying-values
            config[section][key] = str(value)


def _generate_setup_cfg(
    pkg_data: dict,
    keywords: dict,
    config_file_path: Path,
) -> None:
    """Creates a setup.cfg file based on setuptools keywords."""
    setup_keywords = _get_setup_keywords(pkg_data, keywords)

    config = configparser.ConfigParser()

    _write_to_config(config, setup_keywords)

    # Write the config to a file.
    with config_file_path.open('w') as config_file:
        config.write(config_file)


def _import_module_in_package_init(all_files: List[Path]) -> None:
    """Generates an __init__.py that imports the module.

    This makes an individual module usable as a package. This is used for proto
    modules.
    """
    sources = [
        f for f in all_files if f.suffix == '.py' and f.name != '__init__.py'
    ]
    assert len(sources) == 1, (
        'Module as package expects a single .py source file')

    source, = sources
    source.parent.joinpath('__init__.py').write_text(
        f'from {source.stem}.{source.stem} import *\n')


def _load_metadata(label: str,
                   proto_libraries: Iterable[TextIO]) -> Iterator[_ProtoInfo]:
    for proto_library_file in proto_libraries:
        info = json.load(proto_library_file)
        _check_nested_protos(label, info)

        deps = []
        for dep in info['dependencies']:
            with open(dep) as file:
                deps.append(json.load(file)['package'])

        yield _ProtoInfo(Path(info['root']),
                         tuple(Path(p) for p in info['protoc_outputs']), deps)


def main(generated_root: Path, files: List[Path], module_as_package: bool,
         setup_json: TextIO, label: str,
         proto_libraries: Iterable[TextIO]) -> int:
    """Generates a setup.py and other files for a Python package."""
    proto_infos = list(_load_metadata(label, proto_libraries))
    try:
        pkg_data = _collect_all_files(generated_root, files, proto_infos)
    except ValueError as error:
        msg = '\n'.join(textwrap.wrap(str(error), 78))
        print(
            f'ERROR: Failed to generate Python package {label}:\n\n'
            f'{textwrap.indent(msg, "  ")}\n',
            file=sys.stderr)
        return 1

    with setup_json:
        setup_keywords = json.load(setup_json)
        setup_keywords.setdefault('options', {})

    setup_keywords['options'].setdefault('install_requires', [])

    if module_as_package:
        _import_module_in_package_init(files)

    # Create the pyproject.toml and setup.cfg files for this package.
    generated_root.joinpath('pyproject.toml').write_text(PYPROJECT_FILE)
    _generate_setup_cfg(pkg_data,
                        setup_keywords,
                        config_file_path=generated_root.joinpath('setup.cfg'))

    return 0


if __name__ == '__main__':
    sys.exit(main(**vars(_parse_args())))