commit 2dd4dd8517d346457bbd157fca234987b4bc7107
author Wyatt Hepler <hepler@google.com> Wed Jun 24 15:43:38 2020 -0700
committer CQ Bot Account <commit-bot@chromium.org> Thu Jun 25 16:03:29 2020 +0000
tree b1e2be0ff69f563547bb01e184126492888041b0
parent a1c4f670b29b5683733ca3710bd47adb8a714511

pw_presubmit: Avoid duplicate module names in mypy

mypy sometimes complains if multiple modules have the same name, even if
they're in different packages. This change groups Python files so that
each group only contains each module name once.

Change-Id: I81e57ed251c4831db2db7cab11bc074d73ece923
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/12942
Reviewed-by: Rob Mohr <mohrr@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>

pw_presubmit/py/pw_presubmit/python_checks.py

diff --git a/pw_presubmit/py/pw_presubmit/python_checks.py b/pw_presubmit/py/pw_presubmit/python_checks.py
index 2fbb4a0..22083f2 100644
--- a/pw_presubmit/py/pw_presubmit/python_checks.py
+++ b/pw_presubmit/py/pw_presubmit/python_checks.py
@@ -19,9 +19,8 @@
 import logging
 import os
 from pathlib import Path
-import re
 import sys
-from typing import Callable, Iterable, List, Tuple
+from typing import Callable, Iterable, List, Set, Tuple
 
 try:
     import pw_presubmit
@@ -32,7 +31,8 @@
         os.path.abspath(__file__))))
     import pw_presubmit
 
-from pw_presubmit import call, filter_paths, git_repo
+from pw_presubmit import call, filter_paths
+from pw_presubmit.git_repo import find_python_packages, list_files
 
 _LOG = logging.getLogger(__name__)
 
@@ -54,15 +54,14 @@
 
     packages: List[Path] = []
     for repo in ctx.repos:
-        packages += git_repo.find_python_packages(ctx.paths, repo=repo)
+        packages += find_python_packages(ctx.paths, repo=repo)
 
     if not packages:
         _LOG.info('No Python packages were found.')
         return
 
     for package in packages:
-        for test in git_repo.list_files(pathspecs=test_globs,
-                                        repo_path=package):
+        for test in list_files(pathspecs=test_globs, repo_path=package):
             call('python', test)
 
 
@@ -86,26 +85,42 @@
     )
 
 
-_SETUP_PY = re.compile(r'(?:.+/)?setup\.py')
-
-
 @filter_paths(endswith='.py')
 def mypy(ctx: pw_presubmit.PresubmitContext) -> None:
+    # Under some circumstances, mypy cannot check multiple Python files with the
+    # same module name. Group filenames so that no duplicates occur in the same
+    # mypy invocation. Also, omit setup.py from mypy checks.
+    filename_sets: List[Set[str]] = [set()]
+    path_sets: List[List[Path]] = [[]]
+
+    duplicates_ok = '__init__.py', '__main__.py'
+
+    for path in (p for p in ctx.paths if p.name != 'setup.py'):
+        for filenames, paths in zip(filename_sets, path_sets):
+            if path.name in duplicates_ok or path.name not in filenames:
+                paths.append(path)
+                filenames.add(path.name)
+                break
+        else:
+            path_sets.append([path])
+            filename_sets.append({path.name})
+
     env = os.environ.copy()
     # Use this environment variable to force mypy to colorize output.
     # See https://github.com/python/mypy/issues/7771
     env['MYPY_FORCE_COLOR'] = '1'
 
-    run_module(
-        'mypy',
-        *(p for p in ctx.paths if not _SETUP_PY.fullmatch(p.as_posix())),
-        '--pretty',
-        '--color-output',
-        '--show-error-codes',
-        # TODO(pwbug/146): Some imports from installed packages fail. These
-        # imports should be fixed and this option removed.
-        '--ignore-missing-imports',
-        env=env)
+    for paths in path_sets:
+        run_module(
+            'mypy',
+            *paths,
+            '--pretty',
+            '--color-output',
+            '--show-error-codes',
+            # TODO(pwbug/146): Some imports from installed packages fail. These
+            # imports should be fixed and this option removed.
+            '--ignore-missing-imports',
+            env=env)
 
 
 _ALL_CHECKS = (