pw_protobuf_compiler: Protobuf type annotations
- Use the mypy-protobufs protoc plugin to generate type annotations for
Python protobufs.
- Update Python proto package generation to include py.typed files.
Change-Id: I75e658d38b56853135005af6f35624de5df93a7e
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/28960
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Reviewed-by: Alexei Frolov <frolv@google.com>
Reviewed-by: Rob Mohr <mohrr@google.com>
diff --git a/pw_protobuf_compiler/BUILD.gn b/pw_protobuf_compiler/BUILD.gn
index 58dce83..0d678a0 100644
--- a/pw_protobuf_compiler/BUILD.gn
+++ b/pw_protobuf_compiler/BUILD.gn
@@ -14,7 +14,7 @@
import("//build_overrides/pigweed.gni")
-import("$dir_pw_build/input_group.gni")
+import("$dir_pw_build/python.gni")
import("$dir_pw_docgen/docs.gni")
import("$dir_pw_protobuf_compiler/proto.gni")
import("$dir_pw_unit_test/test.gni")
@@ -34,5 +34,17 @@
}
pw_proto_library("nanopb_test_protos") {
- sources = [ "pw_protobuf_compiler_protos/nanopb_test.proto" ]
+ sources = [ "pw_protobuf_compiler_nanopb_protos/nanopb_test.proto" ]
+}
+
+pw_proto_library("test_protos") {
+ sources = [
+ "pw_protobuf_compiler_protos/nested/more_nesting/test.proto",
+ "pw_protobuf_compiler_protos/test.proto",
+ ]
+}
+
+# PyPI Requirements needed to install Python protobuf packages.
+pw_python_requirements("protobuf_requirements") {
+ requirements = [ "mypy-protobuf" ]
}
diff --git a/pw_protobuf_compiler/nanopb_test.cc b/pw_protobuf_compiler/nanopb_test.cc
index 4df3520..e916b49 100644
--- a/pw_protobuf_compiler/nanopb_test.cc
+++ b/pw_protobuf_compiler/nanopb_test.cc
@@ -13,7 +13,7 @@
// the License.
#include "gtest/gtest.h"
-#include "pw_protobuf_compiler_protos/nanopb_test.pb.h"
+#include "pw_protobuf_compiler_nanopb_protos/nanopb_test.pb.h"
TEST(Nanopb, CompilesProtobufs) {
pw_protobuf_compiler_Point point = {4, 8, "point"};
diff --git a/pw_protobuf_compiler/proto.gni b/pw_protobuf_compiler/proto.gni
index 94d0c6c..03ff3c4 100644
--- a/pw_protobuf_compiler/proto.gni
+++ b/pw_protobuf_compiler/proto.gni
@@ -254,9 +254,11 @@
forward_variables_from(invoker, "*", _forwarded_vars)
language = "python"
output_extensions = [ "_pb2.py" ]
+ deps += [ "$dir_pw_protobuf_compiler:protobuf_requirements.install" ]
}
_setup_py = "${invoker.gen_dir}/setup.py"
+ _generated_files = get_target_outputs(":$target_name._gen")
# Create the setup and init files for the Python package.
pw_python_action(target_name + "._package_gen") {
@@ -266,7 +268,8 @@
rebase_path(_setup_py),
"--package",
_package_dir,
- ] + rebase_path(get_path_info(invoker.sources, "dir"), ".")
+ ] + rebase_path(_generated_files, invoker.gen_dir)
+
public_deps = [ ":$_target._gen" ]
stamp = true
}
diff --git a/pw_protobuf_compiler/pw_protobuf_compiler_protos/nanopb_test.proto b/pw_protobuf_compiler/pw_protobuf_compiler_nanopb_protos/nanopb_test.proto
similarity index 100%
rename from pw_protobuf_compiler/pw_protobuf_compiler_protos/nanopb_test.proto
rename to pw_protobuf_compiler/pw_protobuf_compiler_nanopb_protos/nanopb_test.proto
diff --git a/pw_protobuf_compiler/pw_protobuf_compiler_protos/nested/more_nesting/test.proto b/pw_protobuf_compiler/pw_protobuf_compiler_protos/nested/more_nesting/test.proto
new file mode 100644
index 0000000..6bb1417
--- /dev/null
+++ b/pw_protobuf_compiler/pw_protobuf_compiler_protos/nested/more_nesting/test.proto
@@ -0,0 +1,21 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+syntax = "proto3";
+
+package pw.protobuf_compiler.test;
+
+message Message {
+ int32 field = 1;
+}
diff --git a/pw_protobuf_compiler/pw_protobuf_compiler_protos/test.proto b/pw_protobuf_compiler/pw_protobuf_compiler_protos/test.proto
new file mode 100644
index 0000000..27bf6b4
--- /dev/null
+++ b/pw_protobuf_compiler/pw_protobuf_compiler_protos/test.proto
@@ -0,0 +1,22 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+syntax = "proto3";
+
+package pw.protobuf_compiler.test;
+
+enum Enum {
+ FOO = 0;
+ BAR = 1;
+}
diff --git a/pw_protobuf_compiler/py/BUILD.gn b/pw_protobuf_compiler/py/BUILD.gn
index 9c4187d..f2231e0 100644
--- a/pw_protobuf_compiler/py/BUILD.gn
+++ b/pw_protobuf_compiler/py/BUILD.gn
@@ -25,7 +25,11 @@
"pw_protobuf_compiler/proto_target_invalid.py",
"pw_protobuf_compiler/python_protos.py",
]
- tests = [ "python_protos_test.py" ]
+ tests = [
+ "compiled_protos_test.py",
+ "python_protos_test.py",
+ ]
python_deps = [ "$dir_pw_cli/py" ]
+ python_test_deps = [ "..:test_protos.python" ]
pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_protobuf_compiler/py/compiled_protos_test.py b/pw_protobuf_compiler/py/compiled_protos_test.py
new file mode 100755
index 0000000..88f8e62
--- /dev/null
+++ b/pw_protobuf_compiler/py/compiled_protos_test.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests compiling and importing Python protos on the fly."""
+
+import unittest
+
+from pw_protobuf_compiler_protos import test_pb2 as top_level
+from pw_protobuf_compiler_protos.nested.more_nesting import test_pb2
+
+
+class TestCompileAndImport(unittest.TestCase):
+ def test_access_compiled_protobufs(self):
+ self.assertNotEqual(top_level.FOO, top_level.BAR)
+
+ message = test_pb2.Message(field=123)
+ self.assertEqual(message.field, 123)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
index 5c07d2c..7bfd76b 100644
--- a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
+++ b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
@@ -113,7 +113,7 @@
def protoc_python_args(args: argparse.Namespace) -> List[str]:
- return ['--python_out', args.out_dir]
+ return ['--python_out', args.out_dir, '--mypy_out', args.out_dir]
# Default additional protoc arguments for each supported language.
diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py
index b3e1b12..4c3e865 100644
--- a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py
+++ b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py
@@ -14,9 +14,10 @@
"""Generates a setup.py and __init__.py for a Python package."""
import argparse
+from collections import defaultdict
from pathlib import Path
import sys
-from typing import List
+from typing import Dict, List, Set
# Make sure dependencies are optional, since this script may be run when
# installing Python package dependencies through GN.
@@ -29,13 +30,16 @@
import setuptools
setuptools.setup(
- name='<PACKAGE_NAME>',
+ name={name!r},
version='0.0.1',
author='Pigweed Authors',
author_email='pigweed-developers@googlegroups.com',
description='Generated protobuf files',
- packages=setuptools.find_packages(),
+ packages={packages!r},
+ package_data={package_data!r},
+ include_package_data=True,
zip_safe=False,
+ install_requires=['protobuf'],
)
"""
@@ -50,23 +54,47 @@
required=True,
type=Path,
help='Path to setup.py file')
- parser.add_argument('subpackages',
+ parser.add_argument('sources',
type=Path,
nargs='+',
- help='Subpackage paths within the package')
-
+ help='Relative paths to sources in the package')
return parser.parse_args()
-def main(package: str, setup: Path, subpackages: List[Path]) -> int:
- setup.parent.mkdir(exist_ok=True)
+def main(package: str, setup: Path, sources: List[Path]) -> int:
+ """Generates __init__.py and py.typed files and a setup.py."""
+ base = setup.parent.resolve()
+ base.mkdir(exist_ok=True)
- for subpackage in set(subpackages):
- package_dir = setup.parent / subpackage
- package_dir.mkdir(exist_ok=True, parents=True)
- package_dir.joinpath('__init__.py').touch()
+ # Find all directories in the package, including empty ones.
+ subpackages: Set[Path] = set()
+ for source in sources:
+ subpackages.update(base / path for path in source.parents)
+ subpackages.remove(base)
- setup.write_text(_SETUP_TEMPLATE.replace('<PACKAGE_NAME>', package))
+ pkg_data: Dict[str, List[str]] = defaultdict(list)
+
+ # Create __init__.py and py.typed files for each subdirectory.
+ for pkg in subpackages:
+ pkg.mkdir(exist_ok=True, parents=True)
+ pkg.joinpath('__init__.py').touch()
+
+ package_name = '.'.join(pkg.relative_to(base).as_posix().split('/'))
+ pkg.joinpath('py.typed').touch()
+ pkg_data[package_name].append('py.typed')
+
+ # Add the .pyi for each source file.
+ for source in sources:
+ pkg = base / source.parent
+ package_name = '.'.join(pkg.relative_to(base).as_posix().split('/'))
+
+ path = base.joinpath(source).relative_to(pkg).with_suffix('.pyi')
+ pkg_data[package_name].append(str(path))
+
+ setup.write_text(
+ _SETUP_TEMPLATE.format(name=package,
+ packages=list(pkg_data),
+ package_data=dict(pkg_data)))
return 0
diff --git a/pw_protobuf_compiler/py/setup.py b/pw_protobuf_compiler/py/setup.py
index 189992f..b0d8657 100644
--- a/pw_protobuf_compiler/py/setup.py
+++ b/pw_protobuf_compiler/py/setup.py
@@ -24,11 +24,8 @@
packages=setuptools.find_packages(),
package_data={'pw_protobuf_compiler': ['py.typed']},
zip_safe=False,
- entry_points={
- 'console_scripts':
- ['generate_protos = pw_protobuf_compiler.generate_protos:main']
- },
install_requires=[
+ 'mypy-protobuf',
'protobuf',
'pw_cli',
],
diff --git a/pw_unit_test/py/pw_unit_test/rpc.py b/pw_unit_test/py/pw_unit_test/rpc.py
index 1d1cb5f..c2ec22b 100644
--- a/pw_unit_test/py/pw_unit_test/rpc.py
+++ b/pw_unit_test/py/pw_unit_test/rpc.py
@@ -18,9 +18,8 @@
import logging
from typing import Iterable
-from pw_unit_test_proto import unit_test_pb2 # type: ignore
-
import pw_rpc.client
+from pw_unit_test_proto import unit_test_pb2
_LOG = logging.getLogger(__name__)
@@ -66,8 +65,7 @@
"""Called when a new test case is started."""
@abc.abstractmethod
- def test_case_end(self, test_case: TestCase,
- result: unit_test_pb2.TestCaseResult):
+ def test_case_end(self, test_case: TestCase, result: int):
"""Called when a test case completes with its overall result."""
@abc.abstractmethod
@@ -94,8 +92,7 @@
def test_case_start(self, test_case: TestCase):
_LOG.info('[ RUN ] %s', test_case)
- def test_case_end(self, test_case: TestCase,
- result: unit_test_pb2.TestCaseResult):
+ def test_case_end(self, test_case: TestCase, result: int):
if result == unit_test_pb2.TestCaseResult.SUCCESS:
_LOG.info('[ OK ] %s', test_case)
else:
diff --git a/pw_unit_test/py/setup.py b/pw_unit_test/py/setup.py
index 0a7db0b..7ae8df1 100644
--- a/pw_unit_test/py/setup.py
+++ b/pw_unit_test/py/setup.py
@@ -27,5 +27,6 @@
install_requires=[
'pw_cli',
'pw_rpc',
+ 'pw_unit_test_proto',
],
)
