pw_protobuf_compiler: Support proto3 optional fields
proto3 now allows specifying optional fields. These fields behave
equivalently to proto2 optional fields and allow API users to detect
whether the field is present in the proto.
- Have the pw_rpc protoc plugins indicate proto3 optional field support.
- Require mypy-protobuf 1.24 or later for proto3 optional field support.
- Provide the flag for proto3 optional fields to protoc.
Change-Id: I66ba3a280cfc9b5c4d1ff42109a6e1d3d691b271
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/31320
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Alexei Frolov <frolv@google.com>
diff --git a/pw_build/python.gni b/pw_build/python.gni
index 5bb412c..b94a526 100644
--- a/pw_build/python.gni
+++ b/pw_build/python.gni
@@ -243,7 +243,7 @@
if (_should_lint || _test_sources != []) {
# Packages that must be installed to use the package or run its tests.
- _test_install_deps = []
+ _test_install_deps = [ ":$_internal_target.install" ]
foreach(dep, _python_test_deps) {
_test_install_deps += [ "$dep.install" ]
}
diff --git a/pw_protobuf_compiler/BUILD.gn b/pw_protobuf_compiler/BUILD.gn
index 0d678a0..f34438f 100644
--- a/pw_protobuf_compiler/BUILD.gn
+++ b/pw_protobuf_compiler/BUILD.gn
@@ -46,5 +46,6 @@
# PyPI Requirements needed to install Python protobuf packages.
pw_python_requirements("protobuf_requirements") {
- requirements = [ "mypy-protobuf" ]
+ # mypy-protobuf 1.24 is required to support optional fields in proto3.
+ requirements = [ "mypy-protobuf>=1.24" ]
}
diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
index 7bfd76b..3d2932b 100644
--- a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
+++ b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py
@@ -21,7 +21,7 @@
import sys
import tempfile
-from typing import Callable, Dict, List, Optional
+from typing import Callable, Dict, Optional, Tuple
# Make sure dependencies are optional, since this script may be run when
# installing Python package dependencies through GN.
@@ -32,6 +32,8 @@
_LOG = logging.getLogger(__name__)
+_COMMON_FLAGS = ('--experimental_allow_proto3_optional', )
+
def argument_parser(
parser: Optional[argparse.ArgumentParser] = None
@@ -69,56 +71,66 @@
return parser
-def protoc_cc_args(args: argparse.Namespace) -> List[str]:
- return [
+def protoc_cc_args(args: argparse.Namespace) -> Tuple[str, ...]:
+ return _COMMON_FLAGS + (
'--plugin',
f'protoc-gen-custom={args.plugin_path}',
'--custom_out',
args.out_dir,
- ]
+ )
-def protoc_go_args(args: argparse.Namespace) -> List[str]:
- return ['--go_out', f'plugins=grpc:{args.out_dir}']
+def protoc_go_args(args: argparse.Namespace) -> Tuple[str, ...]:
+ return _COMMON_FLAGS + (
+ '--go_out',
+ f'plugins=grpc:{args.out_dir}',
+ )
-def protoc_nanopb_args(args: argparse.Namespace) -> List[str]:
+def protoc_nanopb_args(args: argparse.Namespace) -> Tuple[str, ...]:
# nanopb needs to know of the include path to parse *.options files
- return [
+ return _COMMON_FLAGS + (
'--plugin',
f'protoc-gen-nanopb={args.plugin_path}',
# nanopb_opt provides the flags to use for nanopb_out. Windows doesn't
# like when you merge the two using the `flag,...:out` syntax.
f'--nanopb_opt=-I{args.module_path}',
f'--nanopb_out={args.out_dir}',
- ]
+ )
-def protoc_nanopb_rpc_args(args: argparse.Namespace) -> List[str]:
- return [
+def protoc_nanopb_rpc_args(args: argparse.Namespace) -> Tuple[str, ...]:
+ return _COMMON_FLAGS + (
'--plugin',
f'protoc-gen-custom={args.plugin_path}',
'--custom_out',
args.out_dir,
- ]
+ )
-def protoc_raw_rpc_args(args: argparse.Namespace) -> List[str]:
- return [
+def protoc_raw_rpc_args(args: argparse.Namespace) -> Tuple[str, ...]:
+ return _COMMON_FLAGS + (
'--plugin',
f'protoc-gen-custom={args.plugin_path}',
'--custom_out',
args.out_dir,
- ]
+ )
-def protoc_python_args(args: argparse.Namespace) -> List[str]:
- return ['--python_out', args.out_dir, '--mypy_out', args.out_dir]
+def protoc_python_args(args: argparse.Namespace) -> Tuple[str, ...]:
+ return _COMMON_FLAGS + (
+ '--python_out',
+ args.out_dir,
+ '--mypy_out',
+ args.out_dir,
+ )
+_DefaultArgsFunction = Callable[[argparse.Namespace], Tuple[str, ...]]
+
# Default additional protoc arguments for each supported language.
# TODO(frolv): Make these overridable with a command-line argument.
-DEFAULT_PROTOC_ARGS: Dict[str, Callable[[argparse.Namespace], List[str]]] = {
+DEFAULT_PROTOC_ARGS: Dict[str, _DefaultArgsFunction] = {
'pwpb': protoc_cc_args,
'go': protoc_go_args,
'nanopb': protoc_nanopb_args,
diff --git a/pw_rpc/py/pw_rpc/plugin.py b/pw_rpc/py/pw_rpc/plugin.py
index 595b436..44337d9 100644
--- a/pw_rpc/py/pw_rpc/plugin.py
+++ b/pw_rpc/py/pw_rpc/plugin.py
@@ -63,5 +63,11 @@
request = plugin_pb2.CodeGeneratorRequest.FromString(data)
response = plugin_pb2.CodeGeneratorResponse()
process_proto_request(codegen, request, response)
+
+ # Declare that this plugin supports optional fields in proto3. No proto
+ # message code is generated, so optional in proto3 is supported trivially.
+ response.supported_features |= ( # type: ignore[attr-defined]
+ response.FEATURE_PROTO3_OPTIONAL) # type: ignore[attr-defined]
+
sys.stdout.buffer.write(response.SerializeToString())
return 0
