Replace `Output` with `io::Printer` in upb's reflection code generator. Note that the code has grown a lot in terms of line count -- the original codegen was less than 200 lines, where this refactored version is more than 300 lines. But hopefully it has also gotten more readable and easier to maintain. Feedback welcome about whether it has achieved this goal. PiperOrigin-RevId: 738968810
diff --git a/upb_generator/reflection/BUILD b/upb_generator/reflection/BUILD index 192aad7..e763176 100644 --- a/upb_generator/reflection/BUILD +++ b/upb_generator/reflection/BUILD
@@ -25,16 +25,26 @@ cc_binary( name = "protoc-gen-upbdefs", - srcs = ["generator.cc"], + srcs = [ + "context.h", + "generator.cc", + "header.cc", + "header.h", + "source.cc", + "source.h", + ], copts = UPB_DEFAULT_CPPOPTS, visibility = [ "//pkg:__pkg__", ], deps = [ ":names", + "//src/google/protobuf", "//src/google/protobuf:descriptor_upb_c_proto", "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf/compiler:plugin", + "//src/google/protobuf/io", + "//src/google/protobuf/io:printer", "//upb:mem", "//upb:port", "//upb:reflection", @@ -49,7 +59,9 @@ "@abseil-cpp//absl/memory", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:cord", + "@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", ], )
diff --git a/upb_generator/reflection/context.h b/upb_generator/reflection/context.h new file mode 100644 index 0000000..5657374 --- /dev/null +++ b/upb_generator/reflection/context.h
@@ -0,0 +1,58 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_CONTEXT_H_ +#define THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_CONTEXT_H_ + +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/io/zero_copy_stream.h" + +namespace upb { +namespace generator { +namespace reflection { + +struct Options { + std::string dllexport_decl; +}; + +class Context { + public: + explicit Context(const Options& options, + google::protobuf::io::ZeroCopyOutputStream* stream) + : options_(options), printer_(stream) {} + + // A few convenience wrappers so that users can write `ctx.Emit()` instead of + // `ctx.printer().Emit()`. + void Emit(absl::Span<const google::protobuf::io::Printer::Sub> vars, + absl::string_view format, + google::protobuf::io::Printer::SourceLocation loc = + google::protobuf::io::Printer::SourceLocation::current()) { + printer_.Emit(vars, format, loc); + } + + void Emit(absl::string_view format, + google::protobuf::io::Printer::SourceLocation loc = + google::protobuf::io::Printer::SourceLocation::current()) { + printer_.Emit(format, loc); + } + + const Options& options() const { return options_; } + + private: + const Options& options_; + google::protobuf::io::Printer printer_; +}; + +} // namespace reflection +} // namespace generator +} // namespace upb + +#endif // THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_CONTEXT_H_
diff --git a/upb_generator/reflection/generator.cc b/upb_generator/reflection/generator.cc index a98eb04..dfa4cf0 100644 --- a/upb_generator/reflection/generator.cc +++ b/upb_generator/reflection/generator.cc
@@ -5,167 +5,32 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include <cstddef> #include <cstdint> #include <string> #include <vector> -#include "google/protobuf/descriptor.upb.h" #include "absl/container/flat_hash_set.h" -#include "absl/log/absl_check.h" -#include "absl/memory/memory.h" -#include "absl/strings/cord.h" -#include "absl/strings/escaping.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/plugin.h" +#include "google/protobuf/descriptor.h" #include "upb/mem/arena.hpp" #include "upb/reflection/def.hpp" -#include "upb/util/def_to_proto.h" -#include "upb_generator/common.h" -#include "upb_generator/common/names.h" #include "upb_generator/file_layout.h" -#include "upb_generator/minitable/names.h" #include "upb_generator/plugin.h" -#include "upb_generator/reflection/names.h" - -// Must be last. -#include "upb/port/def.inc" +#include "upb_generator/reflection/context.h" +#include "upb_generator/reflection/header.h" +#include "upb_generator/reflection/source.h" namespace upb { namespace generator { -namespace { - -struct Options { - std::string dllexport_decl; -}; - -std::string DefInitSymbol(upb::FileDefPtr file) { - return ReflectionFileSymbol(file.name()); -} - -static std::string DefHeaderFilename(upb::FileDefPtr file) { - return StripExtension(file.name()) + ".upbdefs.h"; -} - -static std::string DefSourceFilename(upb::FileDefPtr file) { - return StripExtension(file.name()) + ".upbdefs.c"; -} - -void GenerateMessageDefAccessor(upb::MessageDefPtr d, Output& output) { - output("UPB_INLINE const upb_MessageDef *$0(upb_DefPool *s) {\n", - ReflectionGetMessageSymbol(d.full_name())); - output(" _upb_DefPool_LoadDefInit(s, &$0);\n", DefInitSymbol(d.file())); - output(" return upb_DefPool_FindMessageByName(s, \"$0\");\n", d.full_name()); - output("}\n"); - output("\n"); -} - -void WriteDefHeader(upb::FileDefPtr file, const Options& options, - Output& output) { - output(FileWarning(file.name())); - - output( - "#ifndef $0_UPBDEFS_H_\n" - "#define $0_UPBDEFS_H_\n\n" - "#include \"upb/reflection/def.h\"\n" - "#include \"upb/reflection/internal/def_pool.h\"\n" - "\n" - "#include \"upb/port/def.inc\" // Must be last.\n" - "#ifdef __cplusplus\n" - "extern \"C\" {\n" - "#endif\n\n", - IncludeGuard(file.name())); - - output("extern$1 _upb_DefPool_Init $0;\n", DefInitSymbol(file), - PadPrefix(options.dllexport_decl)); - output("\n"); - - for (auto msg : SortedMessages(file)) { - GenerateMessageDefAccessor(msg, output); - } - - output( - "#ifdef __cplusplus\n" - "} /* extern \"C\" */\n" - "#endif\n" - "\n" - "#include \"upb/port/undef.inc\"\n" - "\n" - "#endif /* $0_UPBDEFS_H_ */\n", - IncludeGuard(file.name())); -} - -void WriteDefSource(upb::FileDefPtr file, const Options& options, - Output& output) { - output(FileWarning(file.name())); - - output("#include \"upb/reflection/def.h\"\n"); - output("#include \"$0\"\n", DefHeaderFilename(file)); - output("#include \"$0\"\n", MiniTableHeaderFilename(file.name())); - output("\n"); - - for (int i = 0; i < file.dependency_count(); i++) { - output("extern$1 _upb_DefPool_Init $0;\n", - DefInitSymbol(file.dependency(i)), - PadPrefix(options.dllexport_decl)); - } - - upb::Arena arena; - google_protobuf_FileDescriptorProto* file_proto = - upb_FileDef_ToProto(file.ptr(), arena.ptr()); - size_t serialized_size; - const char* serialized = google_protobuf_FileDescriptorProto_serialize( - file_proto, arena.ptr(), &serialized_size); - absl::string_view file_data(serialized, serialized_size); - - output("static const char descriptor[$0] = {", serialized_size); - - // C90 only guarantees that strings can be up to 509 characters, and some - // implementations have limits here (for example, MSVC only allows 64k: - // https://docs.microsoft.com/en-us/cpp/error-messages/compiler-errors-1/fatal-error-c1091. - // So we always emit an array instead of a string. - for (size_t i = 0; i < serialized_size;) { - for (size_t j = 0; j < 25 && i < serialized_size; ++i, ++j) { - output("'$0', ", absl::CEscape(file_data.substr(i, 1))); - } - output("\n"); - } - output("};\n\n"); - - output("static _upb_DefPool_Init *deps[$0] = {\n", - file.dependency_count() + 1); - for (int i = 0; i < file.dependency_count(); i++) { - output(" &$0,\n", DefInitSymbol(file.dependency(i))); - } - output(" NULL\n"); - output("};\n"); - output("\n"); - - output("_upb_DefPool_Init $0 = {\n", DefInitSymbol(file)); - output(" deps,\n"); - output(" &$0,\n", MiniTableFileVarName(file.name())); - output(" \"$0\",\n", file.name()); - output(" UPB_STRINGVIEW_INIT(descriptor, $0)\n", file_data.size()); - output("};\n"); -} +namespace reflection { void GenerateFile(upb::FileDefPtr file, const Options& options, google::protobuf::compiler::GeneratorContext* context) { - Output h_def_output; - WriteDefHeader(file, options, h_def_output); - { - auto stream = absl::WrapUnique(context->Open(DefHeaderFilename(file))); - ABSL_CHECK(stream->WriteCord(absl::Cord(h_def_output.output()))); - } - - Output c_def_output; - WriteDefSource(file, options, c_def_output); - { - auto stream = absl::WrapUnique(context->Open(DefSourceFilename(file))); - ABSL_CHECK(stream->WriteCord(absl::Cord(c_def_output.output()))); - } + GenerateReflectionHeader(file, options, context); + GenerateReflectionSource(file, options, context); } bool ParseOptions(absl::string_view parameter, Options* options, @@ -182,8 +47,6 @@ return true; } -} // namespace - class ReflectionGenerator : public google::protobuf::compiler::CodeGenerator { bool Generate(const google::protobuf::FileDescriptor* file, const std::string& parameter, @@ -225,10 +88,11 @@ } }; +} // namespace reflection } // namespace generator } // namespace upb int main(int argc, char** argv) { - upb::generator::ReflectionGenerator generator; + upb::generator::reflection::ReflectionGenerator generator; return google::protobuf::compiler::PluginMain(argc, argv, &generator); }
diff --git a/upb_generator/reflection/header.cc b/upb_generator/reflection/header.cc new file mode 100644 index 0000000..ff3bc56 --- /dev/null +++ b/upb_generator/reflection/header.cc
@@ -0,0 +1,98 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Generates foo.upbdefs.h. + +#include <string> + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/printer.h" +#include "upb/reflection/def.hpp" +#include "upb_generator/common/names.h" +#include "upb_generator/file_layout.h" +#include "upb_generator/reflection/context.h" +#include "upb_generator/reflection/names.h" + +namespace upb::generator::reflection { +namespace { + +using Sub = google::protobuf::io::Printer::Sub; + +void GenerateMessageDefAccessor(upb::MessageDefPtr d, Context& ctx) { + ctx.Emit( + {{"def_init_symbol", ReflectionFileSymbol(d.file().name())}, + {"full_name", d.full_name()}, + {"get_message_symbol", ReflectionGetMessageSymbol(d.full_name())}}, + R"cc( + UPB_INLINE const upb_MessageDef *$get_message_symbol$(upb_DefPool *s) { + _upb_DefPool_LoadDefInit(s, &$def_init_symbol$); + return upb_DefPool_FindMessageByName(s, "$full_name$"); + } + )cc"); +} + +void WriteMessages(upb::FileDefPtr file, Context& ctx) { + for (auto msg : SortedMessages(file)) { + GenerateMessageDefAccessor(msg, ctx); + ctx.Emit("\n"); + } +} + +void WriteDefHeader(upb::FileDefPtr file, Context& ctx) { + ctx.Emit( + { + {"def_init_symbol", ReflectionFileSymbol(file.name())}, + {"dllexport_decl", ctx.options().dllexport_decl}, + {"file_warning", FileWarning(file.name())}, + {"include_guard", IncludeGuard(file.name())}, + Sub("messages", [&] { WriteMessages(file, ctx); }).WithSuffix(";"), + }, + R"( + $file_warning$ + + #ifndef $include_guard$_UPBDEFS_H_ + #define $include_guard$_UPBDEFS_H_ + + #include "upb/reflection/def.h" + #include "upb/reflection/internal/def_pool.h" + + #include "upb/port/def.inc" + + #ifdef __cplusplus + extern "C" { + #endif + + extern$ dllexport_decl$ _upb_DefPool_Init $def_init_symbol$; + + $messages$; + + #ifdef __cplusplus + } /* extern "C" */ + #endif + + #include "upb/port/undef.inc" + + #endif /* $include_guard$_UPBDEFS_H_ */ + )"); +} + +} // namespace + +std::string DefHeaderFilename(upb::FileDefPtr file) { + return absl::StrCat(StripExtension(file.name()), ".upbdefs.h"); +} + +void GenerateReflectionHeader(upb::FileDefPtr file, const Options& options, + google::protobuf::compiler::GeneratorContext* context) { + Context h_context(options, context->Open(DefHeaderFilename(file))); + WriteDefHeader(file, h_context); +} + +} // namespace upb::generator::reflection
diff --git a/upb_generator/reflection/header.h b/upb_generator/reflection/header.h new file mode 100644 index 0000000..2310091 --- /dev/null +++ b/upb_generator/reflection/header.h
@@ -0,0 +1,26 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_HEADER_H_ +#define THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_HEADER_H_ + +#include <string> + +#include "google/protobuf/compiler/code_generator.h" +#include "upb/reflection/def.hpp" +#include "upb_generator/reflection/context.h" + +namespace upb::generator::reflection { + +std::string DefHeaderFilename(upb::FileDefPtr file); + +void GenerateReflectionHeader(upb::FileDefPtr file, const Options& options, + google::protobuf::compiler::GeneratorContext* context); + +} // namespace upb::generator::reflection + +#endif // THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_HEADER_H_
diff --git a/upb_generator/reflection/names.h b/upb_generator/reflection/names.h index 70c06d1..c6e2948 100644 --- a/upb_generator/reflection/names.h +++ b/upb_generator/reflection/names.h
@@ -15,8 +15,7 @@ // Must be last. #include "upb/port/def.inc" -namespace upb { -namespace generator { +namespace upb::generator { // These are the publicly visible symbols defined in foo.upbdefs.h. // const upb_MessageDef* <GetMessage>(upb_DefPool *s); @@ -25,8 +24,7 @@ UPBC_API std::string ReflectionGetMessageSymbol(absl::string_view full_name); UPBC_API std::string ReflectionFileSymbol(absl::string_view filename); -} // namespace generator -} // namespace upb +} // namespace upb::generator #include "upb/port/undef.inc"
diff --git a/upb_generator/reflection/source.cc b/upb_generator/reflection/source.cc new file mode 100644 index 0000000..a573f1f --- /dev/null +++ b/upb_generator/reflection/source.cc
@@ -0,0 +1,173 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include <cstddef> +#include <string> + +#include "google/protobuf/descriptor.upb.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/printer.h" +#include "upb/mem/arena.hpp" +#include "upb/reflection/def.hpp" +#include "upb/util/def_to_proto.h" +#include "upb_generator/common/names.h" +#include "upb_generator/minitable/names.h" +#include "upb_generator/reflection/context.h" +#include "upb_generator/reflection/header.h" +#include "upb_generator/reflection/names.h" + +namespace upb::generator::reflection { +namespace { + +using Sub = google::protobuf::io::Printer::Sub; + +static std::string DefSourceFilename(upb::FileDefPtr file) { + return StripExtension(file.name()) + ".upbdefs.c"; +} + +void WriteIncludes(upb::FileDefPtr file, Context& ctx) { + ctx.Emit( + { + {"def_header_filename", DefHeaderFilename(file)}, + {"mini_table_header_filename", MiniTableHeaderFilename(file.name())}, + }, + R"( + #include "upb/reflection/def.h" + #include "$def_header_filename$" + #include "$mini_table_header_filename$" + )"); +} + +void WriteDefPoolFwdDecls(upb::FileDefPtr file, Context& ctx) { + if (file.dependency_count() == 0) return; + + for (int i = 0; i < file.dependency_count(); i++) { + ctx.Emit( + {{"dllexport_decl", ctx.options().dllexport_decl}, + {"def_init_symbol", ReflectionFileSymbol(file.dependency(i).name())}}, + R"cc( + extern$ dllexport_decl$ _upb_DefPool_Init $def_init_symbol$; + )cc"); + } + + ctx.Emit("\n"); +} + +void WriteStringArrayLine(absl::string_view data, Context& ctx) { + std::string line = absl::StrJoin(data, ", ", [](std::string* out, char ch) { + absl::StrAppendFormat(out, "'%v'", + absl::CEscape(absl::string_view(&ch, 1))); + }); + ctx.Emit({{"line", line}}, + R"cc( + $line$, + )cc"); +} + +void WriteStringArray(absl::string_view data, Context& ctx) { + const size_t kMaxLineLength = 12; + for (size_t i = 0; i < data.size(); i += kMaxLineLength) { + WriteStringArrayLine(data.substr(i, kMaxLineLength), ctx); + } +} + +void WriteDescriptor(upb::FileDefPtr file, Context& ctx) { + upb::Arena arena; + google_protobuf_FileDescriptorProto* file_proto = + upb_FileDef_ToProto(file.ptr(), arena.ptr()); + size_t serialized_size; + const char* serialized = google_protobuf_FileDescriptorProto_serialize( + file_proto, arena.ptr(), &serialized_size); + absl::string_view file_data(serialized, serialized_size); + + ctx.Emit({{"serialized_size", file_data.size()}, + {"contents", [&] { WriteStringArray(file_data, ctx); }}}, + R"cc( + static const char descriptor[$serialized_size$] = { + $contents$, + }; + )cc"); + ctx.Emit("\n"); +} + +void WriteDependencies(upb::FileDefPtr file, Context& ctx) { + auto write_dep = [&](int i) { + ctx.Emit({{"sym", ReflectionFileSymbol(file.dependency(i).name())}}, + R"cc( + &$sym$, + )cc"); + }; + + auto write_deps = [&] { + for (int i = 0; i < file.dependency_count(); i++) write_dep(i); + }; + + ctx.Emit({{"dep_count", file.dependency_count() + 1}, + {google::protobuf::io::Printer::Sub("deps", write_deps).WithSuffix(",")}}, + R"cc( + static _upb_DefPool_Init *deps[$dep_count$] = { + $deps$, + NULL, + }; + )cc"); + ctx.Emit("\n"); +} + +void WriteDefPoolInitStruct(upb::FileDefPtr file, Context& ctx) { + ctx.Emit( + { + {"defpool_init_name", ReflectionFileSymbol(file.name())}, + {"file_name", file.name()}, + {"mini_table_file_var_name", MiniTableFileVarName(file.name())}, + }, + R"cc( + _upb_DefPool_Init $defpool_init_name$ = { + deps, + &$mini_table_file_var_name$, + "$file_name$", + UPB_STRINGVIEW_INIT(descriptor, sizeof(descriptor)), + }; + )cc"); +} + +void WriteDefPoolInit(upb::FileDefPtr file, Context& ctx) { + WriteDefPoolFwdDecls(file, ctx); + WriteDescriptor(file, ctx); + WriteDependencies(file, ctx); + WriteDefPoolInitStruct(file, ctx); +} + +void WriteDefSource(upb::FileDefPtr file, Context& ctx) { + ctx.Emit( + { + {Sub("file_warning", FileWarning(file.name())).WithSuffix(";")}, + {Sub("includes", [&] { WriteIncludes(file, ctx); }).WithSuffix(";")}, + {Sub("def_pool_init", [&] { WriteDefPoolInit(file, ctx); }) + .WithSuffix(";")}, + }, + R"cc( + $file_warning$; + $includes$; + + $def_pool_init$; + )cc"); +} + +} // namespace + +void GenerateReflectionSource(upb::FileDefPtr file, const Options& options, + google::protobuf::compiler::GeneratorContext* context) { + Context c_context(options, context->Open(DefSourceFilename(file))); + WriteDefSource(file, c_context); +} + +} // namespace upb::generator::reflection
diff --git a/upb_generator/reflection/source.h b/upb_generator/reflection/source.h new file mode 100644 index 0000000..5c61f19 --- /dev/null +++ b/upb_generator/reflection/source.h
@@ -0,0 +1,26 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_SOURCE_H +#define THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_SOURCE_H + +#include "google/protobuf/compiler/code_generator.h" +#include "upb/reflection/def.hpp" +#include "upb_generator/reflection/context.h" + +namespace upb { +namespace generator { +namespace reflection { + +void GenerateReflectionSource(upb::FileDefPtr file, const Options& options, + google::protobuf::compiler::GeneratorContext* context); + +} // namespace reflection +} // namespace generator +} // namespace upb + +#endif // THIRD_PARTY_UPB_UPB_GENERATOR_REFLECTION_SOURCE_H