Unify dense and chunky enum name lookup caches into a single fast enum cache. PiperOrigin-RevId: 970801564
diff --git a/src/google/protobuf/compiler/cpp/enum.cc b/src/google/protobuf/compiler/cpp/enum.cc index 4caecc9..f09bfe6 100644 --- a/src/google/protobuf/compiler/cpp/enum.cc +++ b/src/google/protobuf/compiler/cpp/enum.cc
@@ -21,9 +21,9 @@ #include "absl/algorithm/container.h" #include "absl/container/btree_map.h" -#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "google/protobuf/compiler/cpp/generator.h" #include "google/protobuf/compiler/cpp/helpers.h" @@ -31,6 +31,7 @@ #include "google/protobuf/compiler/cpp/options.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/generated_enum_util.h" +#include "google/protobuf/generated_message_reflection.h" namespace google { namespace protobuf { @@ -103,16 +104,9 @@ generate_array_size_(ShouldGenerateArraySize(descriptor)), has_reflection_(HasDescriptorMethods(enum_->file(), options_)), limits_(ValueLimits::FromEnum(enum_)) { - // The conditions here for what is "sparse" are not rigorously - // chosen. - size_t values_range = static_cast<size_t>(limits_.max->number()) - - static_cast<size_t>(limits_.min->number()); size_t total_values = static_cast<size_t>(enum_->value_count()); - should_cache_ = has_reflection_ && - (values_range < 16u || values_range < total_values * 2u); - - sorted_unique_values_.reserve(enum_->value_count()); - for (int i = 0; i < enum_->value_count(); ++i) { + sorted_unique_values_.reserve(total_values); + for (size_t i = 0; i < total_values; ++i) { sorted_unique_values_.push_back(enum_->value(i)->number()); } // Sort and deduplicate @@ -120,6 +114,55 @@ sorted_unique_values_.erase( std::unique(sorted_unique_values_.begin(), sorted_unique_values_.end()), sorted_unique_values_.end()); + + if (!has_reflection_) return; + + // Density analysis. The conditions here for what is "sparse" are not + // rigorously chosen. + size_t values_range = static_cast<size_t>(limits_.max->number()) - + static_cast<size_t>(limits_.min->number()); + if (values_range < 16u || values_range < total_values * 2u) { + // If we can fit the whole range in a single chunk, do so. + dense_chunks_.push_back({limits_.min->number(), limits_.max->number(), 0}); + } else if (!sorted_unique_values_.empty()) { + std::vector<internal::ChunkInfo> chunks; + int current_min = sorted_unique_values_[0]; + int current_max = sorted_unique_values_[0]; + uint32_t total_size = 0; + uint32_t current_chunk_size = 0; + auto add_chunk = [&](int min_val, int max_val) { + chunks.push_back({min_val, max_val, total_size}); + // Inclusive size needed for [min, max]. + total_size += + static_cast<uint32_t>(max_val) - static_cast<uint32_t>(min_val) + 1u; + current_chunk_size = 0; + }; + for (int val : sorted_unique_values_) { + uint32_t maybe_chunk_width = + static_cast<uint32_t>(val) - static_cast<uint32_t>(current_min); + current_chunk_size++; + // If adding val to the current chunk would make the chunk too wide or + // sparse, start a new chunk based on the previous value - val will be the + // start of the next chunk. Threshold for each chunk is the same as for + // the single dense cache, above. + if (maybe_chunk_width >= 16u && + maybe_chunk_width >= current_chunk_size * 2u) { + add_chunk(current_min, current_max); + // Bail out if the current list is too big. + if (chunks.size() >= 8 || total_size >= 1024) return; + current_min = val; + } + current_max = val; + } + add_chunk(current_min, current_max); + + // Limit chosen arbitrarily, but too many chunks to check can cost as much + // as the regular hash lookup, and the total size is bounded to avoid + // excessive memory allocation. + if (chunks.size() <= 8 && total_size <= 1024) { + dense_chunks_ = std::move(chunks); + } + } } void EnumGenerator::GenerateDefinition(io::Printer* p) { @@ -234,7 +277,7 @@ )cc"); }; - if (should_cache_ || !has_reflection_) { + if (!dense_chunks_.empty() || !has_reflection_) { p->Emit({{"static_assert", write_assert}}, R"cc( template <typename T> $nodiscard $$return_type$ $Msg_Enum$_Name(T value) { @@ -242,18 +285,28 @@ return $Msg_Enum$_Name(static_cast<$Msg_Enum$>(value)); } )cc"); - if (should_cache_) { + if (!dense_chunks_.empty()) { // Using the NameOfEnum routine can be slow, so we create a small // cache of pointers to the std::string objects that reflection // stores internally. This cache is a simple contiguous array of // pointers, so if the enum values are sparse, it's not worth it. - p->Emit(R"cc( - template <> - $nodiscard $inline $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { - return $pbi$::NameOfDenseEnum<$Msg_Enum$_descriptor, $kMin$, $kMax$>( - static_cast<int>(value)); - } - )cc"); + p->Emit( + { + {"chunks", + absl::StrJoin( + dense_chunks_, ", ", + [](std::string* out, const internal::ChunkInfo& chunk) { + absl::StrAppend(out, "{", chunk.min_val, ", ", + chunk.max_val, ", ", chunk.offset, "}"); + })}, + }, + R"cc( + template <> + $nodiscard $inline $return_type$ $Msg_Enum$_Name($Msg_Enum$ value) { + return $pbi$::NameOfDenseEnum<$Msg_Enum$_descriptor, $chunks$>( + static_cast<int>(value)); + } + )cc"); } } else { p->Emit({{"static_assert", write_assert}}, R"cc(
diff --git a/src/google/protobuf/compiler/cpp/enum.h b/src/google/protobuf/compiler/cpp/enum.h index 5a636ae..e13a0ae 100644 --- a/src/google/protobuf/compiler/cpp/enum.h +++ b/src/google/protobuf/compiler/cpp/enum.h
@@ -12,10 +12,11 @@ #ifndef GOOGLE_PROTOBUF_COMPILER_CPP_ENUM_H__ #define GOOGLE_PROTOBUF_COMPILER_CPP_ENUM_H__ -#include <string> +#include <vector> #include "google/protobuf/compiler/cpp/options.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/generated_message_reflection.h" #include "google/protobuf/io/printer.h" namespace google { @@ -72,9 +73,9 @@ std::vector<int> sorted_unique_values_; bool generate_array_size_; - bool should_cache_; bool has_reflection_; ValueLimits limits_; + std::vector<internal::ChunkInfo> dense_chunks_; }; } // namespace cpp
diff --git a/src/google/protobuf/compiler/java/java_features.pb.h b/src/google/protobuf/compiler/java/java_features.pb.h index dcc255f..7ab873c 100644 --- a/src/google/protobuf/compiler/java/java_features.pb.h +++ b/src/google/protobuf/compiler/java/java_features.pb.h
@@ -108,7 +108,7 @@ } template <> [[nodiscard]] inline const ::std::string& JavaFeatures_NestInFileClassFeature_NestInFileClass_Name(JavaFeatures_NestInFileClassFeature_NestInFileClass value) { - return ::google::protobuf::internal::NameOfDenseEnum<JavaFeatures_NestInFileClassFeature_NestInFileClass_descriptor, 0, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<JavaFeatures_NestInFileClassFeature_NestInFileClass_descriptor, {0, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool JavaFeatures_NestInFileClassFeature_NestInFileClass_Parse( @@ -145,7 +145,7 @@ } template <> [[nodiscard]] inline const ::std::string& JavaFeatures_Utf8Validation_Name(JavaFeatures_Utf8Validation value) { - return ::google::protobuf::internal::NameOfDenseEnum<JavaFeatures_Utf8Validation_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<JavaFeatures_Utf8Validation_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool JavaFeatures_Utf8Validation_Parse(
diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index cb51823..7a6b17b 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h
@@ -108,7 +108,7 @@ } template <> [[nodiscard]] inline const ::std::string& CodeGeneratorResponse_Feature_Name(CodeGeneratorResponse_Feature value) { - return ::google::protobuf::internal::NameOfDenseEnum<CodeGeneratorResponse_Feature_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<CodeGeneratorResponse_Feature_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool CodeGeneratorResponse_Feature_Parse(
diff --git a/src/google/protobuf/cpp_features.pb.h b/src/google/protobuf/cpp_features.pb.h index 683e1d3..5a463e2 100644 --- a/src/google/protobuf/cpp_features.pb.h +++ b/src/google/protobuf/cpp_features.pb.h
@@ -104,7 +104,7 @@ } template <> [[nodiscard]] inline const ::std::string& CppFeatures_StringType_Name(CppFeatures_StringType value) { - return ::google::protobuf::internal::NameOfDenseEnum<CppFeatures_StringType_descriptor, 0, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<CppFeatures_StringType_descriptor, {0, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool CppFeatures_StringType_Parse( @@ -141,7 +141,7 @@ } template <> [[nodiscard]] inline const ::std::string& CppFeatures_RepeatedType_Name(CppFeatures_RepeatedType value) { - return ::google::protobuf::internal::NameOfDenseEnum<CppFeatures_RepeatedType_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<CppFeatures_RepeatedType_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool CppFeatures_RepeatedType_Parse(
diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index b0a1b19..17826b7 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h
@@ -296,7 +296,7 @@ } template <> [[nodiscard]] inline const ::std::string& ExtensionRangeOptions_VerificationState_Name(ExtensionRangeOptions_VerificationState value) { - return ::google::protobuf::internal::NameOfDenseEnum<ExtensionRangeOptions_VerificationState_descriptor, 0, 1>( + return ::google::protobuf::internal::NameOfDenseEnum<ExtensionRangeOptions_VerificationState_descriptor, {0, 1, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool ExtensionRangeOptions_VerificationState_Parse( @@ -348,7 +348,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldDescriptorProto_Type_Name(FieldDescriptorProto_Type value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldDescriptorProto_Type_descriptor, 1, 18>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldDescriptorProto_Type_descriptor, {1, 18, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldDescriptorProto_Type_Parse( @@ -385,7 +385,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldDescriptorProto_Label_Name(FieldDescriptorProto_Label value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldDescriptorProto_Label_descriptor, 1, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldDescriptorProto_Label_descriptor, {1, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldDescriptorProto_Label_Parse( @@ -422,7 +422,7 @@ } template <> [[nodiscard]] inline const ::std::string& FileOptions_OptimizeMode_Name(FileOptions_OptimizeMode value) { - return ::google::protobuf::internal::NameOfDenseEnum<FileOptions_OptimizeMode_descriptor, 1, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<FileOptions_OptimizeMode_descriptor, {1, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FileOptions_OptimizeMode_Parse( @@ -459,7 +459,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldOptions_CType_Name(FieldOptions_CType value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_CType_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_CType_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldOptions_CType_Parse( @@ -496,7 +496,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldOptions_JSType_Name(FieldOptions_JSType value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_JSType_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_JSType_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldOptions_JSType_Parse( @@ -533,7 +533,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldOptions_OptionRetention_Name(FieldOptions_OptionRetention value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_OptionRetention_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_OptionRetention_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldOptions_OptionRetention_Parse( @@ -577,7 +577,7 @@ } template <> [[nodiscard]] inline const ::std::string& FieldOptions_OptionTargetType_Name(FieldOptions_OptionTargetType value) { - return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_OptionTargetType_descriptor, 0, 9>( + return ::google::protobuf::internal::NameOfDenseEnum<FieldOptions_OptionTargetType_descriptor, {0, 9, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FieldOptions_OptionTargetType_Parse( @@ -614,7 +614,7 @@ } template <> [[nodiscard]] inline const ::std::string& MethodOptions_IdempotencyLevel_Name(MethodOptions_IdempotencyLevel value) { - return ::google::protobuf::internal::NameOfDenseEnum<MethodOptions_IdempotencyLevel_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<MethodOptions_IdempotencyLevel_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool MethodOptions_IdempotencyLevel_Parse( @@ -653,7 +653,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_VisibilityFeature_DefaultSymbolVisibility_Name(FeatureSet_VisibilityFeature_DefaultSymbolVisibility value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_VisibilityFeature_DefaultSymbolVisibility_descriptor, 0, 4>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_VisibilityFeature_DefaultSymbolVisibility_descriptor, {0, 4, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_VisibilityFeature_DefaultSymbolVisibility_Parse( @@ -690,7 +690,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_ProtoLimitsFeature_EnforceProtoLimits_Name(FeatureSet_ProtoLimitsFeature_EnforceProtoLimits value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_ProtoLimitsFeature_EnforceProtoLimits_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_ProtoLimitsFeature_EnforceProtoLimits_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_ProtoLimitsFeature_EnforceProtoLimits_Parse( @@ -728,7 +728,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_FieldPresence_Name(FeatureSet_FieldPresence value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_FieldPresence_descriptor, 0, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_FieldPresence_descriptor, {0, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_FieldPresence_Parse( @@ -765,7 +765,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_EnumType_Name(FeatureSet_EnumType value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_EnumType_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_EnumType_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_EnumType_Parse( @@ -802,7 +802,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_RepeatedFieldEncoding_Name(FeatureSet_RepeatedFieldEncoding value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_RepeatedFieldEncoding_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_RepeatedFieldEncoding_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_RepeatedFieldEncoding_Parse( @@ -839,7 +839,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_Utf8Validation_Name(FeatureSet_Utf8Validation value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_Utf8Validation_descriptor, 0, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_Utf8Validation_descriptor, {0, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_Utf8Validation_Parse( @@ -876,7 +876,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_MessageEncoding_Name(FeatureSet_MessageEncoding value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_MessageEncoding_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_MessageEncoding_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_MessageEncoding_Parse( @@ -913,7 +913,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_JsonFormat_Name(FeatureSet_JsonFormat value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_JsonFormat_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_JsonFormat_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_JsonFormat_Parse( @@ -951,7 +951,7 @@ } template <> [[nodiscard]] inline const ::std::string& FeatureSet_EnforceNamingStyle_Name(FeatureSet_EnforceNamingStyle value) { - return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_EnforceNamingStyle_descriptor, 0, 3>( + return ::google::protobuf::internal::NameOfDenseEnum<FeatureSet_EnforceNamingStyle_descriptor, {0, 3, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool FeatureSet_EnforceNamingStyle_Parse( @@ -988,7 +988,7 @@ } template <> [[nodiscard]] inline const ::std::string& GeneratedCodeInfo_Annotation_Semantic_Name(GeneratedCodeInfo_Annotation_Semantic value) { - return ::google::protobuf::internal::NameOfDenseEnum<GeneratedCodeInfo_Annotation_Semantic_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<GeneratedCodeInfo_Annotation_Semantic_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool GeneratedCodeInfo_Annotation_Semantic_Parse( @@ -1031,7 +1031,12 @@ static_assert(::std::is_same<T, Edition>::value || ::std::is_integral<T>::value, "Incorrect type passed to Edition_Name()."); - return ::google::protobuf::internal::NameOfEnum(Edition_descriptor(), value); + return Edition_Name(static_cast<Edition>(value)); +} +template <> +[[nodiscard]] inline const ::std::string& Edition_Name(Edition value) { + return ::google::protobuf::internal::NameOfDenseEnum<Edition_descriptor, {0, 2, 0}, {900, 900, 3}, {998, 1002, 4}, {9999, 9999, 9}, {99997, 99999, 10}, {2147483647, 2147483647, 13}>( + static_cast<int>(value)); } [[nodiscard]] inline bool Edition_Parse( ::absl::string_view name, Edition* PROTOBUF_NONNULL value) { @@ -1067,7 +1072,7 @@ } template <> [[nodiscard]] inline const ::std::string& SymbolVisibility_Name(SymbolVisibility value) { - return ::google::protobuf::internal::NameOfDenseEnum<SymbolVisibility_descriptor, 0, 2>( + return ::google::protobuf::internal::NameOfDenseEnum<SymbolVisibility_descriptor, {0, 2, 0}>( static_cast<int>(value)); } [[nodiscard]] inline bool SymbolVisibility_Parse(
diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 492c210..9f56127 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc
@@ -47,7 +47,6 @@ #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/descriptor_lite.h" #include "google/protobuf/extension_set.h" -#include "google/protobuf/generated_enum_util.h" #include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_tctable_gen.h" #include "google/protobuf/generated_message_tctable_impl.h" @@ -204,43 +203,39 @@ // Internal helper routine for NameOfDenseEnum in the header file. // Allocates and fills a simple array of string pointers, based on -// reflection information about the names of the enums. This routine -// allocates max_val + 1 entries, under the assumption that all the enums -// fall in the range [min_val .. max_val]. -const std::string** MakeDenseEnumCache(const EnumDescriptor* desc, int min_val, - int max_val) { - auto* str_ptrs = - new const std::string*[static_cast<size_t>(max_val - min_val + 1)](); - const int count = desc->value_count(); - for (int i = 0; i < count; ++i) { - const int num = desc->value(i)->number(); - if (str_ptrs[num - min_val] == nullptr) { - // Don't over-write an existing entry, because in case of duplication, the - // first one wins. - str_ptrs[num - min_val] = &internal::NameOfEnumAsString(desc->value(i)); +// reflection information about the names of the enums. +const std::string** InitializeFastEnumCache(FastEnumCacheInfo* info) { + absl::call_once(info->loaded, [info]() { + const ChunkInfo& last_chunk = info->chunks.back(); + const uint32_t total_size = + last_chunk.offset + + static_cast<uint32_t>(last_chunk.max_val - last_chunk.min_val) + 1; + const std::string** str_ptrs = new const std::string*[total_size]; + std::fill_n(str_ptrs, total_size, &GetEmptyStringAlreadyInited()); + const EnumDescriptor* desc = info->descriptor_fn(); + const int count = desc->value_count(); + // Iterate in reverse order to ensure that if there are aliases, the first + // one wins. + for (int i = count - 1; i >= 0; --i) { + const int num = desc->value(i)->number(); + for (const ChunkInfo& chunk : info->chunks) { + if (num >= chunk.min_val) { + if (num <= chunk.max_val) { + str_ptrs[chunk.offset + + static_cast<uint32_t>(num - chunk.min_val)] = + &internal::NameOfEnumAsString(desc->value(i)); + break; + } + } else { + break; + } + } } - } - // Change any unfilled entries to point to the empty string. - for (int i = 0; i < max_val - min_val + 1; ++i) { - if (str_ptrs[i] == nullptr) str_ptrs[i] = &GetEmptyStringAlreadyInited(); - } - return str_ptrs; -} - -PROTOBUF_NOINLINE const std::string& NameOfDenseEnumSlow( - int v, DenseEnumCacheInfo* deci) { - if (v < deci->min_val || v > deci->max_val) - return GetEmptyStringAlreadyInited(); - - // Use run_once to avoid a race condition in initializing the cache. - absl::call_once(deci->loaded, [deci]() { - const std::string** new_cache = - MakeDenseEnumCache(deci->descriptor_fn(), deci->min_val, deci->max_val); // Atomically publish the cache. Doing this inside the call_once ensures // that no thread sees the uninitialized or partially initialized cache. - deci->cache.store(new_cache, std::memory_order_release); + info->flat_cache.store(str_ptrs, std::memory_order_release); }); - return *deci->cache.load(std::memory_order_acquire)[v - deci->min_val]; + return info->flat_cache.load(std::memory_order_acquire); } bool IsMatchingCType(const FieldDescriptor* field, int ctype) {
diff --git a/src/google/protobuf/generated_message_reflection.h b/src/google/protobuf/generated_message_reflection.h index 66494d9..d1455c1 100644 --- a/src/google/protobuf/generated_message_reflection.h +++ b/src/google/protobuf/generated_message_reflection.h
@@ -23,6 +23,8 @@ #include "absl/base/call_once.h" #include "absl/base/optimization.h" #include "absl/log/absl_check.h" +#include "absl/types/span.h" +#include "google/protobuf/class_data.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/generated_enum_reflection.h" #include "google/protobuf/has_bits.h" @@ -356,6 +358,60 @@ return NameOfDenseEnumSlow(v, &deci); } +struct ChunkInfo { + int min_val; + int max_val; + uint32_t offset; // The size of all preceding chunks +}; + +struct FastEnumCacheInfo { + absl::once_flag loaded; + std::atomic<const std::string**> flat_cache{nullptr}; + const EnumDescriptor* (*descriptor_fn)(); + absl::Span<const ChunkInfo> chunks; +}; + +PROTOBUF_EXPORT const std::string** InitializeFastEnumCache( + FastEnumCacheInfo* info); + +// Similar to the routine NameOfEnum, this routine returns the name of an enum. +// Unlike that routine, it allocates, on-demand, a block of pointers to the +// std::string objects allocated by reflection to store the enum names. This +// way, as long as the enum values are fairly dense, looking them up can be +// very fast. +template <const EnumDescriptor* (*descriptor_fn)(), ChunkInfo... chunks> +inline const std::string& NameOfDenseEnum(int v) { + static constexpr ChunkInfo kChunks[] = {chunks...}; + static FastEnumCacheInfo info = {/* once_flag */ {}, + /* atomic ptr */ {}, descriptor_fn, kChunks}; + + const std::string* result = nullptr; + const auto check_chunk = [&](const ChunkInfo& chunk) { + if (v >= chunk.min_val && v <= chunk.max_val) { + const std::string** cache = + info.flat_cache.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(cache == nullptr)) { + cache = InitializeFastEnumCache(&info); + } + result = cache[chunk.offset + static_cast<size_t>(v - chunk.min_val)]; + return true; + } + return false; + }; + + if ((check_chunk(chunks) || ...)) { + return *result; + } + + // Prevent the compiler from pre-loading the empty string address. + // Otherwise, it adds an instruction to every in-bounds-call (adding ~5% + // to cpu). + // We expect to see many more in-bounds calls than out-of-bounds calls, so + // this is a net win. + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); + return GetEmptyStringAlreadyInited(); +} + // Returns whether this type of field is stored in the split struct as a raw // pointer. PROTOBUF_EXPORT bool SplitFieldHasExtraIndirection(