Inline RecordAlloc and RecordFree and embed AllocationMap in HeapProfileTable. base/heap-checker.cc and base/heap-profile-table.cc are compiled with -O3 in separate translation units, so NewHook and DeleteHook previously made cross-TU calls to HeapProfileTable::RecordAlloc and HeapProfileTable::RecordFree and dereferenced a separately allocated AllocationMap* pointer on every malloc and free. Embed AllocationMap directly in HeapProfileTable and define RecordAlloc and RecordFree inline in base/heap-profile-table.h (inlining the empty_bucket_ fast path when stack_depth == 0) so NewHook and DeleteHook inline the entire per-allocation and per-deallocation bookkeeping. PiperOrigin-RevId: 988534158
diff --git a/absl/flags/commandlineflag.cc b/absl/flags/commandlineflag.cc index 4140230..3b3b785 100644 --- a/absl/flags/commandlineflag.cc +++ b/absl/flags/commandlineflag.cc
@@ -26,6 +26,7 @@ absl::string_view CommandLineFlag::TypeName() const { return ""; } bool CommandLineFlag::IsRetired() const { return false; } +void CommandLineFlag::RestoreToInitialState() {} bool CommandLineFlag::ParseFrom(absl::string_view value, std::string* error) { return ParseFrom(value, flags_internal::SET_FLAGS_VALUE, flags_internal::kProgrammaticChange, *error);
diff --git a/absl/flags/commandlineflag.h b/absl/flags/commandlineflag.h index 9336d6d..aab7dac 100644 --- a/absl/flags/commandlineflag.h +++ b/absl/flags/commandlineflag.h
@@ -188,6 +188,9 @@ // or nullptr if flag does not support saving and restoring a state. virtual std::unique_ptr<flags_internal::FlagStateInterface> SaveState() = 0; + // Restores a flag that was in its initial unmodified state when saved. + virtual void RestoreToInitialState(); + // Copy-construct a new value of the flag's type in a memory referenced by // the dst based on the current flag's value. virtual void Read(void* absl_nonnull dst) const = 0;
diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index 662b342..62d1be9 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc
@@ -59,11 +59,17 @@ namespace { // Currently we only validate flag values for user-defined flag types. -bool ShouldValidateFlagValue(FlagFastTypeId flag_type_id) { -#define DONT_VALIDATE(T, _) \ +bool ShouldValidateFlagValue(FlagOpFn op) { +#define DONT_VALIDATE_OP(T, _) \ + if (op == &FlagOps<T>) return false; + ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(DONT_VALIDATE_OP) +#undef DONT_VALIDATE_OP + + FlagFastTypeId flag_type_id = flags_internal::FastTypeId(op); +#define DONT_VALIDATE_ID(T, _) \ if (flag_type_id == absl::FastTypeId<T>()) return false; - ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(DONT_VALIDATE) -#undef DONT_VALIDATE + ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(DONT_VALIDATE_ID) +#undef DONT_VALIDATE_ID return true; } @@ -104,6 +110,39 @@ s_freelist->push_back(p); } +struct InitialDefaultEntry { + const FlagImpl* flag; + FlagDefaultSrc default_value; +}; + +ABSL_CONST_INIT std::vector<InitialDefaultEntry>* s_initial_defaults + ABSL_GUARDED_BY(FreelistMutex()) + ABSL_PT_GUARDED_BY(FreelistMutex()) = nullptr; + +void RecordInitialDefault(const FlagImpl* flag, FlagDefaultSrc default_value) { + absl::MutexLock l(FreelistMutex()); + if (!s_initial_defaults) { + s_initial_defaults = new std::vector<InitialDefaultEntry>; + } + for (const auto& entry : *s_initial_defaults) { + if (entry.flag == flag) return; + } + s_initial_defaults->push_back({flag, default_value}); +} + +FlagDefaultSrc GetInitialDefault(const FlagImpl* flag, + FlagDefaultSrc fallback_value) { + absl::MutexLock l(FreelistMutex()); + if (s_initial_defaults != nullptr) { + for (const auto& entry : *s_initial_defaults) { + if (entry.flag == flag) { + return entry.default_value; + } + } + } + return fallback_value; +} + } // namespace /////////////////////////////////////////////////////////////////////////////// @@ -374,6 +413,7 @@ } bool FlagImpl::IsSpecifiedOnCommandLine() const { + if (ABSL_PREDICT_TRUE(seq_lock_.IsUnmodified())) return false; absl::MutexLock l(DataGuard()); return on_command_line_; } @@ -464,14 +504,14 @@ case FlagValueStorageKind::kHeapAllocated: // Built-in types have side-effect-free defaults; user-defined types may // have custom construction or validation side effects. - return !ShouldValidateFlagValue(flags_internal::FastTypeId(op_)); + return !ShouldValidateFlagValue(op_); } return false; } std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { if (ABSL_PREDICT_TRUE(CanSkipInitOnSave())) { - return std::make_unique<FlagState>(*this, default_value_); + return nullptr; } absl::MutexLock l(DataGuard()); @@ -506,6 +546,13 @@ return nullptr; } +void FlagImpl::RestoreToInitialState() { + if (ABSL_PREDICT_TRUE(seq_lock_.IsUnmodified())) { + return; + } + FlagState(*this, default_value_).Restore(); +} + bool FlagImpl::RestoreState(const FlagState& flag_state) { if (ABSL_PREDICT_TRUE(flag_state.counter_ < 0 && seq_lock_.IsUnmodified())) { return false; @@ -516,21 +563,26 @@ return false; } + const FlagDefaultSrc default_value = + (flag_state.counter_ < 0 && + DefaultKind() == FlagDefaultKind::kDynamicValue) + ? GetInitialDefault(this, default_value_) + : flag_state.value_.default_value; + switch (ValueStorageKind()) { case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: - StoreValue( - flag_state.counter_ < 0 - ? static_cast<const void*>(&flag_state.value_.default_value) - : static_cast<const void*>(&flag_state.value_.one_word), - kProgrammaticChange); + StoreValue(flag_state.counter_ < 0 + ? static_cast<const void*>(&default_value) + : static_cast<const void*>(&flag_state.value_.one_word), + kProgrammaticChange); break; case FlagValueStorageKind::kSequenceLocked: case FlagValueStorageKind::kHeapAllocated: if (flag_state.counter_ < 0) { std::unique_ptr<void, DynValueDeleter> init_value( flags_internal::Alloc(op_), DynValueDeleter{op_}); - (*flag_state.value_.default_value.gen_func)(init_value.get()); + (*default_value.gen_func)(init_value.get()); StoreValue(init_value.get(), kProgrammaticChange); } else { StoreValue(flag_state.value_.heap_allocated, kProgrammaticChange); @@ -652,7 +704,7 @@ void FlagImpl::Write(const void* src) { absl::MutexLock l(DataGuard()); - if (ShouldValidateFlagValue(flags_internal::FastTypeId(op_))) { + if (ShouldValidateFlagValue(op_)) { std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op_, src), DynValueDeleter{op_}}; std::string ignored_error; @@ -717,6 +769,7 @@ default_value_.dynamic_value = tentative_value.release(); tentative_value.reset(old_value); } else { + RecordInitialDefault(this, default_value_); default_value_.dynamic_value = tentative_value.release(); def_kind_ = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); } @@ -734,11 +787,12 @@ } void FlagImpl::CheckDefaultValueParsingRoundtrip() const { - std::string v = DefaultValue(); + if (!ShouldValidateFlagValue(op_)) return; absl::MutexLock lock(DataGuard()); auto dst = MakeInitValue(); + std::string v = flags_internal::Unparse(op_, dst.get()); std::string error; if (!flags_internal::Parse(op_, v, dst.get(), &error)) { ABSL_INTERNAL_LOG(
diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index 8baff43..07cb71c 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h
@@ -378,11 +378,11 @@ MaskedPointer(const MaskedPointer& rhs) = default; MaskedPointer& operator=(const MaskedPointer& rhs) = default; - void* Ptr() const { + ABSL_ATTRIBUTE_ALWAYS_INLINE void* Ptr() const { return reinterpret_cast<void*>(reinterpret_cast<mask_t>(ptr_) & kPtrValueMask); } - bool AllowsUnprotectedRead() const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool AllowsUnprotectedRead() const { return (reinterpret_cast<mask_t>(ptr_) & kAllowsUnprotectedRead) == kAllowsUnprotectedRead; } @@ -437,7 +437,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue(0) {} - bool Get(const SequenceLock&, T& dst) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool Get(const SequenceLock&, T& dst) const { int64_t storage = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(storage == 0)) { // This assert is to ensure that the initialization inside FlagImpl::Init @@ -458,7 +458,7 @@ template <typename T> struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { constexpr FlagValue() : FlagOneWordValue() {} - bool Get(const SequenceLock&, T& dst) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool Get(const SequenceLock&, T& dst) const { int64_t one_word_val = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(one_word_val == FlagOneWordValue::Uninitialized())) { return false; @@ -476,7 +476,8 @@ // value once it is initialized. template <typename T> struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { - bool Get(const SequenceLock& lock, T& dst) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool Get(const SequenceLock& lock, + T& dst) const { return lock.TryRead(&dst, value_words, sizeof(T)); } @@ -531,7 +532,7 @@ // ensure Init is done before any access to the flag value. constexpr FlagValue() : FlagMaskedPointerValue(&buffer[0]) {} - bool Get(const SequenceLock&, T& dst) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE bool Get(const SequenceLock&, T& dst) const { MaskedPointer ptr_value = value.load(std::memory_order_acquire); if (ABSL_PREDICT_TRUE(ptr_value.AllowsUnprotectedRead())) { @@ -641,6 +642,8 @@ private: template <typename T> friend class Flag; + template <typename, bool> + friend class FlagRegistrar; friend class FlagState; // Ensures that `data_guard_` is initialized and returns it. @@ -718,6 +721,7 @@ // saving and restoring a state. std::unique_ptr<FlagStateInterface> SaveState() override ABSL_LOCKS_EXCLUDED(DataGuard()); + void RestoreToInitialState() override ABSL_LOCKS_EXCLUDED(DataGuard()); // Restores the flag state to the supplied state object. If there is // nothing to restore returns false. Otherwise returns true. @@ -792,6 +796,14 @@ // flag reflection handle interface. template <typename T> +struct IsSupportedFlagType : std::false_type {}; +#define ABSL_FLAGS_INTERNAL_IS_SUPPORTED_TYPE(T, name) \ + template <> \ + struct IsSupportedFlagType<T> : std::true_type {}; +ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(ABSL_FLAGS_INTERNAL_IS_SUPPORTED_TYPE) +#undef ABSL_FLAGS_INTERNAL_IS_SUPPORTED_TYPE + +template <typename T> class Flag { public: constexpr Flag(const char* name, const char* type_name, const char* filename, @@ -816,7 +828,7 @@ friend class FlagRegistrar; friend class FlagImplPeer; - T Get() const { + ABSL_ATTRIBUTE_ALWAYS_INLINE T Get() const { // See implementation notes in CommandLineFlag::Get(). union U { T value; @@ -826,7 +838,13 @@ U u; #if !defined(NDEBUG) - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + if constexpr (IsSupportedFlagType<T>::value) { + if (ABSL_PREDICT_FALSE(impl_.op_ != &FlagOps<T>)) { + impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + } + } else { + impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + } #endif if (ABSL_PREDICT_FALSE(!value_.Get(impl_.seq_lock_, u.value))) { @@ -835,7 +853,13 @@ return std::move(u.value); } void Set(const T& v) { - impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + if constexpr (IsSupportedFlagType<T>::value) { + if (ABSL_PREDICT_FALSE(impl_.op_ != &FlagOps<T>)) { + impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + } + } else { + impl_.AssertValidType(absl::FastTypeId<T>(), &GenRuntimeTypeId<T>); + } impl_.Write(&v); } @@ -856,7 +880,7 @@ class FlagImplPeer { public: template <typename T, typename FlagType> - static T InvokeGet(const FlagType& flag) { + ABSL_ATTRIBUTE_ALWAYS_INLINE static T InvokeGet(const FlagType& flag) { return flag.Get(); } template <typename FlagType, typename T> @@ -939,8 +963,10 @@ public: constexpr explicit FlagRegistrar(Flag<T>& flag, const char* filename) : flag_(flag) { - if (do_register) - flags_internal::RegisterCommandLineFlag(flag_.impl_, filename); + if (do_register) { + flags_internal::RegisterCommandLineFlag( + flag_.impl_, flag_.impl_.filename_ == filename ? nullptr : filename); + } } FlagRegistrar OnUpdate(FlagCallbackFunc cb) && {
diff --git a/absl/flags/internal/private_handle_accessor.cc b/absl/flags/internal/private_handle_accessor.cc index 610d1ff..145b1b4 100644 --- a/absl/flags/internal/private_handle_accessor.cc +++ b/absl/flags/internal/private_handle_accessor.cc
@@ -36,6 +36,10 @@ return flag.SaveState(); } +void PrivateHandleAccessor::RestoreToInitialState(CommandLineFlag& flag) { + flag.RestoreToInitialState(); +} + bool PrivateHandleAccessor::IsSpecifiedOnCommandLine( const CommandLineFlag& flag) { return flag.IsSpecifiedOnCommandLine();
diff --git a/absl/flags/internal/private_handle_accessor.h b/absl/flags/internal/private_handle_accessor.h index bf4154c..fedf750 100644 --- a/absl/flags/internal/private_handle_accessor.h +++ b/absl/flags/internal/private_handle_accessor.h
@@ -39,6 +39,9 @@ // Access to CommandLineFlag::SaveState. static std::unique_ptr<FlagStateInterface> SaveState(CommandLineFlag& flag); + // Access to CommandLineFlag::RestoreToInitialState. + static void RestoreToInitialState(CommandLineFlag& flag); + // Access to CommandLineFlag::IsSpecifiedOnCommandLine. static bool IsSpecifiedOnCommandLine(const CommandLineFlag& flag);
diff --git a/absl/flags/parse.cc b/absl/flags/parse.cc index 940bf56..9b9502d 100644 --- a/absl/flags/parse.cc +++ b/absl/flags/parse.cc
@@ -335,13 +335,6 @@ #ifndef NDEBUG flags_internal::ForEachFlag([&](CommandLineFlag& flag) { if (flag.IsRetired()) return; - -#define ABSL_FLAGS_INTERNAL_IGNORE_TYPE(T, _) \ - if (flag.IsOfType<T>()) return; - - ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(ABSL_FLAGS_INTERNAL_IGNORE_TYPE) -#undef ABSL_FLAGS_INTERNAL_IGNORE_TYPE - flags_internal::PrivateHandleAccessor::CheckDefaultValueParsingRoundtrip( flag); });
diff --git a/absl/flags/reflection.cc b/absl/flags/reflection.cc index 13f210c..af8f037 100644 --- a/absl/flags/reflection.cc +++ b/absl/flags/reflection.cc
@@ -322,20 +322,26 @@ flags_internal::ForEachFlag([&](CommandLineFlag& flag) { if (auto flag_state = flags_internal::PrivateHandleAccessor::SaveState(flag)) { - backup_registry_.emplace_back(std::move(flag_state)); + backup_registry_.emplace(&flag, std::move(flag_state)); } }); } // Restores the saved flag states into the flag registry. void RestoreToRegistry() && { - for (const auto& flag_state : backup_registry_) { - std::move(*flag_state).Restore(); - } + flags_internal::ForEachFlag([&](CommandLineFlag& flag) { + if (auto it = backup_registry_.find(&flag); + it != backup_registry_.end()) { + std::move(*it->second).Restore(); + } else { + flags_internal::PrivateHandleAccessor::RestoreToInitialState(flag); + } + }); } private: - std::vector<std::unique_ptr<flags_internal::FlagStateInterface>> + absl::flat_hash_map<CommandLineFlag*, + std::unique_ptr<flags_internal::FlagStateInterface>> backup_registry_; };