Add embedded cleanups
PiperOrigin-RevId: 490480534
diff --git a/src/file_lists.cmake b/src/file_lists.cmake
index 1e326bc..35084d0 100644
--- a/src/file_lists.cmake
+++ b/src/file_lists.cmake
@@ -578,6 +578,7 @@
set(protobuf_test_files
${protobuf_SOURCE_DIR}/src/google/protobuf/any_test.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/arena_align_test.cc
+ ${protobuf_SOURCE_DIR}/src/google/protobuf/arena_cleanup_test.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/arena_unittest.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/arenastring_unittest.cc
${protobuf_SOURCE_DIR}/src/google/protobuf/arenaz_sampler_test.cc
diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel
index 859b953..c77a026 100644
--- a/src/google/protobuf/BUILD.bazel
+++ b/src/google/protobuf/BUILD.bazel
@@ -204,10 +204,27 @@
"//src/google/protobuf:__subpackages__",
],
deps = [
+ ":arena_align",
"@com_google_absl//absl/base:core_headers",
],
)
+cc_test(
+ name = "arena_cleanup_test",
+ srcs = ["arena_cleanup_test.cc"],
+ copts = COPTS + select({
+ "//build_defs:config_msvc": [],
+ "//conditions:default": [
+ "-Wno-error=sign-compare",
+ ],
+ }),
+ deps = [
+ ":arena_cleanup",
+ "@com_google_googletest//:gtest",
+ "@com_google_googletest//:gtest_main",
+ ],
+)
+
cc_library(
name = "arena_config",
srcs = ["arena_config.cc"],
diff --git a/src/google/protobuf/arena.cc b/src/google/protobuf/arena.cc
index 327c26e..68d98ea 100644
--- a/src/google/protobuf/arena.cc
+++ b/src/google/protobuf/arena.cc
@@ -202,6 +202,23 @@
}
PROTOBUF_NOINLINE
+void* SerialArena::AllocateCleanupFallback(size_t size) {
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ AllocateNewBlock(size);
+ return limit_ -= size;
+}
+
+PROTOBUF_NOINLINE
+SerialArena::Memory SerialArena::AllocateCleanupFallback(size_t size,
+ size_t align) {
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(align));
+ AllocateNewBlock(size + align - internal::ArenaAlignDefault::align);
+ size += reinterpret_cast<size_t>(limit_) & (align - 1);
+ return {limit_ -= size, size};
+}
+
+PROTOBUF_NOINLINE
void SerialArena::AddCleanupFallback(void* elem, void (*destructor)(void*)) {
size_t required = cleanup::Size(destructor);
AllocateNewBlock(required);
@@ -277,40 +294,10 @@
char* limit = b->Limit();
char* it = reinterpret_cast<char*>(b->cleanup_nodes);
GOOGLE_DCHECK(!b->IsSentry() || it == limit);
- if (it < limit) {
- // A prefetch distance of 8 here was chosen arbitrarily. It makes the
- // pending nodes fill a cacheline which seemed nice.
- constexpr int kPrefetchDist = 8;
- cleanup::Tag pending_type[kPrefetchDist];
- char* pending_node[kPrefetchDist];
-
- int pos = 0;
- for (; pos < kPrefetchDist && it < limit; ++pos) {
- pending_type[pos] = cleanup::Type(it);
- pending_node[pos] = it;
- it += cleanup::Size(pending_type[pos]);
- }
-
- if (pos < kPrefetchDist) {
- for (int i = 0; i < pos; ++i) {
- cleanup::DestroyNode(pending_type[i], pending_node[i]);
- }
- } else {
- pos = 0;
- while (it < limit) {
- cleanup::PrefetchNode(it);
- cleanup::DestroyNode(pending_type[pos], pending_node[pos]);
- pending_type[pos] = cleanup::Type(it);
- pending_node[pos] = it;
- it += cleanup::Size(pending_type[pos]);
- pos = (pos + 1) % kPrefetchDist;
- }
- for (int i = pos; i < pos + kPrefetchDist; ++i) {
- cleanup::DestroyNode(pending_type[i % kPrefetchDist],
- pending_node[i % kPrefetchDist]);
- }
- }
+ while (it < limit) {
+ it += cleanup::DestroyNodeAt(it);
}
+ GOOGLE_DCHECK_EQ(it, limit);
b = b->next;
} while (b);
}
@@ -749,6 +736,19 @@
}
}
+void* ThreadSafeArena::AllocateCleanupFallback(size_t size) {
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ return GetSerialArenaFallback(size)->AllocateCleanup(size);
+}
+
+ThreadSafeArena::Memory ThreadSafeArena::AllocateCleanupFallback(size_t size,
+ size_t align) {
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(align));
+ size_t required = size + align - ArenaAlignDefault::align;
+ return GetSerialArenaFallback(required)->AllocateCleanup(size, align);
+}
+
void ThreadSafeArena::AddCleanup(void* elem, void (*cleanup)(void*)) {
SerialArena* arena;
if (PROTOBUF_PREDICT_FALSE(!GetSerialArenaFast(&arena))) {
@@ -895,6 +895,38 @@
return impl_.AllocateAlignedWithCleanup(n, align, destructor);
}
+template <typename T, internal::cleanup::Tag tag>
+void* Arena::AllocateWithCleanup() {
+ using internal::ArenaAlignDefault;
+ using internal::cleanup::TaggedNode;
+ static_assert(alignof(T) <= ArenaAlignDefault::align);
+ constexpr size_t n = ArenaAlignDefault::Ceil(sizeof(TaggedNode) + sizeof(T));
+ void* mem = impl_.AllocateCleanup(n);
+ internal::cleanup::CreateNode(mem, tag);
+ return static_cast<TaggedNode*>(mem) + 1;
+}
+
+void* Arena::AllocateWithCleanup(size_t size, void (*dtor)(void*)) {
+ using internal::ArenaAlignDefault;
+ using internal::cleanup::DynamicNode;
+ size_t required = ArenaAlignDefault::Ceil(size + sizeof(DynamicNode));
+ void* mem = impl_.AllocateCleanup(required);
+ internal::cleanup::CreateNode(mem, required, dtor);
+ return static_cast<DynamicNode*>(mem) + 1;
+}
+
+void* Arena::AllocateWithCleanup(size_t size, size_t align,
+ void (*dtor)(void*)) {
+ using internal::cleanup::DynamicNode;
+ size_t required = size + sizeof(DynamicNode);
+ auto mem = impl_.AllocateCleanup(required, align);
+ internal::cleanup::CreateNode(mem.ptr, mem.size, dtor);
+ return static_cast<DynamicNode*>(mem.ptr) + 1;
+}
+
+template void*
+Arena::AllocateWithCleanup<std::string, internal::cleanup::Tag::kString>();
+
} // namespace protobuf
} // namespace google
diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h
index abce219..426a1dd 100644
--- a/src/google/protobuf/arena.h
+++ b/src/google/protobuf/arena.h
@@ -253,6 +253,12 @@
return Arena::CreateMaybeMessage<T>(arena, static_cast<Args&&>(args)...);
}
+ template <typename T>
+ struct ObjectCreator {
+ template <typename... Args>
+ static T* Create(Arena* arena, Args&&... args);
+ };
+
// API to create any objects on the arena. Note that only the object will
// be created on the arena; the underlying ptrs (in case of a proto2 message)
// will be still heap allocated. Proto messages should usually be allocated
@@ -273,11 +279,7 @@
if (arena == nullptr) {
return new T(std::forward<Args>(args)...);
}
- auto destructor =
- internal::ObjectDestructor<std::is_trivially_destructible<T>::value,
- T>::destructor;
- return new (arena->AllocateInternal(sizeof(T), alignof(T), destructor))
- T(std::forward<Args>(args)...);
+ return ObjectCreator<T>::Create(arena, std::forward<Args>(args)...);
}
// API to delete any objects not on an arena. This can be used to safely
@@ -690,6 +692,11 @@
}
}
+ template <typename T, internal::cleanup::Tag tag>
+ void* AllocateWithCleanup();
+ void* AllocateWithCleanup(size_t size, void (*dtor)(void*));
+ void* AllocateWithCleanup(size_t size, size_t align, void (*dtor)(void*));
+
void* Allocate(size_t n);
void* AllocateForArray(size_t n);
void* AllocateAlignedWithCleanup(size_t n, size_t align,
@@ -710,6 +717,45 @@
friend struct internal::ArenaTestPeer;
};
+template <typename T>
+template <typename... Args>
+PROTOBUF_NDEBUG_INLINE T* Arena::ObjectCreator<T>::Create(Arena* arena,
+ Args&&... args) {
+ constexpr auto dtor = &internal::cleanup::arena_destruct_object<T>;
+
+ void* mem;
+ if (std::is_trivially_destructible<T>::value) {
+ // Simple case: trivially destructible
+ mem = arena->AllocateAligned(sizeof(T), alignof(T));
+ } else if (sizeof(T) <= 32) {
+ // The size of this item is small, and locality is important. We allocate
+ // the object inside the cleanup list itself. 32 bytes here is chosen as a
+ // reasonable 'half cache line' size. If we would embed large objects (one
+ // or more cache lines) into the cleanup list, we may actually hurt cleanup
+ // iteration as we then need to skip one or more cache lines of 'dead' data
+ // during cleanup. Custom dtors on large objects are more likely to address
+ // at most a handful of memory locations.
+ mem = (alignof(T) > internal::ArenaAlignDefault::align)
+ ? arena->AllocateWithCleanup(sizeof(T), alignof(T), dtor)
+ : arena->AllocateWithCleanup(sizeof(T), dtor);
+ } else {
+ // Fallback to 'old school' create internal
+ mem = arena->AllocateInternal(sizeof(T), alignof(T), dtor);
+ }
+ return new (mem) T(std::forward<Args>(args)...);
+}
+
+template <>
+struct Arena::ObjectCreator<std::string> {
+ template <typename... Args>
+ PROTOBUF_NDEBUG_INLINE static std::string* Create(Arena* arena,
+ Args&&... args) {
+ using internal::cleanup::Tag;
+ void* mem = arena->AllocateWithCleanup<std::string, Tag::kString>();
+ return new (mem) std::string(std::forward<Args>(args)...);
+ }
+};
+
} // namespace protobuf
} // namespace google
diff --git a/src/google/protobuf/arena_cleanup.h b/src/google/protobuf/arena_cleanup.h
index 0ca60c2..6492cc1 100644
--- a/src/google/protobuf/arena_cleanup.h
+++ b/src/google/protobuf/arena_cleanup.h
@@ -38,6 +38,7 @@
#include "google/protobuf/stubs/logging.h"
#include "google/protobuf/stubs/common.h"
#include "absl/base/attributes.h"
+#include "google/protobuf/arena_align.h"
// Must be included last.
@@ -58,10 +59,14 @@
// lowest 2 bits of the `elem` value identifying the type of node. All node
// types must start with a `uintptr_t` that stores `Tag` in its low two bits.
enum class Tag : uintptr_t {
- kDynamic = 0, // DynamicNode
- kString = 1, // StringNode (std::string)
+ kDynamic = 0, // DynamicNode
+ kEmbedded = 1, // Embedded node
+ kString = 2, // TaggedNode (std::string) (pointer or embedded)
+ kCord = 3, // TaggedNode (absl::Cord) (pointer or embedded)
};
+enum { kTagMask = 0x3 };
+
// DynamicNode contains the object (`elem`) that needs to be
// destroyed, and the function to destroy it (`destructor`)
// elem must be aligned at minimum on a 4 byte boundary.
@@ -70,18 +75,50 @@
void (*destructor)(void*);
};
-// StringNode contains a `std::string` object (`elem`) that needs to be
-// destroyed. The lowest 2 bits of `elem` contain the non-zero kString tag.
-struct StringNode {
+// TaggedNode describes a well known cleanup instance identified by the non-zero
+// tag value stored in the lowest 2 bits of `elem`.
+struct TaggedNode {
uintptr_t elem;
};
-
// EnableSpecializedTags() return true if the alignment of tagged objects
// such as std::string allow us to poke tags in the 2 LSB bits.
inline constexpr bool EnableSpecializedTags() {
// For now we require 2 bits
- return alignof(std::string) >= 8;
+ return alignof(std::string) > kTagMask;
+}
+
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CreateNode(void* pos,
+ const void* object,
+ void (*dtor)(void*)) {
+ auto elem = reinterpret_cast<uintptr_t>(object);
+ DynamicNode n = {elem, dtor};
+ memcpy(pos, &n, sizeof(n));
+}
+
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CreateNode(void* pos, size_t size,
+ void (*dtor)(void*)) {
+ GOOGLE_DCHECK_GT(size, 0);
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ auto elem = static_cast<uintptr_t>(size);
+ DynamicNode n = {elem + static_cast<uintptr_t>(Tag::kEmbedded), dtor};
+ memcpy(pos, &n, sizeof(n));
+}
+
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CreateNode(void* pos,
+ const void* object,
+ Tag tag) {
+ GOOGLE_DCHECK(tag != Tag::kDynamic && tag != Tag::kEmbedded);
+ auto elem = reinterpret_cast<uintptr_t>(object);
+ GOOGLE_DCHECK_EQ(elem & 3, 0ULL); // Must be aligned
+ TaggedNode n = {elem + static_cast<uintptr_t>(tag)};
+ memcpy(pos, &n, sizeof(n));
+}
+
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CreateNode(void* pos, Tag tag) {
+ GOOGLE_DCHECK(tag != Tag::kDynamic && tag != Tag::kEmbedded);
+ TaggedNode n = {static_cast<uintptr_t>(tag)};
+ memcpy(pos, &n, sizeof(n));
}
// Adds a cleanup entry identified by `tag` at memory location `pos`.
@@ -93,7 +130,7 @@
GOOGLE_DCHECK_EQ(elem & 3, 0ULL); // Must be aligned
switch (tag) {
case Tag::kString: {
- StringNode n = {elem | static_cast<uintptr_t>(Tag::kString)};
+ TaggedNode n = {elem | static_cast<uintptr_t>(Tag::kString)};
memcpy(pos, &n, sizeof(n));
return;
}
@@ -105,35 +142,49 @@
memcpy(pos, &n, sizeof(n));
}
-// Optimization: performs a prefetch on `elem_address`.
-inline ABSL_ATTRIBUTE_ALWAYS_INLINE void PrefetchNode(
- const void* elem_address) {
- (void)elem_address;
+template <typename T>
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE size_t DestroyTaggedObject(void* pos,
+ uintptr_t elem) {
+ // elem == 0 identifies embedded, elem != 0 pointer cleanup
+ if (elem) {
+ reinterpret_cast<T*>(elem)->~T();
+ return sizeof(TaggedNode);
+ } else {
+ reinterpret_cast<T*>(static_cast<TaggedNode*>(pos) + 1)->~T();
+ return ArenaAlignDefault::Ceil(sizeof(TaggedNode) + sizeof(T));
+ }
}
// Destroys the node idenitfied by `tag` stored at memory location `pos`.
-inline ABSL_ATTRIBUTE_ALWAYS_INLINE void DestroyNode(Tag tag, const void* pos) {
- if (EnableSpecializedTags()) {
- switch (tag) {
- case Tag::kString: {
- StringNode n;
- memcpy(&n, pos, sizeof(n));
- auto* s = reinterpret_cast<std::string*>(n.elem & ~0x7ULL);
- // Some compilers don't like fully qualified explicit dtor calls,
- // so use an alias to avoid having to type `::`.
- using string_type = std::string;
- s->~string_type();
- return;
- }
- default:
- break;
+inline ABSL_ATTRIBUTE_ALWAYS_INLINE size_t DestroyNodeAt(void* pos) {
+ uintptr_t head = *reinterpret_cast<uintptr_t*>(pos);
+ uintptr_t tag = head & kTagMask;
+ switch (static_cast<Tag>(tag)) {
+ case Tag::kDynamic: {
+ auto* node = static_cast<DynamicNode*>(pos);
+ GOOGLE_DCHECK(node->destructor != nullptr);
+ GOOGLE_DCHECK(head);
+ node->destructor(reinterpret_cast<void*>(head));
+ return sizeof(DynamicNode);
}
+
+ case Tag::kEmbedded: {
+ auto* node = static_cast<DynamicNode*>(pos);
+ GOOGLE_DCHECK(node->destructor != nullptr);
+ GOOGLE_DCHECK_GT(head - tag, 0);
+ node->destructor(node + 1);
+ return head - tag;
+ }
+
+ case Tag::kString:
+ return DestroyTaggedObject<std::string>(pos, head - tag);
+
+ case Tag::kCord:
+ return DestroyTaggedObject<absl::Cord>(pos, head - tag);
}
- DynamicNode n;
- memcpy(&n, pos, sizeof(n));
- n.destructor(reinterpret_cast<void*>(n.elem));
}
+
// Returns the `tag` identifying the type of object for `destructor` or
// kDynamic if `destructor` does not identify a well know object type.
inline ABSL_ATTRIBUTE_ALWAYS_INLINE Tag Type(void (*destructor)(void*)) {
@@ -169,9 +220,11 @@
switch (tag) {
case Tag::kDynamic:
+ case Tag::kEmbedded:
return sizeof(DynamicNode);
case Tag::kString:
- return sizeof(StringNode);
+ case Tag::kCord:
+ return sizeof(TaggedNode);
default:
GOOGLE_LOG(FATAL) << "Corrupted cleanup tag: " << static_cast<int>(tag);
return sizeof(DynamicNode);
diff --git a/src/google/protobuf/arena_cleanup_test.cc b/src/google/protobuf/arena_cleanup_test.cc
new file mode 100644
index 0000000..f9b568d
--- /dev/null
+++ b/src/google/protobuf/arena_cleanup_test.cc
@@ -0,0 +1,106 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2022 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "google/protobuf/arena_cleanup.h"
+
+#include <string>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace internal {
+namespace cleanup {
+namespace {
+
+using testing::Eq;
+
+struct DtorTracker {
+ DtorTracker() {
+ count = 0;
+ object = nullptr;
+ }
+
+ static void dtor(void* obj) {
+ ++count;
+ object = obj;
+ }
+
+ static int count;
+ static void* object;
+};
+
+int DtorTracker::count;
+void* DtorTracker::object;
+
+TEST(CleanupTest, CreateDestroyDynamicNode) {
+ alignas(ArenaAlignDefault::align) char buffer[1024];
+ void* object = &object;
+ DtorTracker dtor_tracker;
+
+ CreateNode(buffer, object, &DtorTracker::dtor);
+ EXPECT_THAT(DestroyNodeAt(buffer), Eq(sizeof(DynamicNode)));
+ EXPECT_THAT(DtorTracker::count, Eq(1));
+ EXPECT_THAT(DtorTracker::object, Eq(object));
+}
+
+TEST(CleanupTest, CreateDestroyEmbeddedNode) {
+ alignas(ArenaAlignDefault::align) char buffer[1024];
+ void* object = buffer + sizeof(DynamicNode);
+ DtorTracker dtor_tracker;
+
+ CreateNode(buffer, sizeof(DynamicNode) + 128, &DtorTracker::dtor);
+ EXPECT_THAT(DestroyNodeAt(buffer), Eq(sizeof(DynamicNode) + 128));
+ EXPECT_THAT(DtorTracker::count, Eq(1));
+ EXPECT_THAT(DtorTracker::object, Eq(object));
+}
+
+TEST(CleanupTest, CreateDestroyStringNode) {
+ alignas(ArenaAlignDefault::align) char buffer[1024];
+ alignas(std::string) char instance[sizeof(std::string)];
+ std::string* s = new (instance) std::string(1000, 'x');
+ CreateNode(buffer, s, Tag::kString);
+ EXPECT_THAT(DestroyNodeAt(buffer), Eq(sizeof(TaggedNode)));
+}
+
+TEST(CleanupTest, CreateDestroyEmbeddedStringNode) {
+ alignas(ArenaAlignDefault::align) char buffer[1024];
+ new (buffer + sizeof(TaggedNode)) std::string(1000, 'x');
+ CreateNode(buffer, Tag::kString);
+ EXPECT_THAT(DestroyNodeAt(buffer),
+ Eq(sizeof(TaggedNode) + sizeof(std::string)));
+}
+
+} // namespace
+} // namespace cleanup
+} // namespace internal
+} // namespace protobuf
+} // namespace google
diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h
index 9d2de93..5b0df53 100644
--- a/src/google/protobuf/arena_impl.h
+++ b/src/google/protobuf/arena_impl.h
@@ -183,6 +183,10 @@
}
uint64_t SpaceUsed() const;
+ inline size_t SpaceAvailable() const {
+ return static_cast<size_t>(limit_ - ptr());
+ }
+
bool HasSpace(size_t n) const {
return n <= static_cast<size_t>(limit_ - ptr());
}
@@ -287,6 +291,25 @@
}
public:
+ PROTOBUF_NDEBUG_INLINE
+ void* AllocateCleanup(size_t size) {
+ GOOGLE_DCHECK(ArenaAlignDefault::IsAligned(size));
+ if (PROTOBUF_PREDICT_TRUE(size <= SpaceAvailable())) {
+ return limit_ -= size;
+ }
+ return AllocateCleanupFallback(size);
+ }
+
+ PROTOBUF_NDEBUG_INLINE
+ Memory AllocateCleanup(size_t size, size_t align) {
+ size_t tail_padding = reinterpret_cast<size_t>(limit_) & (align - 1);
+ if (PROTOBUF_PREDICT_TRUE(size + tail_padding <= SpaceAvailable())) {
+ size += tail_padding;
+ return {limit_ -= size, size};
+ }
+ return AllocateCleanupFallback(size, align);
+ }
+
// Allocate space if the current region provides enough space.
bool MaybeAllocateAligned(size_t n, void** out) {
GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned.
@@ -421,6 +444,8 @@
void* AllocateAlignedWithCleanupFallback(size_t n, size_t align,
void (*destructor)(void*));
void AddCleanupFallback(void* elem, void (*destructor)(void*));
+ void* AllocateCleanupFallback(size_t size);
+ Memory AllocateCleanupFallback(size_t size, size_t align);
inline void AllocateNewBlock(size_t n);
inline void Init(ArenaBlock* b, size_t offset);
@@ -443,6 +468,8 @@
// use #ifdef the select the best implementation based on hardware / OS.
class PROTOBUF_EXPORT ThreadSafeArena {
public:
+ using Memory = SerialArena::Memory;
+
ThreadSafeArena();
// Constructor solely used by message-owned arena.
@@ -507,6 +534,23 @@
// Add object pointer and cleanup function pointer to the list.
void AddCleanup(void* elem, void (*cleanup)(void*));
+ // Allocate cleanup space.
+ PROTOBUF_NDEBUG_INLINE void* AllocateCleanup(size_t size) {
+ SerialArena* arena;
+ if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
+ return arena->AllocateCleanup(size);
+ }
+ return AllocateCleanupFallback(size);
+ }
+
+ PROTOBUF_NDEBUG_INLINE Memory AllocateCleanup(size_t size, size_t align) {
+ SerialArena* arena;
+ if (PROTOBUF_PREDICT_TRUE(GetSerialArenaFast(&arena))) {
+ return arena->AllocateCleanup(size, align);
+ }
+ return AllocateCleanupFallback(size, align);
+ }
+
// Checks whether this arena is message-owned.
PROTOBUF_ALWAYS_INLINE bool IsMessageOwned() const {
return tag_and_id_ & kMessageOwnedArena;
@@ -566,6 +610,8 @@
void InitializeWithPolicy(const AllocationPolicy& policy);
void* AllocateAlignedWithCleanupFallback(size_t n, size_t align,
void (*destructor)(void*));
+ void* AllocateCleanupFallback(size_t size);
+ Memory AllocateCleanupFallback(size_t size, size_t align);
void Init();
