Internal cleanup PiperOrigin-RevId: 1012925172
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index f9f576a..950e9a1 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc
@@ -130,18 +130,19 @@ } // namespace cord_internal +static bool ShouldCopyStringData(const std::string& src) { + // Copy short strings to avoid external block overhead, and copy wasteful + // strings to avoid pinning too much unused memory. + return src.size() <= kMaxBytesToCopy || src.size() < src.capacity() / 2; +} + // Creates a CordRep from the provided string. If the string is large enough, // and not wasteful, we move the string into an external cord rep, preserving // the already allocated string contents. // Requires the provided string length to be larger than `kMaxInline`. static CordRep* absl_nonnull CordRepFromString(std::string&& src) { assert(src.length() > cord_internal::kMaxInline); - if ( - // String is short: copy data to avoid external block overhead. - src.size() <= kMaxBytesToCopy || - // String is wasteful: copy data to avoid pinning too much unused memory. - src.size() < src.capacity() / 2 - ) { + if (ShouldCopyStringData(src)) { return NewTree(src.data(), src.size(), 0); } @@ -183,9 +184,17 @@ void Cord::InlineRep::AppendTreeToTree(CordRep* absl_nonnull tree, MethodIdentifier method) { assert(is_tree()); + CordRep* old_tree = data_.as_tree(); + if (ABSL_PREDICT_FALSE(old_tree->length == 0)) { + { + const CordzUpdateScope scope(data_.cordz_info(), method); + SetTree(tree, scope); + } + CordRep::Unref(old_tree); + return; + } const CordzUpdateScope scope(data_.cordz_info(), method); - tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree); - SetTree(tree, scope); + SetTree(CordRepBtree::Append(ForceBtree(old_tree), tree), scope); } void Cord::InlineRep::AppendTree(CordRep* absl_nonnull tree, @@ -213,9 +222,17 @@ void Cord::InlineRep::PrependTreeToTree(CordRep* absl_nonnull tree, MethodIdentifier method) { assert(is_tree()); + CordRep* old_tree = data_.as_tree(); + if (ABSL_PREDICT_FALSE(old_tree->length == 0)) { + { + const CordzUpdateScope scope(data_.cordz_info(), method); + SetTree(tree, scope); + } + CordRep::Unref(old_tree); + return; + } const CordzUpdateScope scope(data_.cordz_info(), method); - tree = CordRepBtree::Prepend(ForceBtree(data_.as_tree()), tree); - SetTree(tree, scope); + SetTree(CordRepBtree::Prepend(ForceBtree(old_tree), tree), scope); } void Cord::InlineRep::PrependTree(CordRep* absl_nonnull tree, @@ -274,20 +291,28 @@ assert(&src != this); assert(is_tree() || src.is_tree()); auto constexpr method = CordzUpdateTracker::kAssignCord; + CordRep* src_tree = src.tree(); + const bool copy_src_tree = + src_tree != nullptr && + ABSL_PREDICT_TRUE(src_tree->length > 0 || src_tree->IsCrc()); if (ABSL_PREDICT_TRUE(!is_tree())) { - EmplaceTree(CordRep::Ref(src.as_tree()), src.data_, method); + if (copy_src_tree) { + EmplaceTree(CordRep::Ref(src_tree), src.data_, method); + } else { + data_ = {}; + } return; } CordRep* tree = as_tree(); - if (CordRep* src_tree = src.tree()) { + if (copy_src_tree) { // Leave any existing `cordz_info` in place, and let MaybeTrackCord() // decide if this cord should be (or remains to be) sampled or not. data_.set_tree(CordRep::Ref(src_tree)); CordzInfo::MaybeTrackCord(data_, src.data_, method); } else { CordzInfo::MaybeUntrackCord(data_.cordz_info()); - data_ = src.data_; + data_ = src_tree == nullptr ? src.data_ : InlineData{}; } CordRep::Unref(tree); } @@ -341,16 +366,58 @@ } } +namespace cord_internal { +void ClearCordPreservingCapacity(Cord& cord) { + CordRep* tree = cord.contents_.tree(); + if (tree == nullptr) { + cord.contents_.ResetToEmpty(); + return; + } + { + CordzUpdateScope scope(cord.contents_.cordz_info(), + CordzUpdateTracker::kClear); + if (tree->IsFlat() && + tree->flat()->AllocatedSize() <= cord_internal::kMaxFlatSize && + tree->refcount.IsOne()) { + tree->length = 0; + VerifyTree(tree); + return; + } + cord.contents_.SetTreeOrEmpty(nullptr, scope); + } + CordRep::Unref(tree); +} +} // namespace cord_internal + Cord& Cord::AssignLargeString(std::string&& src) { auto constexpr method = CordzUpdateTracker::kAssignString; assert(src.size() > kMaxBytesToCopy); - CordRep* rep = CordRepFromString(std::move(src)); if (CordRep* tree = contents_.tree()) { - CordzUpdateScope scope(contents_.cordz_info(), method); - contents_.SetTree(rep, scope); + CordRep* rep; + if (ShouldCopyStringData(src)) { + { + CordzUpdateScope scope(contents_.cordz_info(), method); + if (tree->IsFlat() && tree->flat()->Capacity() >= src.size() && + tree->refcount.IsOne()) { + memcpy(tree->flat()->Data(), src.data(), src.size()); + tree->length = src.size(); + VerifyTree(tree); + return *this; + } + rep = NewTree(src.data(), src.size(), 0); + contents_.SetTree(rep, scope); + } + CordRep::Unref(tree); + return *this; + } + rep = CordRepFromString(std::move(src)); + { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTree(rep, scope); + } CordRep::Unref(tree); } else { - contents_.EmplaceTree(rep, method); + contents_.EmplaceTree(CordRepFromString(std::move(src)), method); } return *this; } @@ -361,13 +428,28 @@ size_t length = src.size(); CordRep* tree = contents_.tree(); if (length <= InlineRep::kMaxInline) { + if (ABSL_PREDICT_TRUE(tree == nullptr)) { + contents_.set_data(data, length); + return *this; + } + if (ABSL_PREDICT_FALSE(tree->length == 0) && tree->IsFlat()) { + CordzUpdateScope scope(contents_.cordz_info(), method); + if (ABSL_PREDICT_TRUE(tree->refcount.IsOne())) { + if (length > 0) { + memmove(tree->flat()->Data(), data, length); + tree->length = length; + } + VerifyTree(tree); + return *this; + } + } // Embed into this->contents_, which is somewhat subtle: // - MaybeUntrackCord must be called before Unref(tree). // - MaybeUntrackCord must be called before set_data() clobbers cordz_info. // - set_data() must be called before Unref(tree) as it may reference tree. - if (tree != nullptr) CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + CordzInfo::MaybeUntrackCord(contents_.cordz_info()); contents_.set_data(data, length); - if (tree != nullptr) CordRep::Unref(tree); + CordRep::Unref(tree); return *this; } if (tree != nullptr) { @@ -398,45 +480,60 @@ size_t appended = 0; CordRep* rep = tree(); const CordRep* const root = rep; - CordzUpdateScope scope(root ? cordz_info() : nullptr, method); - if (root != nullptr) { - rep = cord_internal::RemoveCrcNode(rep); - char* region; - if (PrepareAppendRegion(rep, ®ion, &appended, src.size())) { - memcpy(region, src.data(), appended); - } - } else { - // Try to fit in the inline buffer if possible. - size_t inline_length = inline_size(); - if (src.size() <= kMaxInline - inline_length) { - // Append new data to embedded array - set_inline_size(inline_length + src.size()); - memcpy(data_.as_chars() + inline_length, src.data(), src.size()); - return; + CordRep* unref_rep = nullptr; + { + CordzUpdateScope scope(root ? cordz_info() : nullptr, method); + if (root != nullptr) { + rep = cord_internal::RemoveCrcNode(rep); + char* region; + if (ABSL_PREDICT_FALSE(rep->length == 0)) { + if (rep->IsFlat() && rep->refcount.IsOne() && + rep->flat()->Capacity() >= + (std::min)(src.size(), cord_internal::kMaxFlatLength)) { + appended = (std::min)(src.size(), rep->flat()->Capacity()); + memcpy(rep->flat()->Data(), src.data(), appended); + rep->length = appended; + } else { + unref_rep = rep; + rep = NewTree(src.data(), src.size(), 0); + appended = src.size(); + } + } else if (PrepareAppendRegion(rep, ®ion, &appended, src.size())) { + memcpy(region, src.data(), appended); + } + } else { + // Try to fit in the inline buffer if possible. + size_t inline_length = inline_size(); + if (src.size() <= kMaxInline - inline_length) { + // Append new data to embedded array + set_inline_size(inline_length + src.size()); + memcpy(data_.as_chars() + inline_length, src.data(), src.size()); + return; + } + + // Allocate flat to be a perfect fit on first append exceeding inlined + // size. Subsequent growth will use amortized growth until we reach + // maximum flat size. + rep = CordRepFlat::New(inline_length + src.size()); + appended = + (std::min)(src.size(), rep->flat()->Capacity() - inline_length); + memcpy(rep->flat()->Data(), data_.as_chars(), inline_length); + memcpy(rep->flat()->Data() + inline_length, src.data(), appended); + rep->length = inline_length + appended; } - // Allocate flat to be a perfect fit on first append exceeding inlined size. - // Subsequent growth will use amortized growth until we reach maximum flat - // size. - rep = CordRepFlat::New(inline_length + src.size()); - appended = std::min(src.size(), rep->flat()->Capacity() - inline_length); - memcpy(rep->flat()->Data(), data_.as_chars(), inline_length); - memcpy(rep->flat()->Data() + inline_length, src.data(), appended); - rep->length = inline_length + appended; - } - - src.remove_prefix(appended); - if (src.empty()) { + src.remove_prefix(appended); + if (!src.empty()) { + // TODO(b/192061034): keep legacy 10% growth rate: consider other rates. + rep = ForceBtree(rep); + const size_t min_growth = (std::max)(rep->length / 10, src.size()); + rep = CordRepBtree::Append(rep->btree(), src, min_growth - src.size()); + } CommitTree(root, rep, scope, method); - return; } - - // TODO(b/192061034): keep legacy 10% growth rate: consider other rates. - rep = ForceBtree(rep); - const size_t min_growth = std::max<size_t>(rep->length / 10, src.size()); - rep = CordRepBtree::Append(rep->btree(), src, min_growth - src.size()); - - CommitTree(root, rep, scope, method); + if (ABSL_PREDICT_FALSE(unref_rep != nullptr)) { + CordRep::Unref(unref_rep); + } } inline CordRep* absl_nonnull Cord::TakeRep() const& { @@ -459,22 +556,47 @@ std::numeric_limits<size_t>::max() - contents_.size(), "Cord length overflow"); + const size_t src_size = src.contents_.size(); if (empty()) { - // Since destination is empty, we can avoid allocating a node, - if (src.contents_.is_tree()) { - // by taking the tree directly - CordRep* rep = - cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); - contents_.EmplaceTree(rep, method); + if (ABSL_PREDICT_FALSE(contents_.is_tree())) { + CordRep* old_tree = contents_.as_tree(); + if (!old_tree->IsFlat() || !old_tree->refcount.IsOne() || + old_tree->flat()->Capacity() < src_size || + src_size > kMaxBytesToCopy) { + if (src.contents_.is_tree()) { + CordRep* rep = + cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTree(rep, scope); + } + } else { + { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTreeOrEmpty(nullptr, scope); + } + contents_.data_ = src.contents_.data_; + } + CordRep::Unref(old_tree); + return; + } + // Fall through to copy `src` into the unshared cached flat in place. } else { - // or copying over inline data - contents_.data_ = src.contents_.data_; + // Since destination is empty, we can avoid allocating a node, + if (src.contents_.is_tree()) { + // by taking the tree directly + CordRep* rep = + cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + contents_.EmplaceTree(rep, method); + } else { + // or copying over inline data + contents_.data_ = src.contents_.data_; + } + return; } - return; } // For short cords, it is faster to copy data if there is room in dst. - const size_t src_size = src.contents_.size(); if (src_size <= kMaxBytesToCopy) { CordRep* src_tree = src.contents_.tree(); if (src_tree == nullptr) { @@ -538,11 +660,42 @@ auto constexpr method = CordzUpdateTracker::kGetAppendBuffer; CordRep* tree = contents_.tree(); if (tree != nullptr) { - CordzUpdateScope scope(contents_.cordz_info(), method); - CordRep::ExtractResult result = ExtractAppendBuffer(tree, min_capacity); - if (result.extracted != nullptr) { - contents_.SetTreeOrEmpty(result.tree, scope); - return CordBuffer(result.extracted->flat()); + if (ABSL_PREDICT_FALSE(tree->length == 0)) { + if (tree->IsFlat()) { + const size_t desired = + block_size == 0 + ? (std::min)(capacity, CordBuffer::MaximumPayload()) + : CordBuffer::DesiredCustomSize(block_size, capacity); + bool reuse = false; + { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTreeOrEmpty(nullptr, scope); + reuse = tree->refcount.IsOne() && + tree->flat()->Capacity() >= desired && + tree->flat()->Capacity() >= min_capacity && + (block_size == 0 || + tree->flat()->AllocatedSize() <= block_size); + } + if (reuse) { + return CordBuffer(tree->flat()); + } + CordRep::Unref(tree); + } + return block_size + ? CordBuffer::CreateWithCustomLimit(block_size, capacity) + : CordBuffer::CreateWithDefaultLimit(capacity); + } + CordRep* extracted = nullptr; + { + CordzUpdateScope scope(contents_.cordz_info(), method); + CordRep::ExtractResult result = ExtractAppendBuffer(tree, min_capacity); + if (result.extracted != nullptr) { + contents_.SetTreeOrEmpty(result.tree, scope); + extracted = result.extracted; + } + } + if (extracted != nullptr) { + return CordBuffer(extracted->flat()); } return block_size ? CordBuffer::CreateWithCustomLimit(block_size, capacity) : CordBuffer::CreateWithDefaultLimit(capacity); @@ -556,7 +709,10 @@ template <typename T, Cord::EnableIfString<T>> void Cord::Append(T&& src) { - if (src.size() <= kMaxBytesToCopy) { + if (src.size() <= kMaxBytesToCopy || + (ABSL_PREDICT_FALSE(contents_.is_tree() && + contents_.as_tree()->length == 0) && + ShouldCopyStringData(src))) { Append(absl::string_view(src)); } else { contents_.MaybeRemoveEmptyCrcNode(); @@ -602,6 +758,9 @@ contents_.data_ = data; return; } + } else if (ABSL_PREDICT_FALSE(contents_.as_tree()->length == 0)) { + contents_.AppendArray(src, method); + return; } CordRep* rep = NewTree(src.data(), src.size(), 0); contents_.PrependTree(rep, method); @@ -614,6 +773,9 @@ const size_t inline_length = contents_.inline_size(); contents_.set_inline_size(inline_length + src.size()); memcpy(contents_.data_.as_chars() + inline_length, src.data(), src.size()); + } else if (ABSL_PREDICT_FALSE(contents_.is_tree() && + contents_.as_tree()->length == 0)) { + contents_.AppendArray(src, method); } else { contents_.AppendTree(CordRepFlat::Create(src), method); } @@ -629,6 +791,9 @@ memcpy(data.as_chars(), src.data(), src.size()); memcpy(data.as_chars() + src.size(), contents_.data(), cur_size); contents_.data_ = data; + } else if (ABSL_PREDICT_FALSE(contents_.is_tree() && + contents_.as_tree()->length == 0)) { + contents_.AppendArray(src, method); } else { contents_.PrependTree(CordRepFlat::Create(src), method); } @@ -636,7 +801,10 @@ template <typename T, Cord::EnableIfString<T>> inline void Cord::Prepend(T&& src) { - if (src.size() <= kMaxBytesToCopy) { + if (src.size() <= kMaxBytesToCopy || + (ABSL_PREDICT_FALSE(contents_.is_tree() && + contents_.as_tree()->length == 0) && + ShouldCopyStringData(src))) { Prepend(absl::string_view(src)); } else { contents_.MaybeRemoveEmptyCrcNode(); @@ -834,7 +1002,9 @@ void Cord::SetCrcCordState(crc_internal::CrcCordState state) { auto constexpr method = CordzUpdateTracker::kSetExpectedChecksum; if (empty()) { - contents_.MaybeRemoveEmptyCrcNode(); + if (CordRep* old_tree = contents_.clear()) { + CordRep::Unref(old_tree); + } CordRep* rep = CordRepCrc::New(nullptr, std::move(state)); contents_.EmplaceTree(rep, method); } else if (!contents_.is_tree()) { @@ -1053,8 +1223,10 @@ void Cord::CopyToArraySlowPath(char* absl_nonnull dst) const { assert(contents_.is_tree()); absl::string_view fragment; - if (GetFlatAux(contents_.tree(), &fragment) && !fragment.empty()) { - memcpy(dst, fragment.data(), fragment.size()); + if (GetFlatAux(contents_.tree(), &fragment)) { + if (!fragment.empty()) { + memcpy(dst, fragment.data(), fragment.size()); + } return; } for (absl::string_view chunk : Chunks()) { @@ -1064,6 +1236,7 @@ } size_t CopyCordToSpan(const Cord& src, absl::Span<char> dst) { + if (src.empty()) return 0; if (src.size() <= dst.size()) { src.CopyToArrayImpl(dst.data()); return src.size(); @@ -1087,6 +1260,9 @@ Cord subcord; auto constexpr method = CordzUpdateTracker::kCordReader; + if (n == 0) { + return subcord; + } if (n <= InlineRep::kMaxInline) { // Range to read fits in inline data. Flatten it. char* data = subcord.contents_.set_data(n); @@ -1510,31 +1686,31 @@ /* static */ bool Cord::GetFlatAux(CordRep* absl_nonnull rep, absl::string_view* absl_nonnull fragment) { assert(rep != nullptr); - if (rep->length == 0) { - *fragment = absl::string_view(); + CordRep* node = cord_internal::SkipCrcNode(rep); + if (node == nullptr) { + *fragment = absl::string_view(""); return true; } - rep = cord_internal::SkipCrcNode(rep); - if (rep->IsFlat()) { - *fragment = absl::string_view(rep->flat()->Data(), rep->length); + if (node->IsFlat()) { + *fragment = absl::string_view(node->flat()->Data(), node->length); return true; - } else if (rep->IsExternal()) { - *fragment = absl::string_view(rep->external()->base, rep->length); + } else if (node->IsExternal()) { + *fragment = absl::string_view(node->external()->base, node->length); return true; - } else if (rep->IsBtree()) { - return rep->btree()->IsFlat(fragment); - } else if (rep->IsSubstring()) { - CordRep* child = rep->substring()->child; + } else if (node->IsBtree()) { + return node->btree()->IsFlat(fragment); + } else if (node->IsSubstring()) { + CordRep* child = node->substring()->child; if (child->IsFlat()) { *fragment = absl::string_view( - child->flat()->Data() + rep->substring()->start, rep->length); + child->flat()->Data() + node->substring()->start, node->length); return true; } else if (child->IsExternal()) { *fragment = absl::string_view( - child->external()->base + rep->substring()->start, rep->length); + child->external()->base + node->substring()->start, node->length); return true; } else if (child->IsBtree()) { - return child->btree()->IsFlat(rep->substring()->start, rep->length, + return child->btree()->IsFlat(node->substring()->start, node->length, fragment); } } @@ -1635,6 +1811,14 @@ ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node)); ABSL_INTERNAL_CHECK(!node->IsCrc(), ReportError(root, node)); } + if (node->length == 0) { + ABSL_INTERNAL_CHECK( + node == root && + (node->IsCrc() || + (node->IsFlat() && + node->flat()->AllocatedSize() <= cord_internal::kMaxFlatSize)), + ReportError(root, node)); + } if (node->IsFlat()) { ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(),
diff --git a/absl/strings/cord.h b/absl/strings/cord.h index 8fe8e9f..e712038 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h
@@ -114,6 +114,18 @@ void AppendCordToString(const Cord& src, std::string* absl_nonnull dst); [[nodiscard]] size_t CopyCordToSpan(const Cord& src, absl::Span<char> dst); +#define ABSL_INTERNAL_HAVE_CLEAR_CORD_PRESERVING_CAPACITY 1 + +namespace cord_internal { +// Clears `cord` to empty (`size() == 0`), retaining its backing `CordRepFlat` +// buffer when `cord` uniquely owns a flat node of at most `kMaxFlatSize` +// bytes so subsequent writes can reuse the allocation in place. While +// `cord.empty()` is `true` and `cord.size() == 0`, +// `cord.EstimatedMemoryUsage()` continues to report the retained flat's +// capacity until explicitly cleared via `Cord::Clear()` or replaced. +void ClearCordPreservingCapacity(Cord& cord); +} // namespace cord_internal + // Cord memory accounting modes enum class CordMemoryAccounting { // Counts the *approximate* number of bytes held in full or in part by this @@ -866,6 +878,7 @@ friend class ::strings::CordReader; friend class CordTestPeer; + friend void cord_internal::ClearCordPreservingCapacity(Cord& cord); friend bool operator==(const Cord& lhs, const Cord& rhs); friend bool operator==(const Cord& lhs, absl::string_view rhs); @@ -1004,6 +1017,7 @@ void CopyToArray(char* absl_nonnull dst) const; bool is_tree() const { return data_.is_tree(); } + bool is_inline_empty() const { return data_.is_empty(); } // Returns true if the Cord is being profiled by cordz. bool is_profiled() const { return data_.is_tree() && data_.is_profiled(); } @@ -1029,6 +1043,7 @@ private: friend class Cord; + friend void cord_internal::ClearCordPreservingCapacity(Cord& cord); void AssignSlow(const InlineRep& src); // Unrefs the tree and stops profiling. @@ -1042,8 +1057,10 @@ // Empty cords that carry a checksum have a CordRepCrc node with a null // child node. The code can avoid lots of special cases where it would // otherwise transition from tree to inline storage if we just remove the - // CordRepCrc node before mutations. Must never be called inside a - // CordzUpdateScope since it untracks the cordz info. + // CordRepCrc node before mutations. 0-length CordRepFlat nodes from + // ClearCordPreservingCapacity are intentionally kept so mutators can reuse + // their capacity. Must never be called inside a CordzUpdateScope since it + // untracks the cordz info. void MaybeRemoveEmptyCrcNode(); cord_internal::InlineData data_; @@ -1185,8 +1202,12 @@ inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) { if (CordRep* tree = src.tree()) { - EmplaceTree(CordRep::Ref(tree), src.data_, - CordzUpdateTracker::kConstructorCord); + if (ABSL_PREDICT_TRUE(tree->length > 0 || tree->IsCrc())) { + EmplaceTree(CordRep::Ref(tree), src.data_, + CordzUpdateTracker::kConstructorCord); + } else { + data_ = {}; + } } else { data_ = src.data_; } @@ -1355,10 +1376,9 @@ inline void Cord::InlineRep::MaybeRemoveEmptyCrcNode() { CordRep* rep = tree(); - if (rep == nullptr || ABSL_PREDICT_TRUE(rep->length > 0)) { + if (rep == nullptr || ABSL_PREDICT_TRUE(rep->length > 0) || !rep->IsCrc()) { return; } - assert(rep->IsCrc()); assert(rep->crc()->child == nullptr); CordzInfo::MaybeUntrackCord(cordz_info()); CordRep::Unref(rep); @@ -1477,14 +1497,16 @@ } inline CordBuffer Cord::GetAppendBuffer(size_t capacity, size_t min_capacity) { - if (empty()) return CordBuffer::CreateWithDefaultLimit(capacity); + if (contents_.is_inline_empty()) { + return CordBuffer::CreateWithDefaultLimit(capacity); + } return GetAppendBufferSlowPath(0, capacity, min_capacity); } inline CordBuffer Cord::GetCustomAppendBuffer(size_t block_size, size_t capacity, size_t min_capacity) { - if (empty()) { + if (contents_.is_inline_empty()) { return block_size ? CordBuffer::CreateWithCustomLimit(block_size, capacity) : CordBuffer::CreateWithDefaultLimit(capacity); } @@ -1519,7 +1541,7 @@ inline void Cord::CopyToArrayImpl(char* absl_nonnull dst) const { if (!contents_.is_tree()) { if (!empty()) contents_.CopyToArray(dst); - } else { + } else if (!empty()) { CopyToArraySlowPath(dst); } }
diff --git a/absl/strings/cord_buffer.h b/absl/strings/cord_buffer.h index c3803eb..b03337c 100644 --- a/absl/strings/cord_buffer.h +++ b/absl/strings/cord_buffer.h
@@ -419,6 +419,9 @@ return static_cast<size_t>(absl::bit_width(size - 1)); } + // Returns the desired payload size for `CreateWithCustomLimit()`. + static size_t DesiredCustomSize(size_t block_size, size_t capacity); + // Implementation of `CreateWithCustomLimit()`. // This implementation allows for future memory allocation hints to // be passed down into the CordRepFlat allocation function. @@ -473,9 +476,8 @@ return CordBuffer(); } -template <typename... AllocationHints> -inline CordBuffer CordBuffer::CreateWithCustomLimitImpl( - size_t block_size, size_t capacity, AllocationHints... hints) { +inline size_t CordBuffer::DesiredCustomSize(size_t block_size, + size_t capacity) { assert(IsPow2(block_size)); capacity = (std::min)(capacity, kCustomLimit); block_size = (std::min)(block_size, kCustomLimit); @@ -498,7 +500,13 @@ capacity = rounded_down; } } - const size_t length = capacity - kOverhead; + return capacity > kOverhead ? capacity - kOverhead : 0; +} + +template <typename... AllocationHints> +inline CordBuffer CordBuffer::CreateWithCustomLimitImpl( + size_t block_size, size_t capacity, AllocationHints... hints) { + const size_t length = DesiredCustomSize(block_size, capacity); auto* rep = CordRepFlat::New(CordRepFlat::Large(), length, hints...); rep->length = 0; return CordBuffer(rep);
diff --git a/absl/strings/cord_test.cc b/absl/strings/cord_test.cc index aed23ef..027fc33 100644 --- a/absl/strings/cord_test.cc +++ b/absl/strings/cord_test.cc
@@ -3848,6 +3848,779 @@ EXPECT_EQ(absl::HashOf(c3), absl::HashOf(absl::string_view())); } +#ifndef ABSL_INTERNAL_HAVE_CLEAR_CORD_PRESERVING_CAPACITY +#error ABSL_INTERNAL_HAVE_CLEAR_CORD_PRESERVING_CAPACITY must be defined. +#endif +static_assert(ABSL_INTERNAL_HAVE_CLEAR_CORD_PRESERVING_CAPACITY == 1, ""); + +namespace { + +void ExpectEmptyClearedFlat(absl::Cord& mutable_cord, + const char* expected_ptr) { + const absl::Cord& cord = mutable_cord; + EXPECT_TRUE(absl::CordTestPeer::IsTree(cord)); + EXPECT_GT(cord.EstimatedMemoryUsage(), sizeof(absl::Cord)); + EXPECT_TRUE(cord.empty()); + EXPECT_EQ(cord.size(), 0); + EXPECT_EQ(cord, ""); + EXPECT_EQ(cord, absl::Cord()); + EXPECT_EQ(absl::Cord(), cord); + EXPECT_EQ(cord.Compare(""), 0); + EXPECT_EQ(cord.Compare(absl::Cord()), 0); + EXPECT_LT(cord.Compare("a"), 0); + EXPECT_GT(absl::Cord("a").Compare(cord), 0); + EXPECT_TRUE(cord.StartsWith("")); + EXPECT_TRUE(cord.StartsWith(absl::Cord())); + EXPECT_FALSE(cord.StartsWith("a")); + EXPECT_TRUE(cord.EndsWith("")); + EXPECT_TRUE(cord.EndsWith(absl::Cord())); + EXPECT_FALSE(cord.EndsWith("a")); + EXPECT_TRUE(cord.Contains("")); + EXPECT_TRUE(cord.Contains(absl::Cord())); + EXPECT_FALSE(cord.Contains("a")); + EXPECT_EQ(cord.Find(""), cord.char_begin()); + EXPECT_EQ(cord.Find(absl::Cord()), cord.char_begin()); + EXPECT_EQ(cord.Find("a"), cord.char_end()); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_NE(cord.TryFlat()->data(), nullptr); + if (expected_ptr != nullptr) { + EXPECT_EQ(cord.TryFlat()->data(), expected_ptr); + } + EXPECT_EQ(cord.TryFlat()->size(), 0); + EXPECT_EQ(cord.TryFlat(), ""); + absl::string_view flattened = mutable_cord.Flatten(); + EXPECT_NE(flattened.data(), nullptr); + if (expected_ptr != nullptr) { + EXPECT_EQ(flattened.data(), expected_ptr); + } + EXPECT_EQ(flattened.size(), 0); + EXPECT_EQ(flattened, ""); + char zero_dst = 'z'; + std::memcpy(&zero_dst, flattened.data(), flattened.size()); + EXPECT_EQ(zero_dst, 'z'); + EXPECT_EQ(std::string(cord), ""); + EXPECT_EQ(absl::StrCat(cord), ""); + EXPECT_EQ(absl::StrCat("pre:", cord, ":post"), "pre::post"); + EXPECT_EQ(absl::StrFormat("%v", cord), ""); + EXPECT_EQ(absl::StrFormat("[%v]", cord), "[]"); + + std::string copied = "non-empty"; + absl::CopyCordToString(cord, &copied); + EXPECT_EQ(copied, ""); + copied = "prefix"; + absl::AppendCordToString(cord, &copied); + EXPECT_EQ(copied, "prefix"); + EXPECT_EQ(absl::CopyCordToSpan(cord, absl::Span<char>()), 0); + char span_buf[8] = {'x'}; + EXPECT_EQ(absl::CopyCordToSpan(cord, absl::MakeSpan(span_buf)), 0); + EXPECT_EQ(span_buf[0], 'x'); + + int chunk_count = 0; + for (absl::string_view chunk : cord.Chunks()) { + static_cast<void>(chunk); + ++chunk_count; + } + EXPECT_EQ(chunk_count, 0); + EXPECT_EQ(cord.chunk_begin(), cord.chunk_end()); + int chunk_callbacks = 0; + absl::CordTestPeer::ForEachChunk( + cord, [&](absl::string_view) { ++chunk_callbacks; }); + EXPECT_EQ(chunk_callbacks, 0); + + int char_count = 0; + for (char ch : cord.Chars()) { + static_cast<void>(ch); + ++char_count; + } + EXPECT_EQ(char_count, 0); + EXPECT_EQ(cord.char_begin(), cord.char_end()); + auto char_it = cord.char_begin(); + absl::Cord::Advance(&char_it, 0); + EXPECT_EQ(char_it, cord.char_end()); + EXPECT_EQ(absl::Cord::AdvanceAndRead(&char_it, 0), ""); + EXPECT_EQ(char_it, cord.char_end()); + + EXPECT_EQ(absl::HashOf(cord), absl::HashOf(absl::Cord())); + EXPECT_EQ(absl::HashOf(cord), absl::HashOf(absl::string_view())); +} + +void ExpectResetToInlineEmpty(const absl::Cord& cord) { + EXPECT_TRUE(cord.empty()); + EXPECT_FALSE(absl::CordTestPeer::IsTree(cord)); + EXPECT_EQ(cord.EstimatedMemoryUsage(), sizeof(absl::Cord)); +} + +absl::Cord MakeClearedFlat(size_t size, const char** data_ptr = nullptr) { + const std::string data(size, 'a'); + absl::Cord cord(data); + if (data_ptr != nullptr) *data_ptr = cord.TryFlat()->data(); + absl::cord_internal::ClearCordPreservingCapacity(cord); + return cord; +} + +} // namespace + +TEST_P(CordTest, ClearCordPreservingCapacityBasicAndReuse) { + for (size_t flat_size : {size_t{16}, size_t{19}, size_t{64}, size_t{256}, + size_t{1024}, kMaxFlatLength}) { + SCOPED_TRACE(flat_size); + const std::string data(flat_size, 'a'); + absl::Cord cord(data); + ASSERT_TRUE(absl::CordTestPeer::IsTree(cord)); + absl::cord_internal::CordRep* tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(tree, nullptr); + ASSERT_TRUE(tree->IsFlat()); + const size_t capacity = tree->flat()->Capacity(); + ASSERT_GE(capacity, flat_size); + const char* old_ptr = cord.TryFlat()->data(); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + ExpectEmptyClearedFlat(cord, old_ptr); + + // Subsequent string_view assignments up to Capacity() reuse `old_ptr`. + std::string rewrite1(flat_size, 'b'); + cord = absl::string_view(rewrite1); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, rewrite1); + EXPECT_EQ(absl::StrCat(cord), rewrite1); + EXPECT_EQ(absl::StrFormat("%v", cord), rewrite1); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + std::string rewrite2(capacity, 'c'); + cord = absl::string_view(rewrite2); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, rewrite2); + + MaybeHarden(cord); + EXPECT_EQ(cord, rewrite2); + } +} + +TEST(CordClearPreservingCapacityTest, DoesNotRetainOtherReps) { + // 1. Shared flat (refcount > 1): `copy` keeps old data, `cord` resets inline. + { + std::string payload(128, 's'); + absl::Cord cord(payload); + const char* old_ptr = cord.TryFlat()->data(); + absl::Cord copy = cord; + absl::cord_internal::ClearCordPreservingCapacity(cord); + ExpectResetToInlineEmpty(cord); + EXPECT_EQ(copy, payload); + EXPECT_EQ(copy.TryFlat()->data(), old_ptr); + } + + // 2. EXTERNAL cord: releaser runs immediately during clear. + { + bool released = false; + std::string external_data(64, 'e'); + absl::Cord cord = absl::MakeCordFromExternal( + external_data, [&](absl::string_view) { released = true; }); + ASSERT_FALSE(released); + absl::cord_internal::ClearCordPreservingCapacity(cord); + EXPECT_TRUE(released); + ExpectResetToInlineEmpty(cord); + } + + // 3. SUBSTRING cord (over external and over flat): backing node is released. + { + bool released = false; + std::string external_data(128, 'u'); + absl::Cord sub; + { + absl::Cord ext = absl::MakeCordFromExternal( + external_data, [&](absl::string_view) { released = true; }); + sub = ext.Subcord(8, 64); + } + ASSERT_FALSE(released); + absl::cord_internal::CordRep* sub_tree = absl::CordTestPeer::Tree(sub); + ASSERT_NE(sub_tree, nullptr); + ASSERT_TRUE(sub_tree->IsSubstring()); + absl::cord_internal::ClearCordPreservingCapacity(sub); + EXPECT_TRUE(released); + ExpectResetToInlineEmpty(sub); + + absl::Cord flat_sub = absl::CordTestPeer::MakeSubstring( + absl::Cord(std::string(64, 'f')), 4, 32); + absl::cord_internal::CordRep* flat_sub_tree = + absl::CordTestPeer::Tree(flat_sub); + ASSERT_NE(flat_sub_tree, nullptr); + ASSERT_TRUE(flat_sub_tree->IsSubstring()); + absl::cord_internal::ClearCordPreservingCapacity(flat_sub); + ExpectResetToInlineEmpty(flat_sub); + } + + // 4. BTREE cord (> kMaxFlatLength, e.g. 16 KiB). + { + const std::string btree_data(16 << 10, 't'); + absl::Cord btree_cord(btree_data); + absl::cord_internal::CordRep* btree = absl::CordTestPeer::Tree(btree_cord); + ASSERT_NE(btree, nullptr); + ASSERT_TRUE(btree->IsBtree()); + absl::cord_internal::ClearCordPreservingCapacity(btree_cord); + ExpectResetToInlineEmpty(btree_cord); + } + + // 5. Large flat (> kMaxFlatSize, e.g. 32 KiB via CreateWithCustomLimit). + { + absl::CordBuffer large_buf = + absl::CordBuffer::CreateWithCustomLimit(64 << 10, 32 << 10); + const size_t fill_len = large_buf.capacity(); + std::memset(large_buf.data(), 'L', fill_len); + large_buf.SetLength(fill_len); + absl::Cord large_flat; + large_flat.Append(std::move(large_buf)); + absl::cord_internal::CordRep* large_tree = + absl::CordTestPeer::Tree(large_flat); + ASSERT_NE(large_tree, nullptr); + ASSERT_TRUE(large_tree->IsFlat()); + ASSERT_GT(large_tree->flat()->AllocatedSize(), + absl::cord_internal::kMaxFlatSize); + absl::cord_internal::ClearCordPreservingCapacity(large_flat); + ExpectResetToInlineEmpty(large_flat); + } + + // 6. Immortal MakeStringConstant cords (short inline and long external). + { + constexpr auto kShortConst = + absl::strings_internal::MakeStringConstant(ShortView{}); + constexpr auto kLongConst = + absl::strings_internal::MakeStringConstant(LongView{}); + absl::Cord c_short(kShortConst); + absl::Cord c_long(kLongConst); + ASSERT_TRUE(absl::CordTestPeer::IsTree(c_long)); + absl::cord_internal::ClearCordPreservingCapacity(c_short); + absl::cord_internal::ClearCordPreservingCapacity(c_long); + ExpectResetToInlineEmpty(c_short); + ExpectResetToInlineEmpty(c_long); + } + + // 7. Inline cords (0..15 bytes) and CRC cords. + { + absl::Cord empty_cord; + absl::Cord inline_cord("0123456789abcde"); + absl::cord_internal::ClearCordPreservingCapacity(empty_cord); + absl::cord_internal::ClearCordPreservingCapacity(inline_cord); + ExpectResetToInlineEmpty(empty_cord); + ExpectResetToInlineEmpty(inline_cord); + + absl::Cord crc_flat(std::string(64, 'k')); + crc_flat.SetExpectedChecksum(12345); + absl::cord_internal::ClearCordPreservingCapacity(crc_flat); + ExpectResetToInlineEmpty(crc_flat); + EXPECT_FALSE(crc_flat.ExpectedChecksum().has_value()); + + absl::Cord crc_empty; + crc_empty.SetExpectedChecksum(67890); + absl::cord_internal::ClearCordPreservingCapacity(crc_empty); + ExpectResetToInlineEmpty(crc_empty); + EXPECT_FALSE(crc_empty.ExpectedChecksum().has_value()); + } +} + +TEST(CordClearPreservingCapacityTest, CopyMoveSwapAndExplicitClear) { + // Explicit Clear() releases the retained flat. + { + absl::Cord cord = MakeClearedFlat(128); + ASSERT_TRUE(absl::CordTestPeer::IsTree(cord)); + ASSERT_GT(cord.EstimatedMemoryUsage(), sizeof(absl::Cord)); + cord.Clear(); + ExpectResetToInlineEmpty(cord); + } + + // Copy construction and copy assignment do NOT share the empty flat or bump + // its refcount, preserving unique ownership for subsequent in-place reuse. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + + absl::Cord copy_constructed(cord); + ExpectResetToInlineEmpty(copy_constructed); + + absl::Cord copy_dsts[] = { + absl::Cord(), + absl::Cord("inline"), + absl::Cord(std::string(200, 'x')), + MakeClearedFlat(96), + }; + ASSERT_TRUE(absl::CordTestPeer::IsTree(copy_dsts[3])); + for (absl::Cord& dst : copy_dsts) { + dst = cord; + ExpectResetToInlineEmpty(dst); + } + + absl::cord_internal::CordRep* cord_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(cord_tree, nullptr); + ASSERT_TRUE(cord_tree->refcount.IsOne()); + std::string next_payload(100, 'z'); + cord = absl::string_view(next_payload); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, next_payload); + } + + // Short (0..15 B) string_view assignments on a cleared Cord retain `old_ptr`, + // while non-cleared Cords still drop their tree and transition to inline. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(256, &old_ptr); + + cord = ""; + EXPECT_TRUE(cord.empty()); + EXPECT_TRUE(absl::CordTestPeer::IsTree(cord)); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + + cord = "0123456789"; + EXPECT_EQ(cord, "0123456789"); + EXPECT_TRUE(absl::CordTestPeer::IsTree(cord)); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + + for (absl::string_view short_val : {"", "0123456789"}) { + absl::cord_internal::ClearCordPreservingCapacity(cord); + cord = short_val; + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + cord = std::string(200, 'z'); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, std::string(200, 'z')); + } + + // Non-cleared Cords (size == 200) assigned short or empty strings (even + // self-aliasing views) drop their tree and become inline. + cord = cord.TryFlat()->substr(0, 10); + EXPECT_EQ(cord, std::string(10, 'z')); + EXPECT_FALSE(absl::CordTestPeer::IsTree(cord)); + + absl::Cord non_cleared(std::string(200, 'z')); + ASSERT_TRUE(absl::CordTestPeer::IsTree(non_cleared)); + non_cleared = ""; + ExpectResetToInlineEmpty(non_cleared); + } + + // Assigning inline/tree Cords and rvalue strings onto a cleared Cord. + { + absl::Cord c1 = MakeClearedFlat(64); + c1 = absl::Cord("short"); + EXPECT_EQ(c1, "short"); + + absl::Cord c2 = MakeClearedFlat(64); + c2 = absl::Cord(std::string(200, 'b')); + EXPECT_EQ(c2, std::string(200, 'b')); + + absl::Cord c3 = MakeClearedFlat(64); + c3 = std::string(2048, 'm'); + EXPECT_EQ(c3, std::string(2048, 'm')); + + // Large rvalue string with wasteful capacity (size < capacity / 2) is not + // stolen as external and copies into the cached flat in place. + const char* old_ptr = nullptr; + absl::Cord c4 = MakeClearedFlat(2048, &old_ptr); + std::string wasteful(1024, 'q'); + wasteful.reserve(8192); + c4 = std::move(wasteful); + ASSERT_TRUE(c4.TryFlat().has_value()); + EXPECT_EQ(c4.TryFlat()->data(), old_ptr); + EXPECT_EQ(c4, std::string(1024, 'q')); + } + + // Move construction, move assignment, and self-move assignment. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + + absl::Cord move_constructed(std::move(cord)); + // NOLINTNEXTLINE(bugprone-use-after-move) + EXPECT_FALSE(absl::CordTestPeer::IsTree(cord)); + EXPECT_TRUE(move_constructed.empty()); + EXPECT_TRUE(absl::CordTestPeer::IsTree(move_constructed)); + + absl::Cord move_assigned(std::string(256, 'q')); + move_assigned = std::move(move_constructed); + EXPECT_TRUE(move_assigned.empty()); + EXPECT_TRUE(absl::CordTestPeer::IsTree(move_assigned)); + + std::string payload(100, 'r'); + move_assigned = absl::string_view(payload); + EXPECT_EQ(move_assigned.TryFlat()->data(), old_ptr); + EXPECT_EQ(move_assigned, payload); + + absl::Cord self_cord = MakeClearedFlat(128); + ASSERT_TRUE(absl::CordTestPeer::IsTree(self_cord)); + absl::Cord& ref = self_cord; + self_cord = std::move(ref); + EXPECT_TRUE(self_cord.empty()); + self_cord = "after_self_move"; + EXPECT_EQ(self_cord, "after_self_move"); + } + + // Swap with empty, inline, and tree cords transfers the retained flat. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + + absl::Cord empty_cord; + cord.swap(empty_cord); + EXPECT_FALSE(absl::CordTestPeer::IsTree(cord)); + EXPECT_TRUE(absl::CordTestPeer::IsTree(empty_cord)); + + absl::Cord inline_cord("hello"); + swap(empty_cord, inline_cord); + EXPECT_EQ(empty_cord, "hello"); + EXPECT_TRUE(inline_cord.empty()); + + const std::string tree_data(512, 't'); + absl::Cord tree_cord(tree_data); + swap(inline_cord, tree_cord); + EXPECT_EQ(inline_cord, std::string(512, 't')); + EXPECT_TRUE(tree_cord.empty()); + + std::string payload(96, 'w'); + tree_cord = absl::string_view(payload); + EXPECT_EQ(tree_cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(tree_cord, payload); + } +} + +TEST(CordClearPreservingCapacityTest, AppendAndPrepend) { + // Append(string_view): empty, fitting capacity (reuses old_ptr), exceeding + // capacity into B-tree, and 2600 B / 5000 B onto undersized or max flats. + { + absl::Cord cord(std::string(128, 'a')); + const char* old_ptr = cord.TryFlat()->data(); + absl::cord_internal::CordRep* cord_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(cord_tree, nullptr); + ASSERT_TRUE(cord_tree->IsFlat()); + const size_t cap = cord_tree->flat()->Capacity(); + absl::cord_internal::ClearCordPreservingCapacity(cord); + + cord.Append(absl::string_view()); + EXPECT_TRUE(cord.empty()); + cord.Append("hello"); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + cord.Append(std::string(cap - 5, 'x')); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord.size(), cap); + + std::string extra(4096, 'y'); + cord.Append(extra); + EXPECT_EQ(cord, absl::StrCat("hello", std::string(cap - 5, 'x'), extra)); + + const std::string s2600(2600, 'x'); + absl::Cord small_cleared(std::string(100, 'a')); + absl::cord_internal::CordRep* small_tree = + absl::CordTestPeer::Tree(small_cleared); + ASSERT_NE(small_tree, nullptr); + ASSERT_TRUE(small_tree->IsFlat()); + ASSERT_LT(small_tree->flat()->Capacity(), s2600.size()); + absl::cord_internal::ClearCordPreservingCapacity(small_cleared); + small_cleared.Append(absl::string_view(s2600)); + EXPECT_TRUE(small_cleared.TryFlat().has_value()); + small_tree = absl::CordTestPeer::Tree(small_cleared); + ASSERT_NE(small_tree, nullptr); + EXPECT_TRUE(small_tree->IsFlat()); + EXPECT_EQ(small_cleared, s2600); + + const std::string s5000(5000, 'y'); + absl::Cord small_to_btree = MakeClearedFlat(100); + small_to_btree.Append(absl::string_view(s5000)); + EXPECT_EQ(small_to_btree.chunk_begin()->size(), kMaxFlatLength); + EXPECT_EQ(small_to_btree, s5000); + + const std::string w5000(5000, 'w'); + const char* max_ptr = nullptr; + absl::Cord max_cleared = MakeClearedFlat(kMaxFlatLength, &max_ptr); + max_cleared.Append(absl::string_view(w5000)); + EXPECT_EQ(max_cleared.chunk_begin()->data(), max_ptr); + EXPECT_EQ(max_cleared.chunk_begin()->size(), kMaxFlatLength); + EXPECT_EQ(max_cleared, w5000); + } + + // Append(const Cord&) and Append(Cord&&): empty, self-append, inline, + // fitting flat, fragmented small, larger flat, and large B-tree. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + + cord.Append(absl::Cord()); + absl::Cord empty_crc; + empty_crc.SetExpectedChecksum(42); + cord.Append(empty_crc); + absl::Cord other_cleared = MakeClearedFlat(64); + cord.Append(other_cleared); + cord.Append(std::move(other_cleared)); + cord.Append(cord); + EXPECT_TRUE(cord.empty()); + EXPECT_TRUE(absl::CordTestPeer::IsTree(cord)); + + absl::Cord frag_src = + absl::MakeFragmentedCord({"frag_", "one_", "two_", "three"}); + cord.Append(frag_src); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, "frag_one_two_three"); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + cord.Append(absl::MakeFragmentedCord({"rval_", "four_", "five"})); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, "rval_four_five"); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + absl::Cord inline_src("hello "); + cord.Append(inline_src); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + cord.Append(absl::Cord(std::string(32, 'f'))); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + cord.Append(absl::MakeFragmentedCord({"1234", "5678"})); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, absl::StrCat("hello ", std::string(32, 'f'), "12345678")); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + std::string medium(300, 'm'); + cord.Append(absl::Cord(medium)); + EXPECT_EQ(cord, medium); + absl::cord_internal::CordRep* medium_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(medium_tree, nullptr); + EXPECT_TRUE(medium_tree->IsFlat()); + + std::string big(16 << 10, 'B'); + absl::Cord big_cord(big); + absl::Cord c_lval = MakeClearedFlat(64); + c_lval.Append(big_cord); + EXPECT_EQ(c_lval, big); + + absl::Cord c_rval = MakeClearedFlat(64); + c_rval.Append(std::move(big_cord)); + EXPECT_EQ(c_rval, big); + } + + // Append(std::string&&): short, medium fitting capacity, and large. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + cord.Append(std::string("short")); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + cord.Append(std::string(60, 'm')); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, absl::StrCat("short", std::string(60, 'm'))); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + cord.Append(std::string(4096, 'S')); + EXPECT_EQ(cord, std::string(4096, 'S')); + } + + // Append(CordBuffer) and Prepend(CordBuffer): empty, inline (8 B), flat (64). + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + cord.Append(absl::CordBuffer()); + cord.Prepend(absl::CordBuffer()); + EXPECT_TRUE(cord.empty()); + cord = absl::string_view(std::string(64, 'z')); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + + for (bool prepend : {false, true}) { + SCOPED_TRACE(prepend); + const char* in_old_ptr = nullptr; + absl::Cord c_in = MakeClearedFlat(128, &in_old_ptr); + absl::CordBuffer in_buf = absl::CordBuffer::CreateWithDefaultLimit(8); + std::memcpy(in_buf.data(), "12345678", 8); + in_buf.SetLength(8); + if (prepend) { + c_in.Prepend(std::move(in_buf)); + } else { + c_in.Append(std::move(in_buf)); + } + ASSERT_TRUE(c_in.TryFlat().has_value()); + EXPECT_EQ(c_in.TryFlat()->data(), in_old_ptr); + EXPECT_EQ(c_in, "12345678"); + + absl::Cord c_flat = MakeClearedFlat(128); + absl::CordBuffer flat_buf = absl::CordBuffer::CreateWithDefaultLimit(64); + const char* flat_buf_ptr = flat_buf.data(); + std::memset(flat_buf.data(), 'F', 64); + flat_buf.SetLength(64); + if (prepend) { + c_flat.Prepend(std::move(flat_buf)); + } else { + c_flat.Append(std::move(flat_buf)); + } + ASSERT_TRUE(c_flat.TryFlat().has_value()); + EXPECT_EQ(c_flat.TryFlat()->data(), flat_buf_ptr); + EXPECT_EQ(c_flat, std::string(64, 'F')); + } + } + + // Prepend(string_view), Prepend(const Cord&), Prepend(Cord&&), + // Prepend(std::string&&). + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + + cord.Prepend(absl::string_view()); + cord.Prepend(absl::Cord()); + cord.Prepend(cord); + EXPECT_TRUE(cord.empty()); + + const std::string prep_sv(64, 'p'); + cord.Prepend(absl::string_view(prep_sv)); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, prep_sv); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + cord.Prepend(absl::Cord("inline_prep")); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, "inline_prep"); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + std::string big_sv(8192, 'X'); + cord.Prepend(big_sv); + EXPECT_EQ(cord, big_sv); + + absl::Cord c_flat = MakeClearedFlat(64); + absl::Cord src_flat(std::string(200, 'Q')); + c_flat.Prepend(src_flat); + EXPECT_EQ(c_flat, std::string(200, 'Q')); + + absl::Cord c_btree = MakeClearedFlat(64); + const std::string btree_prep(16 << 10, 'R'); + c_btree.Prepend(absl::Cord(btree_prep)); + EXPECT_EQ(c_btree, std::string(16 << 10, 'R')); + + const char* str_ptr = nullptr; + absl::Cord c_str = MakeClearedFlat(128, &str_ptr); + c_str.Prepend(std::string(40, 's')); + EXPECT_EQ(c_str.TryFlat()->data(), str_ptr); + EXPECT_EQ(c_str, std::string(40, 's')); + + absl::cord_internal::ClearCordPreservingCapacity(c_str); + c_str.Prepend(std::string(4096, 'L')); + EXPECT_EQ(c_str, std::string(4096, 'L')); + } +} + +TEST(CordClearPreservingCapacityTest, GetAppendBufferSubcordAndCrc) { + // GetAppendBuffer with capacity <= cached capacity extracts `old_ptr`. + { + absl::Cord cord(std::string(128, 'a')); + const char* old_ptr = cord.TryFlat()->data(); + absl::cord_internal::CordRep* cord_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(cord_tree, nullptr); + ASSERT_TRUE(cord_tree->IsFlat()); + const size_t old_cap = cord_tree->flat()->Capacity(); + absl::cord_internal::ClearCordPreservingCapacity(cord); + + absl::CordBuffer buf = cord.GetAppendBuffer(64); + ExpectResetToInlineEmpty(cord); + EXPECT_EQ(buf.length(), 0); + EXPECT_EQ(buf.capacity(), old_cap); + EXPECT_EQ(buf.data(), old_ptr); + + std::memset(buf.data(), 'g', 64); + buf.SetLength(64); + cord.Append(std::move(buf)); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(cord, std::string(64, 'g')); + } + + // GetAppendBuffer(10000) and GetCustomAppendBuffer(64 << 10, 5000) on a + // kMaxFlatLength cached flat reuse `old_ptr`. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(kMaxFlatLength, &old_ptr); + absl::CordBuffer buf = cord.GetAppendBuffer(10000); + ExpectResetToInlineEmpty(cord); + EXPECT_EQ(buf.data(), old_ptr); + EXPECT_EQ(buf.capacity(), kMaxFlatLength); + + const char* custom_ptr = nullptr; + absl::Cord cord2 = MakeClearedFlat(kMaxFlatLength, &custom_ptr); + absl::CordBuffer custom_buf = cord2.GetCustomAppendBuffer(64 << 10, 5000); + ExpectResetToInlineEmpty(cord2); + EXPECT_EQ(custom_buf.data(), custom_ptr); + EXPECT_EQ(custom_buf.capacity(), kMaxFlatLength); + } + + // GetAppendBuffer(2600) on a 100-byte cached flat drops the too-small flat + // and allocates a single contiguous 2600+ byte buffer (no B-tree split). + { + absl::Cord cord(std::string(100, 'a')); + absl::cord_internal::CordRep* cord_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(cord_tree, nullptr); + ASSERT_TRUE(cord_tree->IsFlat()); + ASSERT_LT(cord_tree->flat()->Capacity(), 2600); + absl::cord_internal::ClearCordPreservingCapacity(cord); + + absl::CordBuffer buf = cord.GetAppendBuffer(2600); + ExpectResetToInlineEmpty(cord); + EXPECT_EQ(buf.length(), 0); + EXPECT_GE(buf.capacity(), 2600); + + std::memset(buf.data(), 'B', 2600); + buf.SetLength(2600); + cord.Append(std::move(buf)); + EXPECT_EQ(cord.size(), 2600); + cord_tree = absl::CordTestPeer::Tree(cord); + ASSERT_NE(cord_tree, nullptr); + EXPECT_TRUE(cord_tree->IsFlat()); + } + + // GetCustomAppendBuffer reuses when fitting and allocates large when needed. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + absl::CordBuffer buf_fit = cord.GetCustomAppendBuffer(64 << 10, 80); + EXPECT_EQ(buf_fit.data(), old_ptr); + buf_fit.SetLength(80); + std::memset(buf_fit.data(), 'c', 80); + cord.Append(std::move(buf_fit)); + + absl::cord_internal::ClearCordPreservingCapacity(cord); + absl::CordBuffer buf_large = cord.GetCustomAppendBuffer(64 << 10, 16 << 10); + EXPECT_GE(buf_large.capacity(), (16 << 10) - kFlatOverhead); + } + + // RemovePrefix(0), RemoveSuffix(0), and Subcord(0, 0). + { + absl::Cord c1 = MakeClearedFlat(64); + c1.RemovePrefix(0); + ExpectResetToInlineEmpty(c1); + + absl::Cord c2 = MakeClearedFlat(64); + c2.RemoveSuffix(0); + ExpectResetToInlineEmpty(c2); + + const char* old_ptr3 = nullptr; + absl::Cord c3 = MakeClearedFlat(64, &old_ptr3); + EXPECT_EQ(c3.Subcord(0, 0), ""); + EXPECT_EQ(c3.Subcord(10, 10), ""); + ExpectEmptyClearedFlat(c3, old_ptr3); + } + + // SetExpectedChecksum on an empty cleared Cord replaces the empty flat with + // an empty CordRepCrc (child == nullptr), and GetAppendBuffer on that empty + // CRC-hardened Cord clears the checksum upon subsequent Append. + { + const char* old_ptr = nullptr; + absl::Cord cord = MakeClearedFlat(128, &old_ptr); + cord.SetExpectedChecksum(12345); + EXPECT_EQ(cord.ExpectedChecksum(), 12345); + ASSERT_TRUE(cord.TryFlat().has_value()); + EXPECT_NE(cord.TryFlat()->data(), old_ptr); + ExpectEmptyClearedFlat(cord, /*expected_ptr=*/nullptr); + + absl::CordBuffer buf = cord.GetAppendBuffer(64); + EXPECT_TRUE(cord.empty()); + EXPECT_EQ(cord.ExpectedChecksum(), 12345); + EXPECT_GE(buf.capacity(), 64); + std::memset(buf.data(), 'h', 5); + buf.SetLength(5); + cord.Append(std::move(buf)); + EXPECT_EQ(cord, "hhhhh"); + EXPECT_EQ(cord.ExpectedChecksum(), std::nullopt); + } +} + // This must not be static to avoid aggressive optimizations. ABSL_ATTRIBUTE_WEAK size_t FalseReport(const absl::Cord& a, bool f);
diff --git a/absl/strings/cordz_test.cc b/absl/strings/cordz_test.cc index a35493e..b4aa4ab 100644 --- a/absl/strings/cordz_test.cc +++ b/absl/strings/cordz_test.cc
@@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <atomic> #include <cstddef> #include <cstring> #include <ostream> #include <string> +#include <thread> // NOLINT #include <utility> #include "gmock/gmock.h" @@ -460,6 +462,119 @@ EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kSubCord)); } +TEST(CordzTest, ClearCordPreservingCapacitySampledAndConcurrent) { + CordzSamplingIntervalHelper always_sample(/*interval=*/1); + + // Sampled uniquely-owned flat stays tracked across + // ClearCordPreservingCapacity and subsequent mutations. + { + Cord cord(std::string(128, 'a')); + const char* old_ptr = cord.TryFlat()->data(); + const CordzInfo* info = GetCordzInfoForTesting(cord); + ASSERT_NE(info, nullptr); + EXPECT_TRUE(CordzInfoIsListed(info)); + + cord_internal::ClearCordPreservingCapacity(cord); + EXPECT_EQ(GetCordzInfoForTesting(cord), info); + CordzStatistics stats = info->GetCordzStatistics(); + EXPECT_EQ(stats.size, 0); + EXPECT_EQ(stats.update_tracker.Value(Method::kClear), 1); + + cord_internal::ClearCordPreservingCapacity(cord); + cord = ""; + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(GetCordzInfoForTesting(cord), info); + EXPECT_EQ(info->GetCordzStatistics().size, 0); + + cord = "0123456789"; + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(GetCordzInfoForTesting(cord), info); + EXPECT_EQ(info->GetCordzStatistics().size, 10); + + std::string next(96, 'b'); + cord = absl::string_view(next); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(GetCordzInfoForTesting(cord), info); + EXPECT_EQ(info->GetCordzStatistics().size, 96); + + cord_internal::ClearCordPreservingCapacity(cord); + cord.Append("0123456789abcdef0123456789"); + EXPECT_EQ(cord.TryFlat()->data(), old_ptr); + EXPECT_EQ(info->GetCordzStatistics().size, 26); + + // Non-retained sampled cords (shared flat, B-tree) untrack on clear. + Cord copy = cord; + cord_internal::ClearCordPreservingCapacity(cord); + EXPECT_EQ(GetCordzInfoForTesting(cord), nullptr); + EXPECT_EQ(copy, "0123456789abcdef0123456789"); + + const std::string btree_data(16 << 10, 't'); + Cord btree_cord(btree_data); + ASSERT_NE(GetCordzInfoForTesting(btree_cord), nullptr); + cord_internal::ClearCordPreservingCapacity(btree_cord); + EXPECT_EQ(GetCordzInfoForTesting(btree_cord), nullptr); + } + + // Concurrent Cordz collection vs ClearCordPreservingCapacity / reuse under + // TSan (exercises transient RefCordRep() refcount > 1 races). + { + std::atomic<bool> stop{false}; + const std::string init_data(1024, 'x'); + Cord cord(init_data); + std::thread collector([&stop]() { + while (!stop.load(std::memory_order_relaxed)) { + CordzSampleToken token; + for (const CordzInfo& sampled : token) { + (void)sampled.GetCordzStatistics(); + } + } + }); + + std::string payload_a(200, 'a'); + std::string payload_b(120, 'b'); + Cord src_cord(std::string(80, 'c')); + Cord inline_src("inline123"); + for (int i = 0; i < 500; ++i) { + cord_internal::ClearCordPreservingCapacity(cord); + cord = ""; + EXPECT_TRUE(cord.empty()); + cord = "0123456789"; + EXPECT_EQ(cord.size(), 10); + cord = absl::string_view(payload_a); + EXPECT_EQ(cord.size(), payload_a.size()); + + cord_internal::ClearCordPreservingCapacity(cord); + std::string wasteful(600, 'w'); + wasteful.reserve(4096); + cord = std::move(wasteful); + EXPECT_EQ(cord.size(), 600); + + cord_internal::ClearCordPreservingCapacity(cord); + cord.Append(absl::string_view(payload_b)); + EXPECT_EQ(cord.size(), payload_b.size()); + + cord_internal::ClearCordPreservingCapacity(cord); + cord.Append(src_cord); + EXPECT_EQ(cord.size(), src_cord.size()); + + cord_internal::ClearCordPreservingCapacity(cord); + cord.Append(inline_src); + EXPECT_EQ(cord.size(), inline_src.size()); + cord = absl::string_view(payload_a); + + cord_internal::ClearCordPreservingCapacity(cord); + CordBuffer buf = cord.GetAppendBuffer(128); + std::memset(buf.data(), 'd', 64); + buf.SetLength(64); + cord.Append(std::move(buf)); + EXPECT_EQ(cord.size(), 64); + } + + stop.store(true, std::memory_order_relaxed); + collector.join(); + } +} + } // namespace ABSL_NAMESPACE_END