Remove checks covering multiple batch-hasbit-checks.
We used to generate the checks like
```
if ((BatchCheckHasBit(has_bits[1], mask1) || // <- this one
BatchCheckHasBit(has_bits[2], mask2))) { // <- this one
if (BatchCheckHasBit(has_bits[1], mask1)) { ... }
if (BatchCheckHasBit(has_bits[2], mask2)) { ... }
}
```
I suppose they were added when the size of a batch was limited by 8. The current limit is 32. This CL eliminates these multi-batch checks without performance regression.
PiperOrigin-RevId: 1000933241
diff --git a/src/google/protobuf/compiler/cpp/field_chunk.cc b/src/google/protobuf/compiler/cpp/field_chunk.cc
index 8075e4f..d58c677 100644
--- a/src/google/protobuf/compiler/cpp/field_chunk.cc
+++ b/src/google/protobuf/compiler/cpp/field_chunk.cc
@@ -1,11 +1,9 @@
#include "google/protobuf/compiler/cpp/field_chunk.h"
#include <cstdint>
-#include <vector>
#include "absl/log/absl_check.h"
#include "absl/numeric/bits.h"
-#include "absl/types/optional.h"
#include "absl/types/span.h"
#include "google/protobuf/compiler/cpp/field_layout.h"
#include "google/protobuf/descriptor.h"
@@ -20,38 +18,6 @@
namespace {
-// Returns the (common) hasbit index offset. Callers must guarantee "fields" is
-// not empty. Debug-check fails if it sees different hasbit index offset in
-// "fields".
-int GetCommonHasbitIndexOffset(absl::Span<const FieldDescriptor*> fields,
- const FieldLayout& field_layout) {
- ABSL_CHECK(!fields.empty());
-
- int word_idx = field_layout.GetHasWordIndex(fields.front()).value();
- for (auto* field : fields) {
- ABSL_DCHECK_EQ(word_idx, field_layout.GetHasWordIndex(field).value());
- }
- return word_idx;
-}
-
-// Asserts that chunks in [it, end) share hasbit index offset and returns it.
-int GetSameHasbitIndexOffset(ChunkIterator it, ChunkIterator end,
- const FieldLayout& field_layout) {
- ABSL_CHECK(it != end);
- absl::optional<int> prev_offset;
- for (; it != end; ++it) {
- // Skip empty chunks (likely due to extraction).
- if (it->fields.empty()) continue;
-
- int offset =
- GetCommonHasbitIndexOffset(absl::MakeSpan(it->fields), field_layout);
- ABSL_CHECK(!prev_offset.has_value() || *prev_offset == offset);
- prev_offset = offset;
- }
- ABSL_CHECK(prev_offset.has_value());
- return *prev_offset;
-}
-
uint32_t GenRawChunkMask(absl::Span<const FieldDescriptor* const> fields,
int first_index_offset,
const FieldLayout& field_layout) {
@@ -92,22 +58,6 @@
return SimplifyChunkMask(chunk_mask, first_index_offset, field_layout);
}
-// Returns a bit mask based on has_bit index of "fields" in chunks in [it, end).
-// Assumes that all chunks share the same hasbit word.
-uint32_t GenChunkMask(ChunkIterator it, ChunkIterator end,
- const FieldLayout& field_layout) {
- ABSL_CHECK(it != end);
-
- int first_index_offset = GetSameHasbitIndexOffset(it, end, field_layout);
-
- uint32_t chunk_mask = 0;
- for (; it != end; ++it) {
- chunk_mask |= GenRawChunkMask(it->fields, first_index_offset, field_layout);
- }
-
- return SimplifyChunkMask(chunk_mask, first_index_offset, field_layout);
-}
-
} // namespace cpp
} // namespace compiler
} // namespace protobuf
diff --git a/src/google/protobuf/compiler/cpp/field_chunk.h b/src/google/protobuf/compiler/cpp/field_chunk.h
index b8f5be2..2e3a034 100644
--- a/src/google/protobuf/compiler/cpp/field_chunk.h
+++ b/src/google/protobuf/compiler/cpp/field_chunk.h
@@ -19,27 +19,17 @@
namespace cpp {
struct FieldChunk {
- FieldChunk(bool has_hasbit, bool is_rarely_present, bool should_split)
- : has_hasbit(has_hasbit),
- is_rarely_present(is_rarely_present),
- should_split(should_split) {}
+ explicit FieldChunk(bool has_hasbit) : has_hasbit(has_hasbit) {}
bool has_hasbit;
- bool is_rarely_present;
- bool should_split;
std::vector<const FieldDescriptor*> fields;
};
-using ChunkIterator = std::vector<FieldChunk>::iterator;
-
PROTOC_EXPORT uint32_t
GenChunkMask(absl::Span<const FieldDescriptor* const> fields,
const FieldLayout& field_layout);
-PROTOC_EXPORT uint32_t GenChunkMask(ChunkIterator it, ChunkIterator end,
- const FieldLayout& field_layout);
-
struct AlwaysTruePred {
template <typename T>
bool operator()(const T&) const {
@@ -64,48 +54,12 @@
}
if (force_new_chunk || !equivalent(chunks.back().fields.back(), field)) {
force_new_chunk = false;
- chunks.emplace_back(HasHasbit(field, options),
- IsRarelyPresent(field, options),
- ShouldSplit(field, options));
+ chunks.emplace_back(HasHasbit(field, options));
}
chunks.back().fields.push_back(field);
}
return chunks;
}
-
-template <typename Predicate>
-ChunkIterator FindNextUnequalChunk(ChunkIterator start, ChunkIterator end,
- const Predicate& equal) {
- auto it = start;
- while (++it != end) {
- if (!equal(*start, *it)) {
- return it;
- }
- }
- return end;
-}
-
-// Returns an array of field descriptors that meet "predicate" in [it, end). The
-// matched fields are removed from the original range.
-template <typename Predicate>
-std::vector<const FieldDescriptor*> ExtractFields(ChunkIterator it,
- ChunkIterator end,
- const Predicate& predicate) {
- std::vector<const FieldDescriptor*> res;
- for (; it != end; ++it) {
- auto& source = it->fields;
- if (source.empty()) continue;
-
- for (const auto* f : source) {
- if (predicate(f)) {
- res.push_back(f);
- }
- }
- source.erase(std::remove_if(source.begin(), source.end(), predicate),
- source.end());
- }
- return res;
-}
} // namespace cpp
} // namespace compiler
} // namespace protobuf
diff --git a/src/google/protobuf/compiler/cpp/field_chunk_test.cc b/src/google/protobuf/compiler/cpp/field_chunk_test.cc
index b636709..573b756 100644
--- a/src/google/protobuf/compiler/cpp/field_chunk_test.cc
+++ b/src/google/protobuf/compiler/cpp/field_chunk_test.cc
@@ -21,7 +21,6 @@
namespace {
using proto2_unittest::TestAllTypes;
-using testing::IsEmpty;
template <typename ProtoT>
std::vector<const google::protobuf::FieldDescriptor*> CreateFieldArray(
@@ -86,68 +85,6 @@
EXPECT_FALSE(chunks[1].fields.front()->is_repeated());
}
-TEST(FindNextTest, AllEqual) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
- auto it = FindNextUnequalChunk(
- chunks.begin(), chunks.end(),
- [](const FieldChunk&, const FieldChunk&) { return true; });
-
- EXPECT_EQ(it, chunks.end());
-}
-
-TEST(FindNextTest, AllUnequal) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
- auto it = FindNextUnequalChunk(
- chunks.begin(), chunks.end(),
- [](const FieldChunk&, const FieldChunk&) { return false; });
-
- ASSERT_NE(it, chunks.end());
- EXPECT_EQ(it->fields, chunks[1].fields);
-}
-
-TEST(ExtractTest, ExtractNone) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
-
- ASSERT_EQ(chunks.size(), 2);
- auto extracted =
- ExtractFields(chunks.begin(), chunks.end(),
- [](const FieldDescriptor* f) { return false; });
-
- EXPECT_THAT(extracted, IsEmpty());
-}
-
-TEST(ExtractTest, ExtractAll) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
-
- ASSERT_EQ(chunks.size(), 2);
-
- auto extracted = ExtractFields(chunks.begin(), chunks.end(),
- [](const FieldDescriptor* f) { return true; });
-
- ASSERT_EQ(chunks.size(), 2);
- ASSERT_THAT(chunks[0].fields, IsEmpty());
- ASSERT_THAT(chunks[1].fields, IsEmpty());
-
- auto fields = CreateFieldArray<TestAllTypes>(field_names);
- EXPECT_THAT(extracted, testing::ElementsAreArray(fields));
-}
-
TEST(GenChunkMaskTest, ValidMaskFromFields) {
const char* field_names[] = {"optional_int32", "optional_int64"};
@@ -164,28 +101,6 @@
EXPECT_EQ(mask, (1 << kHasbitIdxAt0) | (1 << kHasbitIdxAt1));
}
-TEST(GenChunkMaskTest, ValidMaskFromChunks) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
-
- ASSERT_EQ(chunks.size(), 2);
- ASSERT_EQ(chunks[0].fields.size(), 1);
- ASSERT_EQ(chunks[0].fields[0]->index(), 0);
- ASSERT_EQ(chunks[1].fields.size(), 1);
- ASSERT_EQ(chunks[1].fields[0]->index(), 1);
-
- const uint32_t kHasbitIdxAt0 = 3;
- const uint32_t kHasbitIdxAt1 = 5;
- auto field_layout =
- FieldLayout::BuildForTesting({chunks[0].fields[0], chunks[1].fields[0]},
- {kHasbitIdxAt0, kHasbitIdxAt1});
- uint32_t mask = GenChunkMask(chunks.begin(), chunks.end(), field_layout);
- EXPECT_EQ(mask, (1 << kHasbitIdxAt0) | (1 << kHasbitIdxAt1));
-}
-
TEST(GenChunkMaskTest, FullWordMaskFromFields) {
const Descriptor* descriptor = TestAllTypes::GetDescriptor();
std::vector<const FieldDescriptor*> fields;
@@ -221,24 +136,6 @@
~uint32_t{0});
}
-TEST(GenChunkMaskTest, FullWordMaskFromChunks) {
- const char* field_names[] = {"optional_int32", "optional_int64"};
- auto chunks = CreateAndCollectFields<TestAllTypes>(
- field_names, [](const FieldDescriptor* lhs, const FieldDescriptor* rhs) {
- return false;
- });
- ASSERT_EQ(chunks.size(), 2);
-
- auto field_layout = FieldLayout::BuildForTesting(
- {chunks[0].fields[0], chunks[1].fields[0]}, {0, 1});
- ASSERT_EQ(field_layout.CountFieldsSharingHasWord(0), 2);
-
- EXPECT_EQ(GenChunkMask(chunks.begin(), chunks.end(), field_layout),
- ~uint32_t{0});
- EXPECT_EQ(GenChunkMask(chunks.begin(), chunks.begin() + 1, field_layout),
- uint32_t{1});
-}
-
} // namespace
} // namespace cpp
} // namespace compiler
diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc
index 5a9e56c..2adcaab 100644
--- a/src/google/protobuf/compiler/cpp/message.cc
+++ b/src/google/protobuf/compiler/cpp/message.cc
@@ -435,91 +435,6 @@
return false;
}
-// Returns true if two chunks may be grouped for hasword check to skip multiple
-// cold fields at once. They have to share the following traits:
-// - whether they have hasbits
-// - whether they are rarely present
-// - whether they are split
-bool MayGroupChunksForHaswordsCheck(const FieldChunk& a, const FieldChunk& b) {
- return a.has_hasbit == b.has_hasbit &&
- a.is_rarely_present == b.is_rarely_present &&
- a.should_split == b.should_split;
-}
-
-// Returns true if it emits conditional check against hasbit words. This is
-// useful to skip multiple fields that are unlikely present based on profile
-// (go/pdproto). Assumes that each chunk is limited to one has "word".
-bool MaybeEmitHaswordsCheck(ChunkIterator it, ChunkIterator end,
- const Options& options,
- const FieldLayout& field_layout,
- int cached_has_word_index, const std::string& from,
- io::Printer* p) {
- if (!it->has_hasbit || !IsProfileDriven(options) ||
- std::distance(it, end) < 2 || !it->is_rarely_present) {
- return false;
- }
-
- auto hasbit_word = [&field_layout](const FieldDescriptor* field) {
- return field_layout.GetHasWordIndex(field).value();
- };
- auto is_same_hasword = [&](const FieldChunk& a, const FieldChunk& b) {
- // Empty fields are assumed to have the same haswords.
- if (a.fields.empty() || b.fields.empty()) return true;
-
- return hasbit_word(a.fields.front()) == hasbit_word(b.fields.front());
- };
-
- struct HasWordMask {
- int word;
- uint32_t mask;
- };
-
- std::vector<HasWordMask> hasword_masks;
- while (it != end) {
- auto next = FindNextUnequalChunk(it, end, is_same_hasword);
- for (; it != next; ++it) {
- if (!it->fields.empty()) {
- hasword_masks.push_back({hasbit_word(it->fields.front()),
- GenChunkMask(it, next, field_layout)});
- break;
- }
- }
- // Jump to the next batch instead.
- it = next;
- }
-
- if (hasword_masks.empty()) return false;
-
- // Emit has_bit check for each has_bit_dword index.
- p->Emit({{"cond",
- [&] {
- int first_word = hasword_masks.front().word;
- for (const auto& m : hasword_masks) {
- uint32_t mask = m.mask;
- int this_word = m.word;
- if (this_word != first_word) {
- p->Emit(R"cc(
- ||
- )cc");
- }
- auto v =
- p->WithVars({{"mask", absl::StrFormat("0x%08xU", mask)}});
- if (this_word == cached_has_word_index) {
- p->Emit("BatchCheckHasBit(cached_has_bits, $mask$)");
- } else {
- p->Emit({{"from", from}, {"word", this_word}},
- "BatchCheckHasBit($from$_impl_._has_bits_[$word$], "
- "$mask$)");
- }
- }
- }}},
- R"cc(
- if (ABSL_PREDICT_FALSE($cond$)) {
- )cc");
- p->Indent();
- return true;
-}
-
// Calculates the expected number of branch checks executed for a chunk of
// `size` fields, where `p_none` is the probability that none of the fields
// are present.
@@ -547,7 +462,7 @@
FieldChunk ChunkWithFields(const FieldChunk& chunk,
std::vector<const FieldDescriptor*> fields) {
- FieldChunk res(chunk.has_hasbit, chunk.is_rarely_present, chunk.should_split);
+ FieldChunk res(chunk.has_hasbit);
res.fields = std::move(fields);
return res;
}
@@ -3461,78 +3376,56 @@
enclosing_if_chunks = PartitionToMinimizeExpectedBranches(
std::move(enclosing_if_chunks), options_);
- auto it = enclosing_if_chunks.begin();
- auto end = enclosing_if_chunks.end();
int cached_has_word_index = -1;
- while (it != end) {
- auto next = FindNextUnequalChunk(it, end, MayGroupChunksForHaswordsCheck);
- bool has_haswords_check = MaybeEmitHaswordsCheck(
- it, next, options_, field_layout_, cached_has_word_index, "this_.", p);
+ for (const FieldChunk& chunk : enclosing_if_chunks) {
+ const std::vector<const FieldDescriptor*>& fields = chunk.fields;
- while (it != next) {
- const std::vector<const FieldDescriptor*>& fields = it->fields;
-
- const int has_word_index =
- field_layout_.GetHasWordIndex(fields.front()).value();
- if (cached_has_word_index != has_word_index) {
- cached_has_word_index = has_word_index;
- p->Emit({{"cached_has_word_index", cached_has_word_index}}, R"cc(
- cached_has_bits = this_.$has_bits$[$cached_has_word_index$];
- )cc");
- }
-
- const auto print_clear_field_chunk = [&] {
- for (const auto& field : fields) {
- p->Emit({{"cond",
- [&] {
- PrintPresenceCheckCondition(field, field_layout_, p,
- options_);
- }},
- {"clear_field", [&] { emit_clear_field(field); }}},
- R"cc(
- if ($cond$) {
- $clear_field$;
- }
- )cc");
- }
- };
-
- // Whether we wrap this chunk in:
- // if (cached_has_bits & <chunk hasbits>) { /* chunk. */ }
- if (fields.size() > 1) {
- // Emit an if() that will let us skip the whole chunk if none are set.
- uint32_t chunk_mask = GenChunkMask(fields, field_layout_);
-
- ABSL_DCHECK_GE(absl::popcount(chunk_mask), 2);
-
- const std::string cond = GenerateConditionMaybeWithProbabilityForGroup(
- chunk_mask, fields, options_);
- p->Emit(
- {{"cond", cond}, {"clear_field_chunk", print_clear_field_chunk}},
- R"cc(
- if ($cond$) {
- $clear_field_chunk$;
- }
- )cc");
- } else {
- print_clear_field_chunk();
- }
-
- // To next chunk.
- ++it;
- }
-
- if (has_haswords_check) {
- p->Outdent();
- p->Emit(R"cc(
- }
+ const int has_word_index =
+ field_layout_.GetHasWordIndex(fields.front()).value();
+ if (cached_has_word_index != has_word_index) {
+ cached_has_word_index = has_word_index;
+ p->Emit({{"cached_has_word_index", cached_has_word_index}}, R"cc(
+ cached_has_bits = this_.$has_bits$[$cached_has_word_index$];
)cc");
-
- // Reset here as it may have been updated in just closed if statement.
- cached_has_word_index = -1;
}
- };
+
+ const auto print_clear_field_chunk = [&] {
+ for (const auto& field : fields) {
+ p->Emit({{"cond",
+ [&] {
+ PrintPresenceCheckCondition(field, field_layout_, p,
+ options_);
+ }},
+ {"clear_field", [&] { emit_clear_field(field); }}},
+ R"cc(
+ if ($cond$) {
+ $clear_field$;
+ }
+ )cc");
+ }
+ };
+
+ // Whether we wrap this chunk in:
+ // if (cached_has_bits & <chunk hasbits>) { /* chunk. */ }
+ if (fields.size() > 1) {
+ // Emit an if() that will let us skip the whole chunk if none are set.
+ uint32_t chunk_mask = GenChunkMask(fields, field_layout_);
+
+ ABSL_DCHECK_GE(absl::popcount(chunk_mask), 2);
+
+ const std::string cond = GenerateConditionMaybeWithProbabilityForGroup(
+ chunk_mask, fields, options_);
+ p->Emit({{"cond", cond}, {"clear_field_chunk", print_clear_field_chunk}},
+ R"cc(
+ if ($cond$) {
+ $clear_field_chunk$;
+ }
+ )cc");
+ } else {
+ print_clear_field_chunk();
+ }
+ }
}
void MessageGenerator::GenerateClear(io::Printer* p) {
@@ -4057,24 +3950,10 @@
generator.GenerateMergingCode(p);
},
/*with_enclosing_braces_always=*/true);
- } else if (cached_has_word_index !=
- field_layout_.GetHasWordIndex(field).value()) {
- // Check hasbit, not using cached bits.
- auto v = p->WithVars(HasBitVars(field));
- p->Emit(
- {{"merge_field",
- [&] {
- prepare_split();
- generator.GenerateMergingCode(p);
- }}},
- R"cc(
- if (CheckHasBit(from.$has_bits$[$has_array_index$], $has_mask$)) {
- $merge_field$;
- }
- )cc");
} else {
// Check hasbit, using cached bits.
- ABSL_CHECK(HasHasbit(field, options_));
+ ABSL_CHECK_EQ(field_layout_.GetHasWordIndex(field),
+ cached_has_word_index);
int has_bit_index = field_layout_.GetHasBitIndex(field).value();
p->Emit(
@@ -4123,28 +4002,8 @@
[&](const auto* f) { return ShouldSplit(f, options_) == is_split; });
chunks = PartitionToMinimizeExpectedBranches(std::move(chunks), options_);
- auto it = chunks.begin();
- auto end = chunks.end();
- while (it != end) {
- auto next = FindNextUnequalChunk(it, end, MayGroupChunksForHaswordsCheck);
- bool has_haswords_check = MaybeEmitHaswordsCheck(
- it, next, options_, field_layout_, cached_has_word_index, "from.", p);
-
- while (it != next) {
- emit_merge_chunk(it->fields);
- ++it;
- }
-
- if (has_haswords_check) {
- p->Outdent();
- p->Emit(R"cc(
- }
- )cc");
-
- // Reset here as it may have been updated in just closed if
- // statement.
- cached_has_word_index = -1;
- }
+ for (const FieldChunk& chunk : chunks) {
+ emit_merge_chunk(chunk.fields);
}
return cached_has_word_index >= 0;
@@ -4977,8 +4836,6 @@
chunks.insert(it, std::move(chunk));
}
- auto it = chunks.begin();
- auto end = chunks.end();
int cached_has_word_index = -1;
const auto update_cached_has_bits = [&](auto& fields) {
const int has_word_index =
@@ -4992,95 +4849,75 @@
)cc");
};
- while (it != end) {
- auto next = FindNextUnequalChunk(it, end, MayGroupChunksForHaswordsCheck);
- bool has_haswords_check = MaybeEmitHaswordsCheck(
- it, next, options_, field_layout_, cached_has_word_index, "this_.", p);
+ for (const FieldChunk& chunk : chunks) {
+ const auto& fields = chunk.fields;
- while (it != next) {
- const auto& fields = it->fields;
-
- // If the chunk is a fixed size singular chunk, use a branchless
- // approach for it.
- if (absl::optional<int> fsize = FixedSize(fields[0])) {
- update_cached_has_bits(fields);
- // When a single field is the only field in its has_word, `GenChunkMask`
- // returns ~0u, which would fail the `absl::has_single_bit(mask)` check
- // below, emitting `::absl::popcount` instead of `static_cast<bool>`.
- // Therefore we process the single-field case separately.
- uint32_t mask =
- fields.size() == 1
- ? uint32_t{1}
- << (field_layout_.GetHasBitIndex(fields[0]).value() % 32)
- : GenChunkMask(fields, field_layout_);
- p->Emit({{"mask", absl::StrFormat("0x%08xU", mask)},
- {"popcount", absl::has_single_bit(mask) ? "static_cast<bool>"
- : "::absl::popcount"},
- {"fsize", *fsize}},
- R"cc(
- //~
- total_size += $popcount$($mask$ & cached_has_bits) * $fsize$;
- )cc");
- ++it;
- continue;
- }
-
- const bool check_has_word =
- fields.size() > 1 &&
- field_layout_.GetHasWordIndex(fields[0]).has_value();
- p->Emit({{"update_byte_size_for_chunk",
- [&] {
- // Go back and emit checks for each of the fields we
- // processed.
- for (const auto* field : fields) {
- EmitUpdateByteSizeForField(field, p, cached_has_word_index);
- }
- }},
- {"may_update_cached_has_word_index",
- [&] {
- if (!check_has_word) return;
- update_cached_has_bits(fields);
- }},
- {"check_if_chunk_present",
- [&] {
- if (!check_has_word) {
- return;
- }
-
- // Emit an if() that will let us skip the whole chunk
- // if none are set.
- uint32_t chunk_mask = GenChunkMask(fields, field_layout_);
-
- // Check has_bits at a time if we have more than one field in
- // this chunk.
- ABSL_DCHECK_GE(absl::popcount(chunk_mask), 2);
-
- p->Emit({{"condition",
- GenerateConditionMaybeWithProbabilityForGroup(
- chunk_mask, fields, options_)}},
- "if ($condition$)");
- }}},
+ // If the chunk is a fixed size singular chunk, use a branchless
+ // approach for it.
+ if (absl::optional<int> fsize = FixedSize(fields[0])) {
+ update_cached_has_bits(fields);
+ // When a single field is the only field in its has_word, `GenChunkMask`
+ // returns ~0u, which would fail the `absl::has_single_bit(mask)` check
+ // below, emitting `::absl::popcount` instead of `static_cast<bool>`.
+ // Therefore we process the single-field case separately.
+ uint32_t mask =
+ fields.size() == 1
+ ? uint32_t{1}
+ << (field_layout_.GetHasBitIndex(fields[0]).value() % 32)
+ : GenChunkMask(fields, field_layout_);
+ p->Emit({{"mask", absl::StrFormat("0x%08xU", mask)},
+ {"popcount", absl::has_single_bit(mask) ? "static_cast<bool>"
+ : "::absl::popcount"},
+ {"fsize", *fsize}},
R"cc(
- $may_update_cached_has_word_index$;
- $check_if_chunk_present$ {
- //~ Force newline.
- $update_byte_size_for_chunk$;
- }
+ //~
+ total_size += $popcount$($mask$ & cached_has_bits) * $fsize$;
)cc");
-
- // To next chunk.
- ++it;
+ continue;
}
- if (has_haswords_check) {
- p->Emit(R"cc(
- }
- )cc");
+ const bool check_has_word =
+ fields.size() > 1 &&
+ field_layout_.GetHasWordIndex(fields[0]).has_value();
+ p->Emit({{"update_byte_size_for_chunk",
+ [&] {
+ // Go back and emit checks for each of the fields we
+ // processed.
+ for (const auto* field : fields) {
+ EmitUpdateByteSizeForField(field, p, cached_has_word_index);
+ }
+ }},
+ {"may_update_cached_has_word_index",
+ [&] {
+ if (!check_has_word) return;
+ update_cached_has_bits(fields);
+ }},
+ {"check_if_chunk_present",
+ [&] {
+ if (!check_has_word) {
+ return;
+ }
- // Reset here as it may have been updated in just closed if
- // statement.
- cached_has_word_index = -1;
- }
+ // Emit an if() that will let us skip the whole chunk
+ // if none are set.
+ uint32_t chunk_mask = GenChunkMask(fields, field_layout_);
+
+ // Check has_bits at a time if we have more than one field in
+ // this chunk.
+ ABSL_DCHECK_GE(absl::popcount(chunk_mask), 2);
+
+ p->Emit({{"condition",
+ GenerateConditionMaybeWithProbabilityForGroup(
+ chunk_mask, fields, options_)}},
+ "if ($condition$)");
+ }}},
+ R"cc(
+ $may_update_cached_has_word_index$;
+ $check_if_chunk_present$ {
+ //~ Force newline.
+ $update_byte_size_for_chunk$;
+ }
+ )cc");
}
}
