Carve the read-only FailureList out of the conformance ResultLedger. The ledger did two things: it judged every result against the expected failures and the enforcement level, and it tallied the outcomes. The judging half is now a FailureList, built once from the failure list files and never changed afterwards, which answers VerdictOnSuccess(), VerdictOnFailure() and VerdictOnSkip(). The ledger owns one, applies its verdicts in the Report*() methods and keeps the counting and the --fix bookkeeping. Nothing about how a result is judged changes; the failure list is now a value that can be reasoned about and tested on its own. FailureList gets its own unit test, and the ledger tests that only checked the list's matching rules and messages move there. One consequence: the wildcard expansion cap is no longer enforced by the verdict (a list that never changes can't count matches), so the ledger reports the overexpanded entries at the end of the run through OverexpandedWildcards() and the environment fails the run over them, instead of the 22nd matching test failing on its own. PiperOrigin-RevId: 1001701867
diff --git a/conformance/BUILD b/conformance/BUILD index 5484b0e..9b6d173 100644 --- a/conformance/BUILD +++ b/conformance/BUILD
@@ -123,6 +123,7 @@ tests = [ "binary_test_util_test", "binary_wireformat_test", + "failure_list_test", "failure_list_trie_node_test", "matchers_test", "naming_test", @@ -319,13 +320,44 @@ ) cc_library( + name = "failure_list", + testonly = True, + srcs = ["failure_list.cc"], + hdrs = ["failure_list.h"], + deps = [ + ":failure_list_trie_node", + ":testee", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + ], +) + +cc_test( + name = "failure_list_test", + srcs = ["failure_list_test.cc"], + deps = [ + ":failure_list", + ":testee", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_matchers", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/types:optional", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( name = "result_ledger", testonly = True, srcs = ["result_ledger.cc"], hdrs = ["result_ledger.h"], deps = [ - ":conformance_cc_proto", - ":failure_list_trie_node", + ":failure_list", ":testee", "@abseil-cpp//absl/algorithm:container", "@abseil-cpp//absl/container:btree", @@ -412,6 +444,7 @@ name = "result_ledger_test", srcs = ["result_ledger_test.cc"], deps = [ + ":failure_list", ":result_ledger", ":testee", "//src/google/protobuf/testing",
diff --git a/conformance/failure_list.cc b/conformance/failure_list.cc new file mode 100644 index 0000000..25e656c --- /dev/null +++ b/conformance/failure_list.cc
@@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/failure_list.h" + +#include <algorithm> +#include <string> + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +constexpr int kFailureMessageLengthLimit = 128; + +void Normalize(std::string& input) { + input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); +} + +} // namespace + +std::string FormatFailureMessage(absl::string_view message) { + std::string result(absl::StripAsciiWhitespace(message)); + // Remove newlines + Normalize(result); + // Truncate failure message if needed + if (result.length() > kFailureMessageLengthLimit) { + result = result.substr(0, kFailureMessageLengthLimit); + } + return result; +} + +absl::Status FailureList::Add(absl::string_view test_name, + absl::string_view failure_message) { + if (absl::Status status = entries_.Insert(test_name); !status.ok()) { + return status; + } + // Normalize the message the same way reported failures are normalized so + // that the two compare equal in VerdictOnFailure(). + ABSL_CHECK(expected_messages_ + .emplace(test_name, FormatFailureMessage(failure_message)) + .second); + return absl::OkStatus(); +} + +absl::Status FailureList::VerdictOnSuccess(absl::string_view test_name) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + return absl::OkStatus(); + } + return absl::FailedPreconditionError(absl::StrCat( + "test ", test_name, " (matched to ", *failure_match, + ") is in the failure list, but test succeeded. Remove its match from " + "the failure list.")); +} + +absl::Status FailureList::VerdictOnFailure( + absl::string_view test_name, TestPriority priority, + absl::string_view failure_message) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + if (priority > enforcement_level_) { + return absl::OkStatus(); // Tolerated. + } + return absl::FailedPreconditionError( + absl::StrCat("Unexpected failure for test: ", test_name)); + } + + // Mirror the legacy runner, which only requires the actual failure message to + // start with the expected one. + std::string formatted_failure_message = FormatFailureMessage(failure_message); + const std::string& expected_failure_message = + expected_messages_.at(*failure_match); + if (!absl::StartsWith(formatted_failure_message, expected_failure_message)) { + return absl::FailedPreconditionError(absl::StrCat( + "Unexpected failure message for test: ", test_name, " expected: ", + expected_failure_message, " actual: ", formatted_failure_message)); + } + return absl::OkStatus(); +} + +absl::Status FailureList::VerdictOnSkip(absl::string_view test_name, + absl::string_view skip_reason) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + return absl::OkStatus(); + } + return absl::FailedPreconditionError(absl::StrCat( + "test ", test_name, " (matched to ", *failure_match, + ") is in the failure list but was skipped by the testee: ", skip_reason, + ". Remove its match from the failure list.")); +} + +absl::optional<std::string> FailureList::MatchingEntry( + absl::string_view test_name) const { + return entries_.WalkDownMatch(test_name); +} + +absl::optional<std::string> FailureList::ExpectedMessage( + absl::string_view entry) const { + auto it = expected_messages_.find(entry); + if (it == expected_messages_.end()) return absl::nullopt; + return it->second; +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/failure_list.h b/conformance/failure_list.h new file mode 100644 index 0000000..3aaa900 --- /dev/null +++ b/conformance/failure_list.h
@@ -0,0 +1,98 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__ + +#include <string> + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/failure_list_trie_node.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// The form a failure message takes in a failure list: surrounding whitespace +// and newlines removed, truncated to 128 characters. Expected messages are +// normalized when they are added and actual ones when they are judged, so +// that FailureList::VerdictOnFailure() compares like with like. +std::string FormatFailureMessage(absl::string_view message); + +// The expected failures of a conformance run, as the failure list files list +// them, plus the enforcement level: everything needed to judge one test's +// outcome, and nothing that changes once the tests run. The ResultLedger +// (result_ledger.h) builds it while loading the failure lists and tallies the +// outcomes against it. Yields() (matchers.h) asks the run's list, through +// the global ledger (see global_test_environment.h), for its verdicts. +class FailureList { + public: + FailureList() : entries_("root") {} + + // The lowest priority whose failures fail the suite (see TestPriority in + // testee.h). VerdictOnFailure() tolerates a failing test of a lower priority + // unless it is listed. Defaults to kLowestPriority, which enforces every + // priority. + void set_enforcement_level(TestPriority level) { enforcement_level_ = level; } + TestPriority enforcement_level() const { return enforcement_level_; } + + // Adds the entry `test_name`, a test name or a wildcard, as expected to fail + // with `failure_message` (see VerdictOnFailure()). Returns an error if the + // entry is already present or contains an invalid wildcard. + absl::Status Add(absl::string_view test_name, + absl::string_view failure_message); + + // The verdict on a test that succeeded: an error naming the entry it + // matched if the test was expected to fail. + absl::Status VerdictOnSuccess(absl::string_view test_name) const; + + // The verdict on a test that failed with `failure_message`. OK if the + // failure doesn't fail the suite. That is the case when the test is in the + // failure list with a matching message (an expected failure), or when it + // isn't listed and `priority` is above the enforcement level (a tolerated + // failure). Otherwise the failure is unexpected and the error says why. A + // listed test is checked whatever its priority, so that the failure list + // can't go stale unnoticed. The normalized actual message only needs to + // start with the expected message, so an empty expected message matches any + // failure. + absl::Status VerdictOnFailure(absl::string_view test_name, + TestPriority priority, + absl::string_view failure_message) const; + + // The verdict on a test the testee skipped for `skip_reason`: an error + // naming the entry it matched if the test is in the failure list, since a + // skipped test can't be an expected failure. Whether that error fails the + // test is the caller's policy; Yields() fails such a test. + absl::Status VerdictOnSkip(absl::string_view test_name, + absl::string_view skip_reason) const; + + // The failure list entry `test_name` matches, possibly a wildcard, if any. + absl::optional<std::string> MatchingEntry(absl::string_view test_name) const; + + // The expected failure message of the entry `entry`, normalized (see + // FormatFailureMessage()). `entry` is an entry exactly as listed, wildcards + // included, not a test name to match. Returns nullopt if there is no such + // entry. + absl::optional<std::string> ExpectedMessage(absl::string_view entry) const; + + private: + FailureListTrieNode entries_; + absl::flat_hash_map<std::string, std::string> expected_messages_; + TestPriority enforcement_level_ = kLowestPriority; +}; + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__
diff --git a/conformance/failure_list_test.cc b/conformance/failure_list_test.cc new file mode 100644 index 0000000..68cd931 --- /dev/null +++ b/conformance/failure_list_test.cc
@@ -0,0 +1,268 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/failure_list.h" + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::testing::AllOf; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Optional; + +TEST(FormatFailureMessageTest, StripsSurroundingWhitespace) { + EXPECT_EQ(FormatFailureMessage("abc"), "abc"); + EXPECT_EQ(FormatFailureMessage(" \t\nabc \n"), "abc"); + EXPECT_EQ(FormatFailureMessage(" \n "), ""); + EXPECT_EQ(FormatFailureMessage(""), ""); +} + +TEST(FormatFailureMessageTest, RemovesNewlines) { + // Other whitespace within the message is kept. + EXPECT_EQ(FormatFailureMessage("a\nb \tc\n\nd"), "ab \tcd"); +} + +TEST(FormatFailureMessageTest, TruncatesTo128Characters) { + const std::string limit(128, 'a'); + EXPECT_EQ(FormatFailureMessage(limit), limit); + EXPECT_EQ(FormatFailureMessage(absl::StrCat(limit, "b")), limit); + // Stripped whitespace and removed newlines don't count toward the limit. + const std::string a(64, 'a'); + const std::string b(64, 'b'); + EXPECT_EQ(FormatFailureMessage(absl::StrCat(" ", a, "\n", b, "\n ")), + absl::StrCat(a, b)); +} + +TEST(FailureListTest, EmptyListExpectsNoFailure) { + FailureList failure_list; + EXPECT_EQ(failure_list.MatchingEntry("foo"), absl::nullopt); + EXPECT_EQ(failure_list.ExpectedMessage("foo"), absl::nullopt); + EXPECT_THAT(failure_list.VerdictOnSuccess("foo"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnSkip("foo", "reason"), IsOk()); +} + +TEST(FailureListTest, AddNormalizesTheExpectedMessage) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", " abc\ndef "), IsOk()); + ASSERT_THAT(failure_list.Add("bar", std::string(1000, 'x')), IsOk()); + ASSERT_THAT(failure_list.Add("baz", ""), IsOk()); + + EXPECT_THAT(failure_list.ExpectedMessage("foo"), Optional(Eq("abcdef"))); + EXPECT_THAT(failure_list.ExpectedMessage("bar"), + Optional(std::string(128, 'x'))); + EXPECT_THAT(failure_list.ExpectedMessage("baz"), Optional(IsEmpty())); +} + +TEST(FailureListTest, AddRejectsADuplicateEntry) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + + EXPECT_THAT(failure_list.Add("foo", "xyz"), + StatusIs(absl::StatusCode::kAlreadyExists, + AllOf(HasSubstr("already exists"), HasSubstr("foo")))); + // The first entry is kept. + EXPECT_THAT(failure_list.ExpectedMessage("foo"), Optional(Eq("abc"))); +} + +TEST(FailureListTest, AddRejectsAnEntryOverlappingAWildcard) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo.*.bar", "abc"), IsOk()); + + EXPECT_THAT(failure_list.Add("foo.bar.*", "xyz"), + StatusIs(absl::StatusCode::kAlreadyExists, + AllOf(HasSubstr("foo.bar.*"), HasSubstr("foo.*.bar")))); + EXPECT_THAT( + failure_list.Add("foo.baz.bar", "xyz"), + StatusIs(absl::StatusCode::kAlreadyExists, + AllOf(HasSubstr("foo.baz.bar"), HasSubstr("foo.*.bar")))); + EXPECT_EQ(failure_list.ExpectedMessage("foo.bar.*"), absl::nullopt); + EXPECT_EQ(failure_list.ExpectedMessage("foo.baz.bar"), absl::nullopt); +} + +TEST(FailureListTest, AddRejectsAnInvalidWildcard) { + FailureList failure_list; + + EXPECT_THAT(failure_list.Add("foo.b*r", "abc"), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(failure_list.MatchingEntry("foo.bar"), absl::nullopt); + EXPECT_EQ(failure_list.ExpectedMessage("foo.b*r"), absl::nullopt); +} + +TEST(FailureListTest, MatchingEntryMatchesWholeNameComponents) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + ASSERT_THAT(failure_list.Add("bar.*.baz", "abc"), IsOk()); + + EXPECT_THAT(failure_list.MatchingEntry("foo"), Optional(Eq("foo"))); + EXPECT_THAT(failure_list.MatchingEntry("bar.a.baz"), + Optional(Eq("bar.*.baz"))); + EXPECT_THAT(failure_list.MatchingEntry("bar.b.baz"), + Optional(Eq("bar.*.baz"))); + // Neither a prefix nor an extension of an entry matches it, and a wildcard + // stands for exactly one component. + EXPECT_EQ(failure_list.MatchingEntry("fo"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("foo.bar"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("bar"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("bar.a"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("bar.a.baz.qux"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("other"), absl::nullopt); +} + +TEST(FailureListTest, ExpectedMessageLooksUpEntriesAsListed) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + ASSERT_THAT(failure_list.Add("bar.*", "xyz"), IsOk()); + + EXPECT_THAT(failure_list.ExpectedMessage("foo"), Optional(Eq("abc"))); + EXPECT_THAT(failure_list.ExpectedMessage("bar.*"), Optional(Eq("xyz"))); + // Not matched like a test name. + EXPECT_EQ(failure_list.ExpectedMessage("bar.x"), absl::nullopt); + EXPECT_EQ(failure_list.ExpectedMessage("unlisted"), absl::nullopt); +} + +TEST(FailureListTest, VerdictOnSuccessNamesTheMatchedEntry) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + ASSERT_THAT(failure_list.Add("bar.*.baz", "abc"), IsOk()); + + EXPECT_THAT(failure_list.VerdictOnSuccess("unlisted"), IsOk()); + // The message is the legacy runner's. + EXPECT_THAT(failure_list.VerdictOnSuccess("foo"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test foo (matched to foo) is in the failure list, but " + "test succeeded. Remove its match from the failure " + "list.")); + EXPECT_THAT(failure_list.VerdictOnSuccess("bar.a.baz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("test bar.a.baz (matched to bar.*.baz) is in " + "the failure list"))); +} + +TEST(FailureListTest, VerdictOnSkipNamesTheMatchedEntry) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("bar.*", "abc"), IsOk()); + + EXPECT_THAT(failure_list.VerdictOnSkip("unlisted", "reason"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnSkip("bar.x", "reason"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test bar.x (matched to bar.*) is in the failure list " + "but was skipped by the testee: reason. Remove its " + "match from the failure list.")); +} + +TEST(FailureListTest, VerdictOnFailureChecksTheExpectedMessage) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + ASSERT_THAT(failure_list.Add("bar.*", "abc"), IsOk()); + + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "abc"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("bar.x", kP0, "abc"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "xyz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: foo expected: " + "abc actual: xyz")); + EXPECT_THAT(failure_list.VerdictOnFailure("bar.x", kP0, "xyz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: bar.x expected: " + "abc actual: xyz")); +} + +TEST(FailureListTest, VerdictOnFailureMatchesTheExpectedMessageAsAPrefix) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + ASSERT_THAT(failure_list.Add("bar", ""), IsOk()); + + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "abc: more details"), + IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "xabc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: foo expected: " + "abc actual: xabc")); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "ab"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: foo expected: " + "abc actual: ab")); + // An empty expected message matches any failure. + EXPECT_THAT(failure_list.VerdictOnFailure("bar", kP0, "xyz"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("bar", kP0, ""), IsOk()); +} + +TEST(FailureListTest, VerdictOnFailureComparesNormalizedMessages) { + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc\ndef"), IsOk()); + ASSERT_THAT(failure_list.Add("bar", std::string(1000, 'x')), IsOk()); + + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, " ab\ncdef\n"), + IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("bar", kP0, std::string(1000, 'x')), + IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure( + "foo", kP0, absl::StrCat("\nx", std::string(1000, 'y'))), + StatusIs(absl::StatusCode::kFailedPrecondition, + absl::StrCat("Unexpected failure message for test: foo " + "expected: abcdef actual: x", + std::string(127, 'y')))); +} + +TEST(FailureListTest, EveryPriorityIsEnforcedByDefault) { + FailureList failure_list; + + EXPECT_EQ(failure_list.enforcement_level(), kLowestPriority); + EXPECT_THAT(failure_list.VerdictOnFailure("p0", kP0, "abc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure for test: p0")); + EXPECT_THAT(failure_list.VerdictOnFailure("p1", kP1, "abc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure for test: p1")); +} + +TEST(FailureListTest, EnforcementLevelToleratesUnlistedLowerPriorities) { + FailureList failure_list; + failure_list.set_enforcement_level(kP0); + + EXPECT_EQ(failure_list.enforcement_level(), kP0); + EXPECT_THAT(failure_list.VerdictOnFailure("p0", kP0, "abc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure for test: p0")); + EXPECT_THAT(failure_list.VerdictOnFailure("p1", kP1, "abc"), IsOk()); +} + +TEST(FailureListTest, ListedFailureIsCheckedWhateverItsPriority) { + // So that the failure list can't go stale unnoticed. + FailureList failure_list; + ASSERT_THAT(failure_list.Add("foo", "abc"), IsOk()); + failure_list.set_enforcement_level(kP0); + + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP1, "abc"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP1, "xyz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: foo expected: " + "abc actual: xyz")); +} + +} // namespace +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_ledger.cc b/conformance/result_ledger.cc index 3d7fc84..92aaab5 100644 --- a/conformance/result_ledger.cc +++ b/conformance/result_ledger.cc
@@ -15,11 +15,11 @@ #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" -#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "conformance/failure_list.h" #include "conformance/testee.h" namespace google { @@ -28,13 +28,6 @@ namespace internal { namespace { -// How many failures one wildcard entry may cover. As in the legacy runner the -// check precedes the count of the current match, so the limit is reached one -// failure later than the number says: the (kMaximumWildcardExpansions + 2)th -// is the first one reported. -constexpr int kMaximumWildcardExpansions = 20; -constexpr int kFailureMessageLengthLimit = 128; - bool InsertUniqueTest(absl::flat_hash_set<std::string>& set, absl::string_view value) { if (!set.emplace(value).second) { @@ -49,22 +42,6 @@ } } -void Normalize(std::string& input) { - input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); -} - -// Sets up a failure message properly for our failure lists. -std::string FormatFailureMessage(absl::string_view input) { - std::string result(absl::StripAsciiWhitespace(input)); - // Remove newlines - Normalize(result); - // Truncate failure message if needed - if (result.length() > kFailureMessageLengthLimit) { - result = result.substr(0, kFailureMessageLengthLimit); - } - return result; -} - std::string ReformatLine(size_t alignment, absl::string_view line) { size_t comment_pos = line.find('#'); absl::string_view test_name = @@ -135,15 +112,10 @@ absl::Status ResultLedger::AddExpectedFailure( absl::string_view test_name, absl::string_view failure_message) { - if (absl::Status status = expected_failure_list_.Insert(test_name); + if (absl::Status status = failure_list_.Add(test_name, failure_message); !status.ok()) { return status; } - // Normalize the message the same way reported failures are normalized so - // that the two compare equal in ReportFailure. - ABSL_CHECK(expected_failure_messages_ - .emplace(test_name, FormatFailureMessage(failure_message)) - .second); ABSL_CHECK(unseen_expected_failures_.emplace(test_name).second); ABSL_CHECK(unmatched_expected_failures_.emplace(test_name).second); return absl::OkStatus(); @@ -215,15 +187,11 @@ } unseen_expected_failures_.erase(*failure_match); seen_unexpected_successes_.insert(*failure_match); - return absl::FailedPreconditionError(absl::StrCat( - "test ", test_name, " (matched to ", *failure_match, - ") is in the failure list, but test succeeded. Remove its match from " - "the failure list.")); + } else { + // This wasn't expected to fail. + IncrementIfUnique(unique, expected_successes_); } - - // This wasn't expected to fail. - IncrementIfUnique(unique, expected_successes_); - return absl::OkStatus(); + return failure_list_.VerdictOnSuccess(test_name); } absl::Status ResultLedger::ReportFailure(absl::string_view test_name, @@ -231,6 +199,8 @@ absl::string_view failure_message) { bool unique = InsertUniqueTest(seen_tests_, test_name); absl::optional<std::string> failure_match = MarkMatched(test_name); + absl::Status verdict = + failure_list_.VerdictOnFailure(test_name, priority, failure_message); std::string formatted_failure_message = FormatFailureMessage(failure_message); // Counts the failure as unexpected and records it for UnexpectedFailures(). @@ -242,26 +212,23 @@ }; if (!failure_match.has_value()) { - if (priority > enforcement_level_) { + if (verdict.ok()) { // Tolerated: neither a failure nor a skip, and not written to the // failure list; only counted. IncrementIfUnique(unique, tolerated_failures_); ABSL_LOG(WARNING) << "WARNING, test=" << test_name << ": " << absl::StripTrailingAsciiWhitespace(failure_message); - return absl::OkStatus(); + } else { + // This was not expected to fail. + record_unexpected_failure(); + new_failures_[test_name] = formatted_failure_message; } - // This was not expected to fail. - record_unexpected_failure(); - new_failures_[test_name] = formatted_failure_message; - return absl::FailedPreconditionError( - absl::StrCat("Unexpected failure for test: ", test_name)); + return verdict; } - // Mirror the legacy runner, which only requires the actual failure message to - // start with the expected one. - const std::string& expected_failure_message = - expected_failure_messages_.at(*failure_match); - if (!absl::StartsWith(formatted_failure_message, expected_failure_message)) { + // A listed test's verdict only depends on its message, which the list + // matches as a prefix of the actual one. + if (!verdict.ok()) { record_unexpected_failure(); // TODO: Keying the replacement by the (possibly wildcard) // entry duplicates the line if another test later matches the entry with @@ -270,28 +237,15 @@ // makes SaveFailureList() keep the original line and add a second // `foo.*.bar` line, which LoadFailureList() then rejects. new_failures_[*failure_match] = formatted_failure_message; - return absl::FailedPreconditionError(absl::StrCat( - "Unexpected failure message for test: ", test_name, " expected: ", - expected_failure_message, " actual: ", formatted_failure_message)); + return verdict; } unseen_expected_failures_.erase(*failure_match); - - if (number_of_matches_[*failure_match] > kMaximumWildcardExpansions) { - record_unexpected_failure(); - return absl::FailedPreconditionError( - absl::StrCat("The wildcard ", *failure_match, - " served as matches to too many test " - "names exceeding the max amount of ", - kMaximumWildcardExpansions, " for test: ", test_name)); - } - IncrementIfUnique(unique, number_of_matches_[*failure_match]); IncrementIfUnique(unique, expected_failures_); ABSL_LOG(INFO) << "Ignoring expected failure for test " << test_name << ": " << absl::StripTrailingAsciiWhitespace(failure_message); - - return absl::OkStatus(); + return verdict; } absl::Status ResultLedger::ReportSkip(absl::string_view test_name, @@ -314,10 +268,7 @@ if (unique) { listed_skip_matches_[test_name] = *failure_match; } - return absl::FailedPreconditionError(absl::StrCat( - "test ", test_name, " (matched to ", *failure_match, - ") is in the failure list but was skipped by the testee: ", skip_reason, - ". Remove its match from the failure list.")); + return failure_list_.VerdictOnSkip(test_name, skip_reason); } void ResultLedger::ReportNotSelected(absl::string_view test_name) { @@ -328,13 +279,22 @@ absl::optional<std::string> ResultLedger::MarkMatched( absl::string_view test_name) { absl::optional<std::string> failure_match = - expected_failure_list_.WalkDownMatch(test_name); + failure_list_.MatchingEntry(test_name); if (failure_match.has_value()) { unmatched_expected_failures_.erase(*failure_match); } return failure_match; } +std::vector<std::string> ResultLedger::OverexpandedWildcards() const { + std::vector<std::string> entries; + for (const auto& [entry, matches] : number_of_matches_) { + if (matches > kMaximumWildcardExpansions) entries.push_back(entry); + } + absl::c_sort(entries); + return entries; +} + std::vector<std::string> ResultLedger::UnseenExpectedFailures() const { std::vector<std::string> unseen(unseen_expected_failures_.begin(), unseen_expected_failures_.end()); @@ -368,7 +328,7 @@ for (const auto& [test_name, matched_entry] : unexpected_success_matches_) { successes.push_back( {/*test_name=*/test_name, - /*failure_message=*/expected_failure_messages_.at(matched_entry), + /*failure_message=*/*failure_list_.ExpectedMessage(matched_entry), /*matched_entry=*/matched_entry}); } return successes;
diff --git a/conformance/result_ledger.h b/conformance/result_ledger.h index 51718ad..8036abd 100644 --- a/conformance/result_ledger.h +++ b/conformance/result_ledger.h
@@ -11,8 +11,7 @@ #include "absl/status/status.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" -#include "conformance/conformance.pb.h" -#include "conformance/failure_list_trie_node.h" +#include "conformance/failure_list.h" #include "conformance/testee.h" namespace google { @@ -36,20 +35,29 @@ absl::optional<std::string> matched_entry; }; -// Tracks the expected failures and the actual results of a test suite. The -// conformance matchers (see matchers.h) report the outcome of every test -// here. The test environment reads the results back and reports them as test -// properties. Under --fix the results also populate the new failure list. +// How many tests one wildcard entry may cover with a verdict, that is an +// expected failure or an unexpected success. See OverexpandedWildcards(). +inline constexpr int kMaximumWildcardExpansions = 20; + +// The ledger of a conformance run: tallies the results of its tests against +// its failure list. The failure list is loaded once (see LoadFailureList()) +// into a FailureList, which decides each result; the Report*() methods apply +// its verdicts and count the outcomes. The conformance matchers (see +// matchers.h) report the outcome of every test here. The test environment +// reads the counters back as test properties and, under --fix, writes the new +// failure list from the ledger (see SaveFailureList()). class ResultLedger { public: - ResultLedger() : expected_failure_list_("root") {} + ResultLedger() = default; ~ResultLedger(); // The lowest priority whose failures fail the suite (see TestPriority in // testee.h). ReportFailure() tolerates a failing test of a lower priority // unless it is in the failure list. Defaults to kLowestPriority, which // enforces every priority. - void set_enforcement_level(TestPriority level) { enforcement_level_ = level; } + void set_enforcement_level(TestPriority level) { + failure_list_.set_enforcement_level(level); + } // Loads a failure list from disk and adds its entries to the ones loaded so // far. Each line that isn't blank or a comment names one expected failure, @@ -59,36 +67,32 @@ // invalid wildcard. absl::Status LoadFailureList(absl::string_view filename); + // The failure list as loaded so far, which decides each result. Yields() + // reads the global environment's through GetGlobalResultLedger(). + const FailureList& failure_list() const { return failure_list_; } + // Saves an updated failure list to disk based on the reported results. // Returns an error if `filename` can't be opened for writing or the write // fails. absl::Status SaveFailureList(absl::string_view filename) const; - // Reports a successful test run. This will return an error if the test was - // expected to fail. + // Records a successful test run and returns + // FailureList::VerdictOnSuccess(). absl::Status ReportSuccess(absl::string_view test_name); - // Reports a failed test run along with the failure message. Returns OK if - // the failure doesn't fail the suite. That is the case when the test is in - // the failure list with a matching message (an expected failure, logged at - // INFO), or when it isn't listed and `priority` is above the enforcement - // level (a tolerated failure, logged as a WARNING and only counted by - // tolerated_failures()). Otherwise the failure is unexpected and the error - // says why. A listed test is checked whatever its priority, so that the - // failure list can't go stale unnoticed. The normalized actual message - // only needs to start with the expected message, so an empty expected - // message matches any failure. + // Records a failed test run along with the failure message and returns + // FailureList::VerdictOnFailure(). A tolerated failure is only counted + // (see tolerated_failures()). An unexpected one is also kept for + // UnexpectedFailures() and SaveFailureList(). absl::Status ReportFailure(absl::string_view test_name, TestPriority priority, absl::string_view failure_message); - // Reports a test that the testee skipped for `skip_reason`. A skip is not - // a verdict on the test's failure list entry, if any. The entry counts as - // seen and matched, so that Finalize() doesn't report it and - // SaveFailureList() keeps it, but the skip is not an expected failure. A - // listed test that is skipped is recorded (see ListedSkips()) and returns an - // error that names the matched entry, like an unexpected success. Whether - // that error fails the test is the caller's policy. The matchers fail such - // a test. + // Records a test that the testee skipped for `skip_reason` and returns + // FailureList::VerdictOnSkip(). A skip is not a verdict on the test's + // entry, if any. The entry counts as seen and matched, so that Finalize() + // doesn't report it and SaveFailureList() keeps it, but the skip is not an + // expected failure. A listed test that is skipped is also recorded for + // ListedSkips(). absl::Status ReportSkip(absl::string_view test_name, absl::string_view skip_reason); @@ -107,6 +111,12 @@ return seen_tests_.contains(test_name); } + // The sorted failure list entries that more than kMaximumWildcardExpansions + // reported tests matched with a verdict, that is as an expected failure or + // an unexpected success. Skips don't count. Such a wildcard hides too + // much; the test environment fails the run over it. + std::vector<std::string> OverexpandedWildcards() const; + // Runs sanity checks over the failure list to make sure everything we // expected to run was reported. Returns an error naming the sorted expected // failure entries that have not been reported as a failure, an unexpected @@ -189,13 +199,14 @@ // UnmatchedExpectedFailures()) and returns it. absl::optional<std::string> MarkMatched(absl::string_view test_name); - FailureListTrieNode expected_failure_list_; - absl::flat_hash_map<std::string, std::string> expected_failure_messages_; + FailureList failure_list_; absl::flat_hash_set<std::string> unseen_expected_failures_; // Entries never matched by name by any Report*() call. absl::flat_hash_set<std::string> unmatched_expected_failures_; absl::flat_hash_set<std::string> seen_unexpected_successes_; + // How many tests matched each entry as an expected failure or an unexpected + // success. See OverexpandedWildcards(). absl::flat_hash_map<std::string, int> number_of_matches_; // Every test name reported so far (see WasReported()), so that a test @@ -221,7 +232,6 @@ int expected_successes_ = 0; int unexpected_successes_ = 0; bool finalized_ = false; - TestPriority enforcement_level_ = kLowestPriority; }; } // namespace internal
diff --git a/conformance/result_ledger_test.cc b/conformance/result_ledger_test.cc index 0f6fe97..06a0cd6 100644 --- a/conformance/result_ledger_test.cc +++ b/conformance/result_ledger_test.cc
@@ -15,6 +15,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "conformance/failure_list.h" #include "conformance/testee.h" namespace google { @@ -200,6 +201,37 @@ EXPECT_THAT(ledger.Finalize(), IsOk()); } +TEST_F(ResultLedgerTest, FailureListReadsRecordNothing) { + CreateFailureList({{"foo", "abc"}, {"bar.*", "abc"}, {"baz", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ledger.set_enforcement_level(kP0); + + // The list is the one loaded, at the ledger's enforcement level. + const FailureList& failure_list = ledger.failure_list(); + EXPECT_EQ(failure_list.enforcement_level(), kP0); + failure_list.MatchingEntry("bar.x"); + failure_list.ExpectedMessage("foo"); + failure_list.VerdictOnSuccess("foo").IgnoreError(); + failure_list.VerdictOnFailure("foo", kP0, "abc").IgnoreError(); + failure_list.VerdictOnSkip("bar.x", "reason").IgnoreError(); + + // Nothing was reported. + EXPECT_FALSE(ledger.WasReported("foo")); + EXPECT_FALSE(ledger.WasReported("bar.x")); + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), + ElementsAre("bar.*", "baz", "foo")); + EXPECT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("were not seen: bar.*, baz, foo"))); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); + EXPECT_EQ(ledger.tolerated_failures(), 0); +} + TEST_F(ResultLedgerTest, ReportExpectedFailure) { CreateFailureList({{"foo", "abc"}}); ResultLedger ledger; @@ -373,29 +405,6 @@ EXPECT_EQ(ledger.unexpected_failures(), 1); } -TEST_F(ResultLedgerTest, ReportFailureEmptyExpectedMessageMatchesAnything) { - CreateFailureList({{"foo"}}); - ResultLedger ledger; - ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(ledger.ReportFailure("foo", kP0, "any message at all"), IsOk()); - - EXPECT_THAT(ledger.Finalize(), IsOk()); - EXPECT_EQ(ledger.expected_failures(), 1); - EXPECT_EQ(ledger.unexpected_failures(), 0); -} - -TEST_F(ResultLedgerTest, ReportFailurePrefixMatchIgnoresNewlinesAndWhitespace) { - CreateFailureList({{"foo", "abc def"}}); - ResultLedger ledger; - ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(ledger.ReportFailure("foo", kP0, " abc def\nghi\n"), IsOk()); - - EXPECT_THAT(ledger.Finalize(), IsOk()); - EXPECT_EQ(ledger.expected_failures(), 1); -} - TEST_F(ResultLedgerTest, PrefixMatchedFailureIsNotAnUnexpectedFailure) { CreateFailureList({{"foo", "abc"}}); ResultLedger ledger; @@ -486,7 +495,7 @@ EXPECT_THAT(ledger.Finalize(), IsOk()); } -TEST_F(ResultLedgerTest, UnexpectedFailuresIncludeExceededWildcardMatches) { +TEST_F(ResultLedgerTest, OverexpandedWildcardKeepsMatching) { CreateFailureList({{"foo.*.bar"}}); ResultLedger ledger; ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); @@ -495,11 +504,14 @@ ASSERT_THAT(ledger.ReportSuccess(absl::StrCat("foo.", i, ".bar")), Not(IsOk())); } - ASSERT_THAT(ledger.ReportFailure("foo.baz.bar", kP0, "msg"), Not(IsOk())); + // The entry is far over the cap, but the verdicts don't change: the test + // environment fails the run over OverexpandedWildcards() instead. + ASSERT_THAT(ledger.ReportFailure("foo.baz.bar", kP0, "msg"), IsOk()); - EXPECT_THAT(ledger.UnexpectedFailures(), - ElementsAre(FieldsAre("foo.baz.bar", "msg", absl::nullopt))); - EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_THAT(ledger.OverexpandedWildcards(), ElementsAre("foo.*.bar")); + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); EXPECT_THAT(ledger.UnexpectedSuccesses(), SizeIs(100)); EXPECT_EQ(ledger.unexpected_successes(), 100); EXPECT_THAT(ledger.Finalize(), IsOk()); @@ -546,26 +558,34 @@ EXPECT_THAT(ledger.Finalize(), IsOk()); } -TEST_F(ResultLedgerTest, ReportUnexpectedFailureTooManyWildcardMatches) { - CreateFailureList({{"foo.*.bar"}}); +TEST_F(ResultLedgerTest, OverexpandedWildcards) { + CreateFailureList({{"foo.*.bar", "abc"}, {"baz.*", "abc"}}); ResultLedger ledger; ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - for (int i = 0; i < 100; ++i) { - EXPECT_THAT(ledger.ReportSuccess(absl::StrCat("foo.", i, ".bar")), - StatusIs(absl::StatusCode::kFailedPrecondition)); + // Expected failures and unexpected successes count toward the cap, each + // test once however often it is reported. + for (int i = 0; i < kMaximumWildcardExpansions; ++i) { + ASSERT_THAT( + ledger.ReportFailure(absl::StrCat("foo.", i, ".bar"), kP0, "abc"), + IsOk()); + ASSERT_THAT(ledger.ReportSuccess(absl::StrCat("baz.", i)), Not(IsOk())); } - ASSERT_THAT(ledger.ReportFailure("foo.baz.bar", kP0, ""), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("too many test names"), - HasSubstr("foo.baz.bar")))); + ASSERT_THAT(ledger.ReportFailure("foo.0.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("baz.0"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), IsEmpty()); + // Skips and failures with another message don't count. + ASSERT_THAT(ledger.ReportSkip("foo.skip.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.other.bar", kP0, "xyz"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), IsEmpty()); + + // One more verdict puts an entry over the cap. The list is sorted. + ASSERT_THAT(ledger.ReportFailure("foo.more.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("baz.more"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), + ElementsAre("baz.*", "foo.*.bar")); EXPECT_THAT(ledger.Finalize(), IsOk()); - EXPECT_EQ(ledger.expected_failures(), 0); - EXPECT_EQ(ledger.unexpected_failures(), 1); - EXPECT_EQ(ledger.expected_successes(), 0); - EXPECT_EQ(ledger.unexpected_successes(), 100); - EXPECT_EQ(ledger.skipped(), 0); } TEST_F(ResultLedgerTest, LoadFailureListInvalidFile) { @@ -588,55 +608,6 @@ ledger.Finalize().IgnoreError(); } -TEST_F(ResultLedgerTest, LoadFailureListOverlappingFailure) { - CreateFailureList(R"( - foo.*.bar # abc - foo.bar.* # zyx -)"); - ResultLedger ledger; - - EXPECT_THAT(ledger.LoadFailureList(failure_list()), - StatusIs(absl::StatusCode::kAlreadyExists, - AllOf(HasSubstr("foo.bar.*"), HasSubstr("foo.*.bar")))); - ledger.Finalize().IgnoreError(); -} - -TEST_F(ResultLedgerTest, UnexpectedSuccessMessageNamesTheEntry) { - CreateFailureList({{"foo", "abc"}}); - ResultLedger ledger; - ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - - // The message is the legacy runner's, including the matched entry. - EXPECT_THAT(ledger.ReportSuccess("foo"), - StatusIs(absl::StatusCode::kFailedPrecondition, - "test foo (matched to foo) is in the failure list, but " - "test succeeded. Remove its match from the failure " - "list.")); - EXPECT_THAT(ledger.Finalize(), IsOk()); - EXPECT_EQ(ledger.unexpected_successes(), 1); -} - -TEST_F(ResultLedgerTest, UnexpectedSuccessMessageNamesTheWildcardEntry) { - CreateFailureList({{"foo.*.bar", "abc"}}); - ResultLedger ledger; - ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(ledger.ReportSuccess("foo.a.bar"), - StatusIs(absl::StatusCode::kFailedPrecondition, - HasSubstr("test foo.a.bar (matched to foo.*.bar) is in " - "the failure list"))); - EXPECT_THAT(ledger.Finalize(), IsOk()); -} - -TEST_F(ResultLedgerTest, EveryPriorityIsEnforcedByDefault) { - ResultLedger ledger; - EXPECT_THAT(ledger.ReportFailure("p0", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(ledger.ReportFailure("p1", kP1, "abc"), Not(IsOk())); - EXPECT_EQ(ledger.unexpected_failures(), 2); - EXPECT_EQ(ledger.tolerated_failures(), 0); - EXPECT_THAT(ledger.Finalize(), IsOk()); -} - TEST_F(ResultLedgerTest, EnforcementLevelZeroToleratesP1) { // Only kP0 is enforced: an unlisted kP1 failure is tolerated. ResultLedger ledger; @@ -709,14 +680,6 @@ ledger.Finalize().IgnoreError(); } -TEST_F(ResultLedgerTest, LoadFailureListInvalidWildcard) { - CreateFailureList({{"foo.b*r", "abc"}}); - ResultLedger ledger; - EXPECT_THAT(ledger.LoadFailureList(failure_list()), - StatusIs(absl::StatusCode::kInvalidArgument)); - ledger.Finalize().IgnoreError(); -} - TEST_F(ResultLedgerTest, LoadFailureListTruncatesMessage) { // Like a reported message, so that the two still compare equal. std::string message(1000, 'b'); @@ -748,28 +711,6 @@ EXPECT_EQ(ledger.expected_failures(), 2); } -TEST_F(ResultLedgerTest, FailureListMatchesWholeNameComponents) { - CreateFailureList({{"foo", "abc"}, {"bar.*.baz", "abc"}, {"qux", "abc"}}); - ResultLedger ledger; - ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); - EXPECT_THAT(ledger.ReportFailure("qux", kP0, "abc"), IsOk()); - EXPECT_THAT(ledger.ReportFailure("bar.a.baz", kP0, "abc"), IsOk()); - EXPECT_THAT(ledger.ReportFailure("bar.b.baz", kP0, "abc"), IsOk()); - // Neither a prefix nor an extension of an entry matches it, and a wildcard - // stands for exactly one component. - EXPECT_THAT(ledger.ReportFailure("foo.bar", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(ledger.ReportFailure("bar", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(ledger.ReportFailure("bar.a", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(ledger.ReportFailure("bar.a.baz.qux", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(ledger.ReportFailure("other", kP0, "abc"), Not(IsOk())); - - EXPECT_EQ(ledger.expected_failures(), 4); - EXPECT_EQ(ledger.unexpected_failures(), 5); - EXPECT_THAT(ledger.Finalize(), IsOk()); -} - TEST_F(ResultLedgerTest, SaveFailureListNoop) { CreateFailureList(R"( foo # abc
diff --git a/conformance/test_environment.cc b/conformance/test_environment.cc index b0b1b73..43cc372 100644 --- a/conformance/test_environment.cc +++ b/conformance/test_environment.cc
@@ -233,6 +233,12 @@ void ConformanceEnvironment::TearDown() { Statistics::From(ledger_).RecordProperties(); + for (const std::string& entry : ledger_.OverexpandedWildcards()) { + ADD_FAILURE() << "The failure list entry " << entry << " matched more than " + << kMaximumWildcardExpansions + << " tests, which hides too much. List the failing tests " + "individually, or narrow the wildcard."; + } const bool partial_run = partial_run_override_.value_or(IsPartialRun()); // The path the failure list was rewritten to, if --fix succeeded.