Internal change PiperOrigin-RevId: 496537672
diff --git a/fuzztest/BUILD b/fuzztest/BUILD index 6dbe207..038cfee 100644 --- a/fuzztest/BUILD +++ b/fuzztest/BUILD
@@ -345,6 +345,7 @@ srcs = ["internal/serialization.cc"], hdrs = ["internal/serialization.h"], deps = [ + ":logging", ":meta", "@com_google_absl//absl/numeric:int128", "@com_google_absl//absl/strings",
diff --git a/fuzztest/internal/serialization.cc b/fuzztest/internal/serialization.cc index d276d9c..73dd439 100644 --- a/fuzztest/internal/serialization.cc +++ b/fuzztest/internal/serialization.cc
@@ -18,7 +18,6 @@ #include <cstddef> #include <cstdint> #include <limits> -#include <optional> #include <string> #include <vector> @@ -72,36 +71,12 @@ } }; -constexpr std::string_view kHeader = "FUZZTESTv1"; - -absl::string_view AsAbsl(std::string_view str) { - return {str.data(), str.size()}; -} - -std::string_view ReadToken(std::string_view& in) { - while (!in.empty() && std::isspace(in[0])) in.remove_prefix(1); - if (in.empty()) return in; - size_t end = 1; - const auto is_literal = [](char c) { - return std::isalnum(c) != 0 || c == '+' || c == '-' || c == '.'; - }; - if (is_literal(in[0])) { - while (end < in.size() && is_literal(in[end])) ++end; - } else if (in[0] == '"') { - while (end < in.size() && in[end] != '"') ++end; - if (end < in.size()) ++end; - } - std::string_view res = in.substr(0, end); - in.remove_prefix(end); - return res; -} - bool ReadScalar(uint64_t& out, std::string_view value) { - return absl::SimpleAtoi(AsAbsl(value), &out); + return absl::SimpleAtoi(IRObject{}.AsAbsl(value), &out); } bool ReadScalar(double& out, std::string_view value) { - return absl::SimpleAtod(AsAbsl(value), &out); + return absl::SimpleAtod(IRObject{}.AsAbsl(value), &out); } bool ReadScalar(std::string& out, std::string_view value) { @@ -134,7 +109,31 @@ return true; } -bool ParseImpl(IRObject& obj, std::string_view& str) { +} // namespace + +void IRObject::Visit(std::string& out) const { + absl::visit(OutputVisitor{value.index(), 0, out}, value); +} + +std::string_view IRObject::ReadToken(std::string_view& in) const { + while (!in.empty() && std::isspace(in[0])) in.remove_prefix(1); + if (in.empty()) return in; + size_t end = 1; + const auto is_literal = [](char c) { + return std::isalnum(c) != 0 || c == '+' || c == '-' || c == '.'; + }; + if (is_literal(in[0])) { + while (end < in.size() && is_literal(in[end])) ++end; + } else if (in[0] == '"') { + while (end < in.size() && in[end] != '"') ++end; + if (end < in.size()) ++end; + } + std::string_view res = in.substr(0, end); + in.remove_prefix(end); + return res; +} + +bool IRObject::ParseImpl(IRObject& obj, std::string_view& str) { std::string_view key = ReadToken(str); if (key.empty() || key == "}") { // The object is empty. Put the token back and return. @@ -171,19 +170,4 @@ } } -} // namespace - -std::string IRObject::ToString() const { - std::string out = absl::StrCat(AsAbsl(kHeader), "\n"); - absl::visit(OutputVisitor{value.index(), 0, out}, value); - return out; -} - -std::optional<IRObject> IRObject::FromString(std::string_view str) { - IRObject object; - if (ReadToken(str) != kHeader) return std::nullopt; - if (!ParseImpl(object, str) || !ReadToken(str).empty()) return std::nullopt; - return object; -} - } // namespace fuzztest::internal
diff --git a/fuzztest/internal/serialization.h b/fuzztest/internal/serialization.h index 2ce6092..82c2d65 100644 --- a/fuzztest/internal/serialization.h +++ b/fuzztest/internal/serialization.h
@@ -23,11 +23,15 @@ #include <tuple> #include <type_traits> #include <utility> +#include <variant> #include <vector> #include "absl/numeric/int128.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "absl/types/span.h" #include "absl/types/variant.h" +#include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/meta.h" namespace fuzztest::internal { @@ -260,10 +264,45 @@ } } + static constexpr std::string_view kHeader = "FUZZTESTv1"; + + absl::string_view AsAbsl(std::string_view str) const { + return {str.data(), str.size()}; + } + // Serialize the object as a string. This is used to persist the object on // files for reproducing bugs later. - std::string ToString() const; - static std::optional<IRObject> FromString(std::string_view str); + template <typename ValueType> + std::string ToString() const { + // Return single string-like or proto values as raw strings. + if constexpr (std::is_same_v<ValueType, std::string> || + is_protocol_buffer_v<ValueType>) { + FUZZTEST_INTERNAL_CHECK_PRECONDITION( + absl::holds_alternative<std::string>(value), + "String-like value should hold a string!"); + return absl::get<std::string>(value); + } + std::string out = absl::StrCat(AsAbsl(kHeader), "\n"); + // Construct out using IRObject format. + IRObject::Visit(out); + return out; + } + + template <typename ValueType> + std::optional<IRObject> FromString(std::string_view str) { + IRObject object; + if constexpr (std::is_same_v<ValueType, std::string>) { + object.value.emplace<std::string>(str); + FUZZTEST_INTERNAL_CHECK( + absl::holds_alternative<std::string>(object.value), + "IRObject value should hold a string after deserializing from a " + "single string-like value!"); + return object; + } + if (ReadToken(str) != kHeader) return std::nullopt; + if (!ParseImpl(object, str) || !ReadToken(str).empty()) return std::nullopt; + return object; + } private: template <typename T> @@ -272,6 +311,10 @@ template <typename K, typename V> static std::pair<std::remove_const_t<K>, std::remove_const_t<V>> RemoveConstFromPair(std::pair<K, V>); + + void Visit(std::string& out) const; + std::string_view ReadToken(std::string_view& in) const; + bool ParseImpl(IRObject& obj, std::string_view& str); }; } // namespace fuzztest::internal
diff --git a/fuzztest/internal/serialization_test.cc b/fuzztest/internal/serialization_test.cc index d0113ac..773228c 100644 --- a/fuzztest/internal/serialization_test.cc +++ b/fuzztest/internal/serialization_test.cc
@@ -39,8 +39,8 @@ using testing::_; using testing::ElementsAre; -using testing::Eq; using testing::FieldsAre; +using testing::HasSubstr; using testing::NanSensitiveDoubleEq; using testing::Not; using testing::Optional; @@ -92,7 +92,10 @@ void VerifyProtobufFormat(const IRObject& object) { IRObjectTestProto proto; - std::string s = object.ToString(); + std::string s = object.ToString<IRObject>(); + // TODO(cnie0017) This is not a IRObjectTestProto + // since it is already being parsed to String once. It is now of IRObject form + // Chop the header. s.erase(0, strlen("FUZZTESTv1\n")); @@ -104,14 +107,19 @@ void RoundTripVerify(const T&... values) { IRObject object; object.value = std::vector{IRObject{values}...}; - std::string s = object.ToString(); + std::string s = object.ToString<IRObject>(); SCOPED_TRACE(s); VerifyProtobufFormat(object); - EXPECT_THAT(IRObject::FromString(s), + // TODO(cnie0017): why can't use T? + EXPECT_THAT(IRObject{}.FromString<IRObject>(s), Optional(SubsAre(ValueIs<T>(values)...))); + // EXPECT_THAT(IRObject::FromString<T>(s), + // Optional(SubsAre(ValueIs<T>(values)...))); + // // Initializer contains unexpanded parameter pack + // // 'T'clang(unexpanded_parameter_pack) } template <typename T> @@ -134,13 +142,13 @@ IRObject{"child3.2.2"}}}}}, IRObject{"child4"}}}; - std::string s = root.ToString(); + std::string s = root.ToString<IRObject>(); SCOPED_TRACE(s); VerifyProtobufFormat(root); - std::optional<IRObject> obj = IRObject::FromString(s); + std::optional<IRObject> obj = IRObject{}.FromString<IRObject>(s); EXPECT_THAT( obj, Optional(SubsAre( ValueIs<std::string>("child1"), ValueIs<std::string>("child2"), @@ -151,9 +159,10 @@ } TEST(SerializerTest, EmptyObjectRoundTrips) { - std::string s = IRObject{}.ToString(); + std::string s = IRObject{}.ToString<std::nullptr_t>(); SCOPED_TRACE(s); - EXPECT_THAT(IRObject::FromString(s), Optional(ValueIs<std::monostate>({}))); + EXPECT_THAT(IRObject{}.FromString<std::nullptr_t>(s), + Optional(ValueIs<std::monostate>({}))); } TEST(SerializerTest, IndentationIsCorrect) { @@ -169,7 +178,7 @@ IRObject{uint64_t{322}}}}}}, IRObject{uint64_t{4}}}}; - std::string s = root.ToString(); + std::string s = root.ToString<IRObject>(); EXPECT_EQ(s, R"(FUZZTESTv1 sub { i: 1 } @@ -185,50 +194,67 @@ )"); } +// TODO(cnie0017): ensure the type passed in are correct // We manually write the serialized form to test the error handling of the // parser. The serializer would not generate these, so we can't use it. TEST(SerializerTest, WrongHeaderWontParse) { - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: 0"), Optional(_)); - EXPECT_THAT(IRObject::FromString("FUZZTESTv2"), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("FUZZtESTv1"), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("-FUZZTESTv1"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i: 0"), + Optional(_)); + EXPECT_THAT(IRObject{}.FromString<std::nullptr_t>("FUZZTESTv2"), + Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::nullptr_t>("FUZZtESTv1"), + Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::nullptr_t>("-FUZZTESTv1"), + Not(Optional(_))); } TEST(SerializerTest, HandlesUnterminatedString) { - EXPECT_THAT(IRObject::FromString("FUZZTESTv1\""), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::nullptr_t>("FUZZTESTv1\""), + Not(Optional(_))); } TEST(SerializerTest, BadScalarWontParse) { - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: 1"), + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i: 1"), Optional(ValueIs<uint64_t>(1))); // Out of bounds values - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: 123456789012345678901"), + EXPECT_THAT(IRObject{}.FromString<std::int64_t>( + "FUZZTESTv1 i: 123456789012345678901"), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: -1"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i: -1"), + Not(Optional(_))); // Missing : - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i 1"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i 1"), + Not(Optional(_))); // Bad tag - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 x: 1"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 x: 1"), + Not(Optional(_))); // Wrong separator - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i; 1"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i; 1"), + Not(Optional(_))); // Extra close - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: 1}"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i: 1}"), + Not(Optional(_))); } TEST(SerializerTest, BadSubWontParse) { - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub { i: 0 }"), + EXPECT_THAT(IRObject{}.FromString<IRObject>("FUZZTESTv1 sub { i: 0 }"), Optional(SubsAre(ValueIs<uint64_t>(0)))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub: { }"), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub }"), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub { "), Not(Optional(_))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub { } }"), Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<IRObject>("FUZZTESTv1 sub: { }"), + Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<IRObject>("FUZZTESTv1 sub }"), + Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<IRObject>("FUZZTESTv1 sub { "), + Not(Optional(_))); + EXPECT_THAT(IRObject{}.FromString<IRObject>("FUZZTESTv1 sub { } }"), + Not(Optional(_))); } TEST(SerializerTest, ExtraWhitespaceIsFine) { - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 i: 0 \n "), + EXPECT_THAT(IRObject{}.FromString<std::int64_t>("FUZZTESTv1 i: 0 \n "), Optional(ValueIs<uint64_t>(0))); - EXPECT_THAT(IRObject::FromString("FUZZTESTv1 sub { \n i: 0 \n} \n "), - Optional(SubsAre(ValueIs<uint64_t>(0)))); + EXPECT_THAT( + IRObject{}.FromString<IRObject>("FUZZTESTv1 sub { \n i: 0 \n} \n "), + Optional(SubsAre(ValueIs<uint64_t>(0)))); } template <typename T> @@ -239,7 +265,7 @@ obj.SetScalar(value); EXPECT_THAT(obj.GetScalar<T>(), Optional(value)); - auto roundtrip = IRObject::FromString(obj.ToString()); + auto roundtrip = IRObject{}.FromString<IRObject>(obj.ToString<IRObject>()); EXPECT_THAT(obj.GetScalar<T>(), Optional(value)); } @@ -383,6 +409,51 @@ .ToCorpus<Tuple>()); } +TEST(SerializerTest, SingleStringValueIsSerializedAsRawString) { + std::string s("ABC"); + IRObject obj; + obj.value = s; + std::string s_serialized = obj.ToString<std::string>(); + + // Should have no header. + EXPECT_THAT(s_serialized, Not(HasSubstr("FUZZTESTv1\n"))); + EXPECT_EQ(s, s_serialized); +} + +TEST(SerializerTest, SingleProtoValueIsSerializaedAsRawString) { + IRObjectTestProto proto; + IRObject obj = IRObject::FromCorpus(proto); + std::string proto_serialized = obj.ToString<IRObjectTestProto>(); + + // Should have no header. + EXPECT_THAT(proto_serialized, Not(HasSubstr("FUZZTESTv1\n"))); + ASSERT_TRUE(google::protobuf::TextFormat::ParseFromString(proto_serialized, &proto)); + std::visit(VerifyVisitor{proto}, obj.value); +} + +TEST(SerializerTest, SingleStringSerializationRoundTripVerify) { + std::string s("ABC"); + IRObject obj; + obj.value = s; + std::string s_serialized = obj.ToString<std::string>(); + std::optional<IRObject> obj_from_str = + IRObject{}.FromString<std::string>(s_serialized); + + EXPECT_THAT(obj_from_str, Optional(ValueIs<std::string>(s))); +} + +TEST(SerializerTest, SingleProtoSerializationRoundTripVerify) { + IRObjectTestProto proto; + std::string proto_str; + ASSERT_TRUE(google::protobuf::TextFormat::PrintToString(proto, &proto_str)); + IRObject obj = IRObject::FromCorpus(proto); + std::string proto_serialized = obj.ToString<IRObjectTestProto>(); + std::optional<IRObject> obj_from_str = + IRObject{}.FromString<std::string>(proto_serialized); + + EXPECT_THAT(obj_from_str, Optional(ValueIs<std::string>(proto_str))); +} + // TODO(sbenzaquen): Add tests for failing conditions in the IR->Corpus conversion. } // namespace