Migrate the Go conformance test to the conformance_test() macro. The go_test ran conformance_test_runner against its own binary, which acts as the testee under RUN_AS_CONFORMANCE_PLUGIN=1. It becomes a manual testee target that only provides the binary, and the golang conformance_test() drives the suites against it with that variable set. The old failure lists are split per suite, unchanged, and stay google3-only. PiperOrigin-RevId: 984357922
diff --git a/cmake/conformance.cmake b/cmake/conformance.cmake index 7eedc94..24bd2e2 100644 --- a/cmake/conformance.cmake +++ b/cmake/conformance.cmake
@@ -141,11 +141,33 @@ target_include_directories(conformance_test_runner PRIVATE ${ABSL_ROOT_DIR}) target_include_directories(conformance_cpp PRIVATE ${ABSL_ROOT_DIR}) +# The runner hosts the gtest-based conformance suites (see +# conformance/conformance_test_main.cc), so it needs googletest even when +# protobuf_BUILD_TESTS is off. gtest.cmake is a no-op if GTest::gmock already +# exists, and otherwise finds or fetches googletest the same way the unit tests +# do. +include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake) + target_link_libraries(conformance_test_runner libconformance_common ${protobuf_LIB_PROTOBUF} ${protobuf_ABSL_USED_TARGETS} + GTest::gmock ) +# The runner reads and sets gtest's flags (GTEST_FLAG_GET/GTEST_FLAG_SET), which +# are exported data symbols of the gtest library. When googletest is built as +# a shared library, consumers must compile with GTEST_LINKED_AS_SHARED_LIBRARY +# so that those symbols are declared dllimport on Windows; googletest only +# attaches that definition to its installed (find_package) targets, not to the +# ones FetchContent builds in-tree, so set it here for either case. +if(TARGET GTest::gtest) + get_target_property(_gtest_target_type GTest::gtest TYPE) + if(_gtest_target_type STREQUAL "SHARED_LIBRARY") + target_compile_definitions(conformance_test_runner + PRIVATE GTEST_LINKED_AS_SHARED_LIBRARY=1) + endif() +endif() + target_link_libraries(conformance_cpp libconformance_common ${protobuf_LIB_PROTOBUF}
diff --git a/conformance/BUILD b/conformance/BUILD index 11b0156..d1b764b 100644 --- a/conformance/BUILD +++ b/conformance/BUILD
@@ -7,6 +7,7 @@ # Protobuf conformance tests. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_cc//cc:cc_test.bzl", "cc_test") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "objc_library") load("@rules_java//java:defs.bzl", "java_binary") @@ -23,6 +24,7 @@ load("//bazel:java_proto_library.bzl", "java_proto_library") load("//bazel:proto_library.bzl", "proto_library") load("//build_defs:internal_shell.bzl", "inline_sh_binary") +load("//conformance:conformance.bzl", "conformance_suite", "conformance_test", "failure_lists") exports_files([ "bazel_conformance_test_runner.sh", @@ -49,45 +51,78 @@ "text_format_failure_list_python.txt", ]) +# The command line and the report format of conformance_test_runner; see +# conformance_test_runner.h. cc_library( - name = "conformance_test_lib", + name = "conformance_test_runner_lib", testonly = 1, - srcs = [ - "conformance_test.cc", - "conformance_test_runner.cc", - ], - hdrs = [ - "conformance_test.h", - ], + srcs = ["conformance_test_runner.cc"], + hdrs = ["conformance_test_runner.h"], deps = [ - ":conformance_cc_proto", - ":failure_list_trie_node", - ":fork_pipe_runner", - ":recording_test_runner", - ":test_runner", "//src/google/protobuf", - "//src/google/protobuf:descriptor_legacy", - "//src/google/protobuf:endian", - "//src/google/protobuf/util:differencer", - "@abseil-cpp//absl/container:btree", "@abseil-cpp//absl/container:flat_hash_set", - "@abseil-cpp//absl/log:absl_check", - "@abseil-cpp//absl/log:absl_log", "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", - "@abseil-cpp//absl/types:optional", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", ], ) +# The main of conformance_test_runner: runs the gtest-based conformance suites +# linked into the same binary behind the conformance_test_runner command line; +# see conformance_test_main.cc. cc_library( name = "conformance_test_main", testonly = 1, srcs = ["conformance_test_main.cc"], deps = [ - ":binary_json_conformance_suite", - ":conformance_test_lib", - ":text_format_conformance_suite", + # The gtest-based conformance suites, linked in as they are migrated. + # The performance suite is linked from the start, so that the runner's + # --performance selects its tests (see PerformanceGtestFilter()) as + # soon as there are any. + ":binary_conformance_tests", # buildcleaner: keep + ":json_conformance_tests", # buildcleaner: keep + ":performance_conformance_tests", # buildcleaner: keep + ":conformance_test_runner_lib", + ":filtering_test_runner", + ":fork_pipe_runner", + ":recording_test_runner", + ":test_environment", + ":result_ledger", + ":test_runner", + ":testee", + ":text_conformance_tests", # buildcleaner: keep + "//src/google/protobuf", + "@googletest//:gtest", + "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + "@abseil-cpp//absl/types:span", + ], +) + +cc_test( + name = "conformance_test_runner_test", + srcs = ["conformance_test_runner_test.cc"], + deps = [ + ":conformance_test_runner_lib", + "//src/google/protobuf", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_matchers", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", ], ) @@ -123,58 +158,57 @@ tests = [ "binary_test_util_test", "binary_wireformat_test", + "conformance_cpp_harness_test", + "conformance_test_runner_test", + "cpp_in_process_test_runner_test", + "cpp_in_process_testee_test", "failure_list_trie_node_test", + "filtering_test_runner_test", + "fork_pipe_runner_test", + "forked_testee_test", + "in_process_test_runner_test", + "in_process_testee_test", + "json_test_util_test", "matchers_test", + "message_type_fixtures_test", "naming_test", "recording_test_runner_test", + "recursion_limit_payloads_test", + "result_ledger_test", + "result_listener_test", + "result_record_test", + "sample_conformance_test", + "test_environment_flags_test", + "test_environment_integration_test", "test_environment_test", "test_fixture_test", - "test_manager_test", "testee_test", + "//conformance/bzl_tests:conformance_bzl_test", ], ) -cc_library( - name = "binary_json_conformance_suite", - testonly = 1, - srcs = ["binary_json_conformance_suite.cc"], - hdrs = ["binary_json_conformance_suite.h"], - deps = [ - ":binary_wireformat", - ":conformance_cc_proto", - ":conformance_test_lib", - "//:test_messages_proto2_cc_proto", - "//:test_messages_proto3_cc_proto", - "//conformance/test_protos:test_messages_edition2023_cc_proto", - "//conformance/test_protos:test_messages_edition2026_cc_proto", - "//conformance/test_protos:test_messages_edition_unstable_cc_proto", - "//editions:test_messages_proto2_editions_cc_proto", - "//editions:test_messages_proto3_editions_cc_proto", - "//src/google/protobuf", - "//src/google/protobuf:protobuf_lite", - "//src/google/protobuf/json", - "//src/google/protobuf/util:differencer", - "//src/google/protobuf/util:type_resolver", - "@abseil-cpp//absl/log:absl_check", - "@abseil-cpp//absl/log:absl_log", - "@abseil-cpp//absl/log:die_if_null", - "@abseil-cpp//absl/status", - "@abseil-cpp//absl/strings", - "@abseil-cpp//absl/strings:str_format", - "@abseil-cpp//absl/strings:string_view", - "@jsoncpp", - ], -) - -# Helpers shared by the gtest-based binary conformance tests (the field-type -# tables the tests are parameterized over). +# Helpers shared by the gtest-based binary conformance tests (field lookups, +# wire types, test-message parameter sets). cc_library( name = "binary_test_util", testonly = 1, srcs = ["binary_test_util.cc"], hdrs = ["binary_test_util.h"], deps = [ + ":binary_wireformat", + ":naming", + "//:test_messages_proto2_cc_proto", + "//:test_messages_proto3_cc_proto", + "//editions:test_messages_proto2_editions_cc_proto", + "//editions:test_messages_proto3_editions_cc_proto", "//src/google/protobuf", + "//src/google/protobuf:port", + "//src/google/protobuf:protobuf_lite", + "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/log:absl_log", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", "@abseil-cpp//absl/types:span", ], ) @@ -184,38 +218,286 @@ srcs = ["binary_test_util_test.cc"], deps = [ ":binary_test_util", + ":binary_wireformat", + "//:test_messages_proto2_cc_proto", + "//:test_messages_proto3_cc_proto", + "//conformance/test_protos:test_messages_edition2023_cc_proto", + "//conformance/test_protos:test_messages_edition_unstable_cc_proto", + "//editions:test_messages_proto2_editions_cc_proto", + "//editions:test_messages_proto3_editions_cc_proto", "//src/google/protobuf", + "//src/google/protobuf/util:differencer", + "@abseil-cpp//absl/strings", "@googletest//:gtest", "@googletest//:gtest_main", ], ) cc_library( - name = "text_format_conformance_suite", + name = "recursion_limit_payloads", testonly = 1, - srcs = ["text_format_conformance_suite.cc"], - hdrs = ["text_format_conformance_suite.h"], + srcs = ["recursion_limit_payloads.cc"], + hdrs = ["recursion_limit_payloads.h"], deps = [ - ":conformance_test_lib", + ":binary_wireformat", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings", + ], +) + +cc_test( + name = "recursion_limit_payloads_test", + srcs = ["recursion_limit_payloads_test.cc"], + deps = [ + ":binary_wireformat", + ":recursion_limit_payloads", + "//:test_messages_proto2_cc_proto", + "//conformance/test_protos:test_messages_edition2023_cc_proto", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( + name = "message_type_fixtures", + testonly = 1, + srcs = ["message_type_fixtures.cc"], + hdrs = ["message_type_fixtures.h"], + deps = [ + ":binary_test_util", + ":test_fixture", + "//conformance/test_protos:test_messages_edition2023_cc_proto", + "//src/google/protobuf", + "@abseil-cpp//absl/strings", + "@googletest//:gtest", + ], +) + +cc_test( + name = "message_type_fixtures_test", + srcs = ["message_type_fixtures_test.cc"], + deps = [ + ":binary_test_util", + ":message_type_fixtures", + ":mock_test_runner", + ":test_environment", + ":test_environment_testing", "//:test_messages_proto2_cc_proto", "//:test_messages_proto3_cc_proto", - "//conformance/test_protos:test_messages_edition2023_cc_proto", - "//conformance/test_protos:test_messages_edition2026_cc_proto", - "//conformance/test_protos:test_messages_edition_unstable_cc_proto", "//editions:test_messages_proto2_editions_cc_proto", - "//editions:test_messages_proto3_editions_cc_proto", "//src/google/protobuf", - "@abseil-cpp//absl/log:absl_log", - "@abseil-cpp//absl/log:die_if_null", "@abseil-cpp//absl/strings", - "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# The packages using conformance_test() (conformance.bzl), which links the +# suite libraries below, :test_environment_main and :forked_testee into each +# test. Add a package here when it moves its conformance test to the macro +# (google3-only packages are marked to be stripped from the open-source +# export; upb's is exported along with its tests). +package_group( + name = "conformance_test_users", + packages = [ + "//conformance/...", + "//upb/conformance", + ], +) + +# The gtest-based binary conformance suite (:binary_conformance_tests), one +# file per migrated group. A single alwayslink library so that linking it into +# conformance_test_runner (or any binary using test_environment_main) registers +# every test; each newly migrated group just adds its file here. +conformance_suite( + name = "binary", + srcs = [ + "binary_delimited_field_test.cc", + "binary_illegal_lengths_test.cc", + "binary_illegal_tags_test.cc", + "binary_merge_test.cc", + "binary_message_set_test.cc", + "binary_oneof_test.cc", + "binary_oneof_zero_test.cc", + "binary_premature_eof_test.cc", + "binary_unknown_fields_test.cc", + "binary_unknown_wire_type_test.cc", + "binary_unmatched_group_test.cc", + "binary_unstable_edition_test.cc", + "binary_utf8_test.cc", + "binary_valid_data_map_test.cc", + "binary_valid_data_repeated_test.cc", + "binary_valid_data_scalar_test.cc", + ], + # conformance_test() links :binary_conformance_tests into every test it + # creates, so it must be visible wherever that macro is used (see + # :conformance_test_users; a package moving its conformance test to the + # macro is added there). + visibility = [":conformance_test_users"], + deps = [ + ":binary_test_util", + ":binary_wireformat", + ":matchers", + ":message_type_fixtures", + ":test_fixture", + ":testee", + "//:test_messages_proto2_cc_proto", + "//conformance/test_protos:test_messages_edition_unstable_cc_proto", + "//src/google/protobuf", + "//src/google/protobuf:port", + "//src/google/protobuf:protobuf_lite", + "@abseil-cpp//absl/log:absl_log", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + ], +) + +# The gtest-based performance conformance suite +# (:performance_conformance_tests): the tests that stress the testee with +# deeply nested and very large payloads, for every wire format that has them. +# Every fixture in it must be named `*PerformanceTest`; see conformance.bzl. +conformance_suite( + name = "performance", + srcs = [ + "binary_performance_test.cc", + "binary_recursion_limit_test.cc", + "text_performance_test.cc", + ], + visibility = [":conformance_test_users"], + deps = [ + ":binary_test_util", + ":binary_wireformat", + ":matchers", + ":message_type_fixtures", + ":recursion_limit_payloads", + ":test_fixture", + ":testee", + "//:test_messages_proto2_cc_proto", + "//conformance/test_protos:test_messages_edition2023_cc_proto", + "//src/google/protobuf", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + ], +) + +# The gtest-based text-format conformance suite (:text_conformance_tests); see +# the binary suite above. +conformance_suite( + name = "text", + srcs = [ + "text_any_test.cc", + "text_enum_test.cc", + "text_float_test.cc", + "text_group_test.cc", + "text_hello_world_test.cc", + "text_integer_test.cc", + "text_map_test.cc", + "text_reserved_field_test.cc", + "text_string_test.cc", + "text_syntax_test.cc", + "text_unknown_field_test.cc", + ], + # conformance_test() links :text_conformance_tests into every test it + # creates, so it must be visible wherever that macro is used (see the + # binary suite above). + visibility = [":conformance_test_users"], + deps = [ + ":binary_test_util", + ":binary_wireformat", + ":conformance_cc_proto", + ":matchers", + ":message_type_fixtures", + ":test_fixture", + ":testee", + "//:test_messages_proto2_cc_proto", + "//src/google/protobuf", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + ], +) + +# Matchers and pure values shared by the JSON conformance tests; see +# :binary_test_util. +cc_library( + name = "json_test_util", + testonly = 1, + hdrs = ["json_test_util.h"], + deps = [ + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@jsoncpp", + ], +) + +cc_test( + name = "json_test_util_test", + srcs = ["json_test_util_test.cc"], + deps = [ + ":json_test_util", + "@googletest//:gtest", + "@googletest//:gtest_main", + "@jsoncpp", + ], +) + +# The gtest-based JSON conformance suite (:json_conformance_tests); see the +# binary suite above. +conformance_suite( + name = "json", + srcs = [ + "json_any_test.cc", + "json_bool_test.cc", + "json_duration_test.cc", + "json_enum_test.cc", + "json_field_mask_test.cc", + "json_field_name_test.cc", + "json_float_test.cc", + "json_hello_world_test.cc", + "json_integer_test.cc", + "json_map_test.cc", + "json_message_test.cc", + "json_null_test.cc", + "json_oneof_test.cc", + "json_repeated_test.cc", + "json_string_test.cc", + "json_struct_test.cc", + "json_timestamp_test.cc", + "json_unknown_enum_test.cc", + "json_value_test.cc", + "json_wrapper_test.cc", + ], + # conformance_test() links :json_conformance_tests into every test it + # creates, so it must be visible wherever that macro is used (see the + # binary suite above). + visibility = [":conformance_test_users"], + deps = [ + ":binary_test_util", + ":binary_wireformat", + ":conformance_cc_proto", + ":json_test_util", + ":matchers", + ":message_type_fixtures", + ":test_fixture", + ":testee", + "//src/google/protobuf", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", ], ) cc_library( name = "test_runner", hdrs = ["test_runner.h"], - visibility = ["//src/google/protobuf:__pkg__"], + # An in-process testee library (testee_runner.h) subclasses + # ConformanceTestRunner, so packages that define one need this directly. + visibility = [":conformance_test_users"], deps = ["@abseil-cpp//absl/strings:string_view"], ) @@ -303,6 +585,153 @@ ) cc_library( + name = "filtering_test_runner", + srcs = ["filtering_test_runner.cc"], + hdrs = ["filtering_test_runner.h"], + deps = [ + ":conformance_cc_proto", + ":test_runner", + "@abseil-cpp//absl/algorithm:container", + "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_test( + name = "filtering_test_runner_test", + srcs = ["filtering_test_runner_test.cc"], + deps = [ + ":conformance_cc_proto", + ":filtering_test_runner", + ":mock_test_runner", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( + name = "in_process_test_runner", + testonly = 1, + srcs = ["in_process_test_runner.cc"], + hdrs = ["in_process_test_runner.h"], + deps = [ + ":conformance_cc_proto", + ":test_runner", + "@abseil-cpp//absl/functional:any_invocable", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + ], +) + +cc_test( + name = "in_process_test_runner_test", + srcs = ["in_process_test_runner_test.cc"], + deps = [ + ":binary_wireformat", + ":conformance_cc_proto", + ":in_process_test_runner", + "//src/google/protobuf:test_textproto", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# The request-handling logic of the C++ conformance testee, shared by the +# conformance_cpp binary and by in-process conformance tests. +cc_library( + name = "conformance_cpp_harness", + testonly = 1, + srcs = ["conformance_cpp_harness.cc"], + hdrs = ["conformance_cpp_harness.h"], + deps = [ + ":conformance_cc_proto", + "//:any_cc_proto", + "//:api_cc_proto", + "//:duration_cc_proto", + "//:empty_cc_proto", + "//:field_mask_cc_proto", + "//:struct_cc_proto", + "//:test_messages_proto2_cc_proto", + "//:test_messages_proto3_cc_proto", + "//:timestamp_cc_proto", + "//:type_cc_proto", + "//:wrappers_cc_proto", + "//conformance/test_protos:test_messages_edition2023_cc_proto", + "//conformance/test_protos:test_messages_edition2026_cc_proto", + "//conformance/test_protos:test_messages_edition_unstable_cc_proto", + "//editions:test_messages_proto2_editions_cc_proto", + "//editions:test_messages_proto3_editions_cc_proto", + "//src/google/protobuf", + "//src/google/protobuf/json", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + ], +) + +cc_test( + name = "conformance_cpp_harness_test", + srcs = ["conformance_cpp_harness_test.cc"], + deps = [ + ":binary_wireformat", + ":conformance_cc_proto", + ":conformance_cpp_harness", + "//:test_messages_proto3_cc_proto", + "//src/google/protobuf:test_textproto", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_matchers", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# Glue between the C++ testee (conformance_cpp_harness) and the tester side +# (in_process_test_runner). Kept separate so conformance_cpp links no runner +# code. +cc_library( + name = "cpp_in_process_test_runner", + testonly = 1, + srcs = ["cpp_in_process_test_runner.cc"], + hdrs = ["cpp_in_process_test_runner.h"], + deps = [ + ":conformance_cc_proto", + ":conformance_cpp_harness", + ":in_process_test_runner", + ":test_runner", + ], +) + +cc_test( + name = "cpp_in_process_test_runner_test", + srcs = ["cpp_in_process_test_runner_test.cc"], + deps = [ + ":conformance_cc_proto", + ":conformance_cpp_harness", + ":cpp_in_process_test_runner", + ":test_runner", + "//:test_messages_proto3_cc_proto", + "//src/google/protobuf:test_textproto", + "//src/google/protobuf/util:differencer", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status:status_matchers", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( name = "binary_wireformat", testonly = 1, srcs = ["binary_wireformat.cc"], @@ -316,14 +745,63 @@ ) cc_library( - name = "test_manager", + name = "failure_list", testonly = True, - srcs = ["test_manager.cc"], - hdrs = ["test_manager.h"], + srcs = ["failure_list.cc"], + hdrs = ["failure_list.h"], deps = [ - ":conformance_cc_proto", ":failure_list_trie_node", ":testee", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + ], +) + +cc_library( + name = "result_record", + testonly = True, + srcs = ["result_record.cc"], + hdrs = ["result_record.h"], + deps = [ + ":testee", + "@abseil-cpp//absl/algorithm:container", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + ], +) + +cc_test( + name = "result_record_test", + srcs = ["result_record_test.cc"], + deps = [ + ":result_record", + ":testee", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_matchers", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_library( + name = "result_ledger", + testonly = True, + srcs = ["result_ledger.cc"], + hdrs = ["result_ledger.h"], + deps = [ + ":failure_list", + ":result_record", + ":testee", "@abseil-cpp//absl/algorithm:container", "@abseil-cpp//absl/container:btree", "@abseil-cpp//absl/container:flat_hash_map", @@ -338,6 +816,21 @@ ) cc_library( + name = "result_listener", + testonly = True, + srcs = ["result_listener.cc"], + hdrs = ["result_listener.h"], + deps = [ + ":result_ledger", + ":result_record", + "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", + "@googletest//:gtest", + ], +) + +cc_library( name = "naming", hdrs = ["naming.h"], deps = [ @@ -382,6 +875,8 @@ "@abseil-cpp//absl/log:absl_log", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:optional", + "@abseil-cpp//absl/types:span", ], ) @@ -406,10 +901,12 @@ ) cc_test( - name = "test_manager_test", - srcs = ["test_manager_test.cc"], + name = "result_ledger_test", + srcs = ["result_ledger_test.cc"], deps = [ - ":test_manager", + ":failure_list", + ":result_ledger", + ":result_record", ":testee", "//src/google/protobuf/testing", "//src/google/protobuf/testing:file", @@ -423,16 +920,31 @@ ], ) +cc_test( + name = "result_listener_test", + srcs = ["result_listener_test.cc"], + deps = [ + ":result_ledger", + ":result_listener", + ":result_record", + ":testee", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + cc_library( name = "global_test_environment", testonly = 1, hdrs = ["global_test_environment.h"], - deps = [":test_manager"], + deps = [":failure_list"], ) # gMock matchers for conformance TestResults. # -# NOTE: Consumers must link a definition of GetGlobalTestManager(), declared in +# NOTE: Consumers must link a definition of GetGlobalFailureList(), declared in # global_test_environment.h. The conformance test environment provides it in # the real test binary; matchers_test provides an in-memory stand-in. cc_library( @@ -443,22 +955,43 @@ deps = [ ":binary_wireformat", ":conformance_cc_proto", + ":failure_list", + ":filtering_test_runner", ":global_test_environment", - ":test_manager", + ":result_record", ":testee", "//src/google/protobuf", "//src/google/protobuf:port", + "//src/google/protobuf:protobuf_lite", + "//src/google/protobuf/json", "//src/google/protobuf/util:differencer", - "@abseil-cpp//absl/base:core_headers", "@abseil-cpp//absl/base:nullability", "@abseil-cpp//absl/log:absl_check", "@abseil-cpp//absl/log:absl_log", "@abseil-cpp//absl/memory", "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/strings:string_view", - "@abseil-cpp//absl/synchronization", + "@abseil-cpp//absl/types:optional", + "@googletest//:gtest", + "@jsoncpp", + ], +) + +# Helpers for unit tests of code that checks results with Yields(). +cc_library( + name = "matchers_testing", + testonly = 1, + srcs = ["matchers_testing.cc"], + hdrs = ["matchers_testing.h"], + deps = [ + ":result_record", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", "@abseil-cpp//absl/types:optional", "@googletest//:gtest", ], @@ -470,13 +1003,18 @@ deps = [ ":binary_wireformat", ":conformance_cc_proto", + ":failure_list", ":global_test_environment", + ":json_test_util", ":matchers", + ":matchers_testing", ":mock_test_runner", - ":test_manager", + ":result_ledger", + ":result_record", ":testee", "//:test_messages_proto2_cc_proto", "//src/google/protobuf", + "//src/google/protobuf:protobuf_lite", "@abseil-cpp//absl/base:log_severity", "@abseil-cpp//absl/log:absl_check", "@abseil-cpp//absl/log:scoped_mock_log", @@ -486,6 +1024,7 @@ "@abseil-cpp//absl/types:optional", "@googletest//:gtest", "@googletest//:gtest_main", + "@jsoncpp", ], ) @@ -507,9 +1046,11 @@ srcs = ["test_environment.cc"], hdrs = ["test_environment.h"], deps = [ + ":failure_list", ":fork_pipe_runner", ":global_test_environment", - ":test_manager", + ":result_ledger", + ":result_listener", ":test_runner", ":testee", "//src/google/protobuf", @@ -520,6 +1061,7 @@ "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:string_view", "@abseil-cpp//absl/types:optional", + "@abseil-cpp//absl/types:span", "@googletest//:gtest", ], ) @@ -544,10 +1086,13 @@ ":conformance_cc_proto", ":global_test_environment", ":matchers", + ":matchers_testing", ":mock_test_runner", + ":result_ledger", + ":result_listener", + ":result_record", ":test_environment", ":test_environment_testing", - ":test_manager", ":testee", "//:test_messages_proto3_cc_proto", "//src/google/protobuf", @@ -574,7 +1119,6 @@ "//src/google/protobuf", "//src/google/protobuf:descriptor_legacy", "@abseil-cpp//absl/base:nullability", - "@abseil-cpp//absl/container:flat_hash_map", "@abseil-cpp//absl/log:absl_check", "@abseil-cpp//absl/strings:string_view", "@googletest//:gtest", @@ -606,6 +1150,246 @@ ], ) +# Defines the global --testee_binary, --failure_list, --fix, ... flags of a +# conformance test binary. Only link it into binaries with a +# test_environment_main-style main (never into conformance_test_runner, whose +# command line parses flags of the same names itself). +cc_library( + name = "test_environment_flags", + testonly = 1, + srcs = ["test_environment_flags.cc"], + hdrs = ["test_environment_flags.h"], + deps = [ + ":test_environment", + ":testee", + "//src/google/protobuf", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/types:span", + ], +) + +cc_test( + name = "test_environment_flags_test", + srcs = ["test_environment_flags_test.cc"], + deps = [ + ":test_environment", + ":test_environment_flags", + ":testee", + "//src/google/protobuf", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/flags:parse", + "@abseil-cpp//absl/flags:reflection", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# The main() for gtest-based conformance suites. Depend on this instead of a +# generic gtest main, together with exactly one library defining +# MakeTesteeRunner() (see :testee_runner): :forked_testee, or an in-process +# testee. conformance_test() (conformance.bzl) adds them as implicit deps, so +# they must be visible wherever that macro is used (see +# :conformance_test_users). +cc_library( + name = "test_environment_main", + testonly = 1, + srcs = ["test_environment_main.cc"], + visibility = [":conformance_test_users"], + deps = [ + ":test_environment", + ":test_environment_flags", + ":testee_runner", + "@abseil-cpp//absl/flags:parse", # buildcleaner: keep + "@abseil-cpp//absl/log:initialize", # buildcleaner: keep + "@abseil-cpp//absl/types:span", + "@googletest//:gtest", + ], + alwayslink = 1, +) + +# The link-time seam through which test_environment_main gets its testee: the +# declaration of MakeTesteeRunner(), which an in-process testee library defines. +cc_library( + name = "testee_runner", + testonly = 1, + hdrs = ["testee_runner.h"], + visibility = [":conformance_test_users"], + deps = [ + ":test_runner", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + ], +) + +# The default testee: --testee_binary, spawned as a child process. Like every +# library defining MakeTesteeRunner() it is alwayslink, so that the definition +# is linked in wherever the library appears on the link line. +cc_library( + name = "forked_testee", + testonly = 1, + srcs = ["forked_testee.cc"], + visibility = [":conformance_test_users"], + deps = [ + ":fork_pipe_runner", + ":test_runner", + ":testee_runner", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + ], + alwayslink = 1, +) + +# The C++ runtime's in-process testee: defines MakeTesteeRunner() on top of +# cpp_in_process_test_runner. conformance_test(in_process = True) links it in +# place of :forked_testee; see conformance.bzl for the one-per-test rule. +cc_library( + name = "cpp_in_process_testee", + testonly = 1, + srcs = ["cpp_in_process_testee.cc"], + deps = [ + ":cpp_in_process_test_runner", + ":test_runner", + ":testee_runner", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + ], + alwayslink = 1, +) + +cc_test( + name = "cpp_in_process_testee_test", + srcs = ["cpp_in_process_testee_test.cc"], + deps = [ + ":conformance_cc_proto", + ":cpp_in_process_testee", + ":test_runner", + ":testee_runner", + "//:test_messages_proto3_cc_proto", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +cc_test( + name = "forked_testee_test", + srcs = ["forked_testee_test.cc"], + deps = [ + ":forked_testee", + ":test_runner", + ":testee_runner", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# A conformance test binary whose testee is linked in (the test file defines +# MakeTesteeRunner() itself), run without --testee_binary: checks that +# test_environment_main hosts an in-process testee end to end. +cc_test( + name = "in_process_testee_test", + srcs = ["in_process_testee_test.cc"], + deps = [ + ":binary_wireformat", + ":conformance_cc_proto", + ":matchers", + ":test_environment_main", + ":test_fixture", + ":test_runner", + ":testee_runner", + "//:test_messages_proto3_cc_proto", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + "@googletest//:gtest", + ], +) + +bzl_library( + name = "conformance_bzl", + srcs = ["conformance.bzl"], + visibility = ["//conformance/bzl_tests:__pkg__"], + deps = [ + "@rules_cc//cc:core_rules", + ], +) + +# The expected-failure lists of the conformance_test()s below, one file per +# test and suite (e.g. failure_lists/cpp_binary.txt). Update them with +# `blaze run :<test>_<suite>_test -- --fix`; see conformance.bzl. +failure_lists(name = "failure_lists") + +# The gtest-based conformance tests of the C++ testee: cpp_binary_test, +# cpp_json_test, cpp_performance_test and cpp_text_test, grouped under the :cpp +# test_suite. They host the testee in-process (:cpp_in_process_testee) rather +# than forking :conformance_cpp, so a testee crash or log shows up in the test +# itself. +conformance_test( + name = "cpp", + failure_lists = ":failure_lists", + in_process = True, + maximum_edition = "2023", + testee = ":cpp_in_process_testee", +) + +# A tiny conformance suite exercising the framework end to end. It +# deliberately contains failing tests, which the accompanying failure lists +# account for. +cc_test( + name = "sample_conformance_test", + srcs = ["sample_conformance_test.cc"], + args = [ + "--testee_binary=$(rootpath :conformance_cpp)", + "--failure_list=$(rootpath sample_failure_list_ok.txt)", + ], + data = [ + "sample_failure_list_ok.txt", + ":conformance_cpp", + ], + deps = [ + ":binary_wireformat", + ":forked_testee", + ":matchers", + ":test_environment_main", + ":test_fixture", + ":testee", + "//:test_messages_proto3_cc_proto", + "@googletest//:gtest", + ], +) + +# Runs sample_conformance_test as a subprocess. Only what needs a real test +# process is checked here; see test_environment_test for the rest. +cc_test( + name = "test_environment_integration_test", + srcs = ["test_environment_integration_test.cc"], + data = [ + "sample_failure_list_empty.txt", + "sample_failure_list_ok.txt", + "sample_failure_list_unseen.txt", + ":conformance_cpp", + ":sample_conformance_test", + ], + env = { + "SAMPLE_TEST": "$(rlocationpath :sample_conformance_test)", + "TESTEE": "$(rlocationpath :conformance_cpp)", + }, + deps = [ + "//src/google/protobuf/testing", + "//src/google/protobuf/testing:file", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + "@bazel_tools//tools/cpp/runfiles", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + cc_binary( name = "conformance_test_runner", testonly = 1, @@ -712,35 +1496,15 @@ ], deps = [ ":conformance_cc_proto", - "//:any_cc_proto", - "//:api_cc_proto", - "//:duration_cc_proto", - "//:empty_cc_proto", - "//:field_mask_cc_proto", + ":conformance_cpp_harness", "//:protobuf", - "//:struct_cc_proto", - "//:test_messages_proto2_cc_proto", - "//:test_messages_proto3_cc_proto", - "//:timestamp_cc_proto", - "//:type_cc_proto", - "//:wrappers_cc_proto", - "//conformance/test_protos:test_messages_edition2023_cc_proto", - "//conformance/test_protos:test_messages_edition2026_cc_proto", - "//conformance/test_protos:test_messages_edition_unstable_cc_proto", - "//editions:test_messages_proto2_editions_cc_proto", - "//editions:test_messages_proto3_editions_cc_proto", "//src/google/protobuf", - "//src/google/protobuf:port", - "//src/google/protobuf:protobuf_lite", - "//src/google/protobuf/json", + "//src/google/protobuf:endian", "//src/google/protobuf/stubs", - "//src/google/protobuf/util:json_util", - "//src/google/protobuf/util:type_resolver", "@abseil-cpp//absl/log:absl_check", "@abseil-cpp//absl/log:absl_log", "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", - "@abseil-cpp//absl/strings", ], )
diff --git a/conformance/binary_delimited_field_test.cc b/conformance/binary_delimited_field_test.cc new file mode 100644 index 0000000..52b6324 --- /dev/null +++ b/conformance/binary_delimited_field_test.cc
@@ -0,0 +1,188 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests for TestAllTypesEdition2023: length-prefixed and +// delimited (group-encoded) message fields, maps, and delimited extensions +// under editions. This holds both the binary-output and the JSON-output leg +// of the legacy suite-level +// BinaryAndJsonConformanceSuite::RunDelimitedFieldTests(); the test names and +// the requests sent to the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +// Every test here parses TestAllTypesEdition2023 (message()), so the fixtures +// skip themselves when --maximum_edition doesn't cover editions. +using DelimitedFieldTest = Edition2023ConformanceTest; + +// Field 1 is optional_int32. +TEST_F(DelimitedFieldTest, ValidNonMessage) { + EXPECT_THAT(Testee("ValidNonMessage") + .ParseBinary(message(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +TEST_F(DelimitedFieldTest, ValidNonMessageJson) { + EXPECT_THAT(Testee("ValidNonMessage") + .ParseBinary(message(), VarintField(1, 99)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +// Field 18 is optional_nested_message, whose field 1 is a. +TEST_F(DelimitedFieldTest, ValidLengthPrefixedField) { + EXPECT_THAT( + Testee("ValidLengthPrefixedField") + .ParseBinary(message(), LengthPrefixedField(18, VarintField(1, 99))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_nested_message { a: 99 })pb")))); +} + +TEST_F(DelimitedFieldTest, ValidLengthPrefixedFieldJson) { + EXPECT_THAT( + Testee("ValidLengthPrefixedField") + .ParseBinary(message(), LengthPrefixedField(18, VarintField(1, 99))) + .SerializeJson(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_nested_message { a: 99 })pb")))); +} + +// Field 56 is map_int32_int32; map entries have key = 1 and value = 2. +TEST_F(DelimitedFieldTest, ValidMapInteger) { + EXPECT_THAT( + Testee("ValidMap.Integer") + .ParseBinary(message(), + LengthPrefixedField( + 56, Wire(VarintField(1, 99), VarintField(2, 87)))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(map_int32_int32 { key: 99 value: 87 })pb")))); +} + +TEST_F(DelimitedFieldTest, ValidMapIntegerJson) { + EXPECT_THAT( + Testee("ValidMap.Integer") + .ParseBinary(message(), + LengthPrefixedField( + 56, Wire(VarintField(1, 99), VarintField(2, 87)))) + .SerializeJson(), + Yields(WhenParsed( + EqualsTextProto(R"pb(map_int32_int32 { key: 99 value: 87 })pb")))); +} + +// Field 71 is map_string_nested_message. +TEST_F(DelimitedFieldTest, ValidMapLengthPrefixed) { + EXPECT_THAT(Testee("ValidMap.LengthPrefixed") + .ParseBinary( + message(), + LengthPrefixedField( + 71, Wire(LengthPrefixedField(1, "a"), + LengthPrefixedField(2, VarintField(1, 87))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(map_string_nested_message { + key: "a" + value: { a: 87 } + })pb")))); +} + +TEST_F(DelimitedFieldTest, ValidMapLengthPrefixedJson) { + EXPECT_THAT(Testee("ValidMap.LengthPrefixed") + .ParseBinary( + message(), + LengthPrefixedField( + 71, Wire(LengthPrefixedField(1, "a"), + LengthPrefixedField(2, VarintField(1, 87))))) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(map_string_nested_message { + key: "a" + value: { a: 87 } + })pb")))); +} + +// Field 201 is groupliketype (whose name matches its type, as a proto2 group +// would) and field 202 is delimited_field (whose name doesn't). Both are +// GroupLikeType, whose field 202 is group_int32. +TEST_F(DelimitedFieldTest, GroupLike) { + EXPECT_THAT( + Testee("ValidDelimitedField.GroupLike") + .ParseBinary(message(), DelimitedField(201, VarintField(202, 99))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(groupliketype { group_int32: 99 })pb")))); +} + +TEST_F(DelimitedFieldTest, GroupLikeJson) { + EXPECT_THAT( + Testee("ValidDelimitedField.GroupLike") + .ParseBinary(message(), DelimitedField(201, VarintField(202, 99))) + .SerializeJson(), + Yields(WhenParsed( + EqualsTextProto(R"pb(groupliketype { group_int32: 99 })pb")))); +} + +TEST_F(DelimitedFieldTest, NotGroupLike) { + EXPECT_THAT( + Testee("ValidDelimitedField.NotGroupLike") + .ParseBinary(message(), DelimitedField(202, VarintField(202, 99))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(delimited_field { group_int32: 99 })pb")))); +} + +TEST_F(DelimitedFieldTest, NotGroupLikeJson) { + EXPECT_THAT( + Testee("ValidDelimitedField.NotGroupLike") + .ParseBinary(message(), DelimitedField(202, VarintField(202, 99))) + .SerializeJson(), + Yields(WhenParsed( + EqualsTextProto(R"pb(delimited_field { group_int32: 99 })pb")))); +} + +// Only TestAllTypesEdition2023 declares delimited extensions: field 121 is +// the groupliketype extension (whose name matches its type, as a proto2 group +// would) and field 122 is delimited_ext (whose name doesn't). Both are +// GroupLikeType, whose field 1 is c. Like the legacy suite, these have no +// JSON-output leg. +using DelimitedExtensionTest = Edition2023ConformanceTest; + +TEST_F(DelimitedExtensionTest, GroupLike) { + EXPECT_THAT( + Testee("ValidDelimitedExtension.GroupLike") + .ParseBinary(message(), DelimitedField(121, VarintField(1, 99))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb([protobuf_test_messages.editions.groupliketype] { + c: 99 + })pb")))); +} + +TEST_F(DelimitedExtensionTest, NotGroupLike) { + EXPECT_THAT( + Testee("ValidDelimitedExtension.NotGroupLike") + .ParseBinary(message(), DelimitedField(122, VarintField(1, 99))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb([protobuf_test_messages.editions.delimited_ext] { + c: 99 + })pb")))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_illegal_lengths_test.cc b/conformance/binary_illegal_lengths_test.cc new file mode 100644 index 0000000..023f86f --- /dev/null +++ b/conformance/binary_illegal_lengths_test.cc
@@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that illegal length prefixes are +// rejected. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestIllegalLengths(); the test names +// and the requests sent to the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using IllegalLengthsTest = MessageTypeConformanceTest; + +// A 5-byte varint length whose bit 32 is set (0x10 in the last byte), so that +// it wraps to 0 in 32-bit arithmetic. Parsers must not overflow and must +// reject the length. +TEST_P(IllegalLengthsTest, BadLengthVarint32BitOverflow) { + const FieldDescriptor& string_field = *GetFieldForType( + *message(), FieldDescriptor::TYPE_STRING, /*repeated=*/false); + EXPECT_THAT(Testee("BadLength_Varint32BitOverflow") + .ParseBinary(message(), Wire(Tag(FieldNumber(string_field), + WireType::kLengthPrefixed), + "\x80\x80\x80\x80\x10")) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, IllegalLengthsTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_illegal_tags_test.cc b/conformance/binary_illegal_tags_test.cc new file mode 100644 index 0000000..08b23dc --- /dev/null +++ b/conformance/binary_illegal_tags_test.cc
@@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that illegal field tags are rejected: a +// field number of zero, and tag varints whose field number is out of range or +// that are overlong. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestIllegalTags(); the test names and +// the requests sent to the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// The one-byte tag of field number 1 with wire type varint (0x08) with its +// continuation bit (0x80) set, i.e. the first byte of a longer tag varint. The +// BadTag tests append further bytes to it. +constexpr absl::string_view kTagWithContinuationBit = "\x88"; + +// The last byte of an overlong varint: neither continuation bit nor value +// bits. Spelled with an explicit length since it is a NUL. +constexpr absl::string_view kVarintTerminator("\0", 1); + +using IllegalTagsTest = MessageTypeConformanceTest; + +// Field number 0 is illegal, whatever the wire type. (The leading "\1" etc. +// are octal escapes: the tag bytes 0x01, 0x02, 0x03 and 0x05.) +TEST_P(IllegalTagsTest, IllegalZeroFieldNumCase0) { + EXPECT_THAT(Testee("IllegalZeroFieldNum_Case_0") + .ParseBinary(message(), Wire("\1DEADBEEF")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(IllegalTagsTest, IllegalZeroFieldNumCase1) { + EXPECT_THAT(Testee("IllegalZeroFieldNum_Case_1") + .ParseBinary(message(), Wire("\2\1\1")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(IllegalTagsTest, IllegalZeroFieldNumCase2) { + EXPECT_THAT(Testee("IllegalZeroFieldNum_Case_2") + .ParseBinary(message(), Wire("\3\4")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(IllegalTagsTest, IllegalZeroFieldNumCase3) { + EXPECT_THAT(Testee("IllegalZeroFieldNum_Case_3") + .ParseBinary(message(), Wire("\5DEAD")) + .ParseOnly(), + Yields(IsParseError())); +} + +// A tag varint whose field number is far out of range of the maximum legal +// field number. The lower 5 bytes of the varint do look like a well-formed +// tag for field number 1. +TEST_P(IllegalTagsTest, BadTagFieldNumberTooHigh) { + EXPECT_THAT(Testee("BadTag_FieldNumberTooHigh") + .ParseBinary(message(), + Wire(kTagWithContinuationBit, + "\x80\x80\x80\x80\x80\x0F", Varint(1234))) + .ParseOnly(), + Yields(IsParseError())); +} + +// A 5-byte tag varint whose value is above UINT32_MAX (bit 35 is set). +TEST_P(IllegalTagsTest, BadTagFieldNumberSlightlyTooHigh) { + EXPECT_THAT( + Testee("BadTag_FieldNumberSlightlyTooHigh") + .ParseBinary(message(), Wire(kTagWithContinuationBit, + "\x80\x80\x80\x40", Varint(1234))) + .ParseOnly(), + Yields(IsParseError())); +} + +// A tag varint that is more than 5 bytes only because it is overlong (seven +// empty continuation bytes), so the decoded value is still below UINT32_MAX. +TEST_P(IllegalTagsTest, BadTagOverlongVarint) { + EXPECT_THAT(Testee("BadTag_OverlongVarint") + .ParseBinary(message(), Wire(kTagWithContinuationBit, + "\x80\x80\x80\x80\x80\x80\x80", + kVarintTerminator, Varint(1234))) + .ParseOnly(), + Yields(IsParseError())); +} + +// An overlong tag varint (eleven empty continuation bytes) that is even more +// than 10 bytes long. +TEST_P(IllegalTagsTest, BadTagVarintMoreThanTenBytes) { + EXPECT_THAT( + Testee("BadTag_VarintMoreThanTenBytes") + .ParseBinary(message(), + Wire(kTagWithContinuationBit, + "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80", + kVarintTerminator, Varint(1234))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, IllegalTagsTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_json_conformance_suite.cc b/conformance/binary_json_conformance_suite.cc deleted file mode 100644 index 6d3c285..0000000 --- a/conformance/binary_json_conformance_suite.cc +++ /dev/null
@@ -1,4509 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include "conformance/binary_json_conformance_suite.h" - -#include <cassert> -#include <cctype> -#include <cstddef> -#include <cstdint> -#include <cstring> -#include <memory> -#include <string> -#include <type_traits> -#include <utility> -#include <vector> - -#include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" -#include "absl/log/die_if_null.h" -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "json/config.h" -#include "json/reader.h" -#include "json/value.h" -#include "conformance/binary_wireformat.h" -#include "conformance/conformance.pb.h" -#include "conformance/conformance_test.h" -#include "conformance/test_protos/test_messages_edition2023.pb.h" -#include "conformance/test_protos/test_messages_edition2026.pb.h" -#include "conformance/test_protos/test_messages_edition_unstable.pb.h" -#include "editions/golden/test_messages_proto2_editions.pb.h" -#include "editions/golden/test_messages_proto3_editions.pb.h" -#include "google/protobuf/json/json.h" -#include "google/protobuf/test_messages_proto2.pb.h" -#include "google/protobuf/test_messages_proto3.pb.h" -#include "google/protobuf/text_format.h" -#include "google/protobuf/unknown_field_set.h" -#include "google/protobuf/util/message_differencer.h" -#include "google/protobuf/util/type_resolver_util.h" -#include "google/protobuf/wire_format_lite.h" - -using ::conformance::ConformanceRequest; -using ::conformance::ConformanceResponse; -using ::conformance::TestStatus; -using ::conformance::WireFormat; -using google::protobuf::Descriptor; -using google::protobuf::FieldDescriptor; -using google::protobuf::internal::WireFormatLite; -using google::protobuf::util::NewTypeResolverForDescriptorPool; -using protobuf_test_messages::edition_unstable::TestAllTypesEditionUnstable; -using protobuf_test_messages::editions::TestAllTypesEdition2023; -using protobuf_test_messages::editions::TestAllTypesEdition2026; -using protobuf_test_messages::proto2::TestAllTypesProto2; -using protobuf_test_messages::proto3::TestAllTypesProto3; -using TestAllTypesProto2Editions = - protobuf_test_messages::editions::proto2::TestAllTypesProto2; -using TestAllTypesProto3Editions = - protobuf_test_messages::editions::proto3::TestAllTypesProto3; - -namespace { - -constexpr absl::string_view kTypeUrlPrefix = "type.googleapis.com"; - -// The number of repetitions to use for performance tests. -// Corresponds approx to 500KB wireformat bytes. -const size_t kPerformanceRepeatCount = 50000; - -std::string GetTypeUrl(const Descriptor* message) { - return absl::StrCat(kTypeUrlPrefix, "/", message->full_name()); -} - -std::string str_repeat(absl::string_view s, int count) { - std::string result; - for (int i = 0; i < count; ++i) { - absl::StrAppend(&result, s); - } - return result; -} - -/* Routines for building arbitrary protos *************************************/ - -// We would use CodedOutputStream except that we want more freedom to build -// arbitrary protos (even invalid ones). - -std::string varint(uint64_t x) { return google::protobuf::conformance::Varint(x).str(); } -std::string longvarint(uint64_t x, int extra) { - return google::protobuf::conformance::LongVarint(x, extra).str(); -} -std::string delim(const std::string& buf) { - return google::protobuf::conformance::LengthPrefixed(buf).str(); -} -std::string u32(uint32_t u32) { - return google::protobuf::conformance::Fixed32(u32).str(); -} -std::string u64(uint64_t u64) { - return google::protobuf::conformance::Fixed64(u64).str(); -} -std::string flt(float f) { return google::protobuf::conformance::Float(f).str(); } -std::string dbl(double d) { return google::protobuf::conformance::Double(d).str(); } -std::string zz32(int32_t x) { return google::protobuf::conformance::SInt32(x).str(); } -std::string zz64(int64_t x) { return google::protobuf::conformance::SInt64(x).str(); } - -std::string tag(uint32_t fieldnum, char wire_type) { - return google::protobuf::conformance::Tag( - fieldnum, static_cast<google::protobuf::conformance::WireType>(wire_type)) - .str(); -} -std::string tag(int fieldnum, char wire_type) { - return tag(static_cast<uint32_t>(fieldnum), wire_type); -} - -std::string field(uint32_t fieldnum, char wire_type, std::string content) { - return absl::StrCat(tag(fieldnum, wire_type), content); -} - -std::string group(uint32_t fieldnum, absl::string_view content) { - return google::protobuf::conformance::DelimitedField(fieldnum, - google::protobuf::conformance::Wire(content)) - .str(); -} - -std::string len(uint32_t fieldnum, std::string content) { - return google::protobuf::conformance::LengthPrefixedField(fieldnum, std::move(content)) - .str(); -} - -std::string GetDefaultValue(FieldDescriptor::Type type) { - switch (type) { - case FieldDescriptor::TYPE_INT32: - case FieldDescriptor::TYPE_INT64: - case FieldDescriptor::TYPE_UINT32: - case FieldDescriptor::TYPE_UINT64: - case FieldDescriptor::TYPE_ENUM: - case FieldDescriptor::TYPE_BOOL: - return varint(0); - case FieldDescriptor::TYPE_SINT32: - return zz32(0); - case FieldDescriptor::TYPE_SINT64: - return zz64(0); - case FieldDescriptor::TYPE_FIXED32: - case FieldDescriptor::TYPE_SFIXED32: - return u32(0); - case FieldDescriptor::TYPE_FIXED64: - case FieldDescriptor::TYPE_SFIXED64: - return u64(0); - case FieldDescriptor::TYPE_FLOAT: - return flt(0); - case FieldDescriptor::TYPE_DOUBLE: - return dbl(0); - case FieldDescriptor::TYPE_STRING: - case FieldDescriptor::TYPE_BYTES: - case FieldDescriptor::TYPE_MESSAGE: - return delim(""); - default: - return ""; - } - return ""; -} - -std::string GetNonDefaultValue(FieldDescriptor::Type type) { - switch (type) { - case FieldDescriptor::TYPE_INT32: - case FieldDescriptor::TYPE_INT64: - case FieldDescriptor::TYPE_UINT32: - case FieldDescriptor::TYPE_UINT64: - case FieldDescriptor::TYPE_ENUM: - case FieldDescriptor::TYPE_BOOL: - return varint(1); - case FieldDescriptor::TYPE_SINT32: - return zz32(1); - case FieldDescriptor::TYPE_SINT64: - return zz64(1); - case FieldDescriptor::TYPE_FIXED32: - case FieldDescriptor::TYPE_SFIXED32: - return u32(1); - case FieldDescriptor::TYPE_FIXED64: - case FieldDescriptor::TYPE_SFIXED64: - return u64(1); - case FieldDescriptor::TYPE_FLOAT: - return flt(1); - case FieldDescriptor::TYPE_DOUBLE: - return dbl(1); - case FieldDescriptor::TYPE_STRING: - case FieldDescriptor::TYPE_BYTES: - return delim("a"); - case FieldDescriptor::TYPE_MESSAGE: - return delim( - absl::StrCat(tag(1, WireFormatLite::WIRETYPE_VARINT), varint(1234))); - default: - return ""; - } - return ""; -} - -#define UNKNOWN_FIELD 666 - -std::string UpperCase(std::string str) { - for (size_t i = 0; i < str.size(); i++) { - str[i] = toupper(str[i]); - } - return str; -} - -std::string MismatchedWireTypeField( - int field_number, WireFormatLite::WireType correct_wire_type) { - if (correct_wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - return absl::StrCat(tag(field_number, WireFormatLite::WIRETYPE_VARINT), - varint(1)); - } - return absl::StrCat( - tag(field_number, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), delim("a")); -} - -bool IsProto3Default(FieldDescriptor::Type type, - const std::string& binary_data) { - switch (type) { - case FieldDescriptor::TYPE_DOUBLE: - return binary_data == dbl(0); - case FieldDescriptor::TYPE_FLOAT: - return binary_data == flt(0); - case FieldDescriptor::TYPE_BOOL: - case FieldDescriptor::TYPE_INT64: - case FieldDescriptor::TYPE_UINT64: - case FieldDescriptor::TYPE_INT32: - case FieldDescriptor::TYPE_UINT32: - case FieldDescriptor::TYPE_SINT32: - case FieldDescriptor::TYPE_SINT64: - case FieldDescriptor::TYPE_ENUM: - return binary_data == varint(0); - case FieldDescriptor::TYPE_FIXED64: - case FieldDescriptor::TYPE_SFIXED64: - return binary_data == u64(0); - case FieldDescriptor::TYPE_FIXED32: - case FieldDescriptor::TYPE_SFIXED32: - return binary_data == u32(0); - case FieldDescriptor::TYPE_STRING: - case FieldDescriptor::TYPE_BYTES: - return binary_data == delim(""); - default: - return false; - } -} - -} // anonymous namespace - -namespace google { -namespace protobuf { -namespace conformance { - -bool BinaryAndJsonConformanceSuite::ParseJsonResponse( - const ConformanceResponse& response, Message* test_message) { - json::ParseOptions options; - options.allow_legacy_nonconformant_behavior = false; - absl::Status status = - json::JsonStringToMessage(response.json_payload(), test_message); - if (!status.ok()) { - ABSL_LOG(ERROR) << status; - return false; - } - return true; -} - -bool BinaryAndJsonConformanceSuite::ParseResponse( - const ConformanceResponse& response, - const ConformanceRequestSetting& setting, Message* test_message) { - const ConformanceRequest& request = setting.GetRequest(); - WireFormat requested_output = request.requested_output_format(); - const std::string& test_name = setting.GetTestName(); - ConformanceLevel level = setting.GetLevel(); - - TestStatus test; - test.set_name(test_name); - switch (response.result_case()) { - case ConformanceResponse::kProtobufPayload: { - if (requested_output != ::conformance::PROTOBUF) { - test.set_failure_message(absl::StrCat( - "Test was asked for ", WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); - ReportFailure(test, level, request, response); - return false; - } - - if (!test_message->ParseFromString(response.protobuf_payload())) { - test.set_failure_message( - "Protobuf output we received from test was unparseable."); - ReportFailure(test, level, request, response); - return false; - } - - break; - } - - case ConformanceResponse::kJsonPayload: { - if (requested_output != ::conformance::JSON) { - test.set_failure_message(absl::StrCat( - "Test was asked for ", WireFormatToString(requested_output), - " output but provided JSON instead.")); - ReportFailure(test, level, request, response); - return false; - } - - if (!ParseJsonResponse(response, test_message)) { - test.set_failure_message( - "JSON output we received from test was unparseable."); - ReportFailure(test, level, request, response); - return false; - } - - break; - } - - default: - ABSL_LOG(FATAL) << test_name - << ": unknown payload type: " << response.result_case() - << ", response: " << response; - } - - return true; -} - -void BinaryAndJsonConformanceSuite::RunSuiteImpl() { - type_resolver_.reset(NewTypeResolverForDescriptorPool( - kTypeUrlPrefix, DescriptorPool::generated_pool())); - - BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto3>( - this, /*run_proto3_tests=*/true); - BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto2>( - this, /*run_proto3_tests=*/false); - if (!this->performance_) { - RunMessageSetTests(); - } else { - RunRecursionLimitTests(); - } - if (maximum_edition_ >= Edition::EDITION_2023) { - BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto3Editions>( - this, /*run_proto3_tests=*/true); - BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto2Editions>( - this, /*run_proto3_tests=*/false); - if (!this->performance_) { - RunDelimitedFieldTests(); - RunUnstableTests(); - if (maximum_edition_ >= Edition::EDITION_2026) { - RunEdition2026Tests(); - } - RunUtf8ValidationTests(); - } - } -} - -void BinaryAndJsonConformanceSuite::RunDelimitedFieldTests() { - SetTypeUrl(GetTypeUrl(TestAllTypesEdition2023::GetDescriptor())); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidNonMessage"), REQUIRED, - field(1, WireFormatLite::WIRETYPE_VARINT, varint(99)), - R"pb(optional_int32: 99)pb"); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidLengthPrefixedField"), REQUIRED, - len(18, field(1, WireFormatLite::WIRETYPE_VARINT, varint(99))), - R"pb(optional_nested_message { a: 99 })pb"); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidMap.Integer"), REQUIRED, - len(56, - absl::StrCat(field(1, WireFormatLite::WIRETYPE_VARINT, varint(99)), - field(2, WireFormatLite::WIRETYPE_VARINT, varint(87)))), - R"pb(map_int32_int32 { key: 99 value: 87 })pb"); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidMap.LengthPrefixed"), REQUIRED, - len(71, absl::StrCat(len(1, "a"), - len(2, field(1, WireFormatLite::WIRETYPE_VARINT, - varint(87))))), - R"pb(map_string_nested_message { - key: "a" - value: { a: 87 } - })pb"); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidDelimitedField.GroupLike"), REQUIRED, - group(201, field(202, WireFormatLite::WIRETYPE_VARINT, varint(99))), - R"pb(groupliketype { group_int32: 99 })pb"); - - RunValidProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidDelimitedField.NotGroupLike"), REQUIRED, - group(202, field(202, WireFormatLite::WIRETYPE_VARINT, varint(99))), - R"pb(delimited_field { group_int32: 99 })pb"); - - // Note: extensions don't work with TypeResolver, which is used by - // binary->JSON tests. - RunValidBinaryProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidDelimitedExtension.GroupLike"), REQUIRED, - group(121, field(1, WireFormatLite::WIRETYPE_VARINT, varint(99))), - R"pb([protobuf_test_messages.editions.groupliketype] { c: 99 })pb"); - - RunValidBinaryProtobufTest<TestAllTypesEdition2023>( - absl::StrCat("ValidDelimitedExtension.NotGroupLike"), REQUIRED, - group(122, field(1, WireFormatLite::WIRETYPE_VARINT, varint(99))), - R"pb([protobuf_test_messages.editions.delimited_ext] { c: 99 })pb"); -} - -void BinaryAndJsonConformanceSuite::RunUnstableTests() { - SetTypeUrl(GetTypeUrl(TestAllTypesEditionUnstable::GetDescriptor())); - - RunValidProtobufTest<TestAllTypesEditionUnstable>( - absl::StrCat("ValidBytes"), REQUIRED, len(13, "foo"), - R"pb(optional_bytes: "foo")pb"); - - RunValidProtobufTest<TestAllTypesEditionUnstable>( - absl::StrCat("ValidMap.Bytes"), REQUIRED, - len(15, absl::StrCat(len(1, "foo"), len(2, "barbaz"))), - R"pb(map_string_bytes { key: "foo" value: "barbaz" })pb"); -} - -void BinaryAndJsonConformanceSuite::RunEdition2026Tests() { - SetTypeUrl(GetTypeUrl(TestAllTypesEdition2026::GetDescriptor())); - - RunValidProtobufTest<TestAllTypesEdition2026>( - absl::StrCat("ValidInt32"), REQUIRED, - field(1, WireFormatLite::WIRETYPE_VARINT, varint(99)), - R"pb(optional_int32: 99)pb"); - - // The (pb.enumvalue.json) option overrides the name used for an enum value - // in JSON. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": "customBar"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_BAR"); - // Using the original enum value name in JSON is also allowed. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueCustomJsonNameOriginalName", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_BAR"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_BAR"); - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueCustomJsonNameNumericValue", REQUIRED, - R"({"optionalForeignEnum": 1})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_BAR"); - RunValidJsonTest<TestAllTypesEdition2026>( - "RepeatedEnumValueCustomJsonName", REQUIRED, - R"({"repeatedForeignEnum": ["customBar", "FOREIGN_ENUM_EDITION2026_BAZ", "customBaz"]})", - R"( - repeated_foreign_enum: FOREIGN_ENUM_EDITION2026_BAR - repeated_foreign_enum: FOREIGN_ENUM_EDITION2026_BAZ - repeated_foreign_enum: FOREIGN_ENUM_EDITION2026_BAZ - )"); - RunValidJsonTest<TestAllTypesEdition2026>( - "MapEnumValueCustomJsonName", REQUIRED, - R"({"mapStringForeignEnum": {"a": "customBar", "b": "FOREIGN_ENUM_EDITION2026_BAZ"}})", - R"( - map_string_foreign_enum { key: "a" value: FOREIGN_ENUM_EDITION2026_BAR } - map_string_foreign_enum { key: "b" value: FOREIGN_ENUM_EDITION2026_BAZ } - )"); - // Unknown names that are neither a declared name nor a custom JSON name. - ExpectParseFailureForJson<TestAllTypesEdition2026>( - "EnumValueUnknownCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": "customQux"})"); - ExpectParseFailureForJson<TestAllTypesEdition2026>( - "EnumValueUnknownOriginalName", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_QUX"})"); - // Empty custom JSON name. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueEmptyCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": ""})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_EMPTY"); - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueEmptyCustomJsonNameOutput", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_EMPTY"})", - [](const Json::Value& value) { - return value["optionalForeignEnum"].isString() && - value["optionalForeignEnum"].asString().empty(); - }); - // Custom JSON name with characters that require escaping in JSON. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueEscapedCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": "e\"sc\tap\ne"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_ESCAPES"); - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueEscapedCustomJsonNameOutput", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_ESCAPES"})", - [](const Json::Value& value) { - return value["optionalForeignEnum"].asString() == "e\"sc\tap\ne"; - }); - // Aliased values share a custom JSON name, and their declared names both - // still parse. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueAliasCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": "customAlias"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_ALIAS"); - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueAliasCustomJsonNameOriginalName", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_ALIAS_TOO"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_ALIAS"); - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueAliasCustomJsonNameOutput", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_ALIAS"})", - [](const Json::Value& value) { - return value["optionalForeignEnum"].asString() == "customAlias"; - }); - // A numeric-looking custom JSON name serializes as a JSON string, and both - // the string and the integer forms parse. - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueNumericCustomJsonName", REQUIRED, - R"({"optionalForeignEnum": "6"})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_NUMERIC"); - RunValidJsonTest<TestAllTypesEdition2026>( - "EnumValueNumericCustomJsonNameIntegerValue", REQUIRED, - R"({"optionalForeignEnum": 6})", - "optional_foreign_enum: FOREIGN_ENUM_EDITION2026_NUMERIC"); - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueNumericCustomJsonNameOutput", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_NUMERIC"})", - [](const Json::Value& value) { - return value["optionalForeignEnum"].isString() && - value["optionalForeignEnum"].asString() == "6"; - }); - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueCustomJsonNameOutput", REQUIRED, - R"({"optionalForeignEnum": "FOREIGN_ENUM_EDITION2026_BAR"})", - [](const Json::Value& value) { - return value["optionalForeignEnum"].asString() == "customBar"; - }); - // A value without the option keeps using its declared name in JSON. - RunValidJsonTestWithValidator<TestAllTypesEdition2026>( - "EnumValueWithoutCustomJsonNameOutput", REQUIRED, - R"({"repeatedForeignEnum": ["FOREIGN_ENUM_EDITION2026_FOO"]})", - [](const Json::Value& value) { - return value["repeatedForeignEnum"][0].asString() == - "FOREIGN_ENUM_EDITION2026_FOO"; - }); -} - -void BinaryAndJsonConformanceSuite::RunUtf8ValidationTests() { - ExpectParseFailureForProto<TestAllTypesEdition2023>( - len(133, "\xA0\xB0\xC0\xD0"), "RejectInvalidUtf8.String.Extension", - RECOMMENDED); - RunValidBinaryProtobufTest< - TestAllTypesEdition2023>(absl::StrCat( - "AcceptInvalidUtf8.Bytes.Extension"), - REQUIRED, len(134, "\xA0\xB0\xC0\xD0"), - R"pb([protobuf_test_messages.editions - .extension_bytes]: "\xA0\xB0\xC0\xD0")pb"); -} - -void BinaryAndJsonConformanceSuite::RunMessageSetTests() { - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding"), REQUIRED, - len(500, - group(1, absl::StrCat(field(2, WireFormatLite::WIRETYPE_VARINT, - varint(4135312)), - len(3, field(9, WireFormatLite::WIRETYPE_VARINT, - varint(99)))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.OutOfOrderGroupsEntries"), REQUIRED, - len(500, - group(1, absl::StrCat(len(3, field(9, WireFormatLite::WIRETYPE_VARINT, - varint(99))), - field(2, WireFormatLite::WIRETYPE_VARINT, - varint(4135312))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); - - // Test that an unknown message set extension always goes to unknown fields. - // This is done by poisoning the extension payload with an entry for field 0. - RunValidRoundtripProtobufTest<TestAllTypesProto2>( - "MessageSetEncoding.UnknownExtension", REQUIRED, - len(500, - group(1, absl::StrCat(field(2, WireFormatLite::WIRETYPE_VARINT, - varint(4135300)), - len(3, field(0, WireFormatLite::WIRETYPE_VARINT, - varint(99))))))); - - // If an encoder is unaware of the message_set_wire_format option it will be - // encoded like any other extension submessage. Decoders should be able to - // tolerate this format as well. - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.SubmessageEncoding"), RECOMMENDED, - len(500, - len(4135312, field(9, WireFormatLite::WIRETYPE_VARINT, varint(99)))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); - - // Test again, but this time we'll try to detect if the implementation put the - // submessage encoded entry into the unknown field set. We'll do this by using - // conflicting oneof entries where order matters when the messages are merged. - // - // In a non-compliant implementation submessage encoded messageset entry will - // be moved to unknown fields and then tacked onto the end of the payload. - // Thus we'll see field b set first, and then field a. - // - // In a compliant implementation we expect the submessage encoded messageset - // to be read first with field a set, and then the normal message set entry - // will be read with field b will be set -- thus field b will win. - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.SubmessageEncoding.NotUnknown"), - RECOMMENDED, - len(500, absl::StrCat( - len(123456789, - field(1, WireFormatLite::WIRETYPE_VARINT, varint(42))), - group(1, absl::StrCat( - field(2, WireFormatLite::WIRETYPE_VARINT, - varint(123456789)), - len(3, field(2, WireFormatLite::WIRETYPE_VARINT, - varint(99))))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.ExtensionWithOneof]: { b: 99 } - })pb" - // clang-format on - ); - - // [type_id, value, type_id (different)] -> first type_id and value honored. - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.DuplicateDifferentTypeId"), - RECOMMENDED, - len(500, - group( - 1, - absl::StrCat( - field(2, WireFormatLite::WIRETYPE_VARINT, varint(4135312)), - len(3, field(9, WireFormatLite::WIRETYPE_VARINT, varint(99))), - field(2, WireFormatLite::WIRETYPE_VARINT, varint(1547769))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); - - // [type_id, value, value] -> first value honored, no merge. - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.DuplicateValue"), RECOMMENDED, - len(500, - group( - 1, - absl::StrCat( - field(2, WireFormatLite::WIRETYPE_VARINT, varint(4135312)), - len(3, field(9, WireFormatLite::WIRETYPE_VARINT, varint(99))), - len(3, - field(9, WireFormatLite::WIRETYPE_VARINT, varint(88)))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); - - // [value, type_id, value] -> first value honored, no merge. - RunValidBinaryProtobufTest<TestAllTypesProto2>( - absl::StrCat("ValidMessageSetEncoding.DuplicateValueOutOfOrder"), - RECOMMENDED, - len(500, - group( - 1, - absl::StrCat( - len(3, field(9, WireFormatLite::WIRETYPE_VARINT, varint(99))), - field(2, WireFormatLite::WIRETYPE_VARINT, varint(4135312)), - len(3, - field(9, WireFormatLite::WIRETYPE_VARINT, varint(88)))))), - // clang-format off - R"pb(message_set_correct: { - [protobuf_test_messages.proto2 - .TestAllTypesProto2.MessageSetCorrectExtension2]: { i: 99 } - })pb" - // clang-format on - ); -} - -void BinaryAndJsonConformanceSuite::RunRecursionLimitTests() { - { - TestAllTypesEdition2023 message; - TestAllTypesEdition2023* sub = &message; - // The default recursion limit is 100 for most languages. 10,000 for - // golang. We use a larger number here for test. - for (int i = 0; i < 20000; i++) { - sub = &(*sub->mutable_map_recursive())[0]; - sub->set_optional_int32(123); - } - ExpectParseFailureForProto<TestAllTypesEdition2023>( - message.SerializeAsString(), "EnforceDepthLimit.Map", RECOMMENDED); - } - - { - TestAllTypesEdition2023 message; - TestAllTypesEdition2023* sub = &message; - for (int i = 0; i < 10000; i++) { - sub = - (*sub->mutable_map_string_nested_message())[""].mutable_corecursive(); - sub->set_optional_int32(123); - } - ExpectParseFailureForProto<TestAllTypesEdition2023>( - message.SerializeAsString(), "EnforceDepthLimit.MapStringKey", - RECOMMENDED); - } - - { - TestAllTypesProto2 proto2_msg; - auto sub = proto2_msg.mutable_message_set_correct(); - // The default recursion limit is 100 for most languages. 10,000 for - // golang. We use a larger number here for test. - for (int i = 0; i < 20000; i++) { - sub = sub->MutableExtension( - TestAllTypesProto2::MessageSetCorrectExtension2:: - message_set_extension) - ->mutable_sub_msg(); - } - sub->MutableExtension(TestAllTypesProto2::MessageSetCorrectExtension2:: - message_set_extension) - ->set_i(123); - ExpectParseFailureForProto<TestAllTypesProto2>( - proto2_msg.SerializeAsString(), "EnforceDepthLimit.MessageSetExtension", - RECOMMENDED); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::ExpectParseFailureForProto( - const std::string& proto, const std::string& test_name, - ConformanceLevel level) { - MessageType prototype; - // We don't expect output, but if the program erroneously accepts the protobuf - // we let it send its response as this. We must not leave it unspecified. - ConformanceRequestSetting setting( - level, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, proto); - - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - std::string effective_test_name = - absl::StrCat(setting.ConformanceLevelToString(level), ".", - setting.GetSyntaxIdentifier(), ".ProtobufInput.", test_name); - - if (!RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kParseError) { - ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kSkipped) { - ReportSkip(test, request, response); - } else if (response.result_case() == ConformanceResponse::kRuntimeError) { - test.set_failure_message( - "Should have failed to parse, but raised an error instead."); - ReportFailure(test, level, request, response); - } else { - test.set_failure_message("Should have failed to parse, but didn't."); - ReportFailure(test, level, request, response); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::RunValidJsonTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json, const std::string& equivalent_text_format) { - MessageType prototype; - RunValidJsonTestWithMessage(test_name, level, input_json, - equivalent_text_format, prototype); -} - -void BinaryAndJsonConformanceSuite::RunValidJsonTestWithMessage( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json, const std::string& equivalent_text_format, - const Message& prototype) { - ConformanceRequestSetting setting1( - level, ::conformance::JSON, ::conformance::PROTOBUF, - ::conformance::JSON_TEST, prototype, test_name, input_json); - RunValidInputTest(setting1, equivalent_text_format); - ConformanceRequestSetting setting2( - level, ::conformance::JSON, ::conformance::JSON, ::conformance::JSON_TEST, - prototype, test_name, input_json); - RunValidInputTest(setting2, equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::ExpectParseFailureForJson( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json) { - MessageType prototype; - // We don't expect output, but if the program erroneously accepts the JSON - // we let it send its response as this. We must not leave it unspecified. - ConformanceRequestSetting setting( - level, ::conformance::JSON, ::conformance::JSON, ::conformance::JSON_TEST, - prototype, test_name, input_json); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - std::string effective_test_name = - absl::StrCat(setting.ConformanceLevelToString(level), ".", - setting.GetSyntaxIdentifier(), ".JsonInput.", test_name); - - if (!RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kParseError) { - ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kSkipped) { - ReportSkip(test, request, response); - } else if (response.result_case() == ConformanceResponse::kRuntimeError) { - test.set_failure_message( - "Should have failed to parse, but raised an error instead."); - ReportFailure(test, level, request, response); - } else { - test.set_failure_message("Should have failed to parse, but didn't."); - ReportFailure(test, level, request, response); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::RunValidJsonTestWithValidator( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json, const Validator& validator) { - MessageType prototype; - ConformanceRequestSetting setting( - level, ::conformance::JSON, ::conformance::JSON, ::conformance::JSON_TEST, - prototype, test_name, input_json); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - const std::string& effective_test_name = setting.GetTestName(); - - if (!RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kSkipped) { - ReportSkip(test, request, response); - return; - } - - if (response.result_case() != ConformanceResponse::kJsonPayload) { - test.set_failure_message(absl::StrCat("Expected JSON payload but got type ", - response.result_case())); - ReportFailure(test, level, request, response); - return; - } - Json::CharReaderBuilder builder; - Json::Value value; - Json::String err; - const std::unique_ptr<Json::CharReader> reader(builder.newCharReader()); - if (!reader->parse( - response.json_payload().c_str(), - response.json_payload().c_str() + response.json_payload().length(), - &value, &err)) { - test.set_failure_message( - absl::StrCat("JSON payload cannot be parsed as valid JSON: ", err)); - ReportFailure(test, level, request, response); - return; - } - if (!validator(value)) { - test.set_failure_message("JSON payload validation failed."); - ReportFailure(test, level, request, response); - return; - } - ReportSuccess(test); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - ExpectParseFailureForProtoWithProtoVersion(const std::string& proto, - const std::string& test_name, - ConformanceLevel level) { - suite_.ExpectParseFailureForProto<MessageType>(proto, test_name, level); -} - -// Expect that this precise protobuf will cause a parse error. -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::ExpectParseFailureForProto( - const std::string& proto, const std::string& test_name, - ConformanceLevel level) { - ExpectParseFailureForProtoWithProtoVersion(proto, test_name, level); -} - -// Expect that this protobuf will cause a parse error, even if it is followed -// by valid protobuf data. We can try running this twice: once with this -// data verbatim and once with this data followed by some valid data. -// -// TODO: implement the second of these. -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::ExpectHardParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level) { - return ExpectParseFailureForProto(proto, test_name, level); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunValidJsonTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json, const std::string& equivalent_text_format) { - MessageType prototype; - RunValidJsonTestWithMessage(test_name, level, input_json, - equivalent_text_format, prototype); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunValidJsonTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format, - const Message& prototype) { - suite_.RunValidJsonTestWithMessage(test_name, level, input_json, - equivalent_text_format, prototype); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunValidJsonTestWithProtobufInput( - const std::string& test_name, ConformanceLevel level, - const MessageType& input, const std::string& equivalent_text_format) { - ConformanceRequestSetting setting( - level, ::conformance::PROTOBUF, ::conformance::JSON, - ::conformance::JSON_TEST, input, test_name, input.SerializeAsString()); - suite_.RunValidInputTest(setting, equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunValidJsonIgnoreUnknownTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format) { - MessageType prototype; - ConformanceRequestSetting setting( - level, ::conformance::JSON, ::conformance::PROTOBUF, - ::conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST, prototype, test_name, - input_json); - suite_.RunValidInputTest(setting, equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::RunValidBinaryProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format) { - MessageType prototype; - - ConformanceRequestSetting binary_to_binary( - level, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, input_protobuf); - RunValidInputTest(binary_to_binary, equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::RunValidRoundtripProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf) { - MessageType prototype; - - ConformanceRequestSetting binary_to_binary( - level, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, input_protobuf); - RunValidBinaryInputTest(binary_to_binary, input_protobuf); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuite::RunValidProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format) { - MessageType prototype; - - ConformanceRequestSetting binary_to_binary( - level, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, input_protobuf); - RunValidInputTest(binary_to_binary, equivalent_text_format); - - ConformanceRequestSetting binary_to_json( - level, ::conformance::PROTOBUF, ::conformance::JSON, - ::conformance::BINARY_TEST, prototype, test_name, input_protobuf); - RunValidInputTest(binary_to_json, equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunValidProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format) { - suite_.RunValidProtobufTest<MessageType>(test_name, level, input_protobuf, - equivalent_text_format); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunValidBinaryProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf) { - RunValidBinaryProtobufTest(test_name, level, input_protobuf, input_protobuf); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunValidBinaryProtobufTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_protobuf, const std::string& expected_protobuf) { - MessageType prototype; - ConformanceRequestSetting setting( - level, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, input_protobuf); - suite_.RunValidBinaryInputTest(setting, expected_protobuf, true); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunBinaryPerformanceMergeMessageWithField(const std::string& test_name, - const std::string& field_proto) { - std::string message_tag = tag(27, WireFormatLite::WIRETYPE_LENGTH_DELIMITED); - std::string message_proto = absl::StrCat(message_tag, delim(field_proto)); - - std::string proto; - for (size_t i = 0; i < kPerformanceRepeatCount; i++) { - proto.append(message_proto); - } - - std::string multiple_repeated_field_proto; - for (size_t i = 0; i < kPerformanceRepeatCount; i++) { - multiple_repeated_field_proto.append(field_proto); - } - std::string expected_proto = - absl::StrCat(message_tag, delim(multiple_repeated_field_proto)); - - RunValidBinaryProtobufTest(test_name, RECOMMENDED, proto, expected_proto); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunValidProtobufTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const Message* input, - const std::string& equivalent_text_format) { - RunValidProtobufTest(test_name, level, input->SerializeAsString(), - equivalent_text_format); -} - -// According to proto JSON specification, JSON serializers follow more strict -// rules than parsers (e.g., a serializer must serialize int32 values as JSON -// numbers while the parser is allowed to accept them as JSON strings). This -// method allows strict checking on a proto JSON serializer by inspecting -// the JSON output directly. -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunValidJsonTestWithValidator(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const Validator& validator) { - suite_.RunValidJsonTestWithValidator<MessageType>(test_name, level, - input_json, validator); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::ExpectParseFailureForJson( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json) { - suite_.ExpectParseFailureForJson<MessageType>(test_name, level, input_json); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - RunValidJsonTestOrParseFailure(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format) { - MessageType prototype; - ConformanceRequestSetting setting( - level, ::conformance::JSON, ::conformance::PROTOBUF, - ::conformance::JSON_TEST, prototype, test_name, input_json); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - std::string effective_test_name = - absl::StrCat(setting.ConformanceLevelToString(level), ".", - SyntaxIdentifier(), ".JsonInput.", test_name); - - if (!suite_.RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(test, request, response); - } else { - std::unique_ptr<Message> reference_message(setting.NewTestMessage()); - ABSL_CHECK(TextFormat::ParseFromString(equivalent_text_format, - reference_message.get())) - << "Failed to parse data for test case: " << setting.GetTestName() - << ", data: " << equivalent_text_format; - std::unique_ptr<Message> test_message(setting.NewTestMessage()); - bool parsed = false; - if (response.result_case() == ConformanceResponse::kProtobufPayload) { - parsed = test_message->ParseFromString(response.protobuf_payload()); - } - if (!parsed) { - test.set_failure_message("Malformed protobuf response"); - suite_.ReportFailure(test, level, request, response); - return; - } - - util::MessageDifferencer differencer; - util::DefaultFieldComparator field_comparator; - field_comparator.set_treat_nan_as_equal(true); - differencer.set_field_comparator(&field_comparator); - std::string differences; - differencer.ReportDifferencesToString(&differences); - if (differencer.Compare(*reference_message, *test_message)) { - suite_.ReportSuccess(test); - } else { - test.set_failure_message( - "Should have failed to parse or matched expected output but did " - "not."); - suite_.ReportFailure(test, level, request, response); - } - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - ExpectSerializeFailureForJson(const std::string& test_name, - ConformanceLevel level, - const std::string& text_format) { - MessageType payload_message; - ABSL_CHECK(TextFormat::ParseFromString(text_format, &payload_message)) - << "Failed to parse: " << text_format; - - MessageType prototype; - ConformanceRequestSetting setting( - level, ::conformance::PROTOBUF, ::conformance::JSON, - ::conformance::JSON_TEST, prototype, test_name, - payload_message.SerializeAsString()); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - const std::string& effective_test_name = setting.GetTestName(); - - if (!suite_.RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kSerializeError) { - suite_.ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(test, request, response); - } else if (response.result_case() == ConformanceResponse::kRuntimeError) { - test.set_failure_message( - "Should have failed to serialize, but raised an error instead."); - suite_.ReportFailure(test, level, request, response); - } else { - test.set_failure_message("Should have failed to serialize, but didn't."); - suite_.ReportFailure(test, level, request, response); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestPrematureEOFForType( - FieldDescriptor::Type type) { - // Incomplete values for each wire type. - static constexpr absl::string_view incompletes[6] = { - "\x80", // VARINT - "abcdefg", // 64BIT - "\x80", // DELIMITED (partial length) - "", // START_GROUP (no value required) - "", // END_GROUP (no value required) - "abc" // 32BIT - }; - - const FieldDescriptor* field = GetFieldForType(type, false); - const FieldDescriptor* rep_field = GetFieldForType(type, true); - WireFormatLite::WireType wire_type = WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(type)); - absl::string_view incomplete = incompletes[wire_type]; - const std::string type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(type))); - - ExpectParseFailureForProto( - tag(field->number(), wire_type), - absl::StrCat("PrematureEofBeforeKnownNonRepeatedValue", type_name), - REQUIRED); - - ExpectParseFailureForProto( - tag(rep_field->number(), wire_type), - absl::StrCat("PrematureEofBeforeKnownRepeatedValue", type_name), - REQUIRED); - - ExpectParseFailureForProto( - tag(UNKNOWN_FIELD, wire_type), - absl::StrCat("PrematureEofBeforeUnknownValue", type_name), REQUIRED); - - ExpectParseFailureForProto( - absl::StrCat(tag(field->number(), wire_type), incomplete), - absl::StrCat("PrematureEofInsideKnownNonRepeatedValue", type_name), - REQUIRED); - - ExpectParseFailureForProto( - absl::StrCat(tag(rep_field->number(), wire_type), incomplete), - absl::StrCat("PrematureEofInsideKnownRepeatedValue", type_name), - REQUIRED); - - ExpectParseFailureForProto( - absl::StrCat(tag(UNKNOWN_FIELD, wire_type), incomplete), - absl::StrCat("PrematureEofInsideUnknownValue", type_name), REQUIRED); - - if (wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED) { - ExpectParseFailureForProto( - absl::StrCat(tag(field->number(), wire_type), varint(1)), - absl::StrCat("PrematureEofInDelimitedDataForKnownNonRepeatedValue", - type_name), - REQUIRED); - - ExpectParseFailureForProto( - absl::StrCat(tag(rep_field->number(), wire_type), varint(1)), - absl::StrCat("PrematureEofInDelimitedDataForKnownRepeatedValue", - type_name), - REQUIRED); - - // EOF in the middle of delimited data for unknown value. - ExpectParseFailureForProto( - absl::StrCat(tag(UNKNOWN_FIELD, wire_type), varint(1)), - absl::StrCat("PrematureEofInDelimitedDataForUnknownValue", type_name), - REQUIRED); - - if (type == FieldDescriptor::TYPE_MESSAGE) { - // Submessage ends in the middle of a value. - std::string incomplete_submsg = absl::StrCat( - tag(WireFormatLite::TYPE_INT32, WireFormatLite::WIRETYPE_VARINT), - incompletes[WireFormatLite::WIRETYPE_VARINT]); - ExpectHardParseFailureForProto( - absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - varint(incomplete_submsg.size()), incomplete_submsg), - absl::StrCat("PrematureEofInSubmessageValue", type_name), REQUIRED); - } - } else if (type != FieldDescriptor::TYPE_GROUP) { - // Non-delimited, non-group: eligible for packing. - - // Packed region ends in the middle of a value. - ExpectHardParseFailureForProto( - absl::StrCat( - tag(rep_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - varint(incomplete.size()), incomplete), - absl::StrCat("PrematureEofInPackedFieldValue", type_name), REQUIRED); - - // EOF in the middle of packed region. - ExpectParseFailureForProto( - absl::StrCat( - tag(rep_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - varint(1)), - absl::StrCat("PrematureEofInPackedField", type_name), REQUIRED); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestValidDataForType( - FieldDescriptor::Type type, - std::vector<std::pair<std::string, std::string>> values) { - const std::string type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(type))); - WireFormatLite::WireType wire_type = WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(type)); - const FieldDescriptor* field = GetFieldForType(type, false); - const FieldDescriptor* rep_field = GetFieldForType(type, true); - - // Test singular data for singular fields. - for (size_t i = 0; i < values.size(); i++) { - std::string proto = - absl::StrCat(tag(field->number(), wire_type), values[i].first); - // In proto3, default primitive fields should not be encoded. - std::string expected_proto = - run_proto3_tests_ && IsProto3Default(field->type(), values[i].second) - ? "" - : absl::StrCat(tag(field->number(), wire_type), values[i].second); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest( - absl::StrCat("ValidDataScalar", type_name, "[", i, "]"), REQUIRED, - proto, text); - RunValidBinaryProtobufTest( - absl::StrCat("ValidDataScalarBinary", type_name, "[", i, "]"), - RECOMMENDED, proto, expected_proto); - } - - // Test repeated data for singular fields. - // For scalar message fields, repeated values are merged, which is tested - // separately. - if (type != FieldDescriptor::TYPE_MESSAGE) { - std::string proto; - for (size_t i = 0; i < values.size(); i++) { - absl::StrAppend(&proto, tag(field->number(), wire_type), values[i].first); - } - std::string expected_proto = - absl::StrCat(tag(field->number(), wire_type), values.back().second); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest(absl::StrCat("RepeatedScalarSelectsLast", type_name), - REQUIRED, proto, text); - } - - // Test repeated fields. - if (FieldDescriptor::IsTypePackable(type)) { - const FieldDescriptor* packed_field = - GetFieldForType(type, true, Packed::kTrue); - const FieldDescriptor* unpacked_field = - GetFieldForType(type, true, Packed::kFalse); - - std::string default_proto_packed; - std::string default_proto_unpacked; - std::string default_proto_packed_expected; - std::string default_proto_unpacked_expected; - std::string packed_proto_packed; - std::string packed_proto_unpacked; - std::string packed_proto_expected; - std::string unpacked_proto_packed; - std::string unpacked_proto_unpacked; - std::string unpacked_proto_expected; - - for (size_t i = 0; i < values.size(); i++) { - absl::StrAppend(&default_proto_unpacked, - tag(rep_field->number(), wire_type), values[i].first); - absl::StrAppend(&default_proto_unpacked_expected, - tag(rep_field->number(), wire_type), values[i].second); - default_proto_packed += values[i].first; - default_proto_packed_expected += values[i].second; - absl::StrAppend(&packed_proto_unpacked, - tag(packed_field->number(), wire_type), values[i].first); - packed_proto_packed += values[i].first; - packed_proto_expected += values[i].second; - absl::StrAppend(&unpacked_proto_unpacked, - tag(unpacked_field->number(), wire_type), - values[i].first); - unpacked_proto_packed += values[i].first; - absl::StrAppend(&unpacked_proto_expected, - tag(unpacked_field->number(), wire_type), - values[i].second); - } - default_proto_packed = absl::StrCat( - tag(rep_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(default_proto_packed)); - default_proto_packed_expected = absl::StrCat( - tag(rep_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(default_proto_packed_expected)); - packed_proto_packed = absl::StrCat( - tag(packed_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(packed_proto_packed)); - packed_proto_expected = absl::StrCat( - tag(packed_field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(packed_proto_expected)); - unpacked_proto_packed = - absl::StrCat(tag(unpacked_field->number(), - WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(unpacked_proto_packed)); - - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(default_proto_packed_expected)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - // Ensures both packed and unpacked data can be parsed. - RunValidProtobufTest( - absl::StrCat("ValidDataRepeated", type_name, ".UnpackedInput"), - REQUIRED, default_proto_unpacked, text); - RunValidProtobufTest( - absl::StrCat("ValidDataRepeated", type_name, ".PackedInput"), REQUIRED, - default_proto_packed, text); - - // proto2 should encode as unpacked by default and proto3 should encode as - // packed by default. - std::string expected_proto = rep_field->is_packed() - ? default_proto_packed_expected - : default_proto_unpacked_expected; - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".UnpackedInput.DefaultOutput"), - RECOMMENDED, default_proto_unpacked, - expected_proto); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".PackedInput.DefaultOutput"), - RECOMMENDED, default_proto_packed, - expected_proto); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".UnpackedInput.PackedOutput"), - RECOMMENDED, packed_proto_unpacked, - packed_proto_expected); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".PackedInput.PackedOutput"), - RECOMMENDED, packed_proto_packed, - packed_proto_expected); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".UnpackedInput.UnpackedOutput"), - RECOMMENDED, unpacked_proto_unpacked, - unpacked_proto_expected); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataRepeated", type_name, - ".PackedInput.UnpackedOutput"), - RECOMMENDED, unpacked_proto_packed, - unpacked_proto_expected); - } else { - std::string proto; - std::string expected_proto; - for (size_t i = 0; i < values.size(); i++) { - proto += - absl::StrCat(tag(rep_field->number(), wire_type), values[i].first); - expected_proto += - absl::StrCat(tag(rep_field->number(), wire_type), values[i].second); - } - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest(absl::StrCat("ValidDataRepeated", type_name), REQUIRED, - proto, text); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::TestValidDataForRepeatedScalarMessage() { - std::vector<std::string> values = { - delim(absl::StrCat( - tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat( - tag(1, WireFormatLite::WIRETYPE_VARINT), varint(1234), - tag(2, WireFormatLite::WIRETYPE_VARINT), varint(1234), - tag(31, WireFormatLite::WIRETYPE_VARINT), varint(1234))))), - delim(absl::StrCat( - tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat( - tag(1, WireFormatLite::WIRETYPE_VARINT), varint(4321), - tag(3, WireFormatLite::WIRETYPE_VARINT), varint(4321), - tag(31, WireFormatLite::WIRETYPE_VARINT), varint(4321))))), - }; - - const std::string expected = - R"({ - corecursive: { - optional_int32: 4321, - optional_int64: 1234, - optional_uint32: 4321, - repeated_int32: [1234, 4321], - } - })"; - - std::string proto; - const FieldDescriptor* field = - GetFieldForType(FieldDescriptor::TYPE_MESSAGE, false); - for (size_t i = 0; i < values.size(); i++) { - absl::StrAppend( - &proto, tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - values[i]); - } - - RunValidProtobufTest("RepeatedScalarMessageMerge", REQUIRED, proto, - absl::StrCat(field->name(), ": ", expected)); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestValidDataForMapType( - FieldDescriptor::Type key_type, FieldDescriptor::Type value_type) { - const std::string key_type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(key_type))); - const std::string value_type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(value_type))); - WireFormatLite::WireType key_wire_type = WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(key_type)); - WireFormatLite::WireType value_wire_type = - WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(value_type)); - - std::string key1_data = - absl::StrCat(tag(1, key_wire_type), GetDefaultValue(key_type)); - std::string value1_data = - absl::StrCat(tag(2, value_wire_type), GetDefaultValue(value_type)); - std::string key2_data = - absl::StrCat(tag(1, key_wire_type), GetNonDefaultValue(key_type)); - std::string value2_data = - absl::StrCat(tag(2, value_wire_type), GetNonDefaultValue(value_type)); - - const FieldDescriptor* field = GetFieldForMapType(key_type, value_type); - - { - // Tests map with default key and value. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key1_data, value1_data))); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest(absl::StrCat("ValidDataMap", key_type_name, - value_type_name, ".Default"), - REQUIRED, proto, text); - } - - { - // Tests map with missing default key and value. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim("")); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest(absl::StrCat("ValidDataMap", key_type_name, - value_type_name, ".MissingDefault"), - REQUIRED, proto, text); - } - - { - // Tests map with non-default key and value. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key2_data, value2_data))); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest(absl::StrCat("ValidDataMap", key_type_name, - value_type_name, ".NonDefault"), - REQUIRED, proto, text); - } - - { - // Tests map with unordered key and value. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(value2_data, key2_data))); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest(absl::StrCat("ValidDataMap", key_type_name, - value_type_name, ".Unordered"), - REQUIRED, proto, text); - } - - { - // Tests map with duplicate key. - std::string proto1 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key2_data, value1_data))); - std::string proto2 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key2_data, value2_data))); - std::string proto = absl::StrCat(proto1, proto2); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto2)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest(absl::StrCat("ValidDataMap", key_type_name, - value_type_name, ".DuplicateKey"), - REQUIRED, proto, text); - } - - { - // Tests map with duplicate key in map entry. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key1_data, key2_data, value2_data))); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest( - absl::StrCat("ValidDataMap", key_type_name, value_type_name, - ".DuplicateKeyInMapEntry"), - REQUIRED, proto, text); - } - - { - // Tests map with duplicate value in map entry. - std::string proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key2_data, value1_data, value2_data))); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest( - absl::StrCat("ValidDataMap", key_type_name, value_type_name, - ".DuplicateValueInMapEntry"), - REQUIRED, proto, text); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunMapEntryWireTypeMismatchTest(const std::string& test_name, - const std::string& proto) { - MessageType prototype; - ConformanceRequestSetting setting( - RECOMMENDED, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, test_name, proto); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - if (!suite_.RunTest(setting.GetTestName(), request, &response)) { - return; - } - - TestStatus test; - test.set_name(setting.GetTestName()); - - if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(test, request, response); - } else if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kProtobufPayload) { - MessageType reference; - ABSL_CHECK(reference.MergeFromString(proto)); - MessageType response_message; - if (suite_.ParseResponse(response, setting, &response_message)) { - if (google::protobuf::util::MessageDifferencer::Equals(reference, - response_message)) { - suite_.ReportSuccess(test); - } else { - test.set_failure_message( - "Output was not equivalent to reference message; the mismatched " - "field appears to have been decoded as a value."); - suite_.ReportFailure(test, setting.GetLevel(), request, response); - } - } - } else { - test.set_failure_message( - "Should have failed to parse or matched expected output but did " - "not."); - suite_.ReportFailure(test, setting.GetLevel(), request, response); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - TestMapEntryWireTypeMismatch(FieldDescriptor::Type key_type, - FieldDescriptor::Type value_type) { - const std::string key_type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(key_type))); - const std::string value_type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(value_type))); - const WireFormatLite::WireType key_wire_type = - WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(key_type)); - const WireFormatLite::WireType value_wire_type = - WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(value_type)); - - const std::string good_key_data = - absl::StrCat(tag(1, key_wire_type), GetNonDefaultValue(key_type)); - const std::string good_value_data = - absl::StrCat(tag(2, value_wire_type), GetNonDefaultValue(value_type)); - - const FieldDescriptor* field = GetFieldForMapType(key_type, value_type); - - // A map entry whose key is encoded with the wrong wire type; the value is - // still well formed. - RunMapEntryWireTypeMismatchTest( - absl::StrCat("ValidDataMap", key_type_name, value_type_name, - ".KeyWireTypeMismatch"), - absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(MismatchedWireTypeField(1, key_wire_type), - good_value_data)))); - - // A map entry whose value is encoded with the wrong wire type; the key is - // still well formed. - RunMapEntryWireTypeMismatchTest( - absl::StrCat("ValidDataMap", key_type_name, value_type_name, - ".ValueWireTypeMismatch"), - absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(good_key_data, - MismatchedWireTypeField(2, value_wire_type))))); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::TestOverwriteMessageValueMap() { - std::string key_data = absl::StrCat( - tag(1, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), delim("")); - std::string field1_data = - absl::StrCat(tag(1, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string field2_data = - absl::StrCat(tag(2, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string field31_data = - absl::StrCat(tag(31, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string submsg1_data = delim(absl::StrCat(field1_data, field31_data)); - std::string submsg2_data = delim(absl::StrCat(field2_data, field31_data)); - std::string value1_data = absl::StrCat( - tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - submsg1_data))); - std::string value2_data = absl::StrCat( - tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - submsg2_data))); - - const FieldDescriptor* field = GetFieldForMapType( - FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_MESSAGE); - - std::string proto1 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key_data, value1_data))); - std::string proto2 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(key_data, value2_data))); - std::string proto = absl::StrCat(proto1, proto2); - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto2)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest("ValidDataMap.STRING.MESSAGE.MergeValue", REQUIRED, - proto, text); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestValidDataForOneofType( - FieldDescriptor::Type type) { - const std::string type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(type))); - WireFormatLite::WireType wire_type = WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(type)); - - const FieldDescriptor* field = GetFieldForOneofType(type); - const std::string default_value = - absl::StrCat(tag(field->number(), wire_type), GetDefaultValue(type)); - const std::string non_default_value = - absl::StrCat(tag(field->number(), wire_type), GetNonDefaultValue(type)); - - { - // Tests oneof with default value. - const std::string& proto = default_value; - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest( - absl::StrCat("ValidDataOneof", type_name, ".DefaultValue"), REQUIRED, - proto, text); - RunValidBinaryProtobufTest( - absl::StrCat("ValidDataOneofBinary", type_name, ".DefaultValue"), - RECOMMENDED, proto, proto); - } - - { - // Tests oneof with non-default value. - const std::string& proto = non_default_value; - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest( - absl::StrCat("ValidDataOneof", type_name, ".NonDefaultValue"), REQUIRED, - proto, text); - RunValidBinaryProtobufTest( - absl::StrCat("ValidDataOneofBinary", type_name, ".NonDefaultValue"), - RECOMMENDED, proto, proto); - } - - { - // Tests oneof with multiple values of the same field. - const std::string proto = absl::StrCat(default_value, non_default_value); - const std::string& expected_proto = non_default_value; - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest(absl::StrCat("ValidDataOneof", type_name, - ".MultipleValuesForSameField"), - REQUIRED, proto, text); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataOneofBinary", type_name, - ".MultipleValuesForSameField"), - RECOMMENDED, proto, expected_proto); - } - - { - // Tests oneof with multiple values of the different fields. - const FieldDescriptor* other_field = GetFieldForOneofType(type, true); - FieldDescriptor::Type other_type = other_field->type(); - WireFormatLite::WireType other_wire_type = - WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(other_type)); - const std::string other_value = - absl::StrCat(tag(other_field->number(), other_wire_type), - GetDefaultValue(other_type)); - - const std::string proto = absl::StrCat(other_value, non_default_value); - const std::string& expected_proto = non_default_value; - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - - RunValidProtobufTest(absl::StrCat("ValidDataOneof", type_name, - ".MultipleValuesForDifferentField"), - REQUIRED, proto, text); - RunValidBinaryProtobufTest(absl::StrCat("ValidDataOneofBinary", type_name, - ".MultipleValuesForDifferentField"), - RECOMMENDED, proto, expected_proto); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestMergeOneofMessage() { - std::string field1_data = - absl::StrCat(tag(1, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string field2a_data = - absl::StrCat(tag(2, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string field2b_data = - absl::StrCat(tag(2, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string field89_data = - absl::StrCat(tag(89, WireFormatLite::WIRETYPE_VARINT), varint(1)); - std::string submsg1_data = absl::StrCat( - tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(field1_data, field2a_data, field89_data))); - std::string submsg2_data = - absl::StrCat(tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(field2b_data, field89_data))); - std::string merged_data = - absl::StrCat(tag(2, WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(absl::StrCat(field1_data, field2b_data, field89_data, - field89_data))); - - const FieldDescriptor* field = - GetFieldForOneofType(FieldDescriptor::TYPE_MESSAGE); - - std::string proto1 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(submsg1_data)); - std::string proto2 = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(submsg2_data)); - std::string proto = absl::StrCat(proto1, proto2); - std::string expected_proto = absl::StrCat( - tag(field->number(), WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - delim(merged_data)); - - MessageType test_message; - ABSL_CHECK(test_message.MergeFromString(expected_proto)); - std::string text; - ABSL_CHECK(TextFormat::PrintToString(test_message, &text)); - RunValidProtobufTest("ValidDataOneof.MESSAGE.Merge", REQUIRED, proto, text); - RunValidBinaryProtobufTest("ValidDataOneofBinary.MESSAGE.Merge", RECOMMENDED, - proto, expected_proto); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestIllegalTags() { - // field num 0 is illegal - std::string nullfield[] = {"\1DEADBEEF", "\2\1\1", "\3\4", "\5DEAD"}; - for (int i = 0; i < 4; i++) { - std::string name = "IllegalZeroFieldNum_Case_0"; - name.back() += i; - ExpectParseFailureForProto(nullfield[i], name, REQUIRED); - } - - // Reused for a few cases: this is a single byte tag for a field number 1 - // varint but where the byte has the continuation bit set, so that we can - // easily construct longer tags. - std::string tag_with_continuation_bit = - tag(1, WireFormatLite::WIRETYPE_VARINT); - assert(tag_with_continuation_bit.size() == 1); - tag_with_continuation_bit[0] |= 0b10000000; - - // A varint where field number is far out of range of - // maximum legal field number. The lower 5 bytes of the varint do look like a - // well-formed tag for field number 1. - ExpectParseFailureForProto( - absl::StrCat(tag_with_continuation_bit, "\x80\x80\x80\x80\x80\x0F", - varint(1234)), - "BadTag_FieldNumberTooHigh", REQUIRED); - - // A 5-byte tag varint where the value is above UINT32_MAX (bit 35 is set) - ExpectParseFailureForProto( - absl::StrCat(tag_with_continuation_bit, "\x80\x80\x80\x40", varint(1234)), - "BadTag_FieldNumberSlightlyTooHigh", REQUIRED); - - // A tag where the varint is more than 5 bytes but only because it is an - // overlong varint, so the decoded value is still below UINT32_MAX. - ExpectParseFailureForProto( - absl::StrCat(tag_with_continuation_bit, "\x80\x80\x80\x80\x80\x80\x80", - std::string("\0", 1), varint(1234)), - "BadTag_OverlongVarint", REQUIRED); - - // An overlong varint that is even more than 10 bytes. - ExpectParseFailureForProto( - absl::StrCat(tag_with_continuation_bit, - "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80", - std::string("\0", 1), varint(1234)), - "BadTag_VarintMoreThanTenBytes", REQUIRED); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestIllegalLengths() { - const FieldDescriptor* string_field = - GetFieldForType(FieldDescriptor::TYPE_STRING, false); - - // A 5-byte varint length where bits 32-34 are set (e.g. bit 32 = 0x10), which - // overflows 32-bit arithmetic and wraps to 0 modulo 2^32. Parsers must not - // overflow and must reject this invalid wire format. - ExpectParseFailureForProto( - absl::StrCat(tag(string_field->number(), - WireFormatLite::WIRETYPE_LENGTH_DELIMITED), - "\x80\x80\x80\x80\x10"), - "BadLength_Varint32BitOverflow", REQUIRED); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestUnmatchedGroup() { - ExpectParseFailureForProto(tag(201, WireFormatLite::WIRETYPE_END_GROUP), - "UnmatchedEndGroup", REQUIRED); - ExpectParseFailureForProto(tag(1234, WireFormatLite::WIRETYPE_END_GROUP), - "UnmatchedEndGroupUnknown", REQUIRED); - ExpectParseFailureForProto(tag(1, WireFormatLite::WIRETYPE_END_GROUP), - "UnmatchedEndGroupWrongType", REQUIRED); - ExpectParseFailureForProto( - len(18, tag(1234, WireFormatLite::WIRETYPE_END_GROUP)), - "UnmatchedEndGroupNestedLen", REQUIRED); - ExpectParseFailureForProto( - group(201, tag(202, WireFormatLite::WIRETYPE_END_GROUP)), - "UnmatchedEndGroupNested", REQUIRED); - ExpectParseFailureForProto( - absl::StrCat(tag(1, WireFormatLite::WIRETYPE_END_GROUP), - len(2, "hello world")), - "UnmatchedEndGroupWithData", REQUIRED); - - ExpectParseFailureForProto(tag(201, WireFormatLite::WIRETYPE_START_GROUP), - "UnmatchedStartGroup", REQUIRED); - ExpectParseFailureForProto(tag(1234, WireFormatLite::WIRETYPE_START_GROUP), - "UnmatchedStartGroupUnknown", REQUIRED); - ExpectParseFailureForProto(tag(1, WireFormatLite::WIRETYPE_START_GROUP), - "UnmatchedStartGroupWrongType", REQUIRED); - ExpectParseFailureForProto( - len(18, tag(1234, WireFormatLite::WIRETYPE_START_GROUP)), - "UnmatchedStartGroupNestedLen", REQUIRED); - ExpectParseFailureForProto( - group(201, tag(202, WireFormatLite::WIRETYPE_START_GROUP)), - "UnmatchedStartGroupNested", REQUIRED); - ExpectParseFailureForProto( - absl::StrCat(tag(1, WireFormatLite::WIRETYPE_START_GROUP), - len(2, "hello world")), - "UnmatchedStartGroupWithData", REQUIRED); - - ExpectParseFailureForProto( - absl::StrCat(tag(201, WireFormatLite::WIRETYPE_START_GROUP), - len(2, "hello world"), - tag(202, WireFormatLite::WIRETYPE_END_GROUP)), - "MismatchedGroupTags", REQUIRED); - ExpectParseFailureForProto( - group(201, absl::StrCat(tag(202, WireFormatLite::WIRETYPE_START_GROUP), - len(2, "hello world"), - tag(203, WireFormatLite::WIRETYPE_END_GROUP))), - "MismatchedNestedGroupTags", REQUIRED); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestUnknownWireType() { - for (uint8_t type : {0x6, 0x7}) { - for (uint8_t field = 0; field < 4; ++field) { - for (uint8_t value = 0; value < 4; ++value) { - std::string name = absl::StrFormat( - "UnknownWireType%d_Field%d_Version%d", type, field, value); - - char data[2]; - data[0] = (field << 3) | type; // unknown wire type. - data[1] = value; - std::string proto = {data, 2}; - ExpectParseFailureForProto(proto, name, REQUIRED); - } - } - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestInvalidUtf8String() { - if (run_proto3_tests_) { - ExpectParseFailureForProto(len(14, "\xA0\xB0\xC0\xD0"), - "RejectInvalidUtf8.String.Singular", - RECOMMENDED); - ExpectParseFailureForProto(len(44, "\xA0\xB0\xC0\xD0"), - "RejectInvalidUtf8.String.Repeated", - RECOMMENDED); - ExpectParseFailureForProto(len(113, "\xA0\xB0\xC0\xD0"), - "RejectInvalidUtf8.String.Oneof", RECOMMENDED); - ExpectParseFailureForProto( - len(69, absl::StrCat(len(1, "\xA0\xB0\xC0\xD0"), len(2, "foo"))), - "RejectInvalidUtf8.String.MapKey", RECOMMENDED); - ExpectParseFailureForProto( - len(69, absl::StrCat(len(1, "foo"), len(2, "\xA0\xB0\xC0\xD0"))), - "RejectInvalidUtf8.String.MapValue", RECOMMENDED); - } else { - // TODO - Once conformance tests can express failures that are - // not expected to be fixed, add proto2 coverage here. - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestOneofMessage() { - MessageType message; - message.set_oneof_uint32(0); - RunValidProtobufTestWithMessage("OneofZeroUint32", RECOMMENDED, &message, - "oneof_uint32: 0"); - message.mutable_oneof_nested_message()->set_a(0); - RunValidProtobufTestWithMessage("OneofZeroMessage", RECOMMENDED, &message, - run_proto3_tests_ - ? "oneof_nested_message: {}" - : "oneof_nested_message: {a: 0}"); - message.mutable_oneof_nested_message()->set_a(1); - RunValidProtobufTestWithMessage("OneofZeroMessageSetTwice", RECOMMENDED, - &message, "oneof_nested_message: {a: 1}"); - message.set_oneof_string(""); - RunValidProtobufTestWithMessage("OneofZeroString", RECOMMENDED, &message, - "oneof_string: \"\""); - message.set_oneof_bytes(""); - RunValidProtobufTestWithMessage("OneofZeroBytes", RECOMMENDED, &message, - "oneof_bytes: \"\""); - message.set_oneof_bool(false); - RunValidProtobufTestWithMessage("OneofZeroBool", RECOMMENDED, &message, - "oneof_bool: false"); - message.set_oneof_uint64(0); - RunValidProtobufTestWithMessage("OneofZeroUint64", RECOMMENDED, &message, - "oneof_uint64: 0"); - message.set_oneof_float(0.0f); - RunValidProtobufTestWithMessage("OneofZeroFloat", RECOMMENDED, &message, - "oneof_float: 0"); - message.set_oneof_double(0.0); - RunValidProtobufTestWithMessage("OneofZeroDouble", RECOMMENDED, &message, - "oneof_double: 0"); - message.set_oneof_enum(MessageType::FOO); - RunValidProtobufTestWithMessage("OneofZeroEnum", RECOMMENDED, &message, - "oneof_enum: FOO"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestUnknownMessage() { - MessageType message; - ABSL_CHECK(message.ParseFromString("\xA8\x1F\x01")); - RunValidBinaryProtobufTest("UnknownVarint", REQUIRED, - message.SerializeAsString()); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::TestUnknownOrdering() { - // Implementations must preserve the ordering of different unknown fields for - // the same field number. This is because some field types will accept - // multiple wire types for the same field. For example, repeated primitive - // fields will accept both length-prefixed (packed) and - // varint/fixed32/fixed64 (unpacked) wire types, and reordering these could - // reorder the elements of the repeated field. - MessageType message; - MessageType prototype; - message.mutable_unknown_fields()->AddLengthDelimited(UNKNOWN_FIELD, "abc"); - message.mutable_unknown_fields()->AddVarint(UNKNOWN_FIELD, 123); - message.mutable_unknown_fields()->AddLengthDelimited(UNKNOWN_FIELD, "def"); - message.mutable_unknown_fields()->AddVarint(UNKNOWN_FIELD, 456); - std::string serialized = message.SerializeAsString(); - - ConformanceRequestSetting setting( - REQUIRED, ::conformance::PROTOBUF, ::conformance::PROTOBUF, - ::conformance::BINARY_TEST, prototype, "UnknownOrdering", serialized); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - if (!suite_.RunTest(setting.GetTestName(), request, &response)) { - return; - } - - MessageType response_message; - TestStatus test; - test.set_name(setting.GetTestName()); - if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(test, request, response); - return; - } - - suite_.ParseResponse(response, setting, &response_message); - - const UnknownFieldSet& ufs = response_message.unknown_fields(); - if (ufs.field_count() != 4 || ufs.field(0).number() != UNKNOWN_FIELD || - ufs.field(1).number() != UNKNOWN_FIELD || - ufs.field(2).number() != UNKNOWN_FIELD || - ufs.field(3).number() != UNKNOWN_FIELD || - ufs.field(0).type() != UnknownField::Type::TYPE_LENGTH_DELIMITED || - ufs.field(1).type() != UnknownField::Type::TYPE_VARINT || - ufs.field(2).type() != UnknownField::Type::TYPE_LENGTH_DELIMITED || - ufs.field(3).type() != UnknownField::Type::TYPE_VARINT || - ufs.field(0).length_delimited() != "abc" || - ufs.field(1).varint() != 123 || - ufs.field(2).length_delimited() != "def" || - ufs.field(3).varint() != 456) { - test.set_failure_message("Unknown field mismatch"); - suite_.ReportFailure(test, setting.GetLevel(), request, response); - } else { - suite_.ReportSuccess(test); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::TestBinaryPerformanceForAlternatingUnknownFields() { - std::string unknown_field_1 = absl::StrCat( - tag(UNKNOWN_FIELD, WireFormatLite::WIRETYPE_VARINT), varint(1234)); - std::string unknown_field_2 = absl::StrCat( - tag(UNKNOWN_FIELD + 1, WireFormatLite::WIRETYPE_VARINT), varint(5678)); - std::string proto; - for (size_t i = 0; i < kPerformanceRepeatCount; i++) { - proto.append(unknown_field_1); - proto.append(unknown_field_2); - } - - RunValidBinaryProtobufTest("TestBinaryPerformanceForAlternatingUnknownFields", - RECOMMENDED, proto); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::Type type) { - const std::string type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(type))); - int field_number = GetFieldForType(type, true, Packed::kFalse)->number(); - std::string rep_field_proto = absl::StrCat( - tag(field_number, WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(type))), - GetNonDefaultValue(type)); - - RunBinaryPerformanceMergeMessageWithField( - absl::StrCat("TestBinaryPerformanceMergeMessageWithRepeatedFieldForType", - type_name), - rep_field_proto); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>:: - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::Type type) { - const std::string type_name = - UpperCase(absl::StrCat(".", FieldDescriptor::TypeName(type))); - std::string unknown_field_proto = absl::StrCat( - tag(UNKNOWN_FIELD, WireFormatLite::WireTypeForFieldType( - static_cast<WireFormatLite::FieldType>(type))), - GetNonDefaultValue(type)); - RunBinaryPerformanceMergeMessageWithField( - absl::StrCat("TestBinaryPerformanceMergeMessageWithUnknownFieldForType", - type_name), - unknown_field_proto); -} - -template <typename MessageType> -BinaryAndJsonConformanceSuiteImpl<MessageType>:: - BinaryAndJsonConformanceSuiteImpl(BinaryAndJsonConformanceSuite* suite, - bool run_proto3_tests) - : suite_(*ABSL_DIE_IF_NULL(suite)), run_proto3_tests_(run_proto3_tests) { - suite_.SetTypeUrl(GetTypeUrl(MessageType::GetDescriptor())); - RunAllTests(); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunAllTests() { - if (!suite_.performance_) { - for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) { - if (i == FieldDescriptor::TYPE_GROUP) continue; - TestPrematureEOFForType(static_cast<FieldDescriptor::Type>(i)); - } - - TestIllegalTags(); - TestIllegalLengths(); - TestUnmatchedGroup(); - TestUnknownWireType(); - TestInvalidUtf8String(); - - int64_t kInt64Min = -9223372036854775808ULL; - int64_t kInt64Max = 9223372036854775807ULL; - uint64_t kUint64Max = 18446744073709551615ULL; - int32_t kInt32Max = 2147483647; - int32_t kInt32Min = -2147483648; - uint32_t kUint32Max = 4294967295UL; - - TestValidDataForType( - FieldDescriptor::TYPE_DOUBLE, - { - {dbl(0), dbl(0)}, - {dbl(0.1), dbl(0.1)}, - {dbl(1.7976931348623157e+308), dbl(1.7976931348623157e+308)}, - {dbl(2.22507385850720138309e-308), - dbl(2.22507385850720138309e-308)}, - }); - TestValidDataForType( - FieldDescriptor::TYPE_FLOAT, - { - {flt(0), flt(0)}, - {flt(0.1), flt(0.1)}, - {flt(1.00000075e-36), flt(1.00000075e-36)}, - {flt(3.402823e+38), flt(3.402823e+38)}, // 3.40282347e+38 - {flt(1.17549435e-38f), flt(1.17549435e-38)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_INT64, - { - {varint(0), varint(0)}, - {varint(12345), varint(12345)}, - {varint(kInt64Max), varint(kInt64Max)}, - {varint(kInt64Min), varint(kInt64Min)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_UINT64, - { - {varint(0), varint(0)}, - {varint(12345), varint(12345)}, - {varint(kUint64Max), varint(kUint64Max)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_INT32, - { - {varint(0), varint(0)}, - {varint(12345), varint(12345)}, - {longvarint(12345, 2), varint(12345)}, - {longvarint(12345, 7), varint(12345)}, - {varint(kInt32Max), varint(kInt32Max)}, - {varint(kInt32Min), varint(kInt32Min)}, - {varint(1LL << 33), varint(0)}, - {varint((1LL << 33) - 1), varint(-1)}, - {varint(kInt64Max), varint(-1)}, - {varint(kInt64Min + 1), varint(1)}, - }); - TestValidDataForType( - FieldDescriptor::TYPE_UINT32, - { - {varint(0), varint(0)}, - {varint(12345), varint(12345)}, - {longvarint(12345, 2), varint(12345)}, - {longvarint(12345, 7), varint(12345)}, - {varint(kUint32Max), varint(kUint32Max)}, // UINT32_MAX - {varint(1LL << 33), varint(0)}, - {varint((1LL << 33) + 1), varint(1)}, - {varint((1LL << 33) - 1), varint((1LL << 32) - 1)}, - {varint(kInt64Max), varint((1LL << 32) - 1)}, - {varint(kInt64Min + 1), varint(1)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_FIXED64, - { - {u64(0), u64(0)}, - {u64(12345), u64(12345)}, - {u64(kUint64Max), u64(kUint64Max)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_FIXED32, - { - {u32(0), u32(0)}, - {u32(12345), u32(12345)}, - {u32(kUint32Max), u32(kUint32Max)}, // UINT32_MAX - }); - TestValidDataForType(FieldDescriptor::TYPE_SFIXED64, - { - {u64(0), u64(0)}, - {u64(12345), u64(12345)}, - {u64(kInt64Max), u64(kInt64Max)}, - {u64(kInt64Min), u64(kInt64Min)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_SFIXED32, - { - {u32(0), u32(0)}, - {u32(12345), u32(12345)}, - {u32(kInt32Max), u32(kInt32Max)}, - {u32(kInt32Min), u32(kInt32Min)}, - }); - // Bools should be serialized as 0 for false and 1 for true. Parsers should - // also interpret any nonzero value as true. - TestValidDataForType(FieldDescriptor::TYPE_BOOL, - { - {varint(0), varint(0)}, - {varint(1), varint(1)}, - {varint(-1), varint(1)}, - {varint(12345678), varint(1)}, - {varint(1LL << 33), varint(1)}, - {varint(kInt64Max), varint(1)}, - {varint(kInt64Min), varint(1)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_SINT32, - { - {zz32(0), zz32(0)}, - {zz32(12345), zz32(12345)}, - {zz32(kInt32Max), zz32(kInt32Max)}, - {zz32(kInt32Min), zz32(kInt32Min)}, - {zz64(kInt32Max + 2LL), zz32(1)}, - }); - TestValidDataForType(FieldDescriptor::TYPE_SINT64, - { - {zz64(0), zz64(0)}, - {zz64(12345), zz64(12345)}, - {zz64(kInt64Max), zz64(kInt64Max)}, - {zz64(kInt64Min), zz64(kInt64Min)}, - }); - TestValidDataForType( - FieldDescriptor::TYPE_STRING, - { - {delim(""), delim("")}, - {delim("Hello world!"), delim("Hello world!")}, - {delim("\'\"\?\\\a\b\f\n\r\t\v"), - delim("\'\"\?\\\a\b\f\n\r\t\v")}, // escape - // U+8C37 U+6B4C ("Google" in Chinese), as UTF-8. - {delim("\xE8\xB0\xB7\xE6\xAD\x8C"), - delim("\xE8\xB0\xB7\xE6\xAD\x8C")}, - // U+1F601 (grinning face with smiling eyes), as UTF-8. - {delim("\xF0\x9F\x98\x81"), delim("\xF0\x9F\x98\x81")}, - }); - TestValidDataForType(FieldDescriptor::TYPE_BYTES, - { - {delim(""), delim("")}, - {delim("Hello world!"), delim("Hello world!")}, - {delim("\x01\x02"), delim("\x01\x02")}, - {delim("\xfb"), delim("\xfb")}, - }); - TestValidDataForType(FieldDescriptor::TYPE_ENUM, - { - {varint(0), varint(0)}, - {varint(1), varint(1)}, - {varint(2), varint(2)}, - {varint(-1), varint(-1)}, - {varint(kInt64Max), varint(-1)}, - {varint(kInt64Min + 1), varint(1)}, - }); - TestValidDataForRepeatedScalarMessage(); - TestValidDataForType( - FieldDescriptor::TYPE_MESSAGE, - { - {delim(""), delim("")}, - {delim(absl::StrCat(tag(1, WireFormatLite::WIRETYPE_VARINT), - varint(1234))), - delim(absl::StrCat(tag(1, WireFormatLite::WIRETYPE_VARINT), - varint(1234)))}, - }); - - TestValidDataForMapType(FieldDescriptor::TYPE_INT32, - FieldDescriptor::TYPE_INT32); - TestValidDataForMapType(FieldDescriptor::TYPE_INT64, - FieldDescriptor::TYPE_INT64); - TestValidDataForMapType(FieldDescriptor::TYPE_UINT32, - FieldDescriptor::TYPE_UINT32); - TestValidDataForMapType(FieldDescriptor::TYPE_UINT64, - FieldDescriptor::TYPE_UINT64); - TestValidDataForMapType(FieldDescriptor::TYPE_SINT32, - FieldDescriptor::TYPE_SINT32); - TestValidDataForMapType(FieldDescriptor::TYPE_SINT64, - FieldDescriptor::TYPE_SINT64); - TestValidDataForMapType(FieldDescriptor::TYPE_FIXED32, - FieldDescriptor::TYPE_FIXED32); - TestValidDataForMapType(FieldDescriptor::TYPE_FIXED64, - FieldDescriptor::TYPE_FIXED64); - TestValidDataForMapType(FieldDescriptor::TYPE_SFIXED32, - FieldDescriptor::TYPE_SFIXED32); - TestValidDataForMapType(FieldDescriptor::TYPE_SFIXED64, - FieldDescriptor::TYPE_SFIXED64); - TestValidDataForMapType(FieldDescriptor::TYPE_INT32, - FieldDescriptor::TYPE_FLOAT); - TestValidDataForMapType(FieldDescriptor::TYPE_INT32, - FieldDescriptor::TYPE_DOUBLE); - TestValidDataForMapType(FieldDescriptor::TYPE_BOOL, - FieldDescriptor::TYPE_BOOL); - TestValidDataForMapType(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_STRING); - TestValidDataForMapType(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_BYTES); - TestValidDataForMapType(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_ENUM); - TestValidDataForMapType(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_MESSAGE); - // Additional test to check overwriting message value map. - TestOverwriteMessageValueMap(); - - TestMapEntryWireTypeMismatch(FieldDescriptor::TYPE_INT32, - FieldDescriptor::TYPE_INT32); - TestMapEntryWireTypeMismatch(FieldDescriptor::TYPE_FIXED32, - FieldDescriptor::TYPE_FIXED32); - TestMapEntryWireTypeMismatch(FieldDescriptor::TYPE_BOOL, - FieldDescriptor::TYPE_BOOL); - TestMapEntryWireTypeMismatch(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_STRING); - TestMapEntryWireTypeMismatch(FieldDescriptor::TYPE_STRING, - FieldDescriptor::TYPE_MESSAGE); - - TestValidDataForOneofType(FieldDescriptor::TYPE_UINT32); - TestValidDataForOneofType(FieldDescriptor::TYPE_BOOL); - TestValidDataForOneofType(FieldDescriptor::TYPE_UINT64); - TestValidDataForOneofType(FieldDescriptor::TYPE_FLOAT); - TestValidDataForOneofType(FieldDescriptor::TYPE_DOUBLE); - TestValidDataForOneofType(FieldDescriptor::TYPE_STRING); - TestValidDataForOneofType(FieldDescriptor::TYPE_BYTES); - TestValidDataForOneofType(FieldDescriptor::TYPE_ENUM); - TestValidDataForOneofType(FieldDescriptor::TYPE_MESSAGE); - // Additional test to check merging oneof message. - TestMergeOneofMessage(); - - // TODO: - // TestValidDataForType(FieldDescriptor::TYPE_GROUP - - // Unknown fields. - // TODO: update this behavior when unknown field's behavior - // changed in open source. Also delete - // Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput - // from failure list of python_cpp python java - TestUnknownMessage(); - TestUnknownOrdering(); - TestOneofMessage(); - - RunJsonTests(); - } - // Flag control performance tests to keep them internal and opt-in only - if (suite_.performance_) { - RunBinaryPerformanceTests(); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunBinaryPerformanceTests() { - TestBinaryPerformanceForAlternatingUnknownFields(); - - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_BOOL); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_DOUBLE); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_FLOAT); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_UINT32); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_UINT64); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_STRING); - TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - FieldDescriptor::TYPE_BYTES); - - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_BOOL); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_DOUBLE); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_FLOAT); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_UINT32); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_UINT64); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_STRING); - TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - FieldDescriptor::TYPE_BYTES); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTests() { - RunValidJsonTest("HelloWorld", REQUIRED, - "{\"optionalString\":\"Hello, World!\"}", - "optional_string: 'Hello, World!'"); - - RunJsonTestsForFieldNameConvention(); - RunJsonTestsForNonRepeatedTypes(); - RunJsonTestsForRepeatedTypes(); - RunJsonTestsForNullTypes(); - - if (run_proto3_tests_) { - RunJsonTestsForWrapperTypes(); - RunJsonTestsForFieldMask(); - RunJsonTestsForStruct(); - RunJsonTestsForValue(); - RunJsonTestsForAny(); - } else { - // Currently Proto2 only, but should also be run on Proto3-optional. - RunJsonTestsForStoresDefaultPrimitive(); - } - - RunJsonTestsForUnknownEnumStringValues(); - - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonNumber", REQUIRED, - R"({"unknown": 1})", ""); - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonString", REQUIRED, - R"({"unknown": "a"})", ""); - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonTrue", REQUIRED, - R"({"unknown": true})", ""); - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonFalse", REQUIRED, - R"({"unknown": false})", ""); - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonNull", REQUIRED, - R"({"unknown": null})", ""); - RunValidJsonIgnoreUnknownTest("IgnoreUnknownJsonObject", REQUIRED, - R"({"unknown": {"a": 1}})", ""); - - ExpectParseFailureForJson("RejectTopLevelNull", REQUIRED, "null"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForStoresDefaultPrimitive() { - RunValidJsonTestWithValidator( - "StoresDefaultPrimitive", REQUIRED, - R"({ - "FieldName13": 0 - })", - [](const Json::Value& value) { return value.isMember("FieldName13"); }); - std::vector<const FieldDescriptor*> extensions; - MessageType::GetDescriptor()->file()->pool()->FindAllExtensions( - MessageType::GetDescriptor(), &extensions); - RunValidJsonTestWithValidator("FieldNameExtension", RECOMMENDED, - absl::Substitute(R"({ - "[$0]": 1 - })", - extensions[0]->full_name()), - [&](const Json::Value& value) { - return value.isMember(absl::StrCat( - "[", extensions[0]->full_name(), "]")); - }); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForReservedFields() { - for (const auto& test_case : std::vector<std::pair<std::string, std::string>>{ - {"Boolean", "true"}, - {"Number", "1"}, - {"String", "\"hello\""}, - {"Message", R"json({ "a": 1 })json"}, - }) { - ExpectParseFailureForJson( - absl::StrCat("RejectReservedFieldName.", test_case.first), REQUIRED, - absl::Substitute(R"json({ - "reserved_field": $0 - })json", - test_case.second)); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForUnknownEnumStringValues() { - // Tests the handling of unknown enum values when encoded as string labels. - // The expected behavior depends on whether unknown fields are ignored: - // * when ignored, the parser should ignore the unknown enum string value. - // * when not ignored, the parser should fail. - struct TestCase { - // Used in the test name. - std::string enum_location; - // JSON input which will contain the unknown field. - std::string input_json; - }; - const std::vector<TestCase> test_cases = { - {"InOptionalField", R"json({ - "optional_nested_enum": "UNKNOWN_ENUM_VALUE" - })json"}, - {"InRepeatedField", R"json({ - "repeated_nested_enum": ["UNKNOWN_ENUM_VALUE"] - })json"}, - {"InMapValue", R"json({ - "map_string_nested_enum": {"key": "UNKNOWN_ENUM_VALUE"} - })json"}, - }; - for (const TestCase& test_case : test_cases) { - // Unknown enum string value is a parse failure when not ignoring unknown - // fields. - ExpectParseFailureForJson( - absl::StrCat("RejectUnknownEnumStringValue", test_case.enum_location), - RECOMMENDED, test_case.input_json); - // Unknown enum string value is ignored when ignoring unknown fields. - RunValidJsonIgnoreUnknownTest( - absl::StrCat("IgnoreUnknownEnumStringValue", test_case.enum_location), - RECOMMENDED, test_case.input_json, ""); - } - - // This test is similar to "InRepeatedField" from above, but it highlights the - // potentially unexpected behavior in an array with mixed known and unknown - // enum string values. - RunValidJsonIgnoreUnknownTest("IgnoreUnknownEnumStringValueInRepeatedPart", - RECOMMENDED, - R"json({ - "repeated_nested_enum": [ - "FOO", - "UNKNOWN_ENUM_VALUE", - "FOO" - ]})json", - R"( - repeated_nested_enum: FOO - repeated_nested_enum: FOO - )"); - - // This test is similar to "InMapValue" from above with mixture of known and - // unknown enum string values in the map. - RunValidJsonIgnoreUnknownTest("IgnoreUnknownEnumStringValueInMapPart", - RECOMMENDED, - R"json({ - "map_string_nested_enum": { - "key1": "FOO", - "key2": "UNKNOWN_ENUM_VALUE" - }})json", - R"( - map_string_nested_enum: { - key: "key1" - value: FOO - } - )"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForFieldNameConvention() { - RunValidJsonTest("FieldNameInSnakeCase", REQUIRED, - R"({ - "fieldname1": 1, - "fieldName2": 2, - "FieldName3": 3, - "fieldName4": 4 - })", - R"( - fieldname1: 1 - field_name2: 2 - _field_name3: 3 - field__name4_: 4 - )"); - RunValidJsonTest("FieldNameWithNumbers", REQUIRED, - R"({ - "field0name5": 5, - "field0Name6": 6 - })", - R"( - field0name5: 5 - field_0_name6: 6 - )"); - RunValidJsonTest("FieldNameWithMixedCases", REQUIRED, - R"({ - "fieldName7": 7, - "FieldName8": 8, - "fieldName9": 9, - "FieldName10": 10, - "FIELDNAME11": 11, - "FIELDName12": 12 - })", - R"( - fieldName7: 7 - FieldName8: 8 - field_Name9: 9 - Field_Name10: 10 - FIELD_NAME11: 11 - FIELD_name12: 12 - )"); - RunValidJsonTest("FieldNameWithDoubleUnderscores", RECOMMENDED, - R"({ - "FieldName13": 13, - "FieldName14": 14, - "fieldName15": 15, - "fieldName16": 16, - "fieldName17": 17, - "FieldName18": 18 - })", - R"( - __field_name13: 13 - __Field_name14: 14 - field__name15: 15 - field__Name16: 16 - field_name17__: 17 - Field_name18__: 18 - )"); - // Using the original proto field name in JSON is also allowed. - RunValidJsonTest("OriginalProtoFieldName", REQUIRED, - R"({ - "fieldname1": 1, - "field_name2": 2, - "_field_name3": 3, - "field__name4_": 4, - "field0name5": 5, - "field_0_name6": 6, - "fieldName7": 7, - "FieldName8": 8, - "field_Name9": 9, - "Field_Name10": 10, - "FIELD_NAME11": 11, - "FIELD_name12": 12, - "__field_name13": 13, - "__Field_name14": 14, - "field__name15": 15, - "field__Name16": 16, - "field_name17__": 17, - "Field_name18__": 18 - })", - R"( - fieldname1: 1 - field_name2: 2 - _field_name3: 3 - field__name4_: 4 - field0name5: 5 - field_0_name6: 6 - fieldName7: 7 - FieldName8: 8 - field_Name9: 9 - Field_Name10: 10 - FIELD_NAME11: 11 - FIELD_name12: 12 - __field_name13: 13 - __Field_name14: 14 - field__name15: 15 - field__Name16: 16 - field_name17__: 17 - Field_name18__: 18 - )"); - // Field names can be escaped. - RunValidJsonTest("FieldNameEscaped", REQUIRED, R"({"fieldn\u0061me1": 1})", - "fieldname1: 1"); - // String ends with escape character. - ExpectParseFailureForJson("StringEndsWithEscapeChar", RECOMMENDED, - "{\"optionalString\": \"abc\\"); - // Field names must be quoted (or it's not valid JSON). - ExpectParseFailureForJson("FieldNameNotQuoted", RECOMMENDED, - "{fieldname1: 1}"); - // Trailing comma is not allowed (not valid JSON). - ExpectParseFailureForJson("TrailingCommaInAnObject", RECOMMENDED, - R"({"fieldname1":1,})"); - ExpectParseFailureForJson("TrailingCommaInAnObjectWithSpace", RECOMMENDED, - R"({"fieldname1":1 ,})"); - ExpectParseFailureForJson("TrailingCommaInAnObjectWithSpaceCommaSpace", - RECOMMENDED, R"({"fieldname1":1 , })"); - ExpectParseFailureForJson("TrailingCommaInAnObjectWithNewlines", RECOMMENDED, - R"({ - "fieldname1":1, - })"); - // JSON doesn't support comments. - ExpectParseFailureForJson("JsonWithComments", RECOMMENDED, - R"({ - // This is a comment. - "fieldname1": 1 - })"); - // JSON spec says whitespace doesn't matter, so try a few spacings to be sure. - RunValidJsonTest("OneLineNoSpaces", RECOMMENDED, - "{\"optionalInt32\":1,\"optionalInt64\":2}", - R"( - optional_int32: 1 - optional_int64: 2 - )"); - RunValidJsonTest("OneLineWithSpaces", RECOMMENDED, - "{ \"optionalInt32\" : 1 , \"optionalInt64\" : 2 }", - R"( - optional_int32: 1 - optional_int64: 2 - )"); - RunValidJsonTest("MultilineNoSpaces", RECOMMENDED, - "{\n\"optionalInt32\"\n:\n1\n,\n\"optionalInt64\"\n:\n2\n}", - R"( - optional_int32: 1 - optional_int64: 2 - )"); - RunValidJsonTest( - "MultilineWithSpaces", RECOMMENDED, - "{\n \"optionalInt32\" : 1\n ,\n \"optionalInt64\" : 2\n}\n", - R"( - optional_int32: 1 - optional_int64: 2 - )"); - // Missing comma between key/value pairs. - ExpectParseFailureForJson("MissingCommaOneLine", RECOMMENDED, - "{ \"optionalInt32\": 1 \"optionalInt64\": 2 }"); - ExpectParseFailureForJson( - "MissingCommaMultiline", RECOMMENDED, - "{\n \"optionalInt32\": 1\n \"optionalInt64\": 2\n}"); - // Duplicated field names have either last-wins or parse failure. - RunValidJsonTestOrParseFailure("FieldNameDuplicate", RECOMMENDED, - R"({ - "optionalNestedMessage": {"a": 1}, - "optionalNestedMessage": {} - })", - "optional_nested_message: {}"); - RunValidJsonTestOrParseFailure("FieldNameDuplicateDifferentCasing1", - RECOMMENDED, - R"({ - "optional_nested_message": {"a": 1}, - "optionalNestedMessage": {} - })", - "optional_nested_message: {}"); - RunValidJsonTestOrParseFailure("FieldNameDuplicateDifferentCasing2", - RECOMMENDED, - R"({ - "optionalNestedMessage": {"a": 1}, - "optional_nested_message": {} - })", - "optional_nested_message: {}"); - // Serializers should use lowerCamelCase by default. - RunValidJsonTestWithValidator("FieldNameInLowerCamelCase", REQUIRED, - R"({ - "fieldname1": 1, - "fieldName2": 2, - "FieldName3": 3, - "fieldName4": 4 - })", - [](const Json::Value& value) { - return value.isMember("fieldname1") && - value.isMember("fieldName2") && - value.isMember("FieldName3") && - value.isMember("fieldName4"); - }); - RunValidJsonTestWithValidator("FieldNameWithNumbersValidator", REQUIRED, - R"({ - "field0name5": 5, - "field0Name6": 6 - })", - [](const Json::Value& value) { - return value.isMember("field0name5") && - value.isMember("field0Name6"); - }); - RunValidJsonTestWithValidator( - "FieldNameWithMixedCasesValidator", REQUIRED, - R"({ - "fieldName7": 7, - "FieldName8": 8, - "fieldName9": 9, - "FieldName10": 10, - "FIELDNAME11": 11, - "FIELDName12": 12 - })", - [](const Json::Value& value) { - return value.isMember("fieldName7") && value.isMember("FieldName8") && - value.isMember("fieldName9") && value.isMember("FieldName10") && - value.isMember("FIELDNAME11") && value.isMember("FIELDName12"); - }); - RunValidJsonTestWithValidator( - "FieldNameWithDoubleUnderscoresValidator", RECOMMENDED, - R"({ - "FieldName13": 13, - "FieldName14": 14, - "fieldName15": 15, - "fieldName16": 16, - "fieldName17": 17, - "FieldName18": 18 - })", - [](const Json::Value& value) { - return value.isMember("FieldName13") && value.isMember("FieldName14") && - value.isMember("fieldName15") && value.isMember("fieldName16") && - value.isMember("fieldName17") && value.isMember("FieldName18"); - }); - - if (run_proto3_tests_) { - RunValidJsonTestWithValidator("SkipsDefaultPrimitive", REQUIRED, - R"({"FieldName13": 0})", - [](const Json::Value& value) { - return !value.isMember("FieldName13"); - }); - } -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForNonRepeatedTypes() { - // Integer fields. - RunValidJsonTest("Int32FieldMaxValue", REQUIRED, - R"({"optionalInt32": 2147483647})", - "optional_int32: 2147483647"); - RunValidJsonTest("Int32FieldMinValue", REQUIRED, - R"({"optionalInt32": -2147483648})", - "optional_int32: -2147483648"); - RunValidJsonTest("Uint32FieldMaxValue", REQUIRED, - R"({"optionalUint32": 4294967295})", - "optional_uint32: 4294967295"); - RunValidJsonTest("Int64FieldMaxValue", REQUIRED, - R"({"optionalInt64": "9223372036854775807"})", - "optional_int64: 9223372036854775807"); - RunValidJsonTest("Int64FieldMinValue", REQUIRED, - R"({"optionalInt64": "-9223372036854775808"})", - "optional_int64: -9223372036854775808"); - RunValidJsonTest("Uint64FieldMaxValue", REQUIRED, - R"({"optionalUint64": "18446744073709551615"})", - "optional_uint64: 18446744073709551615"); - // While not the largest Int64, this is the largest - // Int64 which can be exactly represented within an - // IEEE-754 64-bit float, which is the expected level - // of interoperability guarantee. Larger values may - // work in some implementations, but should not be - // relied upon. - RunValidJsonTest("Int64FieldMaxValueNotQuoted", REQUIRED, - R"({"optionalInt64": 9223372036854774784})", - "optional_int64: 9223372036854774784"); - RunValidJsonTest("Int64FieldMinValueNotQuoted", REQUIRED, - R"({"optionalInt64": -9223372036854775808})", - "optional_int64: -9223372036854775808"); - // Largest interoperable Uint64; see comment above - // for Int64FieldMaxValueNotQuoted. - RunValidJsonTest("Uint64FieldMaxValueNotQuoted", REQUIRED, - R"({"optionalUint64": 18446744073709549568})", - "optional_uint64: 18446744073709549568"); - // Values can be represented as JSON strings. - RunValidJsonTest("Int32FieldStringValue", REQUIRED, - R"({"optionalInt32": "2147483647"})", - "optional_int32: 2147483647"); - RunValidJsonTest("Int32FieldStringValueEscaped", REQUIRED, - R"({"optionalInt32": "2\u003147483647"})", - "optional_int32: 2147483647"); - RunValidJsonTest("Int32FieldStringValueZero", REQUIRED, - R"({"optionalInt32": "0"})", "optional_int32: 0"); - RunValidJsonTest("Int32FieldQuotedExponentialValue", REQUIRED, - R"({"optionalInt32": "1e5"})", "optional_int32: 100000"); - - // Parsers reject out-of-bound integer values. - ExpectParseFailureForJson("Int32FieldTooLarge", REQUIRED, - R"({"optionalInt32": 2147483648})"); - ExpectParseFailureForJson("Int32FieldTooSmall", REQUIRED, - R"({"optionalInt32": -2147483649})"); - ExpectParseFailureForJson("Uint32FieldTooLarge", REQUIRED, - R"({"optionalUint32": 4294967296})"); - ExpectParseFailureForJson("Int64FieldTooLarge", REQUIRED, - R"({"optionalInt64": "9223372036854775808"})"); - ExpectParseFailureForJson("Int64FieldTooSmall", REQUIRED, - R"({"optionalInt64": "-9223372036854775809"})"); - ExpectParseFailureForJson("Uint64FieldTooLarge", REQUIRED, - R"({"optionalUint64": "18446744073709551616"})"); - ExpectParseFailureForJson("Uint64QuotedExponentFieldTooLarge", REQUIRED, - R"({"optionalUint64": "1e536870000"})"); - - // Parser reject non-integer numeric values. - ExpectParseFailureForJson("Int32FieldNotInteger", REQUIRED, - R"({"optionalInt32": 0.5})"); - ExpectParseFailureForJson("Uint32FieldNotInteger", REQUIRED, - R"({"optionalUint32": 0.5})"); - ExpectParseFailureForJson("Int64FieldNotInteger", REQUIRED, - R"({"optionalInt64": "0.5"})"); - ExpectParseFailureForJson("Uint64FieldNotInteger", REQUIRED, - R"({"optionalUint64": "0.5"})"); - - // Parser reject boolean values for integer fields. - ExpectParseFailureForJson("Int32FieldTrueValue", REQUIRED, - R"({"optionalInt32": true})"); - ExpectParseFailureForJson("Int32FieldFalseValue", REQUIRED, - R"({"optionalInt32": false})"); - - // Parser reject non-numeric string values. - ExpectParseFailureForJson("Int32FieldStringValuePartiallyNumeric", REQUIRED, - R"({"optionalInt32": "12abc"})"); - ExpectParseFailureForJson("Int32FieldStringValuePartiallyNumericSpace", - REQUIRED, R"({"optionalInt32": "12 34"})"); - ExpectParseFailureForJson("Int32FieldStringValuePartiallyNumericComma", - REQUIRED, R"({"optionalInt32": "12,34"})"); - // Not a raw string literal: the payload holds the UTF-8 bytes of U+8C37 - // U+6B4C between the digits. - ExpectParseFailureForJson("Int32FieldStringValuePartiallyNumericUnicode", - REQUIRED, - "{\"optionalInt32\": \"12\xE8\xB0\xB7\xE6\xAD\x8C" - "34\"}"); - ExpectParseFailureForJson("Int32FieldStringValueNonNumeric", REQUIRED, - R"({"optionalInt32": "abc"})"); - - // Only ASCII digits 0-9 are digits; other Unicode decimal digits (Nd) are - // rejected. Not raw string literals: the payloads hold UTF-8 bytes of - // Arabic-Indic digits (U+0660-U+0669) and fullwidth digits (U+FF10-U+FF19). - // "١٢٣" - ExpectParseFailureForJson( - "Int32FieldStringValueArabicIndicDigits", REQUIRED, - "{\"optionalInt32\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"); - ExpectParseFailureForJson( - "Int64FieldStringValueFullwidthDigits", REQUIRED, - "{\"optionalInt64\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"); - - // Parser reject empty string values. - ExpectParseFailureForJson("Int32FieldEmptyString", REQUIRED, - R"({"optionalInt32": ""})"); - ExpectParseFailureForJson("Uint32FieldEmptyString", REQUIRED, - R"({"optionalUint32": ""})"); - ExpectParseFailureForJson("Int64FieldEmptyString", REQUIRED, - R"({"optionalInt64": ""})"); - ExpectParseFailureForJson("Uint64FieldEmptyString", REQUIRED, - R"({"optionalUint64": ""})"); - - // Integers but represented as float values are accepted. - RunValidJsonTest("Int32FieldFloatTrailingZero", REQUIRED, - R"({"optionalInt32": 100000.000})", - "optional_int32: 100000"); - RunValidJsonTest("Int32FieldExponentialFormat", REQUIRED, - R"({"optionalInt32": 1e5})", "optional_int32: 100000"); - RunValidJsonTest("Int32FieldMaxFloatValue", REQUIRED, - R"({"optionalInt32": 2.147483647e9})", - "optional_int32: 2147483647"); - RunValidJsonTest("Int32FieldMinFloatValue", REQUIRED, - R"({"optionalInt32": -2.147483648e9})", - "optional_int32: -2147483648"); - RunValidJsonTest("Uint32FieldMaxFloatValue", REQUIRED, - R"({"optionalUint32": 4.294967295e9})", - "optional_uint32: 4294967295"); - - // Parser reject non-numeric values. - ExpectParseFailureForJson("Int32FieldNotNumber", REQUIRED, - R"({"optionalInt32": "3x3"})"); - ExpectParseFailureForJson("Uint32FieldNotNumber", REQUIRED, - R"({"optionalUint32": "3x3"})"); - ExpectParseFailureForJson("Int64FieldNotNumber", REQUIRED, - R"({"optionalInt64": "3x3"})"); - ExpectParseFailureForJson("Uint64FieldNotNumber", REQUIRED, - R"({"optionalUint64": "3x3"})"); - // JSON does not allow "+" on numeric values. - ExpectParseFailureForJson("Int32FieldPlusSign", REQUIRED, - R"({"optionalInt32": +1})"); - // JSON doesn't allow leading 0s. - ExpectParseFailureForJson("Int32FieldLeadingZero", REQUIRED, - R"({"optionalInt32": 01})"); - ExpectParseFailureForJson("Int32FieldNegativeWithLeadingZero", REQUIRED, - R"({"optionalInt32": -01})"); - // String values must follow the same syntax rule. Specifically leading - // or trailing spaces are not allowed. - ExpectParseFailureForJson("Int32FieldLeadingSpace", REQUIRED, - R"({"optionalInt32": " 1"})"); - ExpectParseFailureForJson("Int32FieldTrailingSpace", REQUIRED, - R"({"optionalInt32": "1 "})"); - - // 64-bit values are serialized as strings. - RunValidJsonTestWithValidator( - "Int64FieldBeString", RECOMMENDED, R"({"optionalInt64": 1})", - [](const Json::Value& value) { - return value["optionalInt64"].type() == Json::stringValue && - value["optionalInt64"].asString() == "1"; - }); - RunValidJsonTestWithValidator( - "Uint64FieldBeString", RECOMMENDED, R"({"optionalUint64": 1})", - [](const Json::Value& value) { - return value["optionalUint64"].type() == Json::stringValue && - value["optionalUint64"].asString() == "1"; - }); - - // Bool fields. - RunValidJsonTest("BoolFieldTrue", REQUIRED, R"({"optionalBool":true})", - "optional_bool: true"); - RunValidJsonTest("BoolFieldFalse", REQUIRED, R"({"optionalBool":false})", - "optional_bool: false"); - - // Other forms are not allowed. - ExpectParseFailureForJson("BoolFieldIntegerZero", RECOMMENDED, - R"({"optionalBool":0})"); - ExpectParseFailureForJson("BoolFieldIntegerOne", RECOMMENDED, - R"({"optionalBool":1})"); - ExpectParseFailureForJson("BoolFieldCamelCaseTrue", RECOMMENDED, - R"({"optionalBool":True})"); - ExpectParseFailureForJson("BoolFieldCamelCaseFalse", RECOMMENDED, - R"({"optionalBool":False})"); - ExpectParseFailureForJson("BoolFieldAllCapitalTrue", RECOMMENDED, - R"({"optionalBool":TRUE})"); - ExpectParseFailureForJson("BoolFieldAllCapitalFalse", RECOMMENDED, - R"({"optionalBool":FALSE})"); - ExpectParseFailureForJson("BoolFieldDoubleQuotedTrue", RECOMMENDED, - R"({"optionalBool":"true"})"); - ExpectParseFailureForJson("BoolFieldDoubleQuotedFalse", RECOMMENDED, - R"({"optionalBool":"false"})"); - - // Float fields. - RunValidJsonTest("FloatFieldMinPositiveValue", REQUIRED, - R"({"optionalFloat": 1.175494e-38})", - "optional_float: 1.175494e-38"); - RunValidJsonTest("FloatFieldMaxNegativeValue", REQUIRED, - R"({"optionalFloat": -1.175494e-38})", - "optional_float: -1.175494e-38"); - RunValidJsonTest("FloatFieldMaxPositiveValue", REQUIRED, - R"({"optionalFloat": 3.402823e+38})", - "optional_float: 3.402823e+38"); - RunValidJsonTest("FloatFieldMinNegativeValue", REQUIRED, - R"({"optionalFloat": 3.402823e+38})", - "optional_float: 3.402823e+38"); - // Values can be quoted. - RunValidJsonTest("FloatFieldQuotedValue", REQUIRED, - R"({"optionalFloat": "1"})", "optional_float: 1"); - RunValidJsonTest("FloatFieldQuotedExponentialValue", REQUIRED, - R"({"optionalFloat": "1.175494e-38"})", - "optional_float: 1.175494e-38"); - // Special values. - RunValidJsonTest("FloatFieldNan", REQUIRED, R"({"optionalFloat": "NaN"})", - "optional_float: nan"); - RunValidJsonTest("FloatFieldInfinity", REQUIRED, - R"({"optionalFloat": "Infinity"})", "optional_float: inf"); - RunValidJsonTest("FloatFieldNegativeInfinity", REQUIRED, - R"({"optionalFloat": "-Infinity"})", "optional_float: -inf"); - // Non-canonical Nan will be correctly normalized. - { - MessageType message; - // IEEE floating-point standard 32-bit quiet NaN: - // 0111 1111 1xxx xxxx xxxx xxxx xxxx xxxx - message.set_optional_float(WireFormatLite::DecodeFloat(0x7FA12345)); - RunValidJsonTestWithProtobufInput("FloatFieldNormalizeQuietNan", REQUIRED, - message, "optional_float: nan"); - // IEEE floating-point standard 64-bit signaling NaN: - // 1111 1111 1xxx xxxx xxxx xxxx xxxx xxxx - message.set_optional_float(WireFormatLite::DecodeFloat(0xFFB54321)); - RunValidJsonTestWithProtobufInput("FloatFieldNormalizeSignalingNan", - REQUIRED, message, "optional_float: nan"); - } - - // Special values must be quoted. - ExpectParseFailureForJson("FloatFieldNanNotQuoted", RECOMMENDED, - R"({"optionalFloat": NaN})"); - ExpectParseFailureForJson("FloatFieldInfinityNotQuoted", RECOMMENDED, - R"({"optionalFloat": Infinity})"); - ExpectParseFailureForJson("FloatFieldNegativeInfinityNotQuoted", RECOMMENDED, - R"({"optionalFloat": -Infinity})"); - - // Parsers should reject out-of-bound values. - ExpectParseFailureForJson("FloatFieldTooSmall", REQUIRED, - R"({"optionalFloat": -3.502823e+38})"); - ExpectParseFailureForJson("FloatFieldTooLarge", REQUIRED, - R"({"optionalFloat": 3.502823e+38})"); - - // Parsers should reject empty string values. - ExpectParseFailureForJson("FloatFieldEmptyString", REQUIRED, - R"({"optionalFloat": ""})"); - - // Parser reject non-numeric string values. - ExpectParseFailureForJson("FloatFieldStringValuePartiallyNumeric", REQUIRED, - R"({"optionalFloat": "12abc"})"); - ExpectParseFailureForJson("FloatFieldStringValueNonNumeric", REQUIRED, - R"({"optionalFloat": "abc"})"); - ExpectParseFailureForJson("FloatFieldStringValuePartiallyNumericSpace", - REQUIRED, R"({"optionalFloat": "12 34"})"); - ExpectParseFailureForJson("FloatFieldStringValuePartiallyNumericComma", - REQUIRED, R"({"optionalFloat": "12,34"})"); - // Not a raw string literal: the payload holds the UTF-8 bytes of U+8C37 - // U+6B4C between the digits. - ExpectParseFailureForJson("FloatFieldStringValuePartiallyNumericUnicode", - REQUIRED, - "{\"optionalFloat\": \"12\xE8\xB0\xB7\xE6\xAD\x8C" - "34\"}"); - // Non-ASCII Unicode decimal digits: "١٢٣" (Arabic-Indic) and "１２３" - // (fullwidth). - ExpectParseFailureForJson( - "FloatFieldStringValueArabicIndicDigits", REQUIRED, - "{\"optionalFloat\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"); - ExpectParseFailureForJson( - "FloatFieldStringValueFullwidthDigits", REQUIRED, - "{\"optionalFloat\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"); - - // Parser reject boolean values for float fields. - ExpectParseFailureForJson("FloatFieldTrueValue", REQUIRED, - R"({"optionalFloat": true})"); - ExpectParseFailureForJson("FloatFieldFalseValue", REQUIRED, - R"({"optionalFloat": false})"); - - // Double fields. - RunValidJsonTest("DoubleFieldMinPositiveValue", REQUIRED, - R"({"optionalDouble": 2.22507e-308})", - "optional_double: 2.22507e-308"); - RunValidJsonTest("DoubleFieldMaxNegativeValue", REQUIRED, - R"({"optionalDouble": -2.22507e-308})", - "optional_double: -2.22507e-308"); - RunValidJsonTest("DoubleFieldMaxPositiveValue", REQUIRED, - R"({"optionalDouble": 1.79769e+308})", - "optional_double: 1.79769e+308"); - RunValidJsonTest("DoubleFieldMinNegativeValue", REQUIRED, - R"({"optionalDouble": -1.79769e+308})", - "optional_double: -1.79769e+308"); - // Values can be quoted. - RunValidJsonTest("DoubleFieldQuotedValue", REQUIRED, - R"({"optionalDouble": "1"})", "optional_double: 1"); - RunValidJsonTest("DoubleFieldQuotedExponentialValue", REQUIRED, - R"({"optionalDouble": "2.22507e-308"})", - "optional_double: 2.22507e-308"); - // Special values. - RunValidJsonTest("DoubleFieldNan", REQUIRED, R"({"optionalDouble": "NaN"})", - "optional_double: nan"); - RunValidJsonTest("DoubleFieldInfinity", REQUIRED, - R"({"optionalDouble": "Infinity"})", "optional_double: inf"); - RunValidJsonTest("DoubleFieldNegativeInfinity", REQUIRED, - R"({"optionalDouble": "-Infinity"})", - "optional_double: -inf"); - // Non-canonical Nan will be correctly normalized. - { - MessageType message; - message.set_optional_double( - WireFormatLite::DecodeDouble(int64_t{0x7FFA123456789ABC})); - RunValidJsonTestWithProtobufInput("DoubleFieldNormalizeQuietNan", REQUIRED, - message, "optional_double: nan"); - message.set_optional_double( - WireFormatLite::DecodeDouble(uint64_t{0xFFFBCBA987654321})); - RunValidJsonTestWithProtobufInput("DoubleFieldNormalizeSignalingNan", - REQUIRED, message, - "optional_double: nan"); - } - - // Special values must be quoted. - ExpectParseFailureForJson("DoubleFieldNanNotQuoted", RECOMMENDED, - R"({"optionalDouble": NaN})"); - ExpectParseFailureForJson("DoubleFieldInfinityNotQuoted", RECOMMENDED, - R"({"optionalDouble": Infinity})"); - ExpectParseFailureForJson("DoubleFieldNegativeInfinityNotQuoted", RECOMMENDED, - R"({"optionalDouble": -Infinity})"); - - // Parsers should reject out-of-bound values. - ExpectParseFailureForJson("DoubleFieldTooSmall", REQUIRED, - R"({"optionalDouble": -1.89769e+308})"); - ExpectParseFailureForJson("DoubleFieldTooLarge", REQUIRED, - R"({"optionalDouble": 1.89769e+308})"); - - // Parsers should reject empty string values. - ExpectParseFailureForJson("DoubleFieldEmptyString", REQUIRED, - R"({"optionalDouble": ""})"); - - // Parser reject non-numeric string values. - ExpectParseFailureForJson("DoubleFieldStringValuePartiallyNumeric", REQUIRED, - R"({"optionalDouble": "12abc"})"); - ExpectParseFailureForJson("DoubleFieldStringValueNonNumeric", REQUIRED, - R"({"optionalDouble": "abc"})"); - // Non-ASCII Unicode decimal digits: "١٢٣" (Arabic-Indic) and "１２３" - // (fullwidth). - ExpectParseFailureForJson( - "DoubleFieldStringValueArabicIndicDigits", REQUIRED, - "{\"optionalDouble\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"); - ExpectParseFailureForJson( - "DoubleFieldStringValueFullwidthDigits", REQUIRED, - "{\"optionalDouble\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"); - - // Parser reject boolean values for double fields. - ExpectParseFailureForJson("DoubleFieldTrueValue", REQUIRED, - R"({"optionalDouble": true})"); - ExpectParseFailureForJson("DoubleFieldFalseValue", REQUIRED, - R"({"optionalDouble": false})"); - - // Enum fields. - RunValidJsonTest("EnumField", REQUIRED, R"({"optionalNestedEnum": "FOO"})", - "optional_nested_enum: FOO"); - - // Enum fields with alias - if (run_proto3_tests_) { - RunValidJsonTest("EnumFieldWithAlias", REQUIRED, - R"({"optionalAliasedEnum": "ALIAS_BAZ"})", - "optional_aliased_enum: ALIAS_BAZ"); - RunValidJsonTest("EnumFieldWithAliasUseAlias", REQUIRED, - R"({"optionalAliasedEnum": "MOO"})", - "optional_aliased_enum: ALIAS_BAZ"); - RunValidJsonTest("EnumFieldWithAliasLowerCase", REQUIRED, - R"({"optionalAliasedEnum": "moo"})", - "optional_aliased_enum: ALIAS_BAZ"); - RunValidJsonTest("EnumFieldWithAliasDifferentCase", REQUIRED, - R"({"optionalAliasedEnum": "bAz"})", - "optional_aliased_enum: ALIAS_BAZ"); - } - - // Enum values must be represented as strings. - ExpectParseFailureForJson("EnumFieldNotQuoted", REQUIRED, - R"({"optionalNestedEnum": FOO})"); - // Numeric values are allowed. - RunValidJsonTest("EnumFieldNumericValueZero", REQUIRED, - R"({"optionalNestedEnum": 0})", "optional_nested_enum: FOO"); - RunValidJsonTest("EnumFieldNumericValueNonZero", REQUIRED, - R"({"optionalNestedEnum": 1})", "optional_nested_enum: BAR"); - // Arrays are not allowed for non-repeated enum fields. - ExpectParseFailureForJson("EnumFieldSingleElementArrayEnumName", REQUIRED, - R"({"optionalNestedEnum": ["FOO"]})"); - ExpectParseFailureForJson("EnumFieldSingleElementArrayNumericValue", REQUIRED, - R"({"optionalNestedEnum": [2]})"); - - // Booleans are not allowed for enum fields. - ExpectParseFailureForJson("EnumFieldTrueValue", REQUIRED, - R"({"optionalNestedEnum": true})"); - ExpectParseFailureForJson("EnumFieldFalseValue", REQUIRED, - R"({"optionalNestedEnum": false})"); - - if (run_proto3_tests_) { - // Unknown enum values are represented as numeric values. - RunValidJsonTestWithValidator( - "EnumFieldUnknownValue", REQUIRED, R"({"optionalNestedEnum": 123})", - [](const Json::Value& value) { - return value["optionalNestedEnum"].type() == Json::intValue && - value["optionalNestedEnum"].asInt() == 123; - }); - } - - // String fields. - RunValidJsonTest("StringField", REQUIRED, - R"({"optionalString": "Hello world!"})", - R"(optional_string: "Hello world!")"); - // Non-ASCII characters in the JSON text itself, as opposed to the \uXXXX - // escapes tested below. Note that this is deliberately not a raw string - // literal: the compiler resolves the \x escapes, so the payload holds the - // UTF-8 bytes of U+8C37 U+6B4C ("Google" in Chinese). In the raw string - // literals below, the \u escapes reach the testee's JSON parser verbatim. - RunValidJsonTest("StringFieldUnicode", REQUIRED, - "{\"optionalString\": \"\xE8\xB0\xB7\xE6\xAD\x8C\"}", - R"(optional_string: "\xE8\xB0\xB7\xE6\xAD\x8C")"); - RunValidJsonTest("StringFieldEscape", REQUIRED, - R"({"optionalString": "\"\\\/\b\f\n\r\t"})", - R"(optional_string: "\"\\/\b\f\n\r\t")"); - RunValidJsonTest("StringFieldUnicodeEscape", REQUIRED, - R"({"optionalString": "\u8C37\u6B4C"})", - R"(optional_string: "\xE8\xB0\xB7\xE6\xAD\x8C")"); - RunValidJsonTest("StringFieldSurrogatePair", REQUIRED, - // U+1F601 (grinning face with smiling eyes). - R"({"optionalString": "\uD83D\uDE01"})", - R"(optional_string: "\xF0\x9F\x98\x81")"); - RunValidJsonTest("StringFieldEmbeddedNull", REQUIRED, - R"({"optionalString": "Hello\u0000world!"})", - R"(optional_string: "Hello\000world!")"); - - // Unicode escapes must start with "\u" (lowercase u). - ExpectParseFailureForJson("StringFieldUppercaseEscapeLetter", RECOMMENDED, - R"({"optionalString": "\U8C37\U6b4C"})"); - ExpectParseFailureForJson("StringFieldInvalidEscape", RECOMMENDED, - R"({"optionalString": "\uXXXX\u6B4C"})"); - ExpectParseFailureForJson("StringFieldUnterminatedEscape", RECOMMENDED, - R"({"optionalString": "\u8C3"})"); - ExpectParseFailureForJson("StringFieldUnpairedHighSurrogate", RECOMMENDED, - R"({"optionalString": "\uD800"})"); - ExpectParseFailureForJson("StringFieldUnpairedLowSurrogate", RECOMMENDED, - R"({"optionalString": "\uDC00"})"); - ExpectParseFailureForJson("StringFieldSurrogateInWrongOrder", RECOMMENDED, - R"({"optionalString": "\uDE01\uD83D"})"); - ExpectParseFailureForJson("StringFieldNotAString", REQUIRED, - R"({"optionalString": 12345})"); - - // Bytes fields. - RunValidJsonTest("BytesField", REQUIRED, R"({"optionalBytes": "AQI="})", - R"(optional_bytes: "\x01\x02")"); - RunValidJsonTest("BytesFieldBase64Url", RECOMMENDED, - R"({"optionalBytes": "-_"})", R"(optional_bytes: "\xfb")"); - - // Message fields. - RunValidJsonTest("MessageField", REQUIRED, - R"({"optionalNestedMessage": {"a": 1234}})", - "optional_nested_message: {a: 1234}"); - - // Oneof fields. - RunValidJsonTestOrParseFailure("OneofFieldDuplicate", REQUIRED, - R"({"oneofUint32": 1, "oneofString": "test"})", - "oneof_string: \"test\""); - RunValidJsonTestOrParseFailure("OneofFieldDuplicate2", REQUIRED, - R"({"oneofString": "test", "oneofUint32": 1})", - "oneof_uint32: 1"); - RunValidJsonTest("OneofFieldNullFirst", REQUIRED, - R"({"oneofUint32": null, "oneofString": "test"})", - "oneof_string: \"test\""); - RunValidJsonTest("OneofFieldNullSecond", REQUIRED, - R"({"oneofString": "test", "oneofUint32": null})", - "oneof_string: \"test\""); - RunValidJsonTest("OneofZeroUint32", RECOMMENDED, R"({"oneofUint32": 0})", - "oneof_uint32: 0"); - RunValidJsonTest("OneofZeroMessage", RECOMMENDED, - R"({"oneofNestedMessage": {}})", "oneof_nested_message: {}"); - RunValidJsonTest("OneofZeroString", RECOMMENDED, R"({"oneofString": ""})", - "oneof_string: \"\""); - RunValidJsonTest("OneofZeroBytes", RECOMMENDED, R"({"oneofBytes": ""})", - "oneof_bytes: \"\""); - RunValidJsonTest("OneofZeroBool", RECOMMENDED, R"({"oneofBool": false})", - "oneof_bool: false"); - RunValidJsonTest("OneofZeroUint64", RECOMMENDED, R"({"oneofUint64": 0})", - "oneof_uint64: 0"); - RunValidJsonTest("OneofZeroFloat", RECOMMENDED, R"({"oneofFloat": 0.0})", - "oneof_float: 0"); - RunValidJsonTest("OneofZeroDouble", RECOMMENDED, R"({"oneofDouble": 0.0})", - "oneof_double: 0"); - RunValidJsonTest("OneofZeroEnum", RECOMMENDED, R"({"oneofEnum":"FOO"})", - "oneof_enum: FOO"); - - // Map fields. - RunValidJsonTest("Int32MapField", REQUIRED, - R"({"mapInt32Int32": {"1": 2, "3": 4}})", - "map_int32_int32: {key: 1 value: 2}" - "map_int32_int32: {key: 3 value: 4}"); - ExpectParseFailureForJson("Int32MapFieldKeyNotQuoted", RECOMMENDED, - R"({"mapInt32Int32": {1: 2, 3: 4}})"); - RunValidJsonTest("Uint32MapField", REQUIRED, - R"({"mapUint32Uint32": {"1": 2, "3": 4}})", - "map_uint32_uint32: {key: 1 value: 2}" - "map_uint32_uint32: {key: 3 value: 4}"); - ExpectParseFailureForJson("Uint32MapFieldKeyNotQuoted", RECOMMENDED, - R"({"mapUint32Uint32": {1: 2, 3: 4}})"); - RunValidJsonTest("Int64MapField", REQUIRED, - R"({"mapInt64Int64": {"1": 2, "3": 4}})", - "map_int64_int64: {key: 1 value: 2}" - "map_int64_int64: {key: 3 value: 4}"); - ExpectParseFailureForJson("Int64MapFieldKeyNotQuoted", RECOMMENDED, - R"({"mapInt64Int64": {1: 2, 3: 4}})"); - RunValidJsonTest("Uint64MapField", REQUIRED, - R"({"mapUint64Uint64": {"1": 2, "3": 4}})", - "map_uint64_uint64: {key: 1 value: 2}" - "map_uint64_uint64: {key: 3 value: 4}"); - ExpectParseFailureForJson("Uint64MapFieldKeyNotQuoted", RECOMMENDED, - R"({"mapUint64Uint64": {1: 2, 3: 4}})"); - RunValidJsonTest("BoolMapField", REQUIRED, - R"({"mapBoolBool": {"true": true, "false": false}})", - "map_bool_bool: {key: true value: true}" - "map_bool_bool: {key: false value: false}"); - ExpectParseFailureForJson("BoolMapFieldKeyNotQuoted", RECOMMENDED, - R"({"mapBoolBool": {true: true, false: false}})"); - RunValidJsonTest("MessageMapField", REQUIRED, - R"({ - "mapStringNestedMessage": { - "hello": {"a": 1234}, - "world": {"a": 5678} - } - })", - R"( - map_string_nested_message: { - key: "hello" - value: {a: 1234} - } - map_string_nested_message: { - key: "world" - value: {a: 5678} - } - )"); - // Since Map keys are represented as JSON strings, escaping should be allowed. - RunValidJsonTest("Int32MapEscapedKey", REQUIRED, - R"({"mapInt32Int32": {"\u0031": 2}})", - "map_int32_int32: {key: 1 value: 2}"); - RunValidJsonTest("Int64MapEscapedKey", REQUIRED, - R"({"mapInt64Int64": {"\u0031": 2}})", - "map_int64_int64: {key: 1 value: 2}"); - RunValidJsonTest("BoolMapEscapedKey", REQUIRED, - R"({"mapBoolBool": {"tr\u0075e": true}})", - "map_bool_bool: {key: true value: true}"); - - // http://www.rfc-editor.org/rfc/rfc7159.txt says strings have to use double - // quotes. - ExpectParseFailureForJson("StringFieldSingleQuoteKey", RECOMMENDED, - R"({'optionalString': "Hello world!"})"); - ExpectParseFailureForJson("StringFieldSingleQuoteValue", RECOMMENDED, - R"({"optionalString": 'Hello world!'})"); - ExpectParseFailureForJson("StringFieldSingleQuoteBoth", RECOMMENDED, - R"({'optionalString': 'Hello world!'})"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForRepeatedTypes() { - // Repeated fields. - RunValidJsonTest("PrimitiveRepeatedField", REQUIRED, - R"({"repeatedInt32": [1, 2, 3, 4]})", - "repeated_int32: [1, 2, 3, 4]"); - RunValidJsonTest("EnumRepeatedField", REQUIRED, - R"({"repeatedNestedEnum": ["FOO", "BAR", "BAZ"]})", - "repeated_nested_enum: [FOO, BAR, BAZ]"); - RunValidJsonTest("StringRepeatedField", REQUIRED, - R"({"repeatedString": ["Hello", "world"]})", - R"(repeated_string: ["Hello", "world"])"); - RunValidJsonTest("BytesRepeatedField", REQUIRED, - R"({"repeatedBytes": ["AAEC", "AQI="]})", - R"(repeated_bytes: ["\x00\x01\x02", "\x01\x02"])"); - RunValidJsonTest("MessageRepeatedField", REQUIRED, - R"({"repeatedNestedMessage": [{"a": 1234}, {"a": 5678}]})", - "repeated_nested_message: {a: 1234}" - "repeated_nested_message: {a: 5678}"); - - // Repeated field elements are of incorrect type. - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingIntegersGotBool", REQUIRED, - R"({"repeatedInt32": [1, false, 3, 4]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingIntegersGotString", REQUIRED, - R"({"repeatedInt32": [1, 2, "name", 4]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingIntegersGotMessage", REQUIRED, - R"({"repeatedInt32": [1, 2, 3, {"a": 4}]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingStringsGotInt", REQUIRED, - R"({"repeatedString": ["1", 2, "3", "4"]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingStringsGotBool", REQUIRED, - R"({"repeatedString": ["1", "2", false, "4"]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingStringsGotMessage", REQUIRED, - R"({"repeatedString": ["1", 2, "3", {"a": 4}]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingMessagesGotInt", REQUIRED, - R"({"repeatedNestedMessage": [{"a": 1}, 2]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingMessagesGotBool", REQUIRED, - R"({"repeatedNestedMessage": [{"a": 1}, false]})"); - ExpectParseFailureForJson( - "RepeatedFieldWrongElementTypeExpectingMessagesGotString", REQUIRED, - R"({"repeatedNestedMessage": [{"a": 1}, "2"]})"); - - // A singular field where a repeated field was expected is not allowed, even - // if it is the right type. - ExpectParseFailureForJson("SingleValueForRepeatedFieldInt32", REQUIRED, - R"({"repeatedInt32": 1})"); - ExpectParseFailureForJson("SingleValueForRepeatedFieldMessage", REQUIRED, - R"({"repeatedNestedMessage": {"a": 1}})"); - - // Trailing comma in the repeated field is not allowed. - ExpectParseFailureForJson("RepeatedFieldTrailingComma", RECOMMENDED, - R"({"repeatedInt32": [1, 2, 3, 4,]})"); - ExpectParseFailureForJson("RepeatedFieldTrailingCommaWithSpace", RECOMMENDED, - "{\"repeatedInt32\": [1, 2, 3, 4 ,]}"); - ExpectParseFailureForJson("RepeatedFieldTrailingCommaWithSpaceCommaSpace", - RECOMMENDED, - "{\"repeatedInt32\": [1, 2, 3, 4 , ]}"); - ExpectParseFailureForJson( - "RepeatedFieldTrailingCommaWithNewlines", RECOMMENDED, - "{\"repeatedInt32\": [\n 1,\n 2,\n 3,\n 4,\n]}"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForNullTypes() { - // "null" is accepted for all fields types. - RunValidJsonTest("AllFieldAcceptNull", REQUIRED, - R"({ - "optionalInt32": null, - "optionalInt64": null, - "optionalUint32": null, - "optionalUint64": null, - "optionalSint32": null, - "optionalSint64": null, - "optionalFixed32": null, - "optionalFixed64": null, - "optionalSfixed32": null, - "optionalSfixed64": null, - "optionalFloat": null, - "optionalDouble": null, - "optionalBool": null, - "optionalString": null, - "optionalBytes": null, - "optionalNestedEnum": null, - "optionalNestedMessage": null, - "repeatedInt32": null, - "repeatedInt64": null, - "repeatedUint32": null, - "repeatedUint64": null, - "repeatedSint32": null, - "repeatedSint64": null, - "repeatedFixed32": null, - "repeatedFixed64": null, - "repeatedSfixed32": null, - "repeatedSfixed64": null, - "repeatedFloat": null, - "repeatedDouble": null, - "repeatedBool": null, - "repeatedString": null, - "repeatedBytes": null, - "repeatedNestedEnum": null, - "repeatedNestedMessage": null, - "mapInt32Int32": null, - "mapBoolBool": null, - "mapStringNestedMessage": null - })", - ""); - - // Repeated field elements cannot be null. - ExpectParseFailureForJson("RepeatedFieldPrimitiveElementIsNull", RECOMMENDED, - R"({"repeatedInt32": [1, null, 2]})"); - ExpectParseFailureForJson( - "RepeatedFieldMessageElementIsNull", RECOMMENDED, - R"({"repeatedNestedMessage": [{"a":1}, null, {"a":2}]})"); - // Map field keys cannot be null. - ExpectParseFailureForJson("MapFieldKeyIsNull", RECOMMENDED, - R"({"mapInt32Int32": {null: 1}})"); - // Map field values cannot be null. - ExpectParseFailureForJson("MapFieldValueIsNull", RECOMMENDED, - R"({"mapInt32Int32": {"0": null}})"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForWrapperTypes() { - RunValidJsonTest("OptionalBoolWrapper", REQUIRED, - R"({"optionalBoolWrapper": false})", - "optional_bool_wrapper: {value: false}"); - RunValidJsonTest("OptionalInt32Wrapper", REQUIRED, - R"({"optionalInt32Wrapper": 0})", - "optional_int32_wrapper: {value: 0}"); - RunValidJsonTest("OptionalUint32Wrapper", REQUIRED, - R"({"optionalUint32Wrapper": 0})", - "optional_uint32_wrapper: {value: 0}"); - RunValidJsonTest("OptionalInt64Wrapper", REQUIRED, - R"({"optionalInt64Wrapper": 0})", - "optional_int64_wrapper: {value: 0}"); - RunValidJsonTest("OptionalUint64Wrapper", REQUIRED, - R"({"optionalUint64Wrapper": 0})", - "optional_uint64_wrapper: {value: 0}"); - RunValidJsonTest("OptionalFloatWrapper", REQUIRED, - R"({"optionalFloatWrapper": 0})", - "optional_float_wrapper: {value: 0}"); - RunValidJsonTest("OptionalDoubleWrapper", REQUIRED, - R"({"optionalDoubleWrapper": 0})", - "optional_double_wrapper: {value: 0}"); - RunValidJsonTest("OptionalStringWrapper", REQUIRED, - R"({"optionalStringWrapper": ""})", - R"(optional_string_wrapper: {value: ""})"); - RunValidJsonTest("OptionalBytesWrapper", REQUIRED, - R"({"optionalBytesWrapper": ""})", - R"(optional_bytes_wrapper: {value: ""})"); - RunValidJsonTest("OptionalWrapperTypesWithNonDefaultValue", REQUIRED, - R"({ - "optionalBoolWrapper": true, - "optionalInt32Wrapper": 1, - "optionalUint32Wrapper": 1, - "optionalInt64Wrapper": "1", - "optionalUint64Wrapper": "1", - "optionalFloatWrapper": 1, - "optionalDoubleWrapper": 1, - "optionalStringWrapper": "1", - "optionalBytesWrapper": "AQI=" - })", - R"( - optional_bool_wrapper: {value: true} - optional_int32_wrapper: {value: 1} - optional_uint32_wrapper: {value: 1} - optional_int64_wrapper: {value: 1} - optional_uint64_wrapper: {value: 1} - optional_float_wrapper: {value: 1} - optional_double_wrapper: {value: 1} - optional_string_wrapper: {value: "1"} - optional_bytes_wrapper: {value: "\x01\x02"} - )"); - RunValidJsonTest("RepeatedBoolWrapper", REQUIRED, - R"({"repeatedBoolWrapper": [true, false]})", - "repeated_bool_wrapper: {value: true}" - "repeated_bool_wrapper: {value: false}"); - RunValidJsonTest("RepeatedInt32Wrapper", REQUIRED, - R"({"repeatedInt32Wrapper": [0, 1]})", - "repeated_int32_wrapper: {value: 0}" - "repeated_int32_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedUint32Wrapper", REQUIRED, - R"({"repeatedUint32Wrapper": [0, 1]})", - "repeated_uint32_wrapper: {value: 0}" - "repeated_uint32_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedInt64Wrapper", REQUIRED, - R"({"repeatedInt64Wrapper": [0, 1]})", - "repeated_int64_wrapper: {value: 0}" - "repeated_int64_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedUint64Wrapper", REQUIRED, - R"({"repeatedUint64Wrapper": [0, 1]})", - "repeated_uint64_wrapper: {value: 0}" - "repeated_uint64_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedFloatWrapper", REQUIRED, - R"({"repeatedFloatWrapper": [0, 1]})", - "repeated_float_wrapper: {value: 0}" - "repeated_float_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedDoubleWrapper", REQUIRED, - R"({"repeatedDoubleWrapper": [0, 1]})", - "repeated_double_wrapper: {value: 0}" - "repeated_double_wrapper: {value: 1}"); - RunValidJsonTest("RepeatedStringWrapper", REQUIRED, - R"({"repeatedStringWrapper": ["", "AQI="]})", - R"( - repeated_string_wrapper: {value: ""} - repeated_string_wrapper: {value: "AQI="} - )"); - RunValidJsonTest("RepeatedBytesWrapper", REQUIRED, - R"({"repeatedBytesWrapper": ["", "AQI="]})", - R"( - repeated_bytes_wrapper: {value: ""} - repeated_bytes_wrapper: {value: "\x01\x02"} - )"); - RunValidJsonTest("WrapperTypesWithNullValue", REQUIRED, - R"({ - "optionalBoolWrapper": null, - "optionalInt32Wrapper": null, - "optionalUint32Wrapper": null, - "optionalInt64Wrapper": null, - "optionalUint64Wrapper": null, - "optionalFloatWrapper": null, - "optionalDoubleWrapper": null, - "optionalStringWrapper": null, - "optionalBytesWrapper": null, - "repeatedBoolWrapper": null, - "repeatedInt32Wrapper": null, - "repeatedUint32Wrapper": null, - "repeatedInt64Wrapper": null, - "repeatedUint64Wrapper": null, - "repeatedFloatWrapper": null, - "repeatedDoubleWrapper": null, - "repeatedStringWrapper": null, - "repeatedBytesWrapper": null - })", - ""); - - // Duration - RunValidJsonTest( - "DurationMinValue", REQUIRED, - R"({"optionalDuration": "-315576000000.999999999s"})", - "optional_duration: {seconds: -315576000000 nanos: -999999999}"); - RunValidJsonTest( - "DurationMaxValue", REQUIRED, - R"({"optionalDuration": "315576000000.999999999s"})", - "optional_duration: {seconds: 315576000000 nanos: 999999999}"); - RunValidJsonTest("DurationRepeatedValue", REQUIRED, - R"({"repeatedDuration": ["1.5s", "-1.5s"]})", - "repeated_duration: {seconds: 1 nanos: 500000000}" - "repeated_duration: {seconds: -1 nanos: -500000000}"); - RunValidJsonTest("DurationNull", REQUIRED, R"({"optionalDuration": null})", - ""); - RunValidJsonTest("DurationNegativeSeconds", REQUIRED, - R"({"optionalDuration": "-5s"})", - "optional_duration: {seconds: -5 nanos: 0}"); - RunValidJsonTest("DurationNegativeNanos", REQUIRED, - R"({"optionalDuration": "-0.5s"})", - "optional_duration: {seconds: 0 nanos: -500000000}"); - - ExpectParseFailureForJson("DurationMissingS", REQUIRED, - R"({"optionalDuration": "1"})"); - ExpectParseFailureForJson( - "DurationJsonInputTooSmall", REQUIRED, - R"({"optionalDuration": "-315576000001.000000000s"})"); - ExpectParseFailureForJson( - "DurationJsonInputTooLarge", REQUIRED, - R"({"optionalDuration": "315576000001.000000000s"})"); - ExpectSerializeFailureForJson( - "DurationProtoInputTooSmall", REQUIRED, - "optional_duration: {seconds: -315576000001 nanos: 0}"); - ExpectSerializeFailureForJson( - "DurationProtoInputTooLarge", REQUIRED, - "optional_duration: {seconds: 315576000001 nanos: 0}"); - - ExpectSerializeFailureForJson("DurationProtoNanosWrongSign", REQUIRED, - "optional_duration: {seconds: 1 nanos: -1}"); - ExpectSerializeFailureForJson("DurationProtoNanosWrongSignNegativeSecs", - REQUIRED, - "optional_duration: {seconds: -1 nanos: 1}"); - ExpectSerializeFailureForJson( - "DurationProtoNanosTooSmall", REQUIRED, - "optional_duration: {seconds: -1 nanos: -1000000000}"); - ExpectSerializeFailureForJson( - "DurationProtoNanosTooLarge", REQUIRED, - "optional_duration: {seconds: 1 nanos: 1000000000}"); - - RunValidJsonTestWithValidator( - "DurationHasZeroFractionalDigit", RECOMMENDED, - R"({"optionalDuration": "1.000000000s"})", [](const Json::Value& value) { - return value["optionalDuration"].asString() == "1s"; - }); - RunValidJsonTestWithValidator( - "DurationHas3FractionalDigits", RECOMMENDED, - R"({"optionalDuration": "1.010000000s"})", [](const Json::Value& value) { - return value["optionalDuration"].asString() == "1.010s"; - }); - RunValidJsonTestWithValidator( - "DurationHas6FractionalDigits", RECOMMENDED, - R"({"optionalDuration": "1.000010000s"})", [](const Json::Value& value) { - return value["optionalDuration"].asString() == "1.000010s"; - }); - RunValidJsonTestWithValidator( - "DurationHas9FractionalDigits", RECOMMENDED, - R"({"optionalDuration": "1.000000010s"})", [](const Json::Value& value) { - return value["optionalDuration"].asString() == "1.000000010s"; - }); - - // Timestamp - RunValidJsonTest("TimestampMinValue", REQUIRED, - R"({"optionalTimestamp": "0001-01-01T00:00:00Z"})", - "optional_timestamp: {seconds: -62135596800}"); - RunValidJsonTest( - "TimestampMaxValue", REQUIRED, - R"({"optionalTimestamp": "9999-12-31T23:59:59.999999999Z"})", - "optional_timestamp: {seconds: 253402300799 nanos: 999999999}"); - RunValidJsonTest( - "TimestampRepeatedValue", REQUIRED, - R"({ - "repeatedTimestamp": [ - "0001-01-01T00:00:00Z", - "9999-12-31T23:59:59.999999999Z" - ] - })", - "repeated_timestamp: {seconds: -62135596800}" - "repeated_timestamp: {seconds: 253402300799 nanos: 999999999}"); - RunValidJsonTest("TimestampEpochValue", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.000Z"})", - "optional_timestamp: {seconds: 0}"); - RunValidJsonTest("TimestampNanoAfterEpochlValue", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.000000001Z"})", - "optional_timestamp: {seconds: 0 nanos: 1}"); - RunValidJsonTest("TimestampNanoBeforeEpochValue", REQUIRED, - R"({"optionalTimestamp": "1969-12-31T23:59:59.999999999Z"})", - "optional_timestamp: {seconds: -1 nanos: 999999999}"); - RunValidJsonTest("TimestampLittleAfterEpochlValue", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:01.000000001Z"})", - "optional_timestamp: {seconds: 1 nanos: 1}"); - RunValidJsonTest("TimestampLittleBeforeEpochValue", REQUIRED, - R"({"optionalTimestamp": "1969-12-31T23:59:58.999999999Z"})", - "optional_timestamp: {seconds: -2 nanos: 999999999}"); - RunValidJsonTest("TimestampTenAndHalfSecondsAfterEpochValue", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:10.500Z"})", - "optional_timestamp: {seconds: 10 nanos: 500000000}"); - RunValidJsonTest("TimestampTenAndHalfSecondsBeforeEpochValue", REQUIRED, - R"({"optionalTimestamp": "1969-12-31T23:59:49.500Z"})", - "optional_timestamp: {seconds: -11 nanos: 500000000}"); - RunValidJsonTest("TimestampLeap", REQUIRED, - R"({"optionalTimestamp": "1993-02-10T00:00:00.000Z"})", - "optional_timestamp: {seconds: 729302400}"); - RunValidJsonTest("TimestampWithPositiveOffset", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T08:00:01+08:00"})", - "optional_timestamp: {seconds: 1}"); - RunValidJsonTest("TimestampWithNegativeOffset", REQUIRED, - R"({"optionalTimestamp": "1969-12-31T16:00:01-08:00"})", - "optional_timestamp: {seconds: 1}"); - RunValidJsonTest("TimestampNull", REQUIRED, R"({"optionalTimestamp": null})", - ""); - - ExpectParseFailureForJson("TimestampJsonInputTooSmall", REQUIRED, - R"({"optionalTimestamp": "0000-01-01T00:00:00Z"})"); - ExpectParseFailureForJson( - "TimestampJsonInputTooLarge", REQUIRED, - R"({"optionalTimestamp": "10000-01-01T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputMissingZ", REQUIRED, - R"({"optionalTimestamp": "0001-01-01T00:00:00"})"); - ExpectParseFailureForJson("TimestampJsonInputMissingT", REQUIRED, - R"({"optionalTimestamp": "0001-01-01 00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputLowercaseZ", REQUIRED, - R"({"optionalTimestamp": "0001-01-01T00:00:00z"})"); - ExpectParseFailureForJson("TimestampJsonInputLowercaseT", REQUIRED, - R"({"optionalTimestamp": "0001-01-01t00:00:00Z"})"); - ExpectParseFailureForJson( - "TimestampWithMissingColonInOffset", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T08:00:01+0800"})"); - ExpectSerializeFailureForJson("TimestampProtoInputTooSmall", REQUIRED, - "optional_timestamp: {seconds: -62135596801}"); - ExpectSerializeFailureForJson("TimestampProtoInputTooLarge", REQUIRED, - "optional_timestamp: {seconds: 253402300800}"); - ExpectSerializeFailureForJson( - "TimestampProtoNegativeNanos", REQUIRED, - "optional_timestamp: {seconds: 5000 nanos: -1}"); - ExpectSerializeFailureForJson( - "TimestampProtoNanoTooLarge", REQUIRED, - "optional_timestamp: {seconds: 5000 nanos: 1000000000}"); - RunValidJsonTestWithValidator( - "TimestampZeroNormalized", RECOMMENDED, - R"({"optionalTimestamp": "1969-12-31T16:00:00-08:00"})", - [](const Json::Value& value) { - return value["optionalTimestamp"].asString() == "1970-01-01T00:00:00Z"; - }); - RunValidJsonTestWithValidator( - "TimestampHasZeroFractionalDigit", RECOMMENDED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.000000000Z"})", - [](const Json::Value& value) { - return value["optionalTimestamp"].asString() == "1970-01-01T00:00:00Z"; - }); - RunValidJsonTestWithValidator( - "TimestampHas3FractionalDigits", RECOMMENDED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.010000000Z"})", - [](const Json::Value& value) { - return value["optionalTimestamp"].asString() == - "1970-01-01T00:00:00.010Z"; - }); - RunValidJsonTestWithValidator( - "TimestampHas6FractionalDigits", RECOMMENDED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.000010000Z"})", - [](const Json::Value& value) { - return value["optionalTimestamp"].asString() == - "1970-01-01T00:00:00.000010Z"; - }); - RunValidJsonTestWithValidator( - "TimestampHas9FractionalDigits", RECOMMENDED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.000000010Z"})", - [](const Json::Value& value) { - return value["optionalTimestamp"].asString() == - "1970-01-01T00:00:00.000000010Z"; - }); - - // Out of bounds / invalid components should JSON parse fail - ExpectParseFailureForJson("TimestampJsonInputMonthTooLarge", REQUIRED, - R"({"optionalTimestamp": "1970-13-01T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputMonthZero", REQUIRED, - R"({"optionalTimestamp": "1970-00-01T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputDayTooLarge", REQUIRED, - R"({"optionalTimestamp": "1970-01-32T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputDayZero", REQUIRED, - R"({"optionalTimestamp": "1970-01-00T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputHourTooLarge", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T24:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputHourTooLarge25", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T25:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputMinuteTooLarge", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:60:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputSecondTooLarge", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:60Z"})"); - ExpectParseFailureForJson( - "TimestampJsonInputInvalidOffsetHour", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00+24:00"})"); - ExpectParseFailureForJson( - "TimestampJsonInputInvalidOffsetMinute", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00+00:60"})"); - ExpectParseFailureForJson( - "TimestampJsonInputOffsetBoundaryUnderflow", REQUIRED, - R"({"optionalTimestamp": "0001-01-01T00:00:00+00:01"})"); - ExpectParseFailureForJson( - "TimestampJsonInputOffsetBoundaryOverflow", REQUIRED, - R"({"optionalTimestamp": "9999-12-31T23:59:59-00:01"})"); - ExpectParseFailureForJson( - "TimestampJsonInputInvalidNanos", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.1234567890Z"})"); - ExpectParseFailureForJson( - "TimestampJsonInputInvalidCharsInNanos", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00.123aZ"})"); - ExpectParseFailureForJson("TimestampJsonInputYearTooShort", REQUIRED, - R"({"optionalTimestamp": "999-01-01T00:00:00Z"})"); - ExpectParseFailureForJson( - "TimestampJsonInputYearTooLong", REQUIRED, - R"({"optionalTimestamp": "00001-01-01T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputMonthTooShort", REQUIRED, - R"({"optionalTimestamp": "1970-1-01T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputDayTooShort", REQUIRED, - R"({"optionalTimestamp": "1970-01-1T00:00:00Z"})"); - ExpectParseFailureForJson("TimestampJsonInputNonLeapFeb29", REQUIRED, - R"({"optionalTimestamp": "2001-02-29T00:00:00Z"})"); - - // Honoring time zones correctly - RunValidJsonTest("TimestampJsonInputLeapFeb29", REQUIRED, - R"({"optionalTimestamp": "2000-02-29T00:00:00Z"})", - "optional_timestamp: {seconds: 951782400}"); - RunValidJsonTest("TimestampWithOffsetShiftsDay", REQUIRED, - - R"({"optionalTimestamp": "1970-01-02T01:00:00+02:00"})", - "optional_timestamp: {seconds: 82800}"); - RunValidJsonTest("TimestampWithOffsetBoundaryInBoundsMin", REQUIRED, - R"({"optionalTimestamp": "0001-01-01T00:00:00-00:01"})", - "optional_timestamp: {seconds: -62135596740}"); - RunValidJsonTest("TimestampWithOffsetBoundaryInBoundsMax", REQUIRED, - R"({"optionalTimestamp": "9999-12-31T23:59:59+00:01"})", - "optional_timestamp: {seconds: 253402300739}"); - RunValidJsonTest("TimestampWithComplexOffset", REQUIRED, - R"({"optionalTimestamp": "1970-01-01T00:00:00-11:30"})", - "optional_timestamp: {seconds: 41400}"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl< - MessageType>::RunJsonTestsForFieldMask() { - RunValidJsonTest("FieldMask", REQUIRED, - R"({"optionalFieldMask": "foo,barBaz"})", - R"(optional_field_mask: {paths: "foo" paths: "bar_baz"})"); - RunValidJsonTest("EmptyFieldMask", REQUIRED, R"({"optionalFieldMask": ""})", - R"(optional_field_mask: {})"); - ExpectParseFailureForJson("FieldMaskInvalidCharacter", RECOMMENDED, - R"({"optionalFieldMask": "foo,bar_bar"})"); - ExpectSerializeFailureForJson("FieldMaskPathsDontRoundTrip", RECOMMENDED, - R"(optional_field_mask: {paths: "fooBar"})"); - ExpectSerializeFailureForJson("FieldMaskNumbersDontRoundTrip", RECOMMENDED, - R"(optional_field_mask: {paths: "foo_3_bar"})"); - ExpectSerializeFailureForJson("FieldMaskTooManyUnderscore", RECOMMENDED, - R"(optional_field_mask: {paths: "foo__bar"})"); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTestsForStruct() { - RunValidJsonTest("Struct", REQUIRED, - R"({ - "optionalStruct": { - "nullValue": null, - "intValue": 1234, - "boolValue": true, - "doubleValue": 1234.5678, - "stringValue": "Hello world!", - "listValue": [1234, "5678"], - "objectValue": { - "value": 0 - } - } - })", - R"( - optional_struct: { - fields: { - key: "nullValue" - value: {null_value: NULL_VALUE} - } - fields: { - key: "intValue" - value: {number_value: 1234} - } - fields: { - key: "boolValue" - value: {bool_value: true} - } - fields: { - key: "doubleValue" - value: {number_value: 1234.5678} - } - fields: { - key: "stringValue" - value: {string_value: "Hello world!"} - } - fields: { - key: "listValue" - value: { - list_value: { - values: { - number_value: 1234 - } - values: { - string_value: "5678" - } - } - } - } - fields: { - key: "objectValue" - value: { - struct_value: { - fields: { - key: "value" - value: { - number_value: 0 - } - } - } - } - } - } - )"); - RunValidJsonTest("StructWithEmptyListValue", REQUIRED, - R"({ - "optionalStruct": { - "listValue": [] - } - })", - R"( - optional_struct: { - fields: { - key: "listValue" - value: { - list_value: { - } - } - } - } - )"); - - RunValidJsonTest( - "StructDeepNesting25", RECOMMENDED, - absl::StrCat(R"({"optionalStruct": {)", str_repeat(R"("n": {)", 25), - R"("value": 1)", std::string(25, '}'), "}}"), - absl::StrCat("optional_struct: {\n", - str_repeat(" fields: {\n" - " key: \"n\"\n" - " value: {\n" - " struct_value: {\n", - 25), - " fields: {\n" - " key: \"value\"\n" - " value: { number_value: 1 }\n" - " }\n", - str_repeat(" }\n" - " }\n" - " }\n", - 25), - "}\n")); - - ExpectParseFailureForJson( - "StructDeepNesting200", RECOMMENDED, - absl::StrCat(R"({"optionalStruct": {)", str_repeat(R"("n": {)", 200), - R"("value": 1)", std::string(200, '}'), "}}")); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTestsForValue() { - RunValidJsonTest("ValueAcceptInteger", REQUIRED, R"({"optionalValue": 1})", - "optional_value: { number_value: 1}"); - RunValidJsonTest("ValueAcceptFloat", REQUIRED, R"({"optionalValue": 1.5})", - "optional_value: { number_value: 1.5}"); - RunValidJsonTest("ValueAcceptBool", REQUIRED, R"({"optionalValue": false})", - "optional_value: { bool_value: false}"); - RunValidJsonTest("ValueAcceptNull", REQUIRED, R"({"optionalValue": null})", - "optional_value: { null_value: NULL_VALUE}"); - RunValidJsonTest("ValueAcceptString", REQUIRED, - R"({"optionalValue": "hello"})", - R"(optional_value: { string_value: "hello"})"); - RunValidJsonTest("ValueAcceptList", REQUIRED, - R"({"optionalValue": [0, "hello"]})", - R"( - optional_value: { - list_value: { - values: { - number_value: 0 - } - values: { - string_value: "hello" - } - } - } - )"); - RunValidJsonTest("ValueAcceptObject", REQUIRED, - R"({"optionalValue": {"value": 1}})", - R"( - optional_value: { - struct_value: { - fields: { - key: "value" - value: { - number_value: 1 - } - } - } - } - )"); - RunValidJsonTest("RepeatedValue", REQUIRED, - R"({ - "repeatedValue": [["a"]] - })", - R"( - repeated_value: [ - { - list_value: { - values: [ - { string_value: "a"} - ] - } - } - ] - )"); - RunValidJsonTest("RepeatedListValue", REQUIRED, - R"({ - "repeatedListValue": [["a"]] - })", - R"( - repeated_list_value: [ - { - values: [ - { string_value: "a"} - ] - } - ] - )"); - RunValidJsonTestWithValidator("NullValueInOtherOneofOldFormat", RECOMMENDED, - R"({"oneofNullValue": "NULL_VALUE"})", - [](const Json::Value& value) { - return (value.isMember("oneofNullValue") && - value["oneofNullValue"].isNull()); - }); - RunValidJsonTestWithValidator("NullValueInOtherOneofNewFormat", RECOMMENDED, - R"({"oneofNullValue": null})", - [](const Json::Value& value) { - return (value.isMember("oneofNullValue") && - value["oneofNullValue"].isNull()); - }); - RunValidJsonTestWithValidator( - "NullValueInNormalMessage", RECOMMENDED, R"({"optionalNullValue": null})", - [](const Json::Value& value) { return value.empty(); }); - ExpectSerializeFailureForJson("ValueRejectNanNumberValue", RECOMMENDED, - "optional_value: { number_value: nan}"); - ExpectSerializeFailureForJson("ValueRejectInfNumberValue", RECOMMENDED, - "optional_value: { number_value: inf}"); - RunValidJsonTest("ListValueDeepNesting25", RECOMMENDED, - absl::StrCat(R"({"optionalValue": )", std::string(25, '['), - "1", std::string(25, ']'), "}"), - absl::StrCat("optional_value: {\n", - str_repeat(" list_value: {\n" - " values: {\n", - 25), - " number_value: 1\n", - str_repeat(" }\n" - " }\n", - 25), - "}\n")); - - ExpectParseFailureForJson( - "ListValueDeepNesting200", RECOMMENDED, - absl::StrCat(R"({"optionalValue": )", std::string(200, '['), "1", - std::string(200, ']'), "}")); - RunValidJsonTest( - "ValueDeepNesting25", RECOMMENDED, - absl::StrCat(R"({"optionalValue": {)", str_repeat(R"("n": {)", 25), - R"("value": 1)", std::string(25, '}'), "}}"), - absl::StrCat("optional_value: {\n", " struct_value: {\n", - str_repeat(" fields: {\n" - " key: \"n\"\n" - " value: {\n" - " struct_value: {\n", - 25), - " fields: {\n" - " key: \"value\"\n" - " value: { number_value: 1 }\n" - " }\n", - str_repeat(" }\n" - " }\n" - " }\n", - 25), - " }\n", "}\n")); - - ExpectParseFailureForJson( - "ValueDeepNesting200", RECOMMENDED, - absl::StrCat(R"({"optionalValue": {)", str_repeat(R"("n": {)", 200), - R"("value": 1)", std::string(200, '}'), "}}")); -} - -template <typename MessageType> -void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTestsForAny() { - std::string type_url = GetTypeUrl(MessageType::GetDescriptor()); - RunValidJsonTest("Any", REQUIRED, - absl::Substitute(R"({ - "optionalAny": { - "@type": "$0", - "optionalInt32": 12345 - } - })", - GetTypeUrl(MessageType::GetDescriptor())), - absl::Substitute(R"( - optional_any: { - [$0] { - optional_int32: 12345 - } - } - )", - type_url)); - RunValidJsonTest("AnyNested", REQUIRED, - absl::Substitute(R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Any", - "value": { - "@type": "$0", - "optionalInt32": 12345 - } - } - })", - type_url), - absl::Substitute(R"( - optional_any: { - [type.googleapis.com/google.protobuf.Any] { - [$0] { - optional_int32: 12345 - } - } - } - )", - type_url)); - // The special "@type" tag is not required to appear first. - RunValidJsonTest("AnyUnorderedTypeTag", REQUIRED, - absl::Substitute(R"({ - "optionalAny": { - "optionalInt32": 12345, - "@type": "$0" - } - })", - type_url), - absl::Substitute(R"( - optional_any: { - [$0] { - optional_int32: 12345 - } - } - )", - type_url)); - // Well-known types in Any. - RunValidJsonTest("AnyWithInt32ValueWrapper", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Int32Value", - "value": 12345 - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Int32Value] { - value: 12345 - } - } - )"); - RunValidJsonTest("AnyWithDuration", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Duration", - "value": "1.5s" - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Duration] { - seconds: 1 - nanos: 500000000 - } - } - )"); - RunValidJsonTest("AnyWithTimestamp", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Timestamp", - "value": "1970-01-01T00:00:00Z" - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Timestamp] { - seconds: 0 - nanos: 0 - } - } - )"); - RunValidJsonTest("AnyWithFieldMask", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.FieldMask", - "value": "foo,barBaz" - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.FieldMask] { - paths: ["foo", "bar_baz"] - } - } - )"); - RunValidJsonTest("AnyWithStruct", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Struct", - "value": { - "foo": 1 - } - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Struct] { - fields: { - key: "foo" - value: { - number_value: 1 - } - } - } - } - )"); - RunValidJsonTest("AnyWithValueForJsonObject", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Value", - "value": { - "foo": 1 - } - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Value] { - struct_value: { - fields: { - key: "foo" - value: { - number_value: 1 - } - } - } - } - } - )"); - RunValidJsonTest("AnyWithValueForInteger", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Value", - "value": 1 - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Value] { - number_value: 1 - } - } - )"); - // When the Any is in WKT form (with "@type"), the type_url must be present - // and URL shaped, otherwise it should be a parse error (because it can't be - // parsed into the Any schema). - ExpectParseFailureForJson("AnyWktRepresentationWithEmptyTypeAndValue", - REQUIRED, - R"({ - "optionalAny": { - "@type": "", - "value": "" - } - })"); - ExpectParseFailureForJson("AnyWktRepresentationWithBadType", REQUIRED, - R"({ - "optionalAny": { - "@type": "not_a_url", - "value": "" - } - })"); - // When the Any can be parsed as non-WKT form, the type_url could be missing - // or invalid, since that can still be parsed into the Any schema. - RunValidJsonTest("AnyWithNoType", REQUIRED, - R"({ - "optionalAny": {} - })", - R"( - optional_any: {} - )"); - // `null` where an Any exists should just result in the field being unset. - RunValidJsonTest("AnyNull", REQUIRED, - R"({ - "optionalAny": null - })", - R"( - )"); - - // google.protobuf.Empty packed into an Any, implementations must accept it - // without the "value" field set. This also confirms that what they round trip - // does not have `"value":{}` set on it, since the test harness uses the C++ - // JSON parser which will reject it. - RunValidJsonTest("AnyEmpty", REQUIRED, - R"({ - "optionalAny": { - "@type": "type.googleapis.com/google.protobuf.Empty" - } - })", - R"( - optional_any: { - [type.googleapis.com/google.protobuf.Empty] {} - } - )"); -} - -template <typename MessageType> -const FieldDescriptor* -BinaryAndJsonConformanceSuiteImpl<MessageType>::GetFieldForType( - FieldDescriptor::Type type, bool repeated, Packed packed) const { - const Descriptor* d = MessageType::GetDescriptor(); - for (int i = 0; i < d->field_count(); i++) { - const FieldDescriptor* f = d->field(i); - if (f->type() == type && f->is_repeated() == repeated) { - if ((packed == Packed::kTrue && !f->is_packed()) || - (packed == Packed::kFalse && f->is_packed())) { - continue; - } - return f; - } - } - - absl::string_view packed_string = ""; - const absl::string_view repeated_string = - repeated ? "Repeated " : "Singular "; - if (packed == Packed::kTrue) { - packed_string = "Packed "; - } - if (packed == Packed::kFalse) { - packed_string = "Unpacked "; - } - ABSL_LOG(FATAL) << "Couldn't find field with type: " << repeated_string - << packed_string << FieldDescriptor::TypeName(type) << " for " - << d->full_name(); - return nullptr; -} - -template <typename MessageType> -const FieldDescriptor* -BinaryAndJsonConformanceSuiteImpl<MessageType>::GetFieldForMapType( - FieldDescriptor::Type key_type, FieldDescriptor::Type value_type) const { - const Descriptor* d = MessageType::GetDescriptor(); - for (int i = 0; i < d->field_count(); i++) { - const FieldDescriptor* f = d->field(i); - if (f->is_map()) { - const Descriptor* map_entry = f->message_type(); - const FieldDescriptor* key = map_entry->field(0); - const FieldDescriptor* value = map_entry->field(1); - if (key->type() == key_type && value->type() == value_type) { - return f; - } - } - } - - ABSL_LOG(FATAL) << "Couldn't find map field with type: " - << FieldDescriptor::TypeName(key_type) << " and " - << FieldDescriptor::TypeName(key_type) << " for " - << d->full_name(); - return nullptr; -} - -template <typename MessageType> -const FieldDescriptor* -BinaryAndJsonConformanceSuiteImpl<MessageType>::GetFieldForOneofType( - FieldDescriptor::Type type, bool exclusive) const { - const Descriptor* d = MessageType::GetDescriptor(); - for (int i = 0; i < d->field_count(); i++) { - const FieldDescriptor* f = d->field(i); - if (f->containing_oneof() && ((f->type() == type) ^ exclusive)) { - return f; - } - } - - ABSL_LOG(FATAL) << "Couldn't find oneof field with type: " - << FieldDescriptor::TypeName(type) << " for " - << d->full_name(); - return nullptr; -} - -template <typename MessageType> -std::string BinaryAndJsonConformanceSuiteImpl<MessageType>::SyntaxIdentifier() - const { - if (std::is_same_v<MessageType, TestAllTypesProto2>) { - return "Proto2"; - } else if (std::is_same_v<MessageType, TestAllTypesProto3>) { - return "Proto3"; - } else if (std::is_same_v<MessageType, TestAllTypesProto2Editions>) { - return "Editions_Proto2"; - } else { - return "Editions_Proto3"; - } -} - -} // namespace conformance -} // namespace protobuf -} // namespace google
diff --git a/conformance/binary_json_conformance_suite.h b/conformance/binary_json_conformance_suite.h deleted file mode 100644 index 545b23b..0000000 --- a/conformance/binary_json_conformance_suite.h +++ /dev/null
@@ -1,241 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#ifndef GOOGLE_PROTOBUF_CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H__ -#define GOOGLE_PROTOBUF_CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H__ - -#include <functional> -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "absl/strings/string_view.h" -#include "json/json.h" -#include "conformance/conformance_test.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/util/type_resolver.h" - -namespace google { -namespace protobuf { -namespace conformance { - -class BinaryAndJsonConformanceSuite : public ConformanceTestSuite { - public: - BinaryAndJsonConformanceSuite() = default; - - private: - void RunSuiteImpl() override; - bool ParseJsonResponse(const ::conformance::ConformanceResponse& response, - Message* test_message); - bool ParseResponse(const ::conformance::ConformanceResponse& response, - const ConformanceRequestSetting& setting, - Message* test_message) override; - void SetTypeUrl(absl::string_view type_url) { - type_url_ = std::string(type_url); - } - - template <typename MessageType> - void RunValidBinaryProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format); - - template <typename MessageType> - void RunValidRoundtripProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf); - - template <typename MessageType> - void RunValidProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format); - - template <typename MessageType> - void ExpectParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - - template <typename MessageType> - void RunValidJsonTest(const std::string& test_name, ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format); - - void RunValidJsonTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format, - const Message& prototype); - - template <typename MessageType> - void ExpectParseFailureForJson(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json); - - typedef std::function<bool(const Json::Value&)> Validator; - template <typename MessageType> - void RunValidJsonTestWithValidator(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const Validator& validator); - - void RunDelimitedFieldTests(); - - void RunUnstableTests(); - - void RunEdition2026Tests(); - - void RunUtf8ValidationTests(); - - void RunMessageSetTests(); - - void RunRecursionLimitTests(); - - template <typename MessageType> - friend class BinaryAndJsonConformanceSuiteImpl; - - std::unique_ptr<google::protobuf::util::TypeResolver> type_resolver_; - std::string type_url_; -}; - -template <typename MessageType> -class BinaryAndJsonConformanceSuiteImpl { - public: - explicit BinaryAndJsonConformanceSuiteImpl( - BinaryAndJsonConformanceSuite* suite, bool run_proto3_tests); - - private: - using ConformanceRequestSetting = - BinaryAndJsonConformanceSuite::ConformanceRequestSetting; - using ConformanceLevel = BinaryAndJsonConformanceSuite::ConformanceLevel; - constexpr static ConformanceLevel RECOMMENDED = ConformanceLevel::RECOMMENDED; - constexpr static ConformanceLevel REQUIRED = ConformanceLevel::REQUIRED; - - void RunAllTests(); - - void RunBinaryPerformanceTests(); - void RunJsonTests(); - void RunJsonTestsForStoresDefaultPrimitive(); - void RunJsonTestsForFieldNameConvention(); - void RunJsonTestsForNonRepeatedTypes(); - void RunJsonTestsForRepeatedTypes(); - void RunJsonTestsForNullTypes(); - void RunJsonTestsForWrapperTypes(); - void RunJsonTestsForFieldMask(); - void RunJsonTestsForStruct(); - void RunJsonTestsForValue(); - void RunJsonTestsForAny(); - void RunJsonTestsForUnknownEnumStringValues(); - void RunJsonTestsForReservedFields(); - void RunValidJsonTest(const std::string& test_name, ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format); - void RunValidJsonTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_forma, - const Message& prototype); - void RunValidJsonTestWithProtobufInput( - const std::string& test_name, ConformanceLevel level, - const MessageType& input, const std::string& equivalent_text_format); - void RunValidJsonIgnoreUnknownTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const std::string& equivalent_text_format); - void RunValidProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf, - const std::string& equivalent_text_format); - void RunValidBinaryProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf); - void RunValidBinaryProtobufTest(const std::string& test_name, - ConformanceLevel level, - const std::string& input_protobuf, - const std::string& expected_protobuf); - void RunBinaryPerformanceMergeMessageWithField( - const std::string& test_name, const std::string& field_proto); - - void RunValidProtobufTestWithMessage( - const std::string& test_name, ConformanceLevel level, - const Message* input, const std::string& equivalent_text_format); - - typedef std::function<bool(const Json::Value&)> Validator; - void RunValidJsonTestWithValidator(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json, - const Validator& validator); - void ExpectParseFailureForJson(const std::string& test_name, - ConformanceLevel level, - const std::string& input_json); - void RunValidJsonTestOrParseFailure( - const std::string& test_name, ConformanceLevel level, - const std::string& input_json, const std::string& equivalent_text_format); - void ExpectSerializeFailureForJson(const std::string& test_name, - ConformanceLevel level, - const std::string& text_format); - void ExpectParseFailureForProtoWithProtoVersion(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - void ExpectParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - void ExpectHardParseFailureForProto(const std::string& proto, - const std::string& test_name, - ConformanceLevel level); - void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); - void TestIllegalTags(); - void TestIllegalLengths(); - void TestUnmatchedGroup(); - void TestUnknownWireType(); - void TestInvalidUtf8String(); - void TestOneofMessage(); - void TestUnknownMessage(); - void TestUnknownOrdering(); - void TestValidDataForType( - google::protobuf::FieldDescriptor::Type, - std::vector<std::pair<std::string, std::string>> values); - void TestValidDataForRepeatedScalarMessage(); - void TestValidDataForMapType(google::protobuf::FieldDescriptor::Type, - google::protobuf::FieldDescriptor::Type); - void TestMapEntryWireTypeMismatch(google::protobuf::FieldDescriptor::Type, - google::protobuf::FieldDescriptor::Type); - void RunMapEntryWireTypeMismatchTest(const std::string& test_name, - const std::string& proto); - void TestValidDataForOneofType(google::protobuf::FieldDescriptor::Type); - void TestMergeOneofMessage(); - void TestOverwriteMessageValueMap(); - void TestBinaryPerformanceForAlternatingUnknownFields(); - void TestBinaryPerformanceMergeMessageWithRepeatedFieldForType( - google::protobuf::FieldDescriptor::Type); - void TestBinaryPerformanceMergeMessageWithUnknownFieldForType( - google::protobuf::FieldDescriptor::Type); - - enum class Packed { - kUnspecified = 0, - kTrue = 1, - kFalse = 2, - }; - const FieldDescriptor* GetFieldForType( - FieldDescriptor::Type type, bool repeated, - Packed packed = Packed::kUnspecified) const; - const FieldDescriptor* GetFieldForMapType( - FieldDescriptor::Type key_type, FieldDescriptor::Type value_type) const; - const FieldDescriptor* GetFieldForOneofType(FieldDescriptor::Type type, - bool exclusive = false) const; - std::string SyntaxIdentifier() const; - - BinaryAndJsonConformanceSuite& suite_; - bool run_proto3_tests_; -}; - -} // namespace conformance -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H__
diff --git a/conformance/binary_merge_test.cc b/conformance/binary_merge_test.cc new file mode 100644 index 0000000..0d8e20c --- /dev/null +++ b/conformance/binary_merge_test.cc
@@ -0,0 +1,232 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking how several occurrences of the same field +// in one payload are combined: repeated submessages are merged, a repeated +// map entry replaces the previous value for its key, and repeated occurrences +// of a oneof message field are merged too. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>:: +// TestValidDataForRepeatedScalarMessage(), +// TestOverwriteMessageValueMap() and TestMergeOneofMessage(); the test names +// and the requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidProtobufTest(), each input is sent twice: once to be +// serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"), and both outputs must be equivalent to the same +// reference message. + +#include <cstdint> +#include <utility> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// In every TestAllTypes message, NestedMessage's fields are a (1) and +// corecursive (2), and the TestAllTypes fields used inside the corecursive +// submessages of the merge-test inputs are optional_int32 (1), +// optional_int64 (2), optional_uint32 (3), repeated_int32 (31) and +// unpacked_int32 (89). +constexpr uint32_t kNestedMessageCorecursiveField = 2; +constexpr uint32_t kOptionalInt32Field = 1; +constexpr uint32_t kOptionalInt64Field = 2; +constexpr uint32_t kOptionalUint32Field = 3; +constexpr uint32_t kRepeatedInt32Field = 31; +constexpr uint32_t kUnpackedInt32Field = 89; + +// A NestedMessage whose corecursive submessage is `corecursive`. +Wire NestedMessageWithCorecursive(const Wire& corecursive) { + return LengthPrefixedField(kNestedMessageCorecursiveField, corecursive); +} + +// The input of a merge test together with the single occurrence of the field +// that the parsed input must be equivalent to (and, for the byte-exact test, +// serialize to). +struct MergeTestData { + Wire input; + Wire expected; +}; + +// RepeatedScalarMessageMerge: `message`'s singular message field +// (optional_nested_message) twice, each holding a `corecursive` submessage +// that sets optional_int32, one of optional_int64 / optional_uint32, and +// repeated_int32. A parser must merge the two submessages rather than +// replace the first with the second. +Wire RepeatedScalarMessageMergeInput(const Descriptor& message) { + const uint32_t field = FieldNumber(*GetFieldForType( + message, FieldDescriptor::TYPE_MESSAGE, /*repeated=*/false)); + return Wire( + LengthPrefixedField(field, NestedMessageWithCorecursive(Wire( + VarintField(kOptionalInt32Field, 1234), + VarintField(kOptionalInt64Field, 1234), + VarintField(kRepeatedInt32Field, 1234)))), + LengthPrefixedField(field, NestedMessageWithCorecursive(Wire( + VarintField(kOptionalInt32Field, 4321), + VarintField(kOptionalUint32Field, 4321), + VarintField(kRepeatedInt32Field, 4321))))); +} + +// ValidDataMap.STRING.MESSAGE.MergeValue: two entries of `message`'s +// map_string_nested_message field with the same (empty) key and different +// message values. A later map entry replaces an earlier one with the same +// key, so `expected` is the second entry alone. +MergeTestData MapMessageValueMergeData(const Descriptor& message) { + // Map entries have key = 1 and value = 2. + constexpr uint32_t kKeyField = 1; + constexpr uint32_t kValueField = 2; + const uint32_t field = FieldNumber(*GetFieldForMapType( + message, FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_MESSAGE)); + Wire key = LengthPrefixedField(kKeyField, ""); + Wire first_entry = LengthPrefixedField( + field, + Wire(key, LengthPrefixedField( + kValueField, NestedMessageWithCorecursive(Wire( + VarintField(kOptionalInt32Field, 1), + VarintField(kRepeatedInt32Field, 1)))))); + Wire second_entry = LengthPrefixedField( + field, + Wire(key, LengthPrefixedField( + kValueField, NestedMessageWithCorecursive(Wire( + VarintField(kOptionalInt64Field, 1), + VarintField(kRepeatedInt32Field, 1)))))); + return {/*input=*/Wire(first_entry, second_entry), + /*expected=*/std::move(second_entry)}; +} + +// ValidDataOneof.MESSAGE.Merge: `message`'s oneof message field +// (oneof_nested_message) twice, each holding a `corecursive` submessage. +// Repeated occurrences of a oneof message field are merged like any other +// message field, so `expected` is a single occurrence holding the merged +// submessage. +MergeTestData OneofMessageMergeData(const Descriptor& message) { + const uint32_t field = FieldNumber( + *GetFieldForOneofType(message, FieldDescriptor::TYPE_MESSAGE)); + Wire first = LengthPrefixedField( + field, + NestedMessageWithCorecursive(Wire(VarintField(kOptionalInt32Field, 1), + VarintField(kOptionalInt64Field, 1), + VarintField(kUnpackedInt32Field, 1)))); + Wire second = LengthPrefixedField( + field, + NestedMessageWithCorecursive(Wire(VarintField(kOptionalInt64Field, 1), + VarintField(kUnpackedInt32Field, 1)))); + // The singular fields take the last value seen and the repeated field + // accumulates both. + Wire merged = LengthPrefixedField( + field, + NestedMessageWithCorecursive(Wire(VarintField(kOptionalInt32Field, 1), + VarintField(kOptionalInt64Field, 1), + VarintField(kUnpackedInt32Field, 1), + VarintField(kUnpackedInt32Field, 1)))); + return {/*input=*/Wire(first, second), /*expected=*/std::move(merged)}; +} + +using MergeTest = MessageTypeConformanceTest; + +// Two occurrences of optional_nested_message, each with a corecursive +// submessage, must be merged field by field. optional_nested_message (18) is +// the first singular message field in every TestAllTypes message, which is +// what the legacy test's GetFieldForType(MESSAGE) lookup resolved to. +TEST_P(MergeTest, RepeatedScalarMessageMerge) { + EXPECT_THAT( + Testee("RepeatedScalarMessageMerge") + .ParseBinary(message(), RepeatedScalarMessageMergeInput(*message())) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_nested_message: { + corecursive: { + optional_int32: 4321 + optional_int64: 1234 + optional_uint32: 4321 + repeated_int32: [ 1234, 4321 ] + } + })pb")))); +} + +TEST_P(MergeTest, RepeatedScalarMessageMergeJson) { + EXPECT_THAT( + Testee("RepeatedScalarMessageMerge") + .ParseBinary(message(), RepeatedScalarMessageMergeInput(*message())) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_nested_message: { + corecursive: { + optional_int32: 4321 + optional_int64: 1234 + optional_uint32: 4321 + repeated_int32: [ 1234, 4321 ] + } + })pb")))); +} + +// Two map_string_nested_message entries with the same key: the second entry's +// value replaces the first one's, it isn't merged into it. The legacy test +// computed its expected text by round-tripping the second entry through the +// generated message class and TextFormat; matching the second entry's bytes +// directly is the same equivalence check without the detour. +TEST_P(MergeTest, MapMessageValue) { + MergeTestData data = MapMessageValueMergeData(*message()); + EXPECT_THAT(Testee("ValidDataMap.STRING.MESSAGE.MergeValue") + .ParseBinary(message(), data.input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(data.expected)))); +} + +TEST_P(MergeTest, MapMessageValueJson) { + MergeTestData data = MapMessageValueMergeData(*message()); + EXPECT_THAT(Testee("ValidDataMap.STRING.MESSAGE.MergeValue") + .ParseBinary(message(), data.input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(data.expected)))); +} + +// Two occurrences of oneof_nested_message are merged like any other message +// field. The legacy test compared the parsed output against the merged +// message (via a TextFormat round trip); see MapMessageValue above. +TEST_P(MergeTest, OneofMessage) { + MergeTestData data = OneofMessageMergeData(*message()); + EXPECT_THAT(Testee("ValidDataOneof.MESSAGE.Merge") + .ParseBinary(message(), data.input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(data.expected)))); +} + +TEST_P(MergeTest, OneofMessageJson) { + MergeTestData data = OneofMessageMergeData(*message()); + EXPECT_THAT(Testee("ValidDataOneof.MESSAGE.Merge") + .ParseBinary(message(), data.input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(data.expected)))); +} + +// The same, but the serialized output must be exactly the merged bytes. The +// legacy suite had no JSON leg for this one. +TEST_P(MergeTest, OneofMessageBinary) { + MergeTestData data = OneofMessageMergeData(*message()); + EXPECT_THAT(Testee(kP1, "ValidDataOneofBinary.MESSAGE.Merge") + .ParseBinary(message(), data.input) + .SerializeBinary(), + Yields(RawPayload(data.expected))); +} + +INSTANTIATE_TEST_SUITE_P(All, MergeTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_message_set_test.cc b/conformance/binary_message_set_test.cc new file mode 100644 index 0000000..117509d --- /dev/null +++ b/conformance/binary_message_set_test.cc
@@ -0,0 +1,204 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests for the MessageSet wire format: well-formed items, +// items in unusual orders, duplicated entries, and the plain-submessage +// encoding of an item. This replaces the legacy suite-level +// BinaryAndJsonConformanceSuite::RunMessageSetTests(); the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// These tests only exist for TestAllTypesProto2, the one test message type +// with a message_set_wire_format field. + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/test_messages_proto2.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::proto2::TestAllTypesProto2; + +// Field 500 of TestAllTypesProto2 is message_set_correct, whose type has +// message_set_wire_format. Such a message is a sequence of items, each a +// group (field 1) holding the extension's field number as its type_id (field +// 2, varint) and the extension message itself (field 3, length-prefixed). +constexpr uint32_t kMessageSetField = 500; +constexpr uint32_t kItemField = 1; +constexpr uint32_t kTypeIdField = 2; +constexpr uint32_t kMessageField = 3; + +// The extension numbers, i.e. type_ids, of the MessageSetCorrect extensions +// declared in test_messages_proto2.proto, and one that isn't declared. +constexpr uint32_t kMessageSetCorrectExtension1TypeId = 1547769; +constexpr uint32_t kMessageSetCorrectExtension2TypeId = 4135312; +constexpr uint32_t kExtensionWithOneofTypeId = 123456789; +constexpr uint32_t kUnknownTypeId = 4135300; + +// Field 9 of MessageSetCorrectExtension2 is i. +constexpr uint32_t kExtension2IField = 9; + +// The expected result of every test whose payload decodes to +// MessageSetCorrectExtension2 { i: 99 }. +constexpr absl::string_view kMessageSetCorrectExtension2Text = + R"pb(message_set_correct: { + [protobuf_test_messages.proto2.TestAllTypesProto2 + .MessageSetCorrectExtension2]: { i: 99 } + })pb"; + +// The message_set_correct field, holding the items `content`. +Wire MessageSet(const Wire& content) { + return LengthPrefixedField(kMessageSetField, content); +} + +// A MessageSet item (group 1) with the entries `content`, in that order. +Wire Item(const Wire& content) { return DelimitedField(kItemField, content); } + +// The type_id entry of a MessageSet item. +Wire TypeId(uint32_t type_id) { return VarintField(kTypeIdField, type_id); } + +// The message entry of a MessageSet item. +Wire ItemMessage(const Wire& message) { + return LengthPrefixedField(kMessageField, message); +} + +// A serialized MessageSetCorrectExtension2 with i set to `i`. +Wire Extension2Message(int i) { return VarintField(kExtension2IField, i); } + +class MessageSetTest : public ConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +TEST_F(MessageSetTest, ValidMessageSetEncoding) { + EXPECT_THAT( + Testee(kP0, "ValidMessageSetEncoding") + .ParseBinary( + TestAllTypesProto2::descriptor(), + MessageSet(Item(Wire(TypeId(kMessageSetCorrectExtension2TypeId), + ItemMessage(Extension2Message(99)))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +TEST_F(MessageSetTest, OutOfOrderGroupsEntries) { + EXPECT_THAT( + Testee(kP0, "ValidMessageSetEncoding.OutOfOrderGroupsEntries") + .ParseBinary(TestAllTypesProto2::descriptor(), + MessageSet(Item( + Wire(ItemMessage(Extension2Message(99)), + TypeId(kMessageSetCorrectExtension2TypeId))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +// Test that an unknown message set extension always goes to unknown fields. +// This is done by poisoning the extension payload with an entry for field 0. +TEST_F(MessageSetTest, UnknownExtension) { + Wire input = MessageSet( + Item(Wire(TypeId(kUnknownTypeId), ItemMessage(VarintField(0, 99))))); + EXPECT_THAT(Testee(kP0, "MessageSetEncoding.UnknownExtension") + .ParseBinary(TestAllTypesProto2::descriptor(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// If an encoder is unaware of the message_set_wire_format option it will be +// encoded like any other extension submessage. Decoders should be able to +// tolerate this format as well. +TEST_F(MessageSetTest, SubmessageEncoding) { + EXPECT_THAT( + Testee("ValidMessageSetEncoding.SubmessageEncoding") + .ParseBinary( + TestAllTypesProto2::descriptor(), + MessageSet(LengthPrefixedField(kMessageSetCorrectExtension2TypeId, + Extension2Message(99)))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +// Test again, but this time we'll try to detect if the implementation put the +// submessage encoded entry into the unknown field set. We'll do this by using +// conflicting oneof entries where order matters when the messages are merged. +// +// In a non-compliant implementation submessage encoded messageset entry will +// be moved to unknown fields and then tacked onto the end of the payload. +// Thus we'll see field b set first, and then field a. +// +// In a compliant implementation we expect the submessage encoded messageset +// to be read first with field a set, and then the normal message set entry +// will be read with field b set, so field b wins. +TEST_F(MessageSetTest, SubmessageEncodingNotUnknown) { + // Fields 1 and 2 of ExtensionWithOneof are a and b. + EXPECT_THAT( + Testee("ValidMessageSetEncoding.SubmessageEncoding.NotUnknown") + .ParseBinary( + TestAllTypesProto2::descriptor(), + MessageSet(Wire(LengthPrefixedField(kExtensionWithOneofTypeId, + VarintField(1, 42)), + Item(Wire(TypeId(kExtensionWithOneofTypeId), + ItemMessage(VarintField(2, 99))))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + R"pb(message_set_correct: { + [protobuf_test_messages.proto2.TestAllTypesProto2 + .ExtensionWithOneof]: { b: 99 } + })pb")))); +} + +// [type_id, value, type_id (different)] -> first type_id and value honored. +TEST_F(MessageSetTest, DuplicateDifferentTypeId) { + EXPECT_THAT( + Testee("ValidMessageSetEncoding.DuplicateDifferentTypeId") + .ParseBinary(TestAllTypesProto2::descriptor(), + MessageSet(Item( + Wire(TypeId(kMessageSetCorrectExtension2TypeId), + ItemMessage(Extension2Message(99)), + TypeId(kMessageSetCorrectExtension1TypeId))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +// [type_id, value, value] -> first value honored, no merge. +TEST_F(MessageSetTest, DuplicateValue) { + EXPECT_THAT( + Testee("ValidMessageSetEncoding.DuplicateValue") + .ParseBinary( + TestAllTypesProto2::descriptor(), + MessageSet(Item(Wire(TypeId(kMessageSetCorrectExtension2TypeId), + ItemMessage(Extension2Message(99)), + ItemMessage(Extension2Message(88)))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +// [value, type_id, value] -> first value honored, no merge. +TEST_F(MessageSetTest, DuplicateValueOutOfOrder) { + EXPECT_THAT( + Testee("ValidMessageSetEncoding.DuplicateValueOutOfOrder") + .ParseBinary( + TestAllTypesProto2::descriptor(), + MessageSet(Item(Wire(ItemMessage(Extension2Message(99)), + TypeId(kMessageSetCorrectExtension2TypeId), + ItemMessage(Extension2Message(88)))))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kMessageSetCorrectExtension2Text)))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_oneof_test.cc b/conformance/binary_oneof_test.cc new file mode 100644 index 0000000..fd4d50b --- /dev/null +++ b/conformance/binary_oneof_test.cc
@@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that a oneof member is parsed correctly +// and that later occurrences of a oneof's members replace earlier ones, for +// every member type in ValidDataOneofTypes(): ValidDataOneof.<TYPE>.<Case> +// (REQUIRED, equivalence, with a binary-output and a JSON-output leg) and +// ValidDataOneofBinary.<TYPE>.<Case> (RECOMMENDED, byte-exact) for the cases +// DefaultValue, NonDefaultValue, MultipleValuesForSameField and +// MultipleValuesForDifferentField. +// +// This holds both legs of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestValidDataForOneofType(); the test +// names and the requests sent to the testee are identical to the legacy ones. +// The oneof merge and zero-value tests live in binary_merge_test.cc and +// binary_oneof_zero_test.cc. + +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// Parameterized over (test message type, member type), exercising the +// message's first oneof member of that type. Reporting the message type +// through MessageUnderTest() makes the base SetUp() skip the editions +// instances when --maximum_edition doesn't cover them. +class ValidDataOneofTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, FieldDescriptor::Type>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + FieldDescriptor::Type type() const { return std::get<1>(GetParam()); } + + // The member holding the default and the non-default sample value of + // type(). These scan the descriptor (and check-fail if there is no such + // member), hence the function-style names. + Wire DefaultMember() const { + return FieldWithValue(Member(), GetDefaultValue(type())); + } + Wire NonDefaultMember() const { + return FieldWithValue(Member(), GetNonDefaultValue(type())); + } + + // Another member of the same oneof, of a different type, holding its + // type's default value. + Wire OtherMemberValue() const { + const FieldDescriptor* other = + GetFieldForOneofType(*message(), type(), OneofMember::kOfOtherType); + return FieldWithValue(other, GetDefaultValue(other->type())); + } + + // The legacy test name: "<prefix>.<TYPE>.<suffix>", e.g. + // "ValidDataOneof.UINT32.DefaultValue". + std::string TestName(absl::string_view prefix, + absl::string_view suffix) const { + return absl::StrCat(prefix, ".", UpperCaseTypeName(type()), ".", suffix); + } + + private: + const FieldDescriptor* Member() const { + return GetFieldForOneofType(*message(), type()); + } + + // `payload` (a value without a tag) as `field`. + static Wire FieldWithValue(const FieldDescriptor* field, + const Wire& payload) { + return Wire(Tag(FieldNumber(*field), WireTypeForFieldType(field->type())), + payload); + } + + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// A member set to its default value has presence and must round trip. +TEST_P(ValidDataOneofTest, DefaultValue) { + EXPECT_THAT(Testee(TestName("ValidDataOneof", "DefaultValue")) + .ParseBinary(message(), DefaultMember()) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(DefaultMember())))); +} + +// ... in JSON as well as in binary. +TEST_P(ValidDataOneofTest, DefaultValueJson) { + EXPECT_THAT(Testee(TestName("ValidDataOneof", "DefaultValue")) + .ParseBinary(message(), DefaultMember()) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(DefaultMember())))); +} + +// ... and its binary serialization must be exactly the input. +TEST_P(ValidDataOneofTest, DefaultValueBinary) { + EXPECT_THAT(Testee(kP1, TestName("ValidDataOneofBinary", "DefaultValue")) + .ParseBinary(message(), DefaultMember()) + .SerializeBinary(), + Yields(RawPayload(DefaultMember()))); +} + +// So does a non-default value. +TEST_P(ValidDataOneofTest, NonDefaultValue) { + EXPECT_THAT(Testee(TestName("ValidDataOneof", "NonDefaultValue")) + .ParseBinary(message(), NonDefaultMember()) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... in JSON as well as in binary. +TEST_P(ValidDataOneofTest, NonDefaultValueJson) { + EXPECT_THAT(Testee(TestName("ValidDataOneof", "NonDefaultValue")) + .ParseBinary(message(), NonDefaultMember()) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... and its binary serialization must be exactly the input. +TEST_P(ValidDataOneofTest, NonDefaultValueBinary) { + EXPECT_THAT(Testee(kP1, TestName("ValidDataOneofBinary", "NonDefaultValue")) + .ParseBinary(message(), NonDefaultMember()) + .SerializeBinary(), + Yields(RawPayload(NonDefaultMember()))); +} + +// The same member twice: the last value wins. +TEST_P(ValidDataOneofTest, MultipleValuesForSameField) { + EXPECT_THAT( + Testee(TestName("ValidDataOneof", "MultipleValuesForSameField")) + .ParseBinary(message(), Wire(DefaultMember(), NonDefaultMember())) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... in JSON as well as in binary. +TEST_P(ValidDataOneofTest, MultipleValuesForSameFieldJson) { + EXPECT_THAT( + Testee(TestName("ValidDataOneof", "MultipleValuesForSameField")) + .ParseBinary(message(), Wire(DefaultMember(), NonDefaultMember())) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... and the binary serialization must be exactly the last value. +TEST_P(ValidDataOneofTest, MultipleValuesForSameFieldBinary) { + EXPECT_THAT( + Testee(kP1, + TestName("ValidDataOneofBinary", "MultipleValuesForSameField")) + .ParseBinary(message(), Wire(DefaultMember(), NonDefaultMember())) + .SerializeBinary(), + Yields(RawPayload(NonDefaultMember()))); +} + +// Two different members of the oneof: the last one set is the only one kept. +TEST_P(ValidDataOneofTest, MultipleValuesForDifferentField) { + EXPECT_THAT( + Testee(TestName("ValidDataOneof", "MultipleValuesForDifferentField")) + .ParseBinary(message(), Wire(OtherMemberValue(), NonDefaultMember())) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... in JSON as well as in binary. +TEST_P(ValidDataOneofTest, MultipleValuesForDifferentFieldJson) { + EXPECT_THAT( + Testee(TestName("ValidDataOneof", "MultipleValuesForDifferentField")) + .ParseBinary(message(), Wire(OtherMemberValue(), NonDefaultMember())) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(NonDefaultMember())))); +} + +// ... and the binary serialization must be exactly the last member. +TEST_P(ValidDataOneofTest, MultipleValuesForDifferentFieldBinary) { + EXPECT_THAT( + Testee(kP1, TestName("ValidDataOneofBinary", + "MultipleValuesForDifferentField")) + .ParseBinary(message(), Wire(OtherMemberValue(), NonDefaultMember())) + .SerializeBinary(), + Yields(RawPayload(NonDefaultMember()))); +} + +INSTANTIATE_TEST_SUITE_P(All, ValidDataOneofTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(ValidDataOneofTypes())), + TupleParamName<ValidDataOneofTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_oneof_zero_test.cc b/conformance/binary_oneof_zero_test.cc new file mode 100644 index 0000000..594261a --- /dev/null +++ b/conformance/binary_oneof_zero_test.cc
@@ -0,0 +1,228 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that a oneof member set to its default +// value (zero, empty, or an empty message) survives a round trip to binary and +// to JSON: unlike a plain proto3 field, a oneof member always has presence. +// This holds both legs of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestOneofMessage(); the test names and +// the requests sent to the testee are identical to the legacy ones. +// +// The legacy test set the members one after another on a single generated +// message and sent its serialization each time. Since all of them belong to +// the same oneof, each step left only the member just set, so every test here +// sends the wire encoding of just that member. The oneof members have the +// same field numbers (111-119) in every TestAllTypes message. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +class OneofZeroTest : public MessageTypeConformanceTest { + protected: + // Whether NestedMessage.a has explicit presence (proto2) rather than + // implicit presence (proto3). + bool NestedMessageFieldHasPresence() const { + return message() + ->FindFieldByName("oneof_nested_message") + ->message_type() + ->FindFieldByName("a") + ->has_presence(); + } + + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Field 111 = oneof_uint32. +TEST_P(OneofZeroTest, Uint32) { + EXPECT_THAT(Testee("OneofZeroUint32") + .ParseBinary(message(), VarintField(111, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_uint32: 0)pb")))); +} + +TEST_P(OneofZeroTest, Uint32Json) { + EXPECT_THAT(Testee("OneofZeroUint32") + .ParseBinary(message(), VarintField(111, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_uint32: 0)pb")))); +} + +// Field 112 = oneof_nested_message, whose field 1 is a. The legacy test set a +// to 0: with explicit presence (proto2) a is serialized and must round trip; +// with implicit presence (proto3) it is indistinguishable from unset, so the +// input is the empty message and only that is expected back. +TEST_P(OneofZeroTest, Message) { + const bool has_presence = NestedMessageFieldHasPresence(); + EXPECT_THAT(Testee("OneofZeroMessage") + .ParseBinary(message(), + has_presence + ? LengthPrefixedField(112, VarintField(1, 0)) + : LengthPrefixedField(112, Wire())) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + has_presence ? R"pb(oneof_nested_message: { a: 0 })pb" + : R"pb(oneof_nested_message: {})pb")))); +} + +TEST_P(OneofZeroTest, MessageJson) { + const bool has_presence = NestedMessageFieldHasPresence(); + EXPECT_THAT(Testee("OneofZeroMessage") + .ParseBinary(message(), + has_presence + ? LengthPrefixedField(112, VarintField(1, 0)) + : LengthPrefixedField(112, Wire())) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto( + has_presence ? R"pb(oneof_nested_message: { a: 0 })pb" + : R"pb(oneof_nested_message: {})pb")))); +} + +// Field 112 = oneof_nested_message. The legacy test set a to 0 and then to 1 +// on the same nested message. +TEST_P(OneofZeroTest, MessageSetTwice) { + EXPECT_THAT( + Testee("OneofZeroMessageSetTwice") + .ParseBinary(message(), LengthPrefixedField(112, VarintField(1, 1))) + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(oneof_nested_message: { a: 1 })pb")))); +} + +TEST_P(OneofZeroTest, MessageSetTwiceJson) { + EXPECT_THAT( + Testee("OneofZeroMessageSetTwice") + .ParseBinary(message(), LengthPrefixedField(112, VarintField(1, 1))) + .SerializeJson(), + Yields(WhenParsed( + EqualsTextProto(R"pb(oneof_nested_message: { a: 1 })pb")))); +} + +// Field 113 = oneof_string. +TEST_P(OneofZeroTest, String) { + EXPECT_THAT(Testee("OneofZeroString") + .ParseBinary(message(), LengthPrefixedField(113, "")) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_string: "")pb")))); +} + +TEST_P(OneofZeroTest, StringJson) { + EXPECT_THAT(Testee("OneofZeroString") + .ParseBinary(message(), LengthPrefixedField(113, "")) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_string: "")pb")))); +} + +// Field 114 = oneof_bytes. +TEST_P(OneofZeroTest, Bytes) { + EXPECT_THAT(Testee("OneofZeroBytes") + .ParseBinary(message(), LengthPrefixedField(114, "")) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_bytes: "")pb")))); +} + +TEST_P(OneofZeroTest, BytesJson) { + EXPECT_THAT(Testee("OneofZeroBytes") + .ParseBinary(message(), LengthPrefixedField(114, "")) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_bytes: "")pb")))); +} + +// Field 115 = oneof_bool. +TEST_P(OneofZeroTest, Bool) { + EXPECT_THAT(Testee("OneofZeroBool") + .ParseBinary(message(), VarintField(115, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_bool: false)pb")))); +} + +TEST_P(OneofZeroTest, BoolJson) { + EXPECT_THAT(Testee("OneofZeroBool") + .ParseBinary(message(), VarintField(115, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_bool: false)pb")))); +} + +// Field 116 = oneof_uint64. +TEST_P(OneofZeroTest, Uint64) { + EXPECT_THAT(Testee("OneofZeroUint64") + .ParseBinary(message(), VarintField(116, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_uint64: 0)pb")))); +} + +TEST_P(OneofZeroTest, Uint64Json) { + EXPECT_THAT(Testee("OneofZeroUint64") + .ParseBinary(message(), VarintField(116, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_uint64: 0)pb")))); +} + +// Field 117 = oneof_float. +TEST_P(OneofZeroTest, Float) { + EXPECT_THAT(Testee("OneofZeroFloat") + .ParseBinary(message(), FloatField(117, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_float: 0)pb")))); +} + +TEST_P(OneofZeroTest, FloatJson) { + EXPECT_THAT(Testee("OneofZeroFloat") + .ParseBinary(message(), FloatField(117, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_float: 0)pb")))); +} + +// Field 118 = oneof_double. +TEST_P(OneofZeroTest, Double) { + EXPECT_THAT(Testee("OneofZeroDouble") + .ParseBinary(message(), DoubleField(118, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_double: 0)pb")))); +} + +TEST_P(OneofZeroTest, DoubleJson) { + EXPECT_THAT(Testee("OneofZeroDouble") + .ParseBinary(message(), DoubleField(118, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_double: 0)pb")))); +} + +// Field 119 = oneof_enum; FOO is the zero value of NestedEnum. +TEST_P(OneofZeroTest, Enum) { + EXPECT_THAT(Testee("OneofZeroEnum") + .ParseBinary(message(), VarintField(119, 0)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_enum: FOO)pb")))); +} + +TEST_P(OneofZeroTest, EnumJson) { + EXPECT_THAT(Testee("OneofZeroEnum") + .ParseBinary(message(), VarintField(119, 0)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(oneof_enum: FOO)pb")))); +} + +INSTANTIATE_TEST_SUITE_P(All, OneofZeroTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_performance_test.cc b/conformance/binary_performance_test.cc new file mode 100644 index 0000000..5fb55b9 --- /dev/null +++ b/conformance/binary_performance_test.cc
@@ -0,0 +1,161 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary performance conformance tests: payloads with tens of thousands of +// unknown fields, or of submessages that have to be merged into one. Part of +// the performance suite (see conformance.bzl). This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunBinaryPerformanceTests() +// and its helpers; the test names and the requests sent to the testee are +// identical to the legacy ones. + +#include <cstddef> +#include <cstdint> +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// The number of repetitions to use for performance tests. +// Corresponds approx to 500KB wireformat bytes. +constexpr int kPerformanceRepeatCount = 50000; + +// Field numbers no test message defines. +constexpr uint32_t kUnknownFieldNumber = 666; +constexpr uint32_t kOtherUnknownFieldNumber = 667; + +// Field 27 of every test message is recursive_message, a submessage of the +// message's own type. +constexpr uint32_t kRecursiveMessageField = 27; + +// The field types the merge tests are instantiated with, in the legacy order. +constexpr FieldDescriptor::Type kMergeTypes[] = { + FieldDescriptor::TYPE_BOOL, FieldDescriptor::TYPE_DOUBLE, + FieldDescriptor::TYPE_FLOAT, FieldDescriptor::TYPE_UINT32, + FieldDescriptor::TYPE_UINT64, FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_BYTES, +}; + +// `wire` repeated `count` times. +Wire Repeat(const Wire& wire, int count) { + std::string repeated; + repeated.reserve(wire.size() * static_cast<size_t>(count)); + for (int i = 0; i < count; ++i) { + absl::StrAppend(&repeated, wire.data()); + } + return Wire(repeated); +} + +// Parameterized over the test message type only. +class UnknownFieldsPerformanceTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Many unknown fields alternating between two field numbers must be preserved +// byte for byte. +TEST_P(UnknownFieldsPerformanceTest, AlternatingUnknownFields) { + Wire input = Repeat(Wire(VarintField(kUnknownFieldNumber, 1234), + VarintField(kOtherUnknownFieldNumber, 5678)), + kPerformanceRepeatCount); + EXPECT_THAT(Testee("TestBinaryPerformanceForAlternatingUnknownFields") + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(RawPayload(input))); +} + +INSTANTIATE_TEST_SUITE_P(All, UnknownFieldsPerformanceTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// Parameterized over (test message type, field type): many occurrences of +// recursive_message, each holding one field, must be merged into a single +// recursive_message holding all of them, in order. +class MergeMessagePerformanceTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, FieldDescriptor::Type>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + FieldDescriptor::Type type() const { return std::get<1>(GetParam()); } + + // The legacy test name: "<prefix>.<TYPE>", e.g. + // "TestBinaryPerformanceMergeMessageWithRepeatedFieldForType.BOOL". + std::string TestName(absl::string_view prefix) const { + return absl::StrCat(prefix, ".", UpperCaseTypeName(type())); + } + + private: + TestPriority DefaultPriority() const override { return kP1; } + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// The input of a merge performance test: recursive_message holding `field` (a +// single encoded field, tag and value), kPerformanceRepeatCount times. +Wire RepeatedRecursiveMessageInput(const Wire& field) { + return Repeat(LengthPrefixedField(kRecursiveMessageField, field), + kPerformanceRepeatCount); +} + +// What a parser must merge RepeatedRecursiveMessageInput(field) into: one +// recursive_message holding kPerformanceRepeatCount occurrences of `field`. +Wire MergedRecursiveMessage(const Wire& field) { + return LengthPrefixedField(kRecursiveMessageField, + Repeat(field, kPerformanceRepeatCount)); +} + +// The field is the message's unpacked repeated field of the type. +TEST_P(MergeMessagePerformanceTest, WithRepeatedField) { + const FieldDescriptor* field = GetFieldForType( + *message(), type(), /*repeated=*/true, Packedness::kUnpacked); + Wire entry(Tag(FieldNumber(*field), WireTypeForFieldType(type())), + GetNonDefaultValue(type())); + EXPECT_THAT( + Testee( + TestName("TestBinaryPerformanceMergeMessageWithRepeatedFieldForType")) + .ParseBinary(message(), RepeatedRecursiveMessageInput(entry)) + .SerializeBinary(), + Yields(RawPayload(MergedRecursiveMessage(entry)))); +} + +// The field is unknown, with the wire type of the type. +TEST_P(MergeMessagePerformanceTest, WithUnknownField) { + Wire entry(Tag(kUnknownFieldNumber, WireTypeForFieldType(type())), + GetNonDefaultValue(type())); + EXPECT_THAT( + Testee( + TestName("TestBinaryPerformanceMergeMessageWithUnknownFieldForType")) + .ParseBinary(message(), RepeatedRecursiveMessageInput(entry)) + .SerializeBinary(), + Yields(RawPayload(MergedRecursiveMessage(entry)))); +} + +INSTANTIATE_TEST_SUITE_P( + All, MergeMessagePerformanceTest, + Combine(ValuesIn(AllTestMessageTypes()), ValuesIn(kMergeTypes)), + TupleParamName<MergeMessagePerformanceTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_premature_eof_test.cc b/conformance/binary_premature_eof_test.cc new file mode 100644 index 0000000..3a927f9 --- /dev/null +++ b/conformance/binary_premature_eof_test.cc
@@ -0,0 +1,249 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that a payload truncated before or inside +// a field's value is rejected. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestPrematureEOFForType(), which +// RunAllTests() called for every field type except groups; the test names and +// the requests sent to the testee are identical to the legacy ones. + +#include <cstdint> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/port.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// A field number no test message defines. +constexpr uint32_t kUnknownFieldNumber = 666; + +// An incomplete value for each wire type, exactly as in the legacy suite. +absl::string_view IncompleteValue(WireType wire_type) { + switch (wire_type) { + case WireType::kVarint: + return "\x80"; + case WireType::kFixed64: + return "abcdefg"; + case WireType::kLengthPrefixed: + return "\x80"; // Partial length. + case WireType::kStartGroup: + case WireType::kEndGroup: + return ""; // No value required. + case WireType::kFixed32: + return "abc"; + case WireType::kInvalid: + break; + } + google::protobuf::internal::Unreachable(); +} + +// Parameterized over (test message type, field type). The three test suites +// below only differ in the field types they are instantiated with, so that no +// test ever has to skip itself as "not applicable". Reporting the message +// type through MessageUnderTest() makes the base SetUp() skip the editions +// instances when --maximum_edition doesn't cover them. +class PrematureEofTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, FieldDescriptor::Type>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + FieldDescriptor::Type type() const { return std::get<1>(GetParam()); } + WireType wire_type() const { return WireTypeForFieldType(type()); } + absl::string_view incomplete() const { return IncompleteValue(wire_type()); } + + // The message's singular and repeated field of type(). These scan the + // descriptor (and check-fail if there is no such field), hence the + // function-style names. + const FieldDescriptor* Field() const { + return GetFieldForType(*message(), type(), /*repeated=*/false); + } + const FieldDescriptor* RepeatedField() const { + return GetFieldForType(*message(), type(), /*repeated=*/true); + } + + // The legacy test name: "<prefix>.<TYPE>", e.g. + // "PrematureEofBeforeKnownNonRepeatedValue.INT32". + std::string TestName(absl::string_view prefix) const { + return absl::StrCat(prefix, ".", UpperCaseTypeName(type())); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(PrematureEofTest, BeforeKnownNonRepeatedValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofBeforeKnownNonRepeatedValue")) + .ParseBinary(message(), Tag(FieldNumber(*Field()), wire_type())) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofTest, BeforeKnownRepeatedValue) { + EXPECT_THAT(Testee(TestName("PrematureEofBeforeKnownRepeatedValue")) + .ParseBinary(message(), + Tag(FieldNumber(*RepeatedField()), wire_type())) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofTest, BeforeUnknownValue) { + EXPECT_THAT(Testee(TestName("PrematureEofBeforeUnknownValue")) + .ParseBinary(message(), Tag(kUnknownFieldNumber, wire_type())) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofTest, InsideKnownNonRepeatedValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofInsideKnownNonRepeatedValue")) + .ParseBinary(message(), Wire(Tag(FieldNumber(*Field()), wire_type()), + incomplete())) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofTest, InsideKnownRepeatedValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofInsideKnownRepeatedValue")) + .ParseBinary(message(), + Wire(Tag(FieldNumber(*RepeatedField()), wire_type()), + incomplete())) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofTest, InsideUnknownValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofInsideUnknownValue")) + .ParseBinary(message(), Wire(Tag(kUnknownFieldNumber, wire_type()), + incomplete())) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, PrematureEofTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(AllFieldTypesExceptGroup())), + TupleParamName<PrematureEofTest::ParamType>); + +// A length prefix that promises more data than follows. +class PrematureEofInDelimitedDataTest : public PrematureEofTest {}; + +TEST_P(PrematureEofInDelimitedDataTest, ForKnownNonRepeatedValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofInDelimitedDataForKnownNonRepeatedValue")) + .ParseBinary(message(), Wire(Tag(FieldNumber(*Field()), + WireType::kLengthPrefixed), + Varint(1))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofInDelimitedDataTest, ForKnownRepeatedValue) { + EXPECT_THAT( + Testee(TestName("PrematureEofInDelimitedDataForKnownRepeatedValue")) + .ParseBinary(message(), Wire(Tag(FieldNumber(*RepeatedField()), + WireType::kLengthPrefixed), + Varint(1))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(PrematureEofInDelimitedDataTest, ForUnknownValue) { + EXPECT_THAT(Testee(TestName("PrematureEofInDelimitedDataForUnknownValue")) + .ParseBinary(message(), Wire(Tag(kUnknownFieldNumber, + WireType::kLengthPrefixed), + Varint(1))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P( + All, PrematureEofInDelimitedDataTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(LengthDelimitedFieldTypes())), + TupleParamName<PrematureEofInDelimitedDataTest::ParamType>); + +// A packed repeated field whose payload is truncated. +class PrematureEofInPackedFieldTest : public PrematureEofTest {}; + +// The packed region ends in the middle of one of its values. +// TODO: also run with valid data appended (legacy TODO). +TEST_P(PrematureEofInPackedFieldTest, PackedFieldValue) { + EXPECT_THAT(Testee(TestName("PrematureEofInPackedFieldValue")) + .ParseBinary(message(), + LengthPrefixedField( + FieldNumber(*RepeatedField()), incomplete())) + .ParseOnly(), + Yields(IsParseError())); +} + +// The payload ends before the packed region does. +TEST_P(PrematureEofInPackedFieldTest, PackedField) { + EXPECT_THAT( + Testee(TestName("PrematureEofInPackedField")) + .ParseBinary(message(), Wire(Tag(FieldNumber(*RepeatedField()), + WireType::kLengthPrefixed), + Varint(1))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P( + All, PrematureEofInPackedFieldTest, + Combine(ValuesIn(AllTestMessageTypes()), ValuesIn(PackableFieldTypes())), + TupleParamName<PrematureEofInPackedFieldTest::ParamType>); + +// A submessage that ends in the middle of one of its own values. The legacy +// test only ran this for TYPE_MESSAGE, so the suite is parameterized over the +// message type alone and, as in the other suites here, the gtest name is the +// rest of the conformance name: its legacy ".MESSAGE" suffix. +using PrematureEofInSubmessageValueTest = MessageTypeConformanceTest; + +// TODO: also run with valid data appended (legacy TODO). +TEST_P(PrematureEofInSubmessageValueTest, Message) { + const FieldDescriptor* field = GetFieldForType( + *message(), FieldDescriptor::TYPE_MESSAGE, /*repeated=*/false); + // Field 5 is what the legacy test used: it (accidentally) passed + // WireFormatLite::TYPE_INT32 == 5 as the field number. The bytes are kept + // identical. + Wire incomplete_submessage(Tag(5, WireType::kVarint), + IncompleteValue(WireType::kVarint)); + EXPECT_THAT( + Testee("PrematureEofInSubmessageValue.MESSAGE") + .ParseBinary(message(), LengthPrefixedField(FieldNumber(*field), + incomplete_submessage)) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, PrematureEofInSubmessageValueTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_recursion_limit_test.cc b/conformance/binary_recursion_limit_test.cc new file mode 100644 index 0000000..4d3bd6d --- /dev/null +++ b/conformance/binary_recursion_limit_test.cc
@@ -0,0 +1,85 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary performance conformance tests checking that absurdly deeply nested +// messages are rejected rather than parsed. Part of the performance suite +// (see conformance.bzl). This replaces the legacy suite-level +// BinaryAndJsonConformanceSuite::RunRecursionLimitTests(); the test names and +// the requests sent to the testee are identical to the legacy ones. +// +// Unlike the legacy suite, which ran the TestAllTypesEdition2023 tests here +// regardless of --maximum_edition, these are edition-gated like every other +// editions test (through MessageUnderTest(), see message_type_fixtures.h). + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/recursion_limit_payloads.h" +#include "conformance/test_fixture.h" +#include "conformance/test_protos/test_messages_edition2023.pb.h" +#include "conformance/testee.h" +#include "google/protobuf/test_messages_proto2.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::editions::TestAllTypesEdition2023; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; + +// The default recursion limit is 100 for most languages. 10,000 for golang. +// We use a larger number here for test. +constexpr int kMapDepth = 20000; +constexpr int kMapStringKeyDepth = 10000; +constexpr int kMessageSetDepth = 20000; + +// The payloads are built directly on the wire, see recursion_limit_payloads.h. + +// The TestAllTypesProto2 test, which every --maximum_edition covers. +class RecursionLimitPerformanceTest : public ConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// The TestAllTypesEdition2023 tests, which the base SetUp() skips when +// --maximum_edition doesn't cover editions. +class Edition2023RecursionLimitPerformanceTest + : public Edition2023ConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +TEST_F(Edition2023RecursionLimitPerformanceTest, Map) { + EXPECT_THAT(Testee("EnforceDepthLimit.Map") + .ParseBinary(TestAllTypesEdition2023::descriptor(), + DeepMapPayload(kMapDepth)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_F(Edition2023RecursionLimitPerformanceTest, MapStringKey) { + EXPECT_THAT(Testee("EnforceDepthLimit.MapStringKey") + .ParseBinary(TestAllTypesEdition2023::descriptor(), + DeepMapStringKeyPayload(kMapStringKeyDepth)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_F(RecursionLimitPerformanceTest, MessageSetExtension) { + EXPECT_THAT(Testee("EnforceDepthLimit.MessageSetExtension") + .ParseBinary(TestAllTypesProto2::descriptor(), + DeepMessageSetPayload(kMessageSetDepth)) + .ParseOnly(), + Yields(IsParseError())); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_test_util.cc b/conformance/binary_test_util.cc index c083a80..8cd7830 100644 --- a/conformance/binary_test_util.cc +++ b/conformance/binary_test_util.cc
@@ -7,12 +7,421 @@ #include "conformance/binary_test_util.h" +#include <cstdint> +#include <limits> +#include <string> +#include <vector> + +#include "absl/base/nullability.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "conformance/binary_wireformat.h" +#include "conformance/naming.h" #include "google/protobuf/descriptor.h" +#include "editions/golden/test_messages_proto2_editions.pb.h" +#include "editions/golden/test_messages_proto3_editions.pb.h" +#include "google/protobuf/port.h" +#include "google/protobuf/test_messages_proto2.pb.h" +#include "google/protobuf/test_messages_proto3.pb.h" +#include "google/protobuf/wire_format_lite.h" namespace google { namespace protobuf { namespace conformance { +const FieldDescriptor* absl_nonnull GetFieldForType(const Descriptor& message, + FieldDescriptor::Type type, + bool repeated, + Packedness packedness) { + for (int i = 0; i < message.field_count(); i++) { + const FieldDescriptor* field = message.field(i); + if (field->type() == type && field->is_repeated() == repeated) { + if ((packedness == Packedness::kPacked && !field->is_packed()) || + (packedness == Packedness::kUnpacked && field->is_packed())) { + continue; + } + return field; + } + } + + absl::string_view packed_string; + const absl::string_view repeated_string = + repeated ? "Repeated " : "Singular "; + if (packedness == Packedness::kPacked) { + packed_string = "Packed "; + } + if (packedness == Packedness::kUnpacked) { + packed_string = "Unpacked "; + } + ABSL_LOG(FATAL) << "Couldn't find field with type: " << repeated_string + << packed_string << FieldDescriptor::TypeName(type) << " for " + << message.full_name(); +} + +const FieldDescriptor* absl_nonnull GetFieldForMapType( + const Descriptor& message, FieldDescriptor::Type key_type, + FieldDescriptor::Type value_type) { + for (int i = 0; i < message.field_count(); i++) { + const FieldDescriptor* field = message.field(i); + if (!field->is_map()) continue; + const Descriptor* entry = field->message_type(); + if (entry->map_key()->type() == key_type && + entry->map_value()->type() == value_type) { + return field; + } + } + ABSL_LOG(FATAL) << "Couldn't find map field with type: " + << FieldDescriptor::TypeName(key_type) << " and " + << FieldDescriptor::TypeName(value_type) << " for " + << message.full_name(); +} + +const FieldDescriptor* absl_nonnull GetFieldForOneofType( + const Descriptor& message, FieldDescriptor::Type type, OneofMember member) { + const bool of_type = member == OneofMember::kOfType; + for (int i = 0; i < message.field_count(); i++) { + const FieldDescriptor* field = message.field(i); + if (field->real_containing_oneof() != nullptr && + (field->type() == type) == of_type) { + return field; + } + } + ABSL_LOG(FATAL) << "Couldn't find oneof field with type" + << (of_type ? ": " : " other than: ") + << FieldDescriptor::TypeName(type) << " for " + << message.full_name(); +} + +WireType WireTypeForFieldType(FieldDescriptor::Type type) { + // WireType's enumerators are defined as WireFormatLite's, so the cast is + // value-preserving. + return static_cast<WireType>( + ::google::protobuf::internal::WireFormatLite::WireTypeForFieldType( + static_cast<::google::protobuf::internal::WireFormatLite::FieldType>(type))); +} + +std::string UpperCaseTypeName(FieldDescriptor::Type type) { + return absl::AsciiStrToUpper(FieldDescriptor::TypeName(type)); +} + +Wire GetDefaultValue(FieldDescriptor::Type type) { + switch (type) { + case FieldDescriptor::TYPE_INT32: + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_UINT32: + case FieldDescriptor::TYPE_UINT64: + case FieldDescriptor::TYPE_ENUM: + case FieldDescriptor::TYPE_BOOL: + return Varint(0); + case FieldDescriptor::TYPE_SINT32: + return SInt32(0); + case FieldDescriptor::TYPE_SINT64: + return SInt64(0); + case FieldDescriptor::TYPE_FIXED32: + case FieldDescriptor::TYPE_SFIXED32: + return Fixed32(0); + case FieldDescriptor::TYPE_FIXED64: + case FieldDescriptor::TYPE_SFIXED64: + return Fixed64(0); + case FieldDescriptor::TYPE_FLOAT: + return Float(0); + case FieldDescriptor::TYPE_DOUBLE: + return Double(0); + case FieldDescriptor::TYPE_STRING: + case FieldDescriptor::TYPE_BYTES: + case FieldDescriptor::TYPE_MESSAGE: + return LengthPrefixed(""); + case FieldDescriptor::TYPE_GROUP: + return Wire(); + } + google::protobuf::internal::Unreachable(); +} + +Wire GetNonDefaultValue(FieldDescriptor::Type type) { + switch (type) { + case FieldDescriptor::TYPE_INT32: + case FieldDescriptor::TYPE_INT64: + case FieldDescriptor::TYPE_UINT32: + case FieldDescriptor::TYPE_UINT64: + case FieldDescriptor::TYPE_ENUM: + case FieldDescriptor::TYPE_BOOL: + return Varint(1); + case FieldDescriptor::TYPE_SINT32: + return SInt32(1); + case FieldDescriptor::TYPE_SINT64: + return SInt64(1); + case FieldDescriptor::TYPE_FIXED32: + case FieldDescriptor::TYPE_SFIXED32: + return Fixed32(1); + case FieldDescriptor::TYPE_FIXED64: + case FieldDescriptor::TYPE_SFIXED64: + return Fixed64(1); + case FieldDescriptor::TYPE_FLOAT: + return Float(1); + case FieldDescriptor::TYPE_DOUBLE: + return Double(1); + case FieldDescriptor::TYPE_STRING: + case FieldDescriptor::TYPE_BYTES: + return LengthPrefixed("a"); + case FieldDescriptor::TYPE_MESSAGE: + return LengthPrefixed(VarintField(1, 1234)); + case FieldDescriptor::TYPE_GROUP: + return Wire(); + } + google::protobuf::internal::Unreachable(); +} + +bool IsDefaultValue(FieldDescriptor::Type type, const Wire& payload) { + if (type == FieldDescriptor::TYPE_MESSAGE || + type == FieldDescriptor::TYPE_GROUP) { + return false; + } + return payload == GetDefaultValue(type); +} + +bool HasImplicitPresence(const Descriptor& message) { + return !GetFieldForType(message, FieldDescriptor::TYPE_INT32, + /*repeated=*/false) + ->has_presence(); +} + +absl::Span<const ValidDataCase> ValidDataCases(FieldDescriptor::Type type) { + constexpr int64_t kInt64Min = std::numeric_limits<int64_t>::min(); + constexpr int64_t kInt64Max = std::numeric_limits<int64_t>::max(); + constexpr uint64_t kUint64Max = std::numeric_limits<uint64_t>::max(); + constexpr int32_t kInt32Min = std::numeric_limits<int32_t>::min(); + constexpr int32_t kInt32Max = std::numeric_limits<int32_t>::max(); + constexpr uint32_t kUint32Max = std::numeric_limits<uint32_t>::max(); + + // The tables are the legacy suite's, verbatim (including the entries whose + // input isn't canonical: over-long varints, out-of-range varints that must + // be truncated or clamped, and non-zero bools). Their order is part of the + // test names, so entries may only be appended. + switch (type) { + case FieldDescriptor::TYPE_DOUBLE: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Double(0), Double(0)}, + {Double(0.1), Double(0.1)}, + {Double(1.7976931348623157e+308), Double(1.7976931348623157e+308)}, + {Double(2.22507385850720138309e-308), + Double(2.22507385850720138309e-308)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_FLOAT: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Float(0), Float(0)}, + {Float(0.1), Float(0.1)}, + {Float(1.00000075e-36), Float(1.00000075e-36)}, + {Float(3.402823e+38), Float(3.402823e+38)}, // 3.40282347e+38 + {Float(1.17549435e-38f), Float(1.17549435e-38)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_INT64: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(12345), Varint(12345)}, + {Varint(kInt64Max), Varint(kInt64Max)}, + {Varint(kInt64Min), Varint(kInt64Min)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_UINT64: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(12345), Varint(12345)}, + {Varint(kUint64Max), Varint(kUint64Max)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_INT32: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(12345), Varint(12345)}, + {LongVarint(12345, 2), Varint(12345)}, + {LongVarint(12345, 7), Varint(12345)}, + {Varint(kInt32Max), Varint(kInt32Max)}, + {Varint(kInt32Min), Varint(kInt32Min)}, + {Varint(1LL << 33), Varint(0)}, + {Varint((1LL << 33) - 1), Varint(-1)}, + {Varint(kInt64Max), Varint(-1)}, + {Varint(kInt64Min + 1), Varint(1)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_UINT32: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(12345), Varint(12345)}, + {LongVarint(12345, 2), Varint(12345)}, + {LongVarint(12345, 7), Varint(12345)}, + {Varint(kUint32Max), Varint(kUint32Max)}, // UINT32_MAX + {Varint(1LL << 33), Varint(0)}, + {Varint((1LL << 33) + 1), Varint(1)}, + {Varint((1LL << 33) - 1), Varint((1LL << 32) - 1)}, + {Varint(kInt64Max), Varint((1LL << 32) - 1)}, + {Varint(kInt64Min + 1), Varint(1)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_FIXED64: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Fixed64(0), Fixed64(0)}, + {Fixed64(12345), Fixed64(12345)}, + {Fixed64(kUint64Max), Fixed64(kUint64Max)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_FIXED32: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Fixed32(0), Fixed32(0)}, + {Fixed32(12345), Fixed32(12345)}, + {Fixed32(kUint32Max), Fixed32(kUint32Max)}, // UINT32_MAX + }; + return *kCases; + } + case FieldDescriptor::TYPE_SFIXED64: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Fixed64(0), Fixed64(0)}, + {Fixed64(12345), Fixed64(12345)}, + {Fixed64(kInt64Max), Fixed64(kInt64Max)}, + {Fixed64(kInt64Min), Fixed64(kInt64Min)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_SFIXED32: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Fixed32(0), Fixed32(0)}, + {Fixed32(12345), Fixed32(12345)}, + {Fixed32(kInt32Max), Fixed32(kInt32Max)}, + {Fixed32(kInt32Min), Fixed32(kInt32Min)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_BOOL: { + // Bools should be serialized as 0 for false and 1 for true. Parsers + // should also interpret any nonzero value as true. + // clang-format off + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(1), Varint(1)}, + {Varint(-1), Varint(1)}, + {Varint(12345678), Varint(1)}, + {Varint(1LL << 33), Varint(1)}, + {Varint(kInt64Max), Varint(1)}, + {Varint(kInt64Min), Varint(1)}, + }; + // clang-format on + return *kCases; + } + case FieldDescriptor::TYPE_SINT32: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {SInt32(0), SInt32(0)}, + {SInt32(12345), SInt32(12345)}, + {SInt32(kInt32Max), SInt32(kInt32Max)}, + {SInt32(kInt32Min), SInt32(kInt32Min)}, + {SInt64(kInt32Max + 2LL), SInt32(1)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_SINT64: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {SInt64(0), SInt64(0)}, + {SInt64(12345), SInt64(12345)}, + {SInt64(kInt64Max), SInt64(kInt64Max)}, + {SInt64(kInt64Min), SInt64(kInt64Min)}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_STRING: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {LengthPrefixed(""), LengthPrefixed("")}, + {LengthPrefixed("Hello world!"), LengthPrefixed("Hello world!")}, + {LengthPrefixed("\'\"\?\\\a\b\f\n\r\t\v"), + LengthPrefixed("\'\"\?\\\a\b\f\n\r\t\v")}, // escape + // U+8C37 U+6B4C ("Google" in Chinese), as UTF-8. + {LengthPrefixed("\xE8\xB0\xB7\xE6\xAD\x8C"), + LengthPrefixed("\xE8\xB0\xB7\xE6\xAD\x8C")}, + // U+1F601 (grinning face with smiling eyes), as UTF-8. + {LengthPrefixed("\xF0\x9F\x98\x81"), + LengthPrefixed("\xF0\x9F\x98\x81")}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_BYTES: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {LengthPrefixed(""), LengthPrefixed("")}, + {LengthPrefixed("Hello world!"), LengthPrefixed("Hello world!")}, + {LengthPrefixed("\x01\x02"), LengthPrefixed("\x01\x02")}, + {LengthPrefixed("\xfb"), LengthPrefixed("\xfb")}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_ENUM: { + // clang-format off + static const auto* const kCases = new std::vector<ValidDataCase>{ + {Varint(0), Varint(0)}, + {Varint(1), Varint(1)}, + {Varint(2), Varint(2)}, + {Varint(-1), Varint(-1)}, + {Varint(kInt64Max), Varint(-1)}, + {Varint(kInt64Min + 1), Varint(1)}, + }; + // clang-format on + return *kCases; + } + case FieldDescriptor::TYPE_MESSAGE: { + static const auto* const kCases = new std::vector<ValidDataCase>{ + {LengthPrefixed(""), LengthPrefixed("")}, + {LengthPrefixed(VarintField(1, 1234)), + LengthPrefixed(VarintField(1, 1234))}, + }; + return *kCases; + } + case FieldDescriptor::TYPE_GROUP: + break; + } + ABSL_LOG(FATAL) << "No valid-data cases for field type " + << FieldDescriptor::TypeName(type); +} + +absl::Span<const MapType> ValidDataMapTypes() { + static constexpr MapType kTypes[] = { + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_INT32}, + {FieldDescriptor::TYPE_INT64, FieldDescriptor::TYPE_INT64}, + {FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_UINT32}, + {FieldDescriptor::TYPE_UINT64, FieldDescriptor::TYPE_UINT64}, + {FieldDescriptor::TYPE_SINT32, FieldDescriptor::TYPE_SINT32}, + {FieldDescriptor::TYPE_SINT64, FieldDescriptor::TYPE_SINT64}, + {FieldDescriptor::TYPE_FIXED32, FieldDescriptor::TYPE_FIXED32}, + {FieldDescriptor::TYPE_FIXED64, FieldDescriptor::TYPE_FIXED64}, + {FieldDescriptor::TYPE_SFIXED32, FieldDescriptor::TYPE_SFIXED32}, + {FieldDescriptor::TYPE_SFIXED64, FieldDescriptor::TYPE_SFIXED64}, + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_FLOAT}, + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_DOUBLE}, + {FieldDescriptor::TYPE_BOOL, FieldDescriptor::TYPE_BOOL}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_STRING}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_BYTES}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_ENUM}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_MESSAGE}, + }; + return kTypes; +} + +absl::Span<const FieldDescriptor::Type> ValidDataOneofTypes() { + static constexpr FieldDescriptor::Type kTypes[] = { + FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_BOOL, + FieldDescriptor::TYPE_UINT64, FieldDescriptor::TYPE_FLOAT, + FieldDescriptor::TYPE_DOUBLE, FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_BYTES, FieldDescriptor::TYPE_ENUM, + FieldDescriptor::TYPE_MESSAGE, + }; + return kTypes; +} absl::Span<const FieldDescriptor::Type> AllFieldTypesExceptGroup() { static constexpr FieldDescriptor::Type kTypes[] = { @@ -51,6 +460,62 @@ return kTypes; } +std::vector<const Descriptor*> AllTestMessageTypes() { + return { + protobuf_test_messages::proto3::TestAllTypesProto3::descriptor(), + protobuf_test_messages::proto2::TestAllTypesProto2::descriptor(), + protobuf_test_messages::editions::proto3::TestAllTypesProto3:: + descriptor(), + protobuf_test_messages::editions::proto2::TestAllTypesProto2:: + descriptor(), + }; +} + +std::vector<const Descriptor*> Proto3TestMessageTypes() { + return { + protobuf_test_messages::proto3::TestAllTypesProto3::descriptor(), + protobuf_test_messages::editions::proto3::TestAllTypesProto3:: + descriptor(), + }; +} + +std::vector<const Descriptor*> Proto2TestMessageTypes() { + return { + protobuf_test_messages::proto2::TestAllTypesProto2::descriptor(), + protobuf_test_messages::editions::proto2::TestAllTypesProto2:: + descriptor(), + }; +} + +std::string ParamName(absl::string_view text) { + std::string name; + for (char c : text) { + if (absl::ascii_isalnum(c)) name.push_back(c); + } + ABSL_CHECK(!name.empty()) + << "No letters or digits in parameter name: " << text; + return name; +} + +std::string ParamName(const Descriptor* absl_nonnull message) { + // The edition identifier used in test names ("Editions_Proto2") only differs + // from a valid gtest parameter name by its underscore. + return ParamName(GetEditionIdentifier(*message)); +} + +std::string ParamName(FieldDescriptor::Type type) { + return UpperCaseTypeName(type); +} + +std::string ParamName(int value) { + ABSL_CHECK_GE(value, 0) << "Negative parameters have no valid gtest name"; + return absl::StrCat(value); +} + +std::string ParamName(const MapType& type) { + return absl::StrCat(ParamName(type.key), "_", ParamName(type.value)); +} + } // namespace conformance } // namespace protobuf } // namespace google
diff --git a/conformance/binary_test_util.h b/conformance/binary_test_util.h index 744a5a1..64e61a3 100644 --- a/conformance/binary_test_util.h +++ b/conformance/binary_test_util.h
@@ -8,16 +8,150 @@ #ifndef GOOGLE_PROTOBUF_CONFORMANCE_BINARY_TEST_UTIL_H__ #define GOOGLE_PROTOBUF_CONFORMANCE_BINARY_TEST_UTIL_H__ +#include <cstdint> +#include <string> +#include <vector> + +#include "absl/base/nullability.h" +#include "absl/strings/string_view.h" #include "absl/types/span.h" +#include "conformance/binary_wireformat.h" #include "google/protobuf/descriptor.h" -// Helpers shared by the gtest-based binary conformance tests: the field-type -// tables the tests are parameterized over. Deliberately gtest-free. +// Helpers shared by the gtest-based binary conformance tests. They look up +// the test-message fields exercised for a given field type, encode sample +// values, define the parameter sets the tests are instantiated over, and +// stringify those parameters for gtest names. +// +// This header is deliberately gtest-free. The fixtures and +// INSTANTIATE_TEST_SUITE_P name generators built on top of these helpers live +// in message_type_fixtures.h. namespace google { namespace protobuf { namespace conformance { +// Restricts GetFieldForType() to packed or unpacked repeated fields. Named +// Packedness rather than Packed because Packed() is already the wire-format +// builder in binary_wireformat.h. +enum class Packedness { + kUnspecified = 0, + kPacked = 1, + kUnpacked = 2, +}; + +// Returns the first field of `message`, in declaration order, whose type is +// `type` and whose cardinality matches `repeated`. When `packedness` isn't +// kUnspecified, only fields whose is_packed() matches it are considered. +// That is how proto2 and proto3 messages, with their different packing +// defaults, select their [packed = true] and [packed = false] test fields. +// +// const FieldDescriptor* field = GetFieldForType( +// message, FieldDescriptor::TYPE_INT32, /*repeated=*/true, +// Packedness::kPacked); +// +// Check-fails if `message` has no such field. In particular, kPacked with +// `repeated` false is always fatal, since singular fields are never packed. +const FieldDescriptor* absl_nonnull GetFieldForType( + const Descriptor& message, FieldDescriptor::Type type, bool repeated, + Packedness packedness = Packedness::kUnspecified); + +// Returns `field`'s number as the wire-format builders of binary_wireformat.h +// take it. FieldDescriptor::number() is an int, but field numbers are never +// negative. +inline uint32_t FieldNumber(const FieldDescriptor& field) { + return static_cast<uint32_t>(field.number()); +} + +// Returns the first map field of `message`, in declaration order, whose keys +// are of type `key_type` and whose values are of type `value_type`. For +// example, map_int32_int32 (56) for INT32/INT32 or map_string_nested_message +// (71) for STRING/MESSAGE. Check-fails if `message` has no such field. +const FieldDescriptor* absl_nonnull GetFieldForMapType( + const Descriptor& message, FieldDescriptor::Type key_type, + FieldDescriptor::Type value_type); + +// Which member of a oneof GetFieldForOneofType() selects relative to the +// requested field type. +enum class OneofMember { + kOfType, // The first member of the given type. + kOfOtherType, // The first member of any other type. +}; + +// Returns the first field of `message`, in declaration order, that is a member +// of a non-synthetic oneof and whose type is `type`. For example, oneof_uint32 +// (111) for UINT32. With OneofMember::kOfOtherType, returns the first oneof +// member whose type is not `type` instead. The oneof tests use that to find a +// second member of the same oneof to override. Check-fails if there is no +// such field. +const FieldDescriptor* absl_nonnull GetFieldForOneofType( + const Descriptor& message, FieldDescriptor::Type type, + OneofMember member = OneofMember::kOfType); + +// Returns the wire type a non-packed value of `type` is encoded with. +WireType WireTypeForFieldType(FieldDescriptor::Type type); + +// Returns the upper-case name of `type` as it appears in conformance test +// names, e.g. "INT32", "SFIXED64" or "MESSAGE". +std::string UpperCaseTypeName(FieldDescriptor::Type type); + +// Returns the encoded default value of a field of `type`, without a tag. That +// is zero, or an empty string, bytes or message. TYPE_GROUP has no such +// single-value encoding and yields an empty Wire. +Wire GetDefaultValue(FieldDescriptor::Type type); + +// Returns an encoded non-default value of a field of `type`, without a tag. +// That is 1 for numeric types and enums, "a" for strings and bytes, and a +// message whose field 1 is the varint 1234. TYPE_GROUP yields an empty Wire. +Wire GetNonDefaultValue(FieldDescriptor::Type type); + +// Returns whether `payload`, the encoding of a value of `type` without its tag, +// is the encoding of the type's default value. That is zero, false, the empty +// string or empty bytes, which is what a proto3 serializer omits from a +// singular field. Always false for TYPE_MESSAGE and TYPE_GROUP. Only the +// canonical encoding counts. A ten-byte varint 0 doesn't. +bool IsDefaultValue(FieldDescriptor::Type type, const Wire& payload); + +// Returns whether the singular scalar fields of `message` have implicit +// presence, that is whether it follows proto3 semantics. True for +// TestAllTypesProto3 and its editions equivalent. False for the proto2 ones +// and TestAllTypesEdition2023. Decided by the presence of the message's first +// singular INT32 field, optional_int32 in every TestAllTypes. Check-fails for +// a message without one. +bool HasImplicitPresence(const Descriptor& message); + +// One case of the valid-data tests (ValidDataScalar.<TYPE>[i] and friends). +// `input` is the encoding of a value as sent to the testee. `expected` is the +// canonical encoding a conforming parser must be equivalent to after reading +// it. For example, an over-long varint and its minimal form, or a 64-bit +// varint and its truncation to 32 bits. Both are without a tag. +struct ValidDataCase { + Wire input; + Wire expected; +}; + +// Returns the valid-data cases of `type`. Case i is what the tests named +// ValidDataScalar.<TYPE>[i] and ValidDataScalarBinary.<TYPE>[i] send. The +// repeated-field tests send all of them in this order. Both the binary->binary +// and the binary->JSON legs iterate these tables. Check-fails for TYPE_GROUP, +// which has no cases. +absl::Span<const ValidDataCase> ValidDataCases(FieldDescriptor::Type type); + +// The key and value types of a map field, selecting one of the +// map_<key>_<value> fields of the test messages (see GetFieldForMapType()). +struct MapType { + FieldDescriptor::Type key; + FieldDescriptor::Type value; +}; + +// The key/value type combinations of the map valid-data tests +// (ValidDataMap.<KEY>.<VALUE>.*). +absl::Span<const MapType> ValidDataMapTypes(); + +// The field types of the oneof valid-data tests (ValidDataOneof.<TYPE>.* and +// ValidDataOneofBinary.<TYPE>.*). +absl::Span<const FieldDescriptor::Type> ValidDataOneofTypes(); + // Every field type except TYPE_GROUP (the 17 types the test messages have // singular, repeated, packed and unpacked fields of), in the order the legacy // valid-data tests ran them: DOUBLE, FLOAT, INT64, UINT64, INT32, UINT32, @@ -35,6 +169,62 @@ // non-packable ones: STRING, BYTES, MESSAGE, in the same order. absl::Span<const FieldDescriptor::Type> LengthDelimitedFieldTypes(); +// Returns all four TestAllTypes message types the binary conformance tests run +// over, in this order: Proto3, Proto2, Editions_Proto3, Editions_Proto2. +// +// INSTANTIATE_TEST_SUITE_P(All, FooTest, ValuesIn(AllTestMessageTypes()), +// MessageTypeParamName); +// +// The editions variants are skipped at runtime by the ConformanceTest fixture +// when --maximum_edition doesn't cover them (see MessageUnderTest() in +// test_fixture.h). +// +// Calling this from INSTANTIATE_TEST_SUITE_P is fine. gtest evaluates the +// parameter generators during test registration, not during static +// initialization, so the generated messages' descriptor() accessors are safe +// to call there. +std::vector<const Descriptor*> AllTestMessageTypes(); + +// Returns the proto3-style subset of AllTestMessageTypes(), in this order: +// Proto3, Editions_Proto3. For tests whose expectations depend on proto3 +// semantics, for example UTF-8 validation of string fields. Like +// AllTestMessageTypes(), safe to call from INSTANTIATE_TEST_SUITE_P. +std::vector<const Descriptor*> Proto3TestMessageTypes(); + +// Returns the proto2-style subset of AllTestMessageTypes(), in this order: +// Proto2, Editions_Proto2. For tests whose expectations depend on proto2 +// semantics, for example closed enums or required fields. Like +// AllTestMessageTypes(), safe to call from INSTANTIATE_TEST_SUITE_P. +std::vector<const Descriptor*> Proto2TestMessageTypes(); + +// ParamName() renders one test parameter as a component of a gtest parameter +// name. Every overload yields a non-empty string of letters and digits, with +// "_" between the parts of a compound parameter. Components can therefore be +// joined with "_" (see TupleParamName() in message_type_fixtures.h) and the +// result is always a valid gtest name. Add an overload here for each new +// parameter type the conformance tests are instantiated over. + +// Returns `text` with every character that isn't a letter or digit removed. +// For example, "Editions_Proto2" becomes "EditionsProto2". `text` must contain +// at least one letter or digit. +std::string ParamName(absl::string_view text); + +// Returns the edition identifier of `message` as used in conformance test +// names (see GetEditionIdentifier() in naming.h), minus its underscore. That +// is "Proto3", "Proto2", "EditionsProto3", "EditionsProto2" or +// "EditionUnstable". +std::string ParamName(const Descriptor* absl_nonnull message); + +// The upper-case type name, e.g. "INT32"; see UpperCaseTypeName(). +std::string ParamName(FieldDescriptor::Type type); + +// The decimal representation of `value`, which must not be negative. +std::string ParamName(int value); + +// The key and value type names joined by "_", e.g. "INT32_INT32" or +// "STRING_MESSAGE". +std::string ParamName(const MapType& type); + } // namespace conformance } // namespace protobuf } // namespace google
diff --git a/conformance/binary_test_util_test.cc b/conformance/binary_test_util_test.cc index 305c52b..1d980ff 100644 --- a/conformance/binary_test_util_test.cc +++ b/conformance/binary_test_util_test.cc
@@ -7,21 +7,595 @@ #include "conformance/binary_test_util.h" +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <limits> +#include <map> +#include <memory> #include <set> +#include <string> #include <vector> #include <gmock/gmock.h> #include <gtest/gtest.h> +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "conformance/binary_wireformat.h" +#include "conformance/test_protos/test_messages_edition2023.pb.h" +#include "conformance/test_protos/test_messages_edition_unstable.pb.h" #include "google/protobuf/descriptor.h" +#include "editions/golden/test_messages_proto2_editions.pb.h" +#include "editions/golden/test_messages_proto3_editions.pb.h" +#include "google/protobuf/message.h" +#include "google/protobuf/test_messages_proto2.pb.h" +#include "google/protobuf/test_messages_proto3.pb.h" +#include "google/protobuf/util/message_differencer.h" namespace google { namespace protobuf { namespace conformance { namespace { +using ::protobuf_test_messages::edition_unstable::TestAllTypesEditionUnstable; +using ::protobuf_test_messages::editions::TestAllTypesEdition2023; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using TestAllTypesProto2Editions = + ::protobuf_test_messages::editions::proto2::TestAllTypesProto2; +using TestAllTypesProto3Editions = + ::protobuf_test_messages::editions::proto3::TestAllTypesProto3; +using ::testing::ElementsAre; using ::testing::ElementsAreArray; +using ::testing::FieldsAre; +using ::testing::IsSubsetOf; using ::testing::UnorderedElementsAreArray; +TEST(GetFieldForTypeTest, SingularInt32) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/false) + ->number(), + 1); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/false) + ->number(), + 1); +} + +TEST(GetFieldForTypeTest, RepeatedInt32) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true) + ->number(), + 31); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true) + ->number(), + 31); +} + +// repeated_int32 (31) is packed by default in proto3 but not in proto2, so the +// packed lookup only has to fall through to packed_int32 (75) for proto2. +TEST(GetFieldForTypeTest, PackedInt32) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true, + Packedness::kPacked) + ->number(), + 31); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true, + Packedness::kPacked) + ->number(), + 75); +} + +// ... and conversely the unpacked lookup only has to fall through to +// unpacked_int32 (89) for proto3. +TEST(GetFieldForTypeTest, UnpackedInt32) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true, + Packedness::kUnpacked) + ->number(), + 89); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/true, + Packedness::kUnpacked) + ->number(), + 31); +} + +TEST(GetFieldForTypeTest, Message) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_MESSAGE, /*repeated=*/false) + ->number(), + 18); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_MESSAGE, /*repeated=*/true) + ->number(), + 48); +} + +TEST(GetFieldForTypeTest, Enum) { + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_ENUM, /*repeated=*/false) + ->number(), + 21); + EXPECT_EQ(GetFieldForType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_ENUM, /*repeated=*/true) + ->number(), + 51); +} + +TEST(GetFieldForTypeTest, EditionsMessagesMatchTheirProto2AndProto3Originals) { + for (FieldDescriptor::Type type : + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_MESSAGE, + FieldDescriptor::TYPE_ENUM, FieldDescriptor::TYPE_STRING}) { + for (bool repeated : {false, true}) { + SCOPED_TRACE(absl::StrCat(repeated ? "repeated " : "singular ", + FieldDescriptor::TypeName(type))); + EXPECT_EQ( + GetFieldForType(*TestAllTypesProto3Editions::descriptor(), type, + repeated) + ->number(), + GetFieldForType(*TestAllTypesProto3::descriptor(), type, repeated) + ->number()); + EXPECT_EQ( + GetFieldForType(*TestAllTypesProto2Editions::descriptor(), type, + repeated) + ->number(), + GetFieldForType(*TestAllTypesProto2::descriptor(), type, repeated) + ->number()); + } + } +} + +TEST(GetFieldForTypeDeathTest, MissingFieldIsFatal) { + // TestAllTypesProto3 has no singular group field. + EXPECT_DEATH(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_GROUP, /*repeated=*/false), + "Couldn't find field with type: Singular group for " + "protobuf_test_messages.proto3.TestAllTypesProto3"); +} + +// Singular fields are never packed, so asking for a packed singular field can +// never succeed, even for a type the message does have a singular field of. +TEST(GetFieldForTypeDeathTest, PackedSingularIsFatal) { + EXPECT_DEATH(GetFieldForType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_INT32, /*repeated=*/false, + Packedness::kPacked), + "Couldn't find field with type: Singular Packed int32 for " + "protobuf_test_messages.proto3.TestAllTypesProto3"); +} + +TEST(GetFieldForMapTypeTest, LegacyFieldNumbers) { + for (const Descriptor* message : AllTestMessageTypes()) { + SCOPED_TRACE(message->full_name()); + EXPECT_EQ(GetFieldForMapType(*message, FieldDescriptor::TYPE_INT32, + FieldDescriptor::TYPE_INT32) + ->number(), + 56); + EXPECT_EQ(GetFieldForMapType(*message, FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_STRING) + ->number(), + 69); + EXPECT_EQ(GetFieldForMapType(*message, FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_MESSAGE) + ->number(), + 71); + } +} + +TEST(GetFieldForMapTypeDeathTest, MissingFieldIsFatal) { + // There is no map<bytes, ...> (bytes aren't valid map keys). + EXPECT_DEATH(GetFieldForMapType(*TestAllTypesProto3::descriptor(), + FieldDescriptor::TYPE_BYTES, + FieldDescriptor::TYPE_BYTES), + "Couldn't find map field with type: bytes and bytes for " + "protobuf_test_messages.proto3.TestAllTypesProto3"); +} + +TEST(GetFieldForOneofTypeTest, LegacyFieldNumbers) { + for (const Descriptor* message : AllTestMessageTypes()) { + SCOPED_TRACE(message->full_name()); + EXPECT_EQ( + GetFieldForOneofType(*message, FieldDescriptor::TYPE_UINT32)->number(), + 111); + EXPECT_EQ( + GetFieldForOneofType(*message, FieldDescriptor::TYPE_MESSAGE)->number(), + 112); + EXPECT_EQ( + GetFieldForOneofType(*message, FieldDescriptor::TYPE_ENUM)->number(), + 119); + } +} + +// With kOfOtherType the first oneof member of any *other* type is returned, +// which is oneof_uint32 (111) unless that is the type being excluded. +TEST(GetFieldForOneofTypeTest, OfOtherType) { + const Descriptor& message = *TestAllTypesProto3::descriptor(); + EXPECT_EQ(GetFieldForOneofType(message, FieldDescriptor::TYPE_MESSAGE, + OneofMember::kOfOtherType) + ->number(), + 111); + EXPECT_EQ(GetFieldForOneofType(message, FieldDescriptor::TYPE_UINT32, + OneofMember::kOfOtherType) + ->number(), + 112); +} + +TEST(GetFieldForOneofTypeDeathTest, MissingFieldIsFatal) { + // No oneof member is a group. + EXPECT_DEATH(GetFieldForOneofType(*TestAllTypesProto2::descriptor(), + FieldDescriptor::TYPE_GROUP), + "Couldn't find oneof field with type: group for " + "protobuf_test_messages.proto2.TestAllTypesProto2"); +} + +TEST(WireTypeForFieldTypeTest, AllTypes) { + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_DOUBLE), + WireType::kFixed64); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_FLOAT), + WireType::kFixed32); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_INT64), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_UINT64), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_INT32), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_FIXED64), + WireType::kFixed64); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_FIXED32), + WireType::kFixed32); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_BOOL), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_STRING), + WireType::kLengthPrefixed); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_GROUP), + WireType::kStartGroup); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_MESSAGE), + WireType::kLengthPrefixed); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_BYTES), + WireType::kLengthPrefixed); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_UINT32), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_ENUM), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_SFIXED32), + WireType::kFixed32); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_SFIXED64), + WireType::kFixed64); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_SINT32), + WireType::kVarint); + EXPECT_EQ(WireTypeForFieldType(FieldDescriptor::TYPE_SINT64), + WireType::kVarint); +} + +TEST(UpperCaseTypeNameTest, MatchesLegacyTestNames) { + EXPECT_EQ(UpperCaseTypeName(FieldDescriptor::TYPE_INT32), "INT32"); + EXPECT_EQ(UpperCaseTypeName(FieldDescriptor::TYPE_SFIXED64), "SFIXED64"); + EXPECT_EQ(UpperCaseTypeName(FieldDescriptor::TYPE_MESSAGE), "MESSAGE"); + EXPECT_EQ(UpperCaseTypeName(FieldDescriptor::TYPE_BYTES), "BYTES"); +} + +TEST(GetDefaultValueTest, Varints) { + for (FieldDescriptor::Type type : + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_INT64, + FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_UINT64, + FieldDescriptor::TYPE_ENUM, FieldDescriptor::TYPE_BOOL, + FieldDescriptor::TYPE_SINT32, FieldDescriptor::TYPE_SINT64}) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_EQ(GetDefaultValue(type), Varint(0)); + } +} + +TEST(GetDefaultValueTest, FixedWidth) { + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_FIXED32), Fixed32(0)); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_SFIXED32), Fixed32(0)); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_FLOAT), Float(0)); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_FIXED64), Fixed64(0)); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_SFIXED64), Fixed64(0)); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_DOUBLE), Double(0)); +} + +TEST(GetDefaultValueTest, LengthPrefixed) { + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_STRING), LengthPrefixed("")); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_BYTES), LengthPrefixed("")); + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_MESSAGE), LengthPrefixed("")); +} + +TEST(GetDefaultValueTest, GroupIsEmpty) { + EXPECT_EQ(GetDefaultValue(FieldDescriptor::TYPE_GROUP), Wire()); +} + +TEST(GetNonDefaultValueTest, Varints) { + for (FieldDescriptor::Type type : + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_INT64, + FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_UINT64, + FieldDescriptor::TYPE_ENUM, FieldDescriptor::TYPE_BOOL}) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_EQ(GetNonDefaultValue(type), Varint(1)); + } + // Zig-zag encoded, so 1 becomes 2. + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_SINT32), Varint(2)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_SINT64), Varint(2)); +} + +TEST(GetNonDefaultValueTest, FixedWidth) { + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_FIXED32), Fixed32(1)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_SFIXED32), Fixed32(1)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_FLOAT), Float(1)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_FIXED64), Fixed64(1)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_SFIXED64), Fixed64(1)); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_DOUBLE), Double(1)); +} + +TEST(GetNonDefaultValueTest, LengthPrefixed) { + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_STRING), + LengthPrefixed("a")); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_BYTES), + LengthPrefixed("a")); + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_MESSAGE), + LengthPrefixed(VarintField(1, 1234))); +} + +TEST(GetNonDefaultValueTest, GroupIsEmpty) { + EXPECT_EQ(GetNonDefaultValue(FieldDescriptor::TYPE_GROUP), Wire()); +} + +TEST(GetNonDefaultValueTest, DiffersFromDefaultValue) { + for (FieldDescriptor::Type type : + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_SINT64, + FieldDescriptor::TYPE_FIXED32, FieldDescriptor::TYPE_DOUBLE, + FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_MESSAGE}) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_NE(GetNonDefaultValue(type), GetDefaultValue(type)); + } +} + +// `wire` parsed by the generated class of `message`, or null if it doesn't +// parse. The test inputs are checked against the C++ generated classes, whose +// parsing and merge semantics are the reference the conformance tests hold +// others to. +std::unique_ptr<Message> ParseAs(const Descriptor* message, const Wire& wire) { + std::unique_ptr<Message> parsed( + MessageFactory::generated_factory()->GetPrototype(message)->New()); + if (!parsed->ParseFromString(wire.data())) return nullptr; + return parsed; +} + +// Matches a Message equal to `*expected`. +MATCHER_P(EqualsMessage, expected, "") { + std::string diff; + util::MessageDifferencer differencer; + differencer.ReportDifferencesToString(&diff); + if (differencer.Compare(*expected, arg)) return true; + *result_listener << "which differs: " << diff; + return false; +} + +// Matches a Wire that the generated class of `message` parses into a message +// matching `matcher`. +MATCHER_P2(ParsesAs, message, matcher, "") { + std::unique_ptr<Message> parsed = ParseAs(message, arg); + if (parsed == nullptr) { + *result_listener << "which doesn't parse as " << message->full_name(); + return false; + } + return ExplainMatchResult(matcher, *parsed, result_listener); +} + +// The legacy table sizes; the test names ValidDataScalar.<TYPE>[i] depend on +// them. +TEST(ValidDataCasesTest, LegacyTableSizes) { + const std::map<FieldDescriptor::Type, size_t> kSizes = { + {FieldDescriptor::TYPE_DOUBLE, 4}, {FieldDescriptor::TYPE_FLOAT, 5}, + {FieldDescriptor::TYPE_INT64, 4}, {FieldDescriptor::TYPE_UINT64, 3}, + {FieldDescriptor::TYPE_INT32, 10}, {FieldDescriptor::TYPE_UINT32, 10}, + {FieldDescriptor::TYPE_FIXED64, 3}, {FieldDescriptor::TYPE_FIXED32, 3}, + {FieldDescriptor::TYPE_SFIXED64, 4}, {FieldDescriptor::TYPE_SFIXED32, 4}, + {FieldDescriptor::TYPE_BOOL, 7}, {FieldDescriptor::TYPE_SINT32, 5}, + {FieldDescriptor::TYPE_SINT64, 4}, {FieldDescriptor::TYPE_STRING, 5}, + {FieldDescriptor::TYPE_BYTES, 4}, {FieldDescriptor::TYPE_ENUM, 6}, + {FieldDescriptor::TYPE_MESSAGE, 2}, + }; + ASSERT_EQ(kSizes.size(), AllFieldTypesExceptGroup().size()); + size_t total = 0; + for (const auto& [type, size] : kSizes) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_EQ(ValidDataCases(type).size(), size); + total += size; + } + EXPECT_EQ(total, size_t{83}); +} + +// Every table starts with the type's default value, which is what the proto3 +// rule of the singular-field tests hinges on. +TEST(ValidDataCasesTest, FirstCaseIsTheDefaultValue) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_EQ(ValidDataCases(type).front().input, GetDefaultValue(type)); + EXPECT_EQ(ValidDataCases(type).front().expected, GetDefaultValue(type)); + } +} + +// The RepeatedScalarSelectsLast tests expect the last case of a table without +// applying the proto3 default rule, which relies on no table ending in the +// type's default value. +TEST(ValidDataCasesTest, LastCaseIsNotDefault) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + if (type == FieldDescriptor::TYPE_MESSAGE) continue; + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_FALSE(IsDefaultValue(type, ValidDataCases(type).back().expected)); + } +} + +// A few entries whose input isn't the canonical encoding, spelled out to pin +// the legacy bytes. +TEST(ValidDataCasesTest, NonCanonicalInputs) { + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_INT32)[2].input, + LongVarint(12345, 2)); + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_INT32)[2].expected, + Varint(12345)); + // A 64-bit varint truncated to int32 -1. + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_INT32)[8].input, + Varint(std::numeric_limits<int64_t>::max())); + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_INT32)[8].expected, + Varint(-1)); + // Any non-zero bool is true. + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_BOOL)[3].input, + Varint(12345678)); + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_BOOL)[3].expected, Varint(1)); + // A non-ASCII string (U+8C37 U+6B4C as UTF-8) round-trips unchanged. + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_STRING)[3].input, + LengthPrefixed("\xE8\xB0\xB7\xE6\xAD\x8C")); + EXPECT_EQ(ValidDataCases(FieldDescriptor::TYPE_STRING)[3].expected, + LengthPrefixed("\xE8\xB0\xB7\xE6\xAD\x8C")); +} + +// The invariant every table encodes: for the singular field of the type, +// the C++ generated class parses `input` into a message equivalent to the one +// it parses `expected` into. +TEST(ValidDataCasesTest, InputParsesEquivalentlyToExpected) { + for (const Descriptor* message : AllTestMessageTypes()) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + const Wire tag = + Tag(FieldNumber(*GetFieldForType(*message, type, /*repeated=*/false)), + WireTypeForFieldType(type)); + int i = 0; + for (const ValidDataCase& value : ValidDataCases(type)) { + SCOPED_TRACE(absl::StrCat(message->full_name(), " ", + FieldDescriptor::TypeName(type), "[", i++, + "]")); + std::unique_ptr<Message> expected = + ParseAs(message, Wire(tag, value.expected)); + ASSERT_NE(expected, nullptr); + EXPECT_THAT(Wire(tag, value.input), + ParsesAs(message, EqualsMessage(expected.get()))); + } + } + } +} + +TEST(ValidDataCasesDeathTest, GroupIsFatal) { + EXPECT_DEATH(ValidDataCases(FieldDescriptor::TYPE_GROUP), + "No valid-data cases for field type group"); +} + +// The legacy table; the test names ValidDataMap.<KEY>.<VALUE>.* depend on it. +TEST(ValidDataMapTypesTest, LegacyTable) { + EXPECT_THAT( + ValidDataMapTypes(), + ElementsAre( + FieldsAre(FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_INT32), + FieldsAre(FieldDescriptor::TYPE_INT64, FieldDescriptor::TYPE_INT64), + FieldsAre(FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_UINT32), + FieldsAre(FieldDescriptor::TYPE_UINT64, FieldDescriptor::TYPE_UINT64), + FieldsAre(FieldDescriptor::TYPE_SINT32, FieldDescriptor::TYPE_SINT32), + FieldsAre(FieldDescriptor::TYPE_SINT64, FieldDescriptor::TYPE_SINT64), + FieldsAre(FieldDescriptor::TYPE_FIXED32, + FieldDescriptor::TYPE_FIXED32), + FieldsAre(FieldDescriptor::TYPE_FIXED64, + FieldDescriptor::TYPE_FIXED64), + FieldsAre(FieldDescriptor::TYPE_SFIXED32, + FieldDescriptor::TYPE_SFIXED32), + FieldsAre(FieldDescriptor::TYPE_SFIXED64, + FieldDescriptor::TYPE_SFIXED64), + FieldsAre(FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_FLOAT), + FieldsAre(FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_DOUBLE), + FieldsAre(FieldDescriptor::TYPE_BOOL, FieldDescriptor::TYPE_BOOL), + FieldsAre(FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_STRING), + FieldsAre(FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_BYTES), + FieldsAre(FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_ENUM), + FieldsAre(FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_MESSAGE))); +} + +// Every test message has a map field for every entry whose key and value are +// of the entry's types (GetFieldForMapType() check-fails if there is none). +TEST(ValidDataMapTypesTest, EveryMessageTypeHasEveryMapField) { + for (const Descriptor* message : AllTestMessageTypes()) { + for (const MapType& type : ValidDataMapTypes()) { + SCOPED_TRACE(absl::StrCat(message->full_name(), " ", ParamName(type))); + const Descriptor* entry = + GetFieldForMapType(*message, type.key, type.value)->message_type(); + EXPECT_EQ(entry->map_key()->type(), type.key); + EXPECT_EQ(entry->map_value()->type(), type.value); + } + } +} + +// The legacy table; the test names ValidDataOneof.<TYPE>.* depend on it. +TEST(ValidDataOneofTypesTest, LegacyTable) { + EXPECT_THAT( + ValidDataOneofTypes(), + ElementsAre(FieldDescriptor::TYPE_UINT32, FieldDescriptor::TYPE_BOOL, + FieldDescriptor::TYPE_UINT64, FieldDescriptor::TYPE_FLOAT, + FieldDescriptor::TYPE_DOUBLE, FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_BYTES, FieldDescriptor::TYPE_ENUM, + FieldDescriptor::TYPE_MESSAGE)); +} + +// Every test message has a oneof member of every entry's type, and another +// member of a different type to override it with (GetFieldForOneofType() +// would check-fail otherwise). +TEST(ValidDataOneofTypesTest, EveryMessageTypeHasEveryOneofMember) { + for (const Descriptor* message : AllTestMessageTypes()) { + for (FieldDescriptor::Type type : ValidDataOneofTypes()) { + SCOPED_TRACE(absl::StrCat(message->full_name(), " ", + FieldDescriptor::TypeName(type))); + const FieldDescriptor* field = GetFieldForOneofType(*message, type); + const FieldDescriptor* other = + GetFieldForOneofType(*message, type, OneofMember::kOfOtherType); + EXPECT_EQ(field->type(), type); + EXPECT_NE(other->type(), type); + EXPECT_EQ(field->real_containing_oneof(), other->real_containing_oneof()); + } + } +} + +TEST(IsDefaultValueTest, ScalarTypes) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + if (type == FieldDescriptor::TYPE_MESSAGE) continue; + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_TRUE(IsDefaultValue(type, GetDefaultValue(type))); + EXPECT_FALSE(IsDefaultValue(type, GetNonDefaultValue(type))); + // Case 1 of every table is a non-default value. + EXPECT_FALSE(IsDefaultValue(type, ValidDataCases(type)[1].expected)); + } +} + +// Only the canonical encoding of the default value counts. +TEST(IsDefaultValueTest, OverlongVarintZeroIsNotDefault) { + EXPECT_FALSE(IsDefaultValue(FieldDescriptor::TYPE_INT32, + Wire(std::string("\x80\x00", 2)))); +} + +// Messages and groups have no default value that a serializer would omit. +TEST(IsDefaultValueTest, MessageAndGroupAreNeverDefault) { + EXPECT_FALSE(IsDefaultValue(FieldDescriptor::TYPE_MESSAGE, + GetDefaultValue(FieldDescriptor::TYPE_MESSAGE))); + EXPECT_FALSE(IsDefaultValue(FieldDescriptor::TYPE_GROUP, + GetDefaultValue(FieldDescriptor::TYPE_GROUP))); +} + +TEST(HasImplicitPresenceTest, Proto3FamilyOnly) { + EXPECT_TRUE(HasImplicitPresence(*TestAllTypesProto3::descriptor())); + EXPECT_FALSE(HasImplicitPresence(*TestAllTypesProto2::descriptor())); + EXPECT_TRUE(HasImplicitPresence(*TestAllTypesProto3Editions::descriptor())); + EXPECT_FALSE(HasImplicitPresence(*TestAllTypesProto2Editions::descriptor())); + EXPECT_FALSE(HasImplicitPresence(*TestAllTypesEdition2023::descriptor())); +} + +// ... which is exactly the Proto3TestMessageTypes() subset. +TEST(HasImplicitPresenceTest, MatchesProto3TestMessageTypes) { + const std::vector<const Descriptor*> proto3 = Proto3TestMessageTypes(); + for (const Descriptor* message : AllTestMessageTypes()) { + SCOPED_TRACE(message->full_name()); + EXPECT_EQ(HasImplicitPresence(*message), + std::find(proto3.begin(), proto3.end(), message) != proto3.end()); + } +} + // Enumerates FieldDescriptor::Type independently of the table. TEST(AllFieldTypesExceptGroupTest, IsEveryTypeButGroup) { std::set<FieldDescriptor::Type> all_but_group; @@ -33,6 +607,29 @@ UnorderedElementsAreArray(all_but_group)); } +// Every test message has a singular and a repeated field of every entry's type +// (GetFieldForType() would check-fail otherwise). +TEST(AllFieldTypesExceptGroupTest, + EveryMessageTypeHasSingularAndRepeatedField) { + for (const Descriptor* message : AllTestMessageTypes()) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + SCOPED_TRACE(absl::StrCat(message->full_name(), " ", + FieldDescriptor::TypeName(type))); + EXPECT_EQ(GetFieldForType(*message, type, /*repeated=*/false)->type(), + type); + EXPECT_EQ(GetFieldForType(*message, type, /*repeated=*/true)->type(), + type); + } + } +} + +TEST(AllFieldTypesExceptGroupTest, EveryTypeHasValidDataCases) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_FALSE(ValidDataCases(type).empty()); + } +} + TEST(PackableFieldTypesTest, IsThePackableSubsetInTableOrder) { std::vector<FieldDescriptor::Type> packable; for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { @@ -41,6 +638,24 @@ EXPECT_THAT(PackableFieldTypes(), ElementsAreArray(packable)); } +// Every test message has both a [packed = true] and a [packed = false] +// repeated field of every packable type, which is what the packed-field tests +// select with Packedness::kPacked / kUnpacked. +TEST(PackableFieldTypesTest, EveryMessageTypeHasPackedAndUnpackedField) { + for (const Descriptor* message : AllTestMessageTypes()) { + for (FieldDescriptor::Type type : PackableFieldTypes()) { + SCOPED_TRACE(absl::StrCat(message->full_name(), " ", + FieldDescriptor::TypeName(type))); + const FieldDescriptor* packed = GetFieldForType( + *message, type, /*repeated=*/true, Packedness::kPacked); + const FieldDescriptor* unpacked = GetFieldForType( + *message, type, /*repeated=*/true, Packedness::kUnpacked); + EXPECT_TRUE(packed->is_packed()); + EXPECT_FALSE(unpacked->is_packed()); + } + } +} + TEST(LengthDelimitedFieldTypesTest, IsTheNonPackableSubsetInTableOrder) { std::vector<FieldDescriptor::Type> non_packable; for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { @@ -49,6 +664,121 @@ EXPECT_THAT(LengthDelimitedFieldTypes(), ElementsAreArray(non_packable)); } +TEST(LengthDelimitedFieldTypesTest, AreEncodedLengthPrefixed) { + for (FieldDescriptor::Type type : LengthDelimitedFieldTypes()) { + SCOPED_TRACE(FieldDescriptor::TypeName(type)); + EXPECT_EQ(WireTypeForFieldType(type), WireType::kLengthPrefixed); + } +} + +TEST(AllTestMessageTypesTest, LegacyOrder) { + EXPECT_THAT(AllTestMessageTypes(), + ElementsAre(TestAllTypesProto3::descriptor(), + TestAllTypesProto2::descriptor(), + TestAllTypesProto3Editions::descriptor(), + TestAllTypesProto2Editions::descriptor())); +} + +TEST(Proto3TestMessageTypesTest, LegacyOrder) { + EXPECT_THAT(Proto3TestMessageTypes(), + ElementsAre(TestAllTypesProto3::descriptor(), + TestAllTypesProto3Editions::descriptor())); +} + +TEST(Proto3TestMessageTypesTest, IsSubsetOfAllTestMessageTypes) { + EXPECT_THAT(Proto3TestMessageTypes(), IsSubsetOf(AllTestMessageTypes())); +} + +TEST(Proto2TestMessageTypesTest, LegacyOrder) { + EXPECT_THAT(Proto2TestMessageTypes(), + ElementsAre(TestAllTypesProto2::descriptor(), + TestAllTypesProto2Editions::descriptor())); +} + +TEST(Proto2TestMessageTypesTest, IsSubsetOfAllTestMessageTypes) { + EXPECT_THAT(Proto2TestMessageTypes(), IsSubsetOf(AllTestMessageTypes())); +} + +TEST(Proto2TestMessageTypesTest, TogetherWithProto3CoversAllTestMessageTypes) { + std::vector<const Descriptor*> both = Proto2TestMessageTypes(); + for (const Descriptor* message : Proto3TestMessageTypes()) { + both.push_back(message); + } + EXPECT_THAT(both, UnorderedElementsAreArray(AllTestMessageTypes())); +} + +TEST(ParamNameTest, StringKeepsOnlyLettersAndDigits) { + EXPECT_EQ(ParamName("Proto3"), "Proto3"); + EXPECT_EQ(ParamName("Editions_Proto2"), "EditionsProto2"); + EXPECT_EQ(ParamName("a-b.c d/e"), "abcde"); + const std::string underscored = "x_1"; + EXPECT_EQ(ParamName(underscored), "x1"); +} + +TEST(ParamNameTest, MessageTypeIsTheEditionIdentifierWithoutUnderscore) { + EXPECT_EQ(ParamName(TestAllTypesProto3::descriptor()), "Proto3"); + EXPECT_EQ(ParamName(TestAllTypesProto2::descriptor()), "Proto2"); + EXPECT_EQ(ParamName(TestAllTypesProto3Editions::descriptor()), + "EditionsProto3"); + EXPECT_EQ(ParamName(TestAllTypesProto2Editions::descriptor()), + "EditionsProto2"); + EXPECT_EQ(ParamName(TestAllTypesEditionUnstable::descriptor()), + "EditionUnstable"); +} + +TEST(ParamNameTest, FieldTypeIsTheUpperCaseTypeName) { + EXPECT_EQ(ParamName(FieldDescriptor::TYPE_INT32), "INT32"); + EXPECT_EQ(ParamName(FieldDescriptor::TYPE_SFIXED64), "SFIXED64"); + EXPECT_EQ(ParamName(FieldDescriptor::TYPE_MESSAGE), "MESSAGE"); +} + +TEST(ParamNameTest, IntIsDecimal) { + EXPECT_EQ(ParamName(0), "0"); + EXPECT_EQ(ParamName(7), "7"); + EXPECT_EQ(ParamName(1234), "1234"); +} + +TEST(ParamNameTest, MapTypeIsKeyAndValueJoinedByUnderscore) { + EXPECT_EQ(ParamName(MapType{FieldDescriptor::TYPE_INT32, + FieldDescriptor::TYPE_INT32}), + "INT32_INT32"); + EXPECT_EQ(ParamName(MapType{FieldDescriptor::TYPE_STRING, + FieldDescriptor::TYPE_MESSAGE}), + "STRING_MESSAGE"); +} + +// Matches a non-empty string of ASCII letters and digits. (Not +// MatchesRegex("[A-Za-z0-9]+"): gtest's own regex engine, which it falls back +// to on Windows, has no bracket expressions.) +MATCHER(IsAlphanumeric, negation ? "isn't alphanumeric" : "is alphanumeric") { + return !arg.empty() && std::all_of(arg.begin(), arg.end(), [](char c) { + return absl::ascii_isalnum(c); + }); +} + +// Every ParamName() result must be usable as (part of) a gtest parameter name. +TEST(ParamNameTest, ResultsAreValidGtestNameComponents) { + for (const Descriptor* message : AllTestMessageTypes()) { + EXPECT_THAT(ParamName(message), IsAlphanumeric()); + } + for (int i = 1; i <= FieldDescriptor::MAX_TYPE; ++i) { + EXPECT_THAT(ParamName(static_cast<FieldDescriptor::Type>(i)), + IsAlphanumeric()); + } + for (const MapType& type : ValidDataMapTypes()) { + std::vector<std::string> parts = absl::StrSplit(ParamName(type), '_'); + EXPECT_THAT(parts, ElementsAre(IsAlphanumeric(), IsAlphanumeric())); + } +} + +TEST(ParamNameDeathTest, StringWithoutLettersOrDigitsIsFatal) { + EXPECT_DEATH(ParamName("_-_"), "No letters or digits in parameter name"); +} + +TEST(ParamNameDeathTest, NegativeIntIsFatal) { + EXPECT_DEATH(ParamName(-1), "Negative parameters have no valid gtest name"); +} + } // namespace } // namespace conformance } // namespace protobuf
diff --git a/conformance/binary_unknown_fields_test.cc b/conformance/binary_unknown_fields_test.cc new file mode 100644 index 0000000..4fb5cc6 --- /dev/null +++ b/conformance/binary_unknown_fields_test.cc
@@ -0,0 +1,76 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that unknown fields are preserved, and +// preserved in order. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestUnknownMessage() and +// TestUnknownOrdering(); the test names and the requests sent to the testee +// are identical to the legacy ones. + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/unknown_field_set.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// A field number no test message defines. +constexpr uint32_t kUnknownFieldNumber = 666; + +using UnknownFieldsTest = MessageTypeConformanceTest; + +// An unknown varint field must be round-tripped byte for byte. Field 501 +// isn't defined by any of the test messages; the legacy test spelled the input +// as the bytes "\xA8\x1F\x01". +TEST_P(UnknownFieldsTest, UnknownVarint) { + Wire input = VarintField(501, 1); + EXPECT_THAT( + Testee("UnknownVarint").ParseBinary(message(), input).SerializeBinary(), + Yields(RawPayload(input))); +} + +// Implementations must preserve the ordering of different unknown fields for +// the same field number. This is because some field types will accept +// multiple wire types for the same field. For example, repeated primitive +// fields will accept both length-prefixed (packed) and +// varint/fixed32/fixed64 (unpacked) wire types, and reordering these could +// reorder the elements of the repeated field. +TEST_P(UnknownFieldsTest, UnknownOrdering) { + UnknownFieldSet expected; + expected.AddLengthDelimited(kUnknownFieldNumber, "abc"); + expected.AddVarint(kUnknownFieldNumber, 123); + expected.AddLengthDelimited(kUnknownFieldNumber, "def"); + expected.AddVarint(kUnknownFieldNumber, 456); + // The legacy test sent the serialization of a message holding exactly + // these unknown fields; these are its bytes. + Wire input(LengthPrefixedField(kUnknownFieldNumber, "abc"), + VarintField(kUnknownFieldNumber, 123), + LengthPrefixedField(kUnknownFieldNumber, "def"), + VarintField(kUnknownFieldNumber, 456)); + EXPECT_THAT( + Testee("UnknownOrdering").ParseBinary(message(), input).SerializeBinary(), + Yields(WhenParsed(HasUnknownFieldsInOrder(expected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, UnknownFieldsTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_unknown_wire_type_test.cc b/conformance/binary_unknown_wire_type_test.cc new file mode 100644 index 0000000..49e91fe --- /dev/null +++ b/conformance/binary_unknown_wire_type_test.cc
@@ -0,0 +1,76 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that a tag with one of the two undefined +// wire types (6 and 7) is rejected. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestUnknownWireType(); the test names +// and the requests sent to the testee are identical to the legacy ones. + +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::Range; +using ::testing::Values; +using ::testing::ValuesIn; + +// Parameterized over (test message type, wire type, field number, value byte). +// Reporting the message type through MessageUnderTest() makes the base SetUp() +// skip the editions instances when --maximum_edition doesn't cover them. +class UnknownWireTypeTest : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, int, int, int>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + int wire_type() const { return std::get<1>(GetParam()); } + int field_number() const { return std::get<2>(GetParam()); } + int value() const { return std::get<3>(GetParam()); } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(UnknownWireTypeTest, Rejected) { + // Two bytes: a one-byte tag with the unknown wire type, then one byte of + // "value". Tag() can't encode wire type 7, so the tag byte is assembled by + // hand exactly as the legacy test did. + const std::string payload = { + static_cast<char>((field_number() << 3) | wire_type()), + static_cast<char>(value())}; + EXPECT_THAT(Testee(absl::StrCat("UnknownWireType", wire_type(), "_Field", + field_number(), "_Version", value())) + .ParseBinary(message(), Wire(payload)) + .ParseOnly(), + Yields(IsParseError())); +} + +// Instance names are "<Message>_<wire type>_<field number>_<value>". +INSTANTIATE_TEST_SUITE_P(All, UnknownWireTypeTest, + Combine(ValuesIn(AllTestMessageTypes()), Values(6, 7), + Range(0, 4), Range(0, 4)), + TupleParamName<UnknownWireTypeTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_unmatched_group_test.cc b/conformance/binary_unmatched_group_test.cc new file mode 100644 index 0000000..8dc1232 --- /dev/null +++ b/conformance/binary_unmatched_group_test.cc
@@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that group tags without a matching start +// or end tag, or with mismatched field numbers, are rejected. This replaces +// the legacy BinaryAndJsonConformanceSuiteImpl<M>::TestUnmatchedGroup(); the +// test names and the requests sent to the testee are identical to the legacy +// ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// Field numbers as in the legacy tests: 18 is optional_nested_message, 1 and +// 2 are scalar fields, 1234 is undefined, and 201-203 are the Data group and +// its members in the proto2 messages (wrapper fields in the proto3 ones). +using UnmatchedGroupTest = MessageTypeConformanceTest; + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroup) { + EXPECT_THAT(Testee("UnmatchedEndGroup") + .ParseBinary(message(), Tag(201, WireType::kEndGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroupUnknown) { + EXPECT_THAT(Testee("UnmatchedEndGroupUnknown") + .ParseBinary(message(), Tag(1234, WireType::kEndGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroupWrongType) { + EXPECT_THAT(Testee("UnmatchedEndGroupWrongType") + .ParseBinary(message(), Tag(1, WireType::kEndGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroupNestedLen) { + EXPECT_THAT( + Testee("UnmatchedEndGroupNestedLen") + .ParseBinary(message(), + LengthPrefixedField(18, Tag(1234, WireType::kEndGroup))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroupNested) { + EXPECT_THAT( + Testee("UnmatchedEndGroupNested") + .ParseBinary(message(), + DelimitedField(201, Tag(202, WireType::kEndGroup))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedEndGroupWithData) { + EXPECT_THAT( + Testee("UnmatchedEndGroupWithData") + .ParseBinary(message(), Wire(Tag(1, WireType::kEndGroup), + LengthPrefixedField(2, "hello world"))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroup) { + EXPECT_THAT(Testee("UnmatchedStartGroup") + .ParseBinary(message(), Tag(201, WireType::kStartGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroupUnknown) { + EXPECT_THAT(Testee("UnmatchedStartGroupUnknown") + .ParseBinary(message(), Tag(1234, WireType::kStartGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroupWrongType) { + EXPECT_THAT(Testee("UnmatchedStartGroupWrongType") + .ParseBinary(message(), Tag(1, WireType::kStartGroup)) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroupNestedLen) { + EXPECT_THAT( + Testee("UnmatchedStartGroupNestedLen") + .ParseBinary(message(), LengthPrefixedField( + 18, Tag(1234, WireType::kStartGroup))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroupNested) { + EXPECT_THAT( + Testee("UnmatchedStartGroupNested") + .ParseBinary(message(), + DelimitedField(201, Tag(202, WireType::kStartGroup))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, UnmatchedStartGroupWithData) { + EXPECT_THAT( + Testee("UnmatchedStartGroupWithData") + .ParseBinary(message(), Wire(Tag(1, WireType::kStartGroup), + LengthPrefixedField(2, "hello world"))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, MismatchedGroupTags) { + EXPECT_THAT( + Testee("MismatchedGroupTags") + .ParseBinary(message(), Wire(Tag(201, WireType::kStartGroup), + LengthPrefixedField(2, "hello world"), + Tag(202, WireType::kEndGroup))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(UnmatchedGroupTest, MismatchedNestedGroupTags) { + EXPECT_THAT( + Testee("MismatchedNestedGroupTags") + .ParseBinary( + message(), + DelimitedField(201, Wire(Tag(202, WireType::kStartGroup), + LengthPrefixedField(2, "hello world"), + Tag(203, WireType::kEndGroup)))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, UnmatchedGroupTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_unstable_edition_test.cc b/conformance/binary_unstable_edition_test.cc new file mode 100644 index 0000000..4f2d700 --- /dev/null +++ b/conformance/binary_unstable_edition_test.cc
@@ -0,0 +1,89 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests for TestAllTypesEditionUnstable, the test message +// declared under the unstable edition. This holds both the binary-output and +// the JSON-output leg of the legacy suite-level +// BinaryAndJsonConformanceSuite::RunUnstableTests(); the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// Like the legacy suite, these run whenever --maximum_edition is 2023 or +// newer (see ConformanceTest::MessageUnderTest()), not only when it names +// the unstable edition itself. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/test_fixture.h" +#include "conformance/test_protos/test_messages_edition_unstable.pb.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::edition_unstable::TestAllTypesEditionUnstable; + +// Reporting TestAllTypesEditionUnstable through MessageUnderTest() makes the +// base SetUp() skip these tests when --maximum_edition doesn't cover it. +class UnstableEditionTest : public ConformanceTest { + protected: + static const Descriptor* message() { + return TestAllTypesEditionUnstable::descriptor(); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// Field 13 is optional_bytes. +TEST_F(UnstableEditionTest, ValidBytes) { + EXPECT_THAT( + Testee("ValidBytes") + .ParseBinary(message(), LengthPrefixedField(13, "foo")) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_bytes: "foo")pb")))); +} + +TEST_F(UnstableEditionTest, ValidBytesJson) { + EXPECT_THAT( + Testee("ValidBytes") + .ParseBinary(message(), LengthPrefixedField(13, "foo")) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_bytes: "foo")pb")))); +} + +// Field 15 is map_string_bytes; its entries have key = 1 and value = 2. +TEST_F(UnstableEditionTest, ValidMapBytes) { + EXPECT_THAT(Testee("ValidMap.Bytes") + .ParseBinary(message(), + LengthPrefixedField( + 15, Wire(LengthPrefixedField(1, "foo"), + LengthPrefixedField(2, "barbaz")))) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + R"pb(map_string_bytes { key: "foo" value: "barbaz" })pb")))); +} + +TEST_F(UnstableEditionTest, ValidMapBytesJson) { + EXPECT_THAT(Testee("ValidMap.Bytes") + .ParseBinary(message(), + LengthPrefixedField( + 15, Wire(LengthPrefixedField(1, "foo"), + LengthPrefixedField(2, "barbaz")))) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto( + R"pb(map_string_bytes { key: "foo" value: "barbaz" })pb")))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_utf8_test.cc b/conformance/binary_utf8_test.cc new file mode 100644 index 0000000..24056b8 --- /dev/null +++ b/conformance/binary_utf8_test.cc
@@ -0,0 +1,124 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that invalid UTF-8 in a string field is +// rejected, and that a bytes field accepts it. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestInvalidUtf8String() (which only +// ran for the proto3-style message types) and the suite-level +// BinaryAndJsonConformanceSuite::RunUtf8ValidationTests(); the test names and +// the requests sent to the testee are identical to the legacy ones. +// +// TODO: Once conformance tests can express failures that are +// not expected to be fixed, add proto2 coverage here. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// Four bytes that aren't valid UTF-8. +constexpr absl::string_view kInvalidUtf8 = "\xA0\xB0\xC0\xD0"; + +// Instantiated over the proto3-style test message types only: proto3 rejects +// invalid UTF-8 in string fields, proto2 doesn't validate it. +class Utf8StringTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Field 14 is optional_string. +TEST_P(Utf8StringTest, Singular) { + EXPECT_THAT(Testee("RejectInvalidUtf8.String.Singular") + .ParseBinary(message(), LengthPrefixedField(14, kInvalidUtf8)) + .ParseOnly(), + Yields(IsParseError())); +} + +// Field 44 is repeated_string. +TEST_P(Utf8StringTest, Repeated) { + EXPECT_THAT(Testee("RejectInvalidUtf8.String.Repeated") + .ParseBinary(message(), LengthPrefixedField(44, kInvalidUtf8)) + .ParseOnly(), + Yields(IsParseError())); +} + +// Field 113 is oneof_string. +TEST_P(Utf8StringTest, Oneof) { + EXPECT_THAT( + Testee("RejectInvalidUtf8.String.Oneof") + .ParseBinary(message(), LengthPrefixedField(113, kInvalidUtf8)) + .ParseOnly(), + Yields(IsParseError())); +} + +// Field 69 is map_string_string; its entries have key = 1 and value = 2. +TEST_P(Utf8StringTest, MapKey) { + EXPECT_THAT( + Testee("RejectInvalidUtf8.String.MapKey") + .ParseBinary( + message(), + LengthPrefixedField(69, Wire(LengthPrefixedField(1, kInvalidUtf8), + LengthPrefixedField(2, "foo")))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(Utf8StringTest, MapValue) { + EXPECT_THAT( + Testee("RejectInvalidUtf8.String.MapValue") + .ParseBinary(message(), + LengthPrefixedField( + 69, Wire(LengthPrefixedField(1, "foo"), + LengthPrefixedField(2, kInvalidUtf8)))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, Utf8StringTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +// The extension counterparts only exist for TestAllTypesEdition2023, where +// field 133 is the extension_string extension and field 134 is the +// extension_bytes extension. +using Utf8ExtensionTest = Edition2023ConformanceTest; + +TEST_F(Utf8ExtensionTest, String) { + EXPECT_THAT( + Testee(kP1, "RejectInvalidUtf8.String.Extension") + .ParseBinary(message(), LengthPrefixedField(133, kInvalidUtf8)) + .ParseOnly(), + Yields(IsParseError())); +} + +// Bytes fields aren't validated, so the same bytes must be accepted. +TEST_F(Utf8ExtensionTest, Bytes) { + constexpr absl::string_view kExpected = R"pb( + [protobuf_test_messages.editions.extension_bytes]: "\xA0\xB0\xC0\xD0" + )pb"; + EXPECT_THAT( + Testee("AcceptInvalidUtf8.Bytes.Extension") + .ParseBinary(message(), LengthPrefixedField(134, kInvalidUtf8)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_valid_data_map_test.cc b/conformance/binary_valid_data_map_test.cc new file mode 100644 index 0000000..c4f01b4 --- /dev/null +++ b/conformance/binary_valid_data_map_test.cc
@@ -0,0 +1,320 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests for map fields: ValidDataMap.<KEY>.<VALUE>.<Case> +// (REQUIRED, equivalence) checks that a map entry is parsed correctly whatever +// the order, presence and repetition of its key and value, for every key/value +// type combination in ValidDataMapTypes(); and the RECOMMENDED +// .KeyWireTypeMismatch / .ValueWireTypeMismatch tests check that an entry +// whose key or value is encoded with the wrong wire type is either rejected or +// parsed as if that field were unknown. +// +// This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestValidDataForMapType() and +// TestMapEntryWireTypeMismatch(); the test names and the requests sent to the +// testee are identical to the legacy ones. Like the legacy +// RunValidProtobufTest(), each ValidDataMap input is sent twice: once to be +// serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"), and both outputs must be equivalent to the same +// reference message. The wire-type mismatch tests were binary-only. + +#include <cstdint> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::AnyOf; +using ::testing::Combine; +using ::testing::ValuesIn; + +// Map entries are messages with key = 1 and value = 2. +constexpr uint32_t kKeyField = 1; +constexpr uint32_t kValueField = 2; + +// The five key/value type combinations the legacy suite ran the wire-type +// mismatch tests over, in its order: varint, fixed32 and bool scalars, string, +// and a message-valued map. +constexpr MapType kMapEntryWireTypeMismatchTypes[] = { + {FieldDescriptor::TYPE_INT32, FieldDescriptor::TYPE_INT32}, + {FieldDescriptor::TYPE_FIXED32, FieldDescriptor::TYPE_FIXED32}, + {FieldDescriptor::TYPE_BOOL, FieldDescriptor::TYPE_BOOL}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_STRING}, + {FieldDescriptor::TYPE_STRING, FieldDescriptor::TYPE_MESSAGE}, +}; + +// Field `field_number` encoded with a wire type other than +// `correct_wire_type`: a one-byte length-delimited "a" for scalar fields, and +// the varint 1 for length-delimited ones. +Wire MismatchedWireTypeField(uint32_t field_number, + WireType correct_wire_type) { + if (correct_wire_type == WireType::kLengthPrefixed) { + return VarintField(field_number, 1); + } + return LengthPrefixedField(field_number, "a"); +} + +// Common part of the fixtures below: parameterized over (test message type, +// map type), exercising the message's map field with those key and value +// types. Reporting the message type through MessageUnderTest() makes the base +// SetUp() skip the editions instances when --maximum_edition doesn't cover +// them. +class MapTestBase : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, MapType>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + FieldDescriptor::Type key_type() const { return std::get<1>(GetParam()).key; } + FieldDescriptor::Type value_type() const { + return std::get<1>(GetParam()).value; + } + + // The key and value fields of an entry holding the default and the + // non-default sample value of their types (GetDefaultValue() and + // GetNonDefaultValue() with the entry field's tag in front). + Wire DefaultKeyField() const { return Key(GetDefaultValue(key_type())); } + Wire NonDefaultKeyField() const { + return Key(GetNonDefaultValue(key_type())); + } + Wire DefaultValueField() const { + return Value(GetDefaultValue(value_type())); + } + Wire NonDefaultValueField() const { + return Value(GetNonDefaultValue(value_type())); + } + + // The key and value fields of an entry encoded with the wrong wire type. + Wire MismatchedKeyField() const { + return MismatchedWireTypeField(kKeyField, WireTypeForFieldType(key_type())); + } + Wire MismatchedValueField() const { + return MismatchedWireTypeField(kValueField, + WireTypeForFieldType(value_type())); + } + + // `entry` as one entry of the message's map field for (key_type(), + // value_type()). This scans the descriptor (and check-fails if there is no + // such field), hence the function-style name. + Wire MapEntry(const Wire& entry) const { + return LengthPrefixedField( + FieldNumber(*GetFieldForMapType(*message(), key_type(), value_type())), + entry); + } + + // The legacy test name: "ValidDataMap.<KEY>.<VALUE>.<suffix>", e.g. + // "ValidDataMap.INT32.INT32.Default". + std::string TestName(absl::string_view suffix) const { + return absl::StrCat("ValidDataMap.", UpperCaseTypeName(key_type()), ".", + UpperCaseTypeName(value_type()), ".", suffix); + } + + private: + Wire Key(const Wire& payload) const { + return Wire(Tag(kKeyField, WireTypeForFieldType(key_type())), payload); + } + Wire Value(const Wire& payload) const { + return Wire(Tag(kValueField, WireTypeForFieldType(value_type())), payload); + } + + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// Every entry below must parse to the same map as a conforming parser (the +// C++ implementation) reads from it. +class ValidDataMapTest : public MapTestBase {}; + +// An entry with the default key and value, both present. +TEST_P(ValidDataMapTest, Default) { + const Wire input = MapEntry(Wire(DefaultKeyField(), DefaultValueField())); + EXPECT_THAT(Testee(TestName("Default")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, DefaultJson) { + const Wire input = MapEntry(Wire(DefaultKeyField(), DefaultValueField())); + EXPECT_THAT( + Testee(TestName("Default")).ParseBinary(message(), input).SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// An empty entry: the key and value are both missing and take their defaults. +TEST_P(ValidDataMapTest, MissingDefault) { + const Wire input = MapEntry(Wire()); + EXPECT_THAT(Testee(TestName("MissingDefault")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, MissingDefaultJson) { + const Wire input = MapEntry(Wire()); + EXPECT_THAT(Testee(TestName("MissingDefault")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// An entry with a non-default key and value. +TEST_P(ValidDataMapTest, NonDefault) { + const Wire input = + MapEntry(Wire(NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("NonDefault")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, NonDefaultJson) { + const Wire input = + MapEntry(Wire(NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("NonDefault")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// The value before the key. +TEST_P(ValidDataMapTest, Unordered) { + const Wire input = + MapEntry(Wire(NonDefaultValueField(), NonDefaultKeyField())); + EXPECT_THAT(Testee(TestName("Unordered")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, UnorderedJson) { + const Wire input = + MapEntry(Wire(NonDefaultValueField(), NonDefaultKeyField())); + EXPECT_THAT(Testee(TestName("Unordered")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// Two entries with the same key: the second one replaces the first. +TEST_P(ValidDataMapTest, DuplicateKey) { + const Wire first = MapEntry(Wire(NonDefaultKeyField(), DefaultValueField())); + const Wire second = + MapEntry(Wire(NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateKey")) + .ParseBinary(message(), Wire(first, second)) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(second)))); +} + +TEST_P(ValidDataMapTest, DuplicateKeyJson) { + const Wire first = MapEntry(Wire(NonDefaultKeyField(), DefaultValueField())); + const Wire second = + MapEntry(Wire(NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateKey")) + .ParseBinary(message(), Wire(first, second)) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(second)))); +} + +// The key twice within one entry: the last one wins. +TEST_P(ValidDataMapTest, DuplicateKeyInMapEntry) { + const Wire input = MapEntry( + Wire(DefaultKeyField(), NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateKeyInMapEntry")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, DuplicateKeyInMapEntryJson) { + const Wire input = MapEntry( + Wire(DefaultKeyField(), NonDefaultKeyField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateKeyInMapEntry")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +// The value twice within one entry: the last one wins (or, for message +// values, they merge: the same result here). +TEST_P(ValidDataMapTest, DuplicateValueInMapEntry) { + const Wire input = MapEntry( + Wire(NonDefaultKeyField(), DefaultValueField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateValueInMapEntry")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +TEST_P(ValidDataMapTest, DuplicateValueInMapEntryJson) { + const Wire input = MapEntry( + Wire(NonDefaultKeyField(), DefaultValueField(), NonDefaultValueField())); + EXPECT_THAT(Testee(TestName("DuplicateValueInMapEntry")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(input)))); +} + +INSTANTIATE_TEST_SUITE_P(All, ValidDataMapTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(ValidDataMapTypes())), + TupleParamName<ValidDataMapTest::ParamType>); + +// An entry whose key or value is encoded with the wrong wire type. A parser +// may reject it or treat the mismatched field as unknown, so that the key or +// value takes its default (the C++ implementation does the latter); what it +// must not do is decode the bytes as a value of the declared type. +class MapEntryWireTypeMismatchTest : public MapTestBase { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// The key has the wrong wire type; the value is still well formed. +TEST_P(MapEntryWireTypeMismatchTest, KeyWireTypeMismatch) { + const Wire input = + MapEntry(Wire(MismatchedKeyField(), NonDefaultValueField())); + EXPECT_THAT( + Testee(TestName("KeyWireTypeMismatch")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsBinaryProto(input))))); +} + +// The value has the wrong wire type; the key is still well formed. +TEST_P(MapEntryWireTypeMismatchTest, ValueWireTypeMismatch) { + const Wire input = + MapEntry(Wire(NonDefaultKeyField(), MismatchedValueField())); + EXPECT_THAT( + Testee(TestName("ValueWireTypeMismatch")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsBinaryProto(input))))); +} + +INSTANTIATE_TEST_SUITE_P( + All, MapEntryWireTypeMismatchTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(kMapEntryWireTypeMismatchTypes)), + TupleParamName<MapEntryWireTypeMismatchTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_valid_data_repeated_test.cc b/conformance/binary_valid_data_repeated_test.cc new file mode 100644 index 0000000..ffe3717 --- /dev/null +++ b/conformance/binary_valid_data_repeated_test.cc
@@ -0,0 +1,258 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that a repeated field accepts every valid +// encoding of its values, packed or unpacked, and that the output uses the +// packing the field declares. For every packable type, ValidDataCases(type) +// is sent as a whole to the message's repeated field, in both encodings: +// ValidDataRepeated.<TYPE>.UnpackedInput / .PackedInput (REQUIRED, +// equivalence) and the byte-exact RECOMMENDED variants +// .<Un>PackedInput.DefaultOutput (the field with the message's default +// packing), .<Un>PackedInput.PackedOutput (the [packed = true] field) and +// .<Un>PackedInput.UnpackedOutput (the [packed = false] field). The +// non-packable types (string, bytes, message) only have the REQUIRED +// ValidDataRepeated.<TYPE>. +// +// This replaces the repeated-field part of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestValidDataForType(); the test names +// and the requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidBinaryProtobufTest() + RunValidProtobufToJsonTest() +// pair, each REQUIRED input is sent twice: once to be serialized as binary +// ("<name>.ProtobufOutput") and once as JSON ("<name>.JsonOutput"), both +// expecting the same canonical message. The byte-exact RECOMMENDED tests +// only have a binary leg, as before. + +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// Parameterized over (test message type, field type). Reporting the message +// type through MessageUnderTest() makes the base SetUp() skip the editions +// instances when --maximum_edition doesn't cover them. +class ValidDataRepeatedTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, FieldDescriptor::Type>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + FieldDescriptor::Type type() const { return std::get<1>(GetParam()); } + + // The message's repeated fields of type(): its first repeated field of + // that type, with the message's default packing (e.g. repeated_int32, or + // repeated_nested_enum for ENUM), and the ones declared [packed = true] and + // [packed = false]. These scan the descriptor (and check-fail if + // there is no such field), hence the function-style names. + const FieldDescriptor* RepeatedField() const { + return GetFieldForType(*message(), type(), /*repeated=*/true); + } + const FieldDescriptor* PackedField() const { + return GetFieldForType(*message(), type(), /*repeated=*/true, + Packedness::kPacked); + } + const FieldDescriptor* UnpackedField() const { + return GetFieldForType(*message(), type(), /*repeated=*/true, + Packedness::kUnpacked); + } + + // All cases of type() as `field`, one tag per value, as sent to the testee + // (the inputs) or as a conforming implementation encodes them (the canonical + // forms). + Wire UnpackedInput(const FieldDescriptor* field) const { + return UnpackedCases(field, &ValidDataCase::input); + } + Wire UnpackedExpected(const FieldDescriptor* field) const { + return UnpackedCases(field, &ValidDataCase::expected); + } + + // The same values in one packed `field`. + Wire PackedInput(const FieldDescriptor* field) const { + return PackedCases(field, &ValidDataCase::input); + } + Wire PackedExpected(const FieldDescriptor* field) const { + return PackedCases(field, &ValidDataCase::expected); + } + + // The canonical forms encoded as RepeatedField() declares: packed by + // default in proto3, unpacked in proto2. + Wire DefaultExpected() const { + return RepeatedField()->is_packed() ? PackedExpected(RepeatedField()) + : UnpackedExpected(RepeatedField()); + } + + // The legacy test name: "ValidDataRepeated.<TYPE><suffix>", e.g. + // "ValidDataRepeated.INT32.PackedInput.DefaultOutput". + std::string TestName(absl::string_view suffix = "") const { + return absl::StrCat("ValidDataRepeated.", UpperCaseTypeName(type()), + suffix); + } + + private: + // One `column` of every case of type(), each as its own occurrence of + // `field` ... + Wire UnpackedCases(const FieldDescriptor* field, + Wire ValidDataCase::* column) const { + const Wire tag = Tag(FieldNumber(*field), WireTypeForFieldType(type())); + std::string bytes; + for (const ValidDataCase& value : ValidDataCases(type())) { + absl::StrAppend(&bytes, tag.data(), (value.*column).data()); + } + return Wire(bytes); + } + + // ... or concatenated into one packed `field`. + Wire PackedCases(const FieldDescriptor* field, + Wire ValidDataCase::* column) const { + std::string bytes; + for (const ValidDataCase& value : ValidDataCases(type())) { + absl::StrAppend(&bytes, (value.*column).data()); + } + return LengthPrefixedField(FieldNumber(*field), bytes); + } + + TestPriority DefaultPriority() const override { return kP1; } + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// Both encodings of the default field must be accepted and yield the same +// values, whether serialized as binary or as JSON. +TEST_P(ValidDataRepeatedTest, UnpackedInput) { + EXPECT_THAT( + Testee(kP0, TestName(".UnpackedInput")) + .ParseBinary(message(), UnpackedInput(RepeatedField())) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(PackedExpected(RepeatedField()))))); +} + +TEST_P(ValidDataRepeatedTest, UnpackedInputJson) { + EXPECT_THAT( + Testee(kP0, TestName(".UnpackedInput")) + .ParseBinary(message(), UnpackedInput(RepeatedField())) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(PackedExpected(RepeatedField()))))); +} + +TEST_P(ValidDataRepeatedTest, PackedInput) { + EXPECT_THAT( + Testee(kP0, TestName(".PackedInput")) + .ParseBinary(message(), PackedInput(RepeatedField())) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(PackedExpected(RepeatedField()))))); +} + +TEST_P(ValidDataRepeatedTest, PackedInputJson) { + EXPECT_THAT( + Testee(kP0, TestName(".PackedInput")) + .ParseBinary(message(), PackedInput(RepeatedField())) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(PackedExpected(RepeatedField()))))); +} + +// The default field must be serialized with its declared packing whatever +// the input's encoding was ... +TEST_P(ValidDataRepeatedTest, UnpackedInputDefaultOutput) { + EXPECT_THAT(Testee(TestName(".UnpackedInput.DefaultOutput")) + .ParseBinary(message(), UnpackedInput(RepeatedField())) + .SerializeBinary(), + Yields(RawPayload(DefaultExpected()))); +} + +TEST_P(ValidDataRepeatedTest, PackedInputDefaultOutput) { + EXPECT_THAT(Testee(TestName(".PackedInput.DefaultOutput")) + .ParseBinary(message(), PackedInput(RepeatedField())) + .SerializeBinary(), + Yields(RawPayload(DefaultExpected()))); +} + +// ... and so must the explicitly packed ... +TEST_P(ValidDataRepeatedTest, UnpackedInputPackedOutput) { + EXPECT_THAT(Testee(TestName(".UnpackedInput.PackedOutput")) + .ParseBinary(message(), UnpackedInput(PackedField())) + .SerializeBinary(), + Yields(RawPayload(PackedExpected(PackedField())))); +} + +TEST_P(ValidDataRepeatedTest, PackedInputPackedOutput) { + EXPECT_THAT(Testee(TestName(".PackedInput.PackedOutput")) + .ParseBinary(message(), PackedInput(PackedField())) + .SerializeBinary(), + Yields(RawPayload(PackedExpected(PackedField())))); +} + +// ... and explicitly unpacked fields. +TEST_P(ValidDataRepeatedTest, UnpackedInputUnpackedOutput) { + EXPECT_THAT(Testee(TestName(".UnpackedInput.UnpackedOutput")) + .ParseBinary(message(), UnpackedInput(UnpackedField())) + .SerializeBinary(), + Yields(RawPayload(UnpackedExpected(UnpackedField())))); +} + +TEST_P(ValidDataRepeatedTest, PackedInputUnpackedOutput) { + EXPECT_THAT(Testee(TestName(".PackedInput.UnpackedOutput")) + .ParseBinary(message(), PackedInput(UnpackedField())) + .SerializeBinary(), + Yields(RawPayload(UnpackedExpected(UnpackedField())))); +} + +INSTANTIATE_TEST_SUITE_P(All, ValidDataRepeatedTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(PackableFieldTypes())), + TupleParamName<ValidDataRepeatedTest::ParamType>); + +// The non-packable types only have one encoding; the conformance name has no +// suffix. Of the inherited helpers only RepeatedField(), UnpackedInput() and +// UnpackedExpected() apply here; the packed ones would check-fail. +class ValidDataRepeatedNonPackableTest : public ValidDataRepeatedTest { + private: + TestPriority DefaultPriority() const override { return kP0; } +}; + +TEST_P(ValidDataRepeatedNonPackableTest, Repeated) { + EXPECT_THAT( + Testee(TestName()) + .ParseBinary(message(), UnpackedInput(RepeatedField())) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(UnpackedExpected(RepeatedField()))))); +} + +TEST_P(ValidDataRepeatedNonPackableTest, RepeatedJson) { + EXPECT_THAT( + Testee(TestName()) + .ParseBinary(message(), UnpackedInput(RepeatedField())) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(UnpackedExpected(RepeatedField()))))); +} + +INSTANTIATE_TEST_SUITE_P( + All, ValidDataRepeatedNonPackableTest, + Combine(ValuesIn(AllTestMessageTypes()), + ValuesIn(LengthDelimitedFieldTypes())), + TupleParamName<ValidDataRepeatedNonPackableTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/binary_valid_data_scalar_test.cc b/conformance/binary_valid_data_scalar_test.cc new file mode 100644 index 0000000..cdbe7f2 --- /dev/null +++ b/conformance/binary_valid_data_scalar_test.cc
@@ -0,0 +1,200 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Binary conformance tests checking that every valid encoding of a singular +// field's value is parsed correctly: ValidDataScalar.<TYPE>[i] (REQUIRED, +// equivalence) and ValidDataScalarBinary.<TYPE>[i] (RECOMMENDED, byte-exact) +// for case i of ValidDataCases(type), and RepeatedScalarSelectsLast.<TYPE>, +// which sends all cases for the singular field at once and expects the last +// one. This replaces the singular-field part of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::TestValidDataForType(); the test names +// and the requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidBinaryProtobufTest() + RunValidProtobufToJsonTest() +// pair, each REQUIRED input is sent twice: once to be serialized as binary +// ("<name>.ProtobufOutput") and once as JSON ("<name>.JsonOutput"), both +// expecting the same canonical message. + +#include <cstddef> +#include <tuple> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// AllFieldTypesExceptGroup() without TYPE_MESSAGE: repeated occurrences of a +// singular message field are merged, not replaced, which binary_merge_test.cc +// covers. +std::vector<FieldDescriptor::Type> SelectsLastTypes() { + std::vector<FieldDescriptor::Type> types; + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + if (type != FieldDescriptor::TYPE_MESSAGE) types.push_back(type); + } + return types; +} + +// Common part of the fixtures below: parameterized over a tuple whose first +// two elements are the test message type and the field type, exercising the +// message's singular field of that type. Reporting the message type through +// MessageUnderTest() makes the base SetUp() skip the editions instances when +// --maximum_edition doesn't cover them. +template <typename Param> +class ValidDataTestBase : public ConformanceTest, + public testing::WithParamInterface<Param> { + protected: + const Descriptor* message() const { return std::get<0>(this->GetParam()); } + FieldDescriptor::Type type() const { return std::get<1>(this->GetParam()); } + + // `payload` (a value without a tag) as the message's singular field of + // type(). This scans the descriptor (and check-fails if there is no such + // field), hence the function-style name. + Wire FieldWithValue(const Wire& payload) const { + return Wire(Tag(FieldNumber(*GetFieldForType(*message(), type(), + /*repeated=*/false)), + WireTypeForFieldType(type())), + payload); + } + + // The legacy test name: "<prefix>.<TYPE><suffix>", e.g. + // "ValidDataScalar.INT32[3]". + std::string TestName(absl::string_view prefix, + absl::string_view suffix = "") const { + return absl::StrCat(prefix, ".", UpperCaseTypeName(type()), suffix); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// One gtest per (message type, field type, case index), so that every +// conformance test is its own gtest and no test body loops or skips. +class ValidDataScalarTest + : public ValidDataTestBase< + std::tuple<const Descriptor*, FieldDescriptor::Type, int>> { + protected: + int index() const { return std::get<2>(GetParam()); } + const ValidDataCase& value() const { + return ValidDataCases(type())[static_cast<size_t>(index())]; + } + + // The canonical serialization of the parsed input. A proto3 message + // doesn't serialize a singular scalar field holding its default value, so + // the expected output is then empty; this is the legacy suite's proto3 + // rule. + Wire Expected() const { + if (HasImplicitPresence(*message()) && + IsDefaultValue(type(), value().expected)) { + return Wire(); + } + return FieldWithValue(value().expected); + } + + std::string TestName(absl::string_view prefix) const { + return ValidDataTestBase::TestName(prefix, absl::StrCat("[", index(), "]")); + } +}; + +// AllTestMessageTypes() x AllFieldTypesExceptGroup() x the case indices of each +// type. +std::vector<ValidDataScalarTest::ParamType> ValidDataScalarParams() { + std::vector<ValidDataScalarTest::ParamType> params; + for (const Descriptor* message : AllTestMessageTypes()) { + for (FieldDescriptor::Type type : AllFieldTypesExceptGroup()) { + const int count = static_cast<int>(ValidDataCases(type).size()); + for (int i = 0; i < count; ++i) params.emplace_back(message, type, i); + } + } + return params; +} + +// The parsed value must be equivalent to the case's canonical form ... +TEST_P(ValidDataScalarTest, Scalar) { + EXPECT_THAT(Testee(TestName("ValidDataScalar")) + .ParseBinary(message(), FieldWithValue(value().input)) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto(Expected())))); +} + +// ... also when serialized as JSON ... +TEST_P(ValidDataScalarTest, ScalarJson) { + EXPECT_THAT(Testee(TestName("ValidDataScalar")) + .ParseBinary(message(), FieldWithValue(value().input)) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto(Expected())))); +} + +// ... and its binary serialization must be exactly the canonical form. +TEST_P(ValidDataScalarTest, ScalarBinary) { + EXPECT_THAT(Testee(kP1, TestName("ValidDataScalarBinary")) + .ParseBinary(message(), FieldWithValue(value().input)) + .SerializeBinary(), + Yields(RawPayload(Expected()))); +} + +INSTANTIATE_TEST_SUITE_P(All, ValidDataScalarTest, + ValuesIn(ValidDataScalarParams()), + TupleParamName<ValidDataScalarTest::ParamType>); + +// Several occurrences of a singular scalar field: the last one wins. The +// input is every case of the type in order, so the last case's canonical form +// is expected. Unlike the singular tests above, the legacy test didn't apply +// the proto3 default rule here (no type's last case is a default value +// anyway). +class RepeatedScalarSelectsLastTest + : public ValidDataTestBase< + std::tuple<const Descriptor*, FieldDescriptor::Type>> {}; + +TEST_P(RepeatedScalarSelectsLastTest, SelectsLast) { + Wire input; + for (const ValidDataCase& value : ValidDataCases(type())) { + input = Wire(input, FieldWithValue(value.input)); + } + EXPECT_THAT(Testee(TestName("RepeatedScalarSelectsLast")) + .ParseBinary(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsBinaryProto( + FieldWithValue(ValidDataCases(type()).back().expected))))); +} + +TEST_P(RepeatedScalarSelectsLastTest, SelectsLastJson) { + Wire input; + for (const ValidDataCase& value : ValidDataCases(type())) { + input = Wire(input, FieldWithValue(value.input)); + } + EXPECT_THAT(Testee(TestName("RepeatedScalarSelectsLast")) + .ParseBinary(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsBinaryProto( + FieldWithValue(ValidDataCases(type()).back().expected))))); +} + +INSTANTIATE_TEST_SUITE_P( + All, RepeatedScalarSelectsLastTest, + Combine(ValuesIn(AllTestMessageTypes()), ValuesIn(SelectsLastTypes())), + TupleParamName<RepeatedScalarSelectsLastTest::ParamType>); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/bzl_tests/BUILD b/conformance/bzl_tests/BUILD new file mode 100644 index 0000000..f355b12 --- /dev/null +++ b/conformance/bzl_tests/BUILD
@@ -0,0 +1,59 @@ +# Copyright (c) 2026, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# Analysis tests for //conformance:conformance.bzl. They +# live in this leaf package, which nothing depends on, because they load +# rules_testing: a dev-only dependency of the open-source build that the +# exported conformance package itself must not require. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("//conformance:conformance.bzl", "failure_lists") +load(":conformance_bzl_test.bzl", "conformance_bzl_test_suite") + +package(default_applicable_licenses = ["//:license"]) + +# The analysis tests and their subjects (manual conformance_tests that are +# never run). The subjects' failure lists are the fixtures in +# bzl_test_failure_lists/, and their testees are the stubs below. +conformance_bzl_test_suite(name = "conformance_bzl_test") + +failure_lists(name = "bzl_test_failure_lists") + +cc_binary( + name = "stub_testee", + testonly = 1, + srcs = ["stub_testee.cc"], +) + +# The in_process = True counterpart: a library that would define +# MakeTesteeRunner() (see testee_runner.h) if it were ever linked. +cc_library( + name = "stub_in_process_testee", + testonly = 1, + srcs = ["stub_in_process_testee.cc"], + deps = [ + "//conformance:test_runner", + "//conformance:testee_runner", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + ], + alwayslink = 1, +) + +bzl_library( + name = "conformance_bzl_test_bzl", + srcs = ["conformance_bzl_test.bzl"], + visibility = ["//visibility:private"], + deps = [ + "//conformance:conformance_bzl", + "@rules_testing//lib:analysis_test_bzl", + "@rules_testing//lib:truth_bzl", + "@rules_testing//lib:util_bzl", + ], +)
diff --git a/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_binary.txt b/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_binary.txt new file mode 100644 index 0000000..7e3c8a4 --- /dev/null +++ b/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_binary.txt
@@ -0,0 +1,2 @@ +# Fixture for conformance_bzl_test (see conformance_bzl_test.bzl); the test +# it belongs to is never run.
diff --git a/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_performance.txt b/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_performance.txt new file mode 100644 index 0000000..7e3c8a4 --- /dev/null +++ b/conformance/bzl_tests/bzl_test_failure_lists/conformance_bzl_test_with_failure_list_subject_performance.txt
@@ -0,0 +1,2 @@ +# Fixture for conformance_bzl_test (see conformance_bzl_test.bzl); the test +# it belongs to is never run.
diff --git a/conformance/bzl_tests/conformance_bzl_test.bzl b/conformance/bzl_tests/conformance_bzl_test.bzl new file mode 100644 index 0000000..a6cd8b7 --- /dev/null +++ b/conformance/bzl_tests/conformance_bzl_test.bzl
@@ -0,0 +1,275 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2026 Google LLC. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Analysis tests for the conformance_test() macro in conformance.bzl. + +These check the targets the macro generates and the command lines it passes +to them. The `fail()` paths (unknown suite, misnamed failure list, foreign +failure_lists) happen at loading time and so can't be covered here. + +The tests live in their own leaf package (bzl_tests) because they load +rules_testing, which is only a dev dependency of the open-source build; the +exported conformance package must not depend on it. +""" + +load( + "@rules_testing//lib:analysis_test.bzl", + "analysis_test", +) +load( + "@rules_testing//lib:truth.bzl", + "matching", + "subjects", +) +load("@rules_testing//lib:util.bzl", "util") +load("//conformance:conformance.bzl", "conformance_test") + +# The failure_lists() the subjects use, and the stub testees (a binary and an +# in-process library); see BUILD. +_FAILURE_LISTS = "bzl_test_failure_lists" +_TESTEE = ":stub_testee" +_IN_PROCESS_TESTEE = ":stub_in_process_testee" + +def conformance_bzl_test_suite(name): + """Creates the analysis tests for conformance_test() and a suite of them. + + Every generated target is prefixed with `name`. The `with_failure_list` + test needs the fixtures `<name>_with_failure_list_subject_binary.txt` and + `<name>_with_failure_list_subject_performance.txt` in the package's + failure lists directory. + + Args: + name: The name of the test_suite, and the prefix of its tests. + """ + tests = [] + for suffix, setup in [ + ("with_failure_list", _test_with_failure_list), + ("without_failure_list", _test_without_failure_list), + ("options", _test_options), + ("in_process", _test_in_process), + ]: + test_name = "%s_%s" % (name, suffix) + setup(test_name) + tests.append(":" + test_name) + native.test_suite( + name = name, + tests = tests, + ) + +# The conformance_test() under test can't be named after the analysis test, +# since its test_suite would clash with it. +def _subject(name): + return name + "_subject" + +def _subject_of(target, suite = "binary"): + """The name of the conformance_test() whose `suite` test is `target`.""" + return target.label.name.removesuffix("_%s_test" % suite) + +def _failure_list_path(target, suite = "binary"): + """The workspace-relative path of `target`'s failure list. + + `target` is the `suite` test of a conformance_test(). + """ + return "%s/%s/%s_%s.txt" % ( + target.label.package, + _FAILURE_LISTS, + _subject_of(target, suite), + suite, + ) + +def _label_names(value, *, meta): + """Subject factory for a label list attribute, comparing target names. + + Labels from other repositories are ignored: those are implicit dependencies + added by the rule itself (e.g. @bazel_tools//tools/cpp:link_extra_lib on a + cc_test), not by the macro under test. + """ + return subjects.collection( + [target.label.name for target in value if not target.label.repo_name], + meta = meta, + ) + +def _test_with_failure_list(name): + for suite in ("binary", "performance"): + fixture = "%s/%s_%s.txt" % (_FAILURE_LISTS, _subject(name), suite) + if not native.glob([fixture], allow_empty = True): + fail("%s: the fixture failure list %s doesn't exist." % + (name, fixture)) + util.helper_target( + conformance_test, + name = _subject(name), + testee = _TESTEE, + failure_lists = ":" + _FAILURE_LISTS, + ) + analysis_test( + name = name, + targets = { + "suite": _subject(name), + "test": _subject(name) + "_binary_test", + "text_test": _subject(name) + "_text_test", + "performance_test": _subject(name) + "_performance_test", + "testee": _TESTEE, + }, + impl = _test_with_failure_list_impl, + ) + +def _test_with_failure_list_impl(env, targets): + test = env.expect.that_target(targets.test) + failure_list = _failure_list_path(targets.test) + test.attr("args", factory = subjects.collection).contains_exactly([ + "--testee_binary=$(rootpath %s)" % _TESTEE, + "--failure_list=" + failure_list, + ]).in_order() + test.attr("deps", factory = _label_names).contains_exactly([ + "binary_conformance_tests", + "test_environment_main", + "forked_testee", + ]) + test.tags().contains("conformance") + + # The testee's path is taken from the target rather than spelled out so + # that the check also holds where the binary is stub_testee.exe. + testee = targets.testee[DefaultInfo].files_to_run.executable + test.runfiles().contains_at_least([ + "{workspace}/" + failure_list, + "{workspace}/" + testee.short_path, + ]) + + # The text suite's test links that suite instead; it has no fixture list, + # so --fix is told where to create one (see _test_without_failure_list). + text_test = env.expect.that_target(targets.text_test) + text_failure_list = _failure_list_path(targets.text_test, suite = "text") + text_test.attr("args", factory = subjects.collection).contains_exactly([ + "--testee_binary=$(rootpath %s)" % _TESTEE, + "--fix_output_file=" + text_failure_list, + ]).in_order() + text_test.attr("deps", factory = _label_names).contains_exactly([ + "text_conformance_tests", + "test_environment_main", + "forked_testee", + ]) + + # The performance suite is one of the default suites: it gets a test of + # its own, built like the others from its own library and failure list. + performance_test = env.expect.that_target(targets.performance_test) + performance_failure_list = _failure_list_path( + targets.performance_test, + suite = "performance", + ) + args_subject = performance_test.attr("args", factory = subjects.collection) + args_subject.contains_exactly([ + "--testee_binary=$(rootpath %s)" % _TESTEE, + "--failure_list=" + performance_failure_list, + ]).in_order() + performance_test.attr("deps", factory = _label_names).contains_exactly([ + "performance_conformance_tests", + "test_environment_main", + "forked_testee", + ]) + performance_test.runfiles().contains_at_least([ + "{workspace}/" + performance_failure_list, + ]) + + # No suite is kept off the sanitizers any more: the macro adds no tag but + # "conformance" to the caller's. + performance_test.tags().contains("conformance") + performance_test.tags().not_contains("noasan") + test.tags().not_contains("noasan") + + # One test per known conformance suite (see _SUITES in conformance.bzl). + suite = env.expect.that_target(targets.suite) + suite.attr("tests", factory = _label_names).contains_exactly([ + _subject_of(targets.test) + "_binary_test", + _subject_of(targets.test) + "_text_test", + _subject_of(targets.test) + "_json_test", + _subject_of(targets.test) + "_performance_test", + ]) + + # helper_target tags the tests manual (among others) so that they never run + # outside these analysis tests; the tags must reach the test_suite too. + suite.tags().contains("manual") + +def _test_without_failure_list(name): + util.helper_target( + conformance_test, + name = _subject(name), + testee = _TESTEE, + failure_lists = ":" + _FAILURE_LISTS, + ) + analysis_test( + name = name, + target = _subject(name) + "_binary_test", + impl = _test_without_failure_list_impl, + ) + +def _test_without_failure_list_impl(env, target): + # With no failure list, --fix is told where to create one instead. + test = env.expect.that_target(target) + test.attr("args", factory = subjects.collection).contains_exactly([ + "--testee_binary=$(rootpath %s)" % _TESTEE, + "--fix_output_file=" + _failure_list_path(target), + ]).in_order() + +def _test_options(name): + util.helper_target( + conformance_test, + name = _subject(name), + testee = _TESTEE, + failure_lists = ":" + _FAILURE_LISTS, + suites = ["binary"], + maximum_edition = "2023", + enforcement_level = 0, + tags = ["noasan"], + ) + analysis_test( + name = name, + target = _subject(name) + "_binary_test", + impl = _test_options_impl, + ) + +def _test_options_impl(env, target): + test = env.expect.that_target(target) + test.attr("args", factory = subjects.collection).contains_exactly([ + "--testee_binary=$(rootpath %s)" % _TESTEE, + "--fix_output_file=" + _failure_list_path(target), + "--maximum_edition=EDITION_2023", + "--enforcement_level=0", + ]).in_order() + test.tags().contains_at_least(["conformance", "noasan"]) + +def _test_in_process(name): + util.helper_target( + conformance_test, + name = _subject(name), + testee = _IN_PROCESS_TESTEE, + failure_lists = ":" + _FAILURE_LISTS, + suites = ["binary"], + in_process = True, + ) + analysis_test( + name = name, + target = _subject(name) + "_binary_test", + impl = _test_in_process_impl, + ) + +def _test_in_process_impl(env, target): + # The testee is linked in place of forked_testee and gets no + # --testee_binary; the other flags are unchanged. + test = env.expect.that_target(target) + test.attr("args", factory = subjects.collection).contains_exactly([ + "--fix_output_file=" + _failure_list_path(target), + ]).in_order() + test.attr("deps", factory = _label_names).contains_exactly([ + "binary_conformance_tests", + "test_environment_main", + "stub_in_process_testee", + ]) + test.attr("data", factory = _label_names).contains_exactly([_FAILURE_LISTS]) + test.runfiles().not_contains_predicate( + matching.str_matches("*/stub_testee*"), + ) + test.tags().contains("conformance")
diff --git a/conformance/bzl_tests/stub_in_process_testee.cc b/conformance/bzl_tests/stub_in_process_testee.cc new file mode 100644 index 0000000..e623b7e --- /dev/null +++ b/conformance/bzl_tests/stub_in_process_testee.cc
@@ -0,0 +1,32 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A stand-in in-process testee for the analysis tests of +// conformance_test(in_process = True) (see conformance_bzl_test.bzl). It is +// only ever analyzed, never linked or run. + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +std::unique_ptr<ConformanceTestRunner> MakeTesteeRunner( + absl::string_view testee_binary, + absl::Span<const std::string> testee_args) { + return nullptr; +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/bzl_tests/stub_testee.cc b/conformance/bzl_tests/stub_testee.cc new file mode 100644 index 0000000..eb1be33 --- /dev/null +++ b/conformance/bzl_tests/stub_testee.cc
@@ -0,0 +1,10 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A stand-in testee for the analysis tests of conformance_test() (see +// conformance_bzl_test.bzl). It is only ever analyzed, never run. +int main() { return 0; }
diff --git a/conformance/conformance.bzl b/conformance/conformance.bzl new file mode 100644 index 0000000..e66880b --- /dev/null +++ b/conformance/conformance.bzl
@@ -0,0 +1,308 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2026 Google LLC. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Starlark macros for the gtest-based Protobuf conformance tests. + +A conformance *suite* (e.g. `binary`) is a `cc_library` of gtest-based tests +that drive a testee through the conformance protocol; see +`conformance_suite()`. A conformance *test* runs every suite against one +testee, one `cc_test` per suite, with that testee's failure lists; see +`conformance_test()` and `failure_lists()`. The testee is either a binary +that the test forks and talks to over a pipe, or (`in_process = True`) a +`cc_library` linked into the test binary itself. + +Typical use, in the package holding the testee: + + failure_lists(name = "failure_lists") + + conformance_test( + name = "cpp", + testee = ":conformance_cpp", + failure_lists = ":failure_lists", + ) + +This creates `:cpp_binary_test`, `:cpp_json_test`, `:cpp_performance_test` +and `:cpp_text_test` (one `:cpp_<suite>_test` per suite), plus a `:cpp` +test_suite running all of them. The expected failures for `:cpp_binary_test` +live in `failure_lists/cpp_binary.txt`; when that file doesn't exist, the +testee is expected to pass everything. Either way + + blaze run :cpp_binary_test -- --fix + +creates or rewrites the file to match the observed results. + +conformance_test() supersedes the macro of the same name in defs.bzl, which +wraps the conformance_test_runner command line (legacy `--failure_list` / +`--text_format_failure_list` flags) and stays until the language runtimes +have migrated to this macro. +TODO: retire defs.bzl once they have. + +PLEASE DO NOT DEPEND ON THE CONTENTS OF THIS FILE, IT IS UNSTABLE. +""" + +load("@rules_cc//cc:cc_test.bzl", "cc_test") +load("@rules_cc//cc:defs.bzl", "cc_library") + +# The conformance suites, i.e. the valid `name`s of conformance_suite(). +# conformance_test() creates one test per suite, and failure lists are named +# after them (see failure_lists()). Every suite but `performance` covers one +# wire format; `performance` holds the tests that stress the testee (deeply +# nested and very large payloads) for every format, which are kept apart +# because they are slow and their expected failures are unrelated to the +# format's. Every gtest fixture in the performance suite is named +# `*PerformanceTest` and no fixture outside it is: that is how +# conformance_test_runner's --performance tells them apart (see +# test_fixture.h). +_SUITES = ["binary", "json", "performance", "text"] + +# The gtest main that turns a suite library into a conformance test binary, and +# the testee it links by default: --testee_binary run as a subprocess (see +# testee_runner.h for how the testee is chosen at link time). +_TEST_ENVIRONMENT_MAIN = Label( + "//conformance:test_environment_main", +) +_FORKED_TESTEE = Label("//conformance:forked_testee") + +def _suite_library(suite): + """The label of the cc_library that conformance_suite(suite) creates.""" + return Label( + "//conformance:%s_conformance_tests" % suite, + ) + +def _check_suite(suite, context): + if suite not in _SUITES: + fail("%s: unknown conformance suite %r; the known suites are %s." % + (context, suite, ", ".join(_SUITES))) + +def conformance_suite(name, srcs, deps = [], **kwargs): + """Defines a conformance suite: a library of gtest-based conformance tests. + + Creates `cc_library(name = name + "_conformance_tests", alwayslink = 1, + testonly = 1, ...)`; no target named `name` itself is created, so + depend on `:<name>_conformance_tests`. Linking it into a binary that uses + `:test_environment_main` (which conformance_test() does) registers every + test in `srcs`; the library itself must not depend on any gtest main. + + Must be used in //conformance, which is where + conformance_test() looks the suite libraries up. + + Args: + name: The suite's name, one of the known suites (`binary`, ...). + srcs: The test sources. Fixtures in the `performance` suite must be + named `*PerformanceTest`; fixtures in any other suite must not be. + deps: Their dependencies, e.g. `:test_fixture` and `:matchers`. + **kwargs: Passed through to the cc_library (e.g. `visibility`). + `testonly` and `alwayslink` are set by the macro and can't be given. + """ + _check_suite(name, "conformance_suite(%r)" % name) + library = _suite_library(name) + if native.package_relative_label(":" + library.name) != library: + fail("conformance_suite(%r) must be used in package //%s, not //%s." % + (name, library.package, native.package_name())) + for attr in ("testonly", "alwayslink"): + if attr in kwargs: + fail(("conformance_suite(%r): %s is set by the macro and can't " + + "be overridden.") % (name, attr)) + cc_library( + name = library.name, + testonly = 1, + srcs = srcs, + deps = deps, + alwayslink = 1, + **kwargs + ) + +def failure_lists(name): + """Declares the directory `<name>/` of this package as its failure lists. + + The directory holds the expected-failure lists of the package's + conformance_test()s, one file per (test, suite) pair named + `<test name>_<suite>.txt`, where `<test name>` is the `name` of a + conformance_test() and `<suite>` one of the conformance suites, e.g. + `cpp_binary.txt` for `conformance_test(name = "cpp")` and the `binary` + suite. A file that doesn't exist means "no expected failures". Anything + else in the directory (a file not named that way, a file for an unknown + suite, any file in a subdirectory) fails the load; a well-formed list that + no conformance_test() uses (e.g. `foo_binary.txt` without a `foo` test, or + a list for a suite the test skips) can't be detected, though, and is simply + never read. The directory must not be a package of its own (the lists + are found with `glob`, which doesn't look into subpackages). + + Nothing in the directory should be edited by hand: `blaze run + :<test name>_<suite>_test -- --fix` creates or rewrites the file for that + test to match the observed results. Only the directory itself has to + exist beforehand; create it (`mkdir`) together with the first list. + + Args: + name: The name of the directory, and of the filegroup created for it. + Pass `":" + name` as the `failure_lists` of the package's + conformance_test()s. + """ + if native.subpackages(include = [name, name + "/**"], allow_empty = True): + fail(("failure_lists(%r): %s/ must not contain a BUILD file (it is " + + "a directory of failure lists, not a package).") % (name, name)) + files = native.glob(["%s/**" % name], allow_empty = True) + for file in files: + # The test name may itself contain underscores (e.g. java_lite), so + # only the suffix can be validated. + stem = file[len(name) + 1:] + if "/" in stem or not stem.endswith(".txt"): + stem = "" + else: + stem = stem[:-len(".txt")] + if not [ + s + for s in _SUITES + if stem.endswith("_" + s) and len(stem) > len(s) + 1 + ]: + fail(("failure_lists(%r): %s doesn't correspond to a " + + "conformance suite: failure lists must be named " + + "<test name>_<suite>.txt, where <suite> is one of %s.") % + (name, file, ", ".join(_SUITES))) + native.filegroup( + name = name, + srcs = files, + ) + +def conformance_test( + name, + testee, + failure_lists, + suites = None, + maximum_edition = None, + enforcement_level = None, + in_process = False, + **kwargs): + """Runs the conformance suites against a testee. + + Creates one `cc_test` per suite, named `<name>_<suite>_test` (e.g. + `cpp_binary_test`, `cpp_performance_test`), and a `test_suite` named + `name` running all of them. + Each test links the suite's library with `:test_environment_main` and runs + it against `testee` with the failure list + `<failure_lists directory>/<name>_<suite>.txt` if that file exists; + otherwise the testee is expected to pass every test in the suite, and + `blaze run ... -- --fix` creates the file (see failure_lists()). + + By default `testee` is a binary: the test links `:forked_testee`, which + spawns it as a subprocess speaking the conformance protocol on its + stdin/stdout, and it is passed as `--testee_binary`. With `in_process`, + `testee` is a cc_library that defines MakeTesteeRunner() (see + testee_runner.h) and is linked into the test instead, so the suites run + the testee in their own process: no fork, one process to debug, and a + testee-side crash or log shows up in the test itself. The flip side is + that there is no per-test isolation: a testee crash or hang takes the + whole test binary down instead of failing one test (the forked form + respawns the testee), so only use it for testees that never crash on + conformance input. Both modes take the same failure lists and flags. + + Args: + name: The test's name; also the prefix of its failure lists. + testee: The testee. Without `in_process`, a binary run as a subprocess. + With `in_process`, a `cc_library` defining `MakeTesteeRunner()`, like + this package's own `:cpp_in_process_testee` (private to it). Such a + library must be `alwayslink = 1` and `testonly = 1`, and exactly one + of them may be linked into a test: none fails to link, two is an ODR + violation (a duplicate-symbol error under static linking, silently + ambiguous under dynamic linking). A macro can't check any of this, + nor the kind of a label, so passing a binary here with `in_process` + fails at analysis time (a cc_test can't depend on a binary), and a + library without it fails at analysis time on `$(rootpath)` or, if + that resolves to its single archive, when the test tries to run it. + failure_lists: The failure_lists() of this package. + suites: The suites to run; default all of them, the performance suite + included. Use this to skip a suite the testee doesn't support at all + (e.g. `text` for a runtime without text format). + maximum_edition: The newest edition to test (e.g. `"2023"`); default + proto2 and proto3 only. Passed as `--maximum_edition=EDITION_2023`. + enforcement_level: The lowest priority whose failures fail the run, as its + number (0: only P0, 1: P0 and P1). By default every priority is + enforced; failures of tests of a lower priority are tolerated unless the + test is in the failure list. Passed as `--enforcement_level`. + in_process: Whether `testee` is a cc_library to link into each test + instead of a binary to spawn. + **kwargs: Passed through to every cc_test (e.g. `size`, `timeout`, + `tags`); `tags` also apply to the test_suite. `args`, `deps` and + `data` are set by the macro and can't be given. + """ + if suites == None: + suites = _SUITES + for suite in suites: + _check_suite(suite, "conformance_test(%r)" % name) + + if enforcement_level != None: + if type(enforcement_level) != "int" or enforcement_level < 0: + fail(("conformance_test(%r): enforcement_level must be None or a " + + "non-negative int, not %r.") % (name, enforcement_level)) + + # The failure lists must be in this package, since only the calling + # package's files can be globbed to find out which lists exist. + lists_label = native.package_relative_label(failure_lists) + if native.package_relative_label(":" + lists_label.name) != lists_label: + fail(("conformance_test(%r): failure_lists (%s) must be a " + + "failure_lists() in the same package, //%s.") % + (name, failure_lists, native.package_name())) + + for attr in ("args", "deps", "data"): + if attr in kwargs: + fail(("conformance_test(%r): %s is set by the macro and can't " + + "be overridden.") % (name, attr)) + tags = kwargs.pop("tags", []) + tests = [] + for suite in suites: + test_name = "%s_%s_test" % (name, suite) + failure_list = "%s/%s_%s.txt" % (lists_label.name, name, suite) + + # The runfiles path of the failure list, which is also its path relative + # to the workspace root so that --fix can find the source file under + # $BUILD_WORKSPACE_DIRECTORY. + failure_list_path = failure_list + if lists_label.package: + failure_list_path = "%s/%s" % (lists_label.package, failure_list) + + if in_process: + # The testee is linked in and would reject --testee_binary (see + # testee_runner.h). + args = [] + data = [failure_lists] + deps = [_suite_library(suite), _TEST_ENVIRONMENT_MAIN, testee] + else: + args = ["--testee_binary=$(rootpath %s)" % testee] + data = [testee, failure_lists] + deps = [ + _suite_library(suite), + _TEST_ENVIRONMENT_MAIN, + _FORKED_TESTEE, + ] + + # (native.glob wouldn't see the file if the failure lists directory ever + # became a package of its own; failure_lists() forbids that.) + if native.glob([failure_list], allow_empty = True): + args.append("--failure_list=%s" % failure_list_path) + else: + # No expected failures; tell --fix where to create the list. + args.append("--fix_output_file=%s" % failure_list_path) + if maximum_edition: + args.append("--maximum_edition=EDITION_%s" % maximum_edition) + if enforcement_level != None: + args.append("--enforcement_level=%d" % enforcement_level) + + cc_test( + name = test_name, + args = args, + data = data, + tags = ["conformance"] + tags, + deps = deps, + **kwargs + ) + tests.append(":" + test_name) + + native.test_suite( + name = name, + tests = tests, + tags = tags, + )
diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 4b292dd..edeb7f0 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc
@@ -5,14 +5,15 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd +// The C++ conformance testee. Speaks the length-prefixed stdin/stdout protocol +// described in fork_pipe_runner.cc and delegates each request to +// CppConformanceHarness. + #include <errno.h> -#include <stdarg.h> -#include <stdlib.h> #include <cstddef> #include <cstdint> #include <cstdio> -#include <memory> #include <string> #ifdef _WIN32 @@ -20,60 +21,26 @@ #include <io.h> #endif -#include "google/protobuf/any.pb.h" -#include "google/protobuf/api.pb.h" -#include "google/protobuf/duration.pb.h" -#include "google/protobuf/empty.pb.h" -#include "google/protobuf/field_mask.pb.h" -#include "google/protobuf/struct.pb.h" -#include "google/protobuf/timestamp.pb.h" -#include "google/protobuf/type.pb.h" -#include "google/protobuf/wrappers.pb.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "conformance/conformance.pb.h" -#include "conformance/test_protos/test_messages_edition2023.pb.h" -#include "conformance/test_protos/test_messages_edition2026.pb.h" -#include "conformance/test_protos/test_messages_edition_unstable.pb.h" -#include "editions/golden/test_messages_proto2_editions.pb.h" -#include "editions/golden/test_messages_proto3_editions.pb.h" +#include "conformance/conformance_cpp_harness.h" #include "google/protobuf/endian.h" -#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" -#include "google/protobuf/test_messages_proto2.pb.h" -#include "google/protobuf/test_messages_proto3.pb.h" -#include "google/protobuf/text_format.h" -#include "google/protobuf/util/json_util.h" -#include "google/protobuf/util/type_resolver.h" -#include "google/protobuf/util/type_resolver_util.h" #include "google/protobuf/stubs/status_macros.h" -// Must be included last. -#include "google/protobuf/port_def.inc" - namespace google { namespace protobuf { namespace conformance { namespace { + using ::conformance::ConformanceRequest; using ::conformance::ConformanceResponse; -using ::google::protobuf::util::JsonParseOptions; -using ::google::protobuf::util::JsonStringToMessage; -using ::google::protobuf::util::MessageToJsonString; -using ::google::protobuf::util::NewTypeResolverForDescriptorPool; -using ::google::protobuf::util::TypeResolver; -using ::protobuf_test_messages::edition_unstable::TestAllTypesEditionUnstable; -using ::protobuf_test_messages::editions::TestAllTypesEdition2023; -using ::protobuf_test_messages::editions::TestAllTypesEdition2026; -using ::protobuf_test_messages::proto2::TestAllTypesProto2; -using ::protobuf_test_messages::proto3::TestAllTypesProto3; -using TestAllTypesProto2Editions = - ::protobuf_test_messages::editions::proto2::TestAllTypesProto2; -using TestAllTypesProto3Editions = - ::protobuf_test_messages::editions::proto3::TestAllTypesProto3; + +// Set to true to log every request/response pair. +constexpr bool kVerbose = false; absl::Status ReadAll(FILE* file, char* buf, size_t len) { while (len > 0) { @@ -99,143 +66,16 @@ return absl::OkStatus(); } -class Harness { - public: - Harness() { - google::protobuf::LinkMessageReflection<TestAllTypesProto2>(); - google::protobuf::LinkMessageReflection<TestAllTypesProto3>(); - google::protobuf::LinkMessageReflection<TestAllTypesEdition2023>(); - google::protobuf::LinkMessageReflection<TestAllTypesEdition2026>(); - google::protobuf::LinkMessageReflection<TestAllTypesEditionUnstable>(); - google::protobuf::LinkMessageReflection<TestAllTypesProto2Editions>(); - google::protobuf::LinkMessageReflection<TestAllTypesProto3Editions>(); - - // Force link one wkt from each wkt file. - google::protobuf::LinkMessageReflection<google::protobuf::Any>(); - google::protobuf::LinkMessageReflection<google::protobuf::Api>(); - google::protobuf::LinkMessageReflection<google::protobuf::Duration>(); - google::protobuf::LinkMessageReflection<google::protobuf::Empty>(); - google::protobuf::LinkMessageReflection<google::protobuf::FieldMask>(); - google::protobuf::LinkMessageReflection<google::protobuf::Struct>(); - google::protobuf::LinkMessageReflection<google::protobuf::Timestamp>(); - google::protobuf::LinkMessageReflection<google::protobuf::Type>(); - google::protobuf::LinkMessageReflection<google::protobuf::DoubleValue>(); - - resolver_.reset(NewTypeResolverForDescriptorPool( - "type.googleapis.com", DescriptorPool::generated_pool())); - } - - absl::StatusOr<ConformanceResponse> RunTest( - const ConformanceRequest& request); - - // Returns Ok(true) if we're done processing requests. - absl::StatusOr<bool> ServeConformanceRequest(); - - private: - bool verbose_ = false; - std::unique_ptr<TypeResolver> resolver_; -}; - -absl::StatusOr<ConformanceResponse> Harness::RunTest( - const ConformanceRequest& request) { - const Descriptor* descriptor = - DescriptorPool::generated_pool()->FindMessageTypeByName( - request.message_type()); - if (descriptor == nullptr) { - return absl::NotFoundError( - absl::StrCat("No such message type: ", request.message_type())); - } - std::string type_url = - absl::StrCat("type.googleapis.com/", request.message_type()); - - std::unique_ptr<Message> test_message( - MessageFactory::generated_factory()->GetPrototype(descriptor)->New()); - ConformanceResponse response; - - switch (request.payload_case()) { - case ConformanceRequest::kProtobufPayload: { - if (!test_message->ParseFromString(request.protobuf_payload())) { - response.set_parse_error("parse error (no more details available)"); - return response; - } - break; - } - - case ConformanceRequest::kJsonPayload: { - JsonParseOptions options; - options.ignore_unknown_fields = - (request.test_category() == - ::conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); - absl::Status status = JsonStringToMessage(request.json_payload(), - test_message.get(), options); - if (!status.ok()) { - response.set_parse_error( - absl::StrCat("parse error: ", status.message())); - return response; - } - break; - } - - case ConformanceRequest::kTextPayload: { - if (!TextFormat::ParseFromString(request.text_payload(), - test_message.get())) { - response.set_parse_error("parse error (no more details available)"); - return response; - } - break; - } - - case ConformanceRequest::PAYLOAD_NOT_SET: - return absl::InvalidArgumentError("request didn't have payload"); - - default: - return absl::InvalidArgumentError( - absl::StrCat("unknown payload type", request.payload_case())); - } - - switch (request.requested_output_format()) { - case ::conformance::UNSPECIFIED: - return absl::InvalidArgumentError("unspecified output format"); - - case ::conformance::PROTOBUF: { - ABSL_CHECK( - test_message->SerializeToString(response.mutable_protobuf_payload())); - break; - } - - case ::conformance::JSON: { - absl::Status status = - MessageToJsonString(*test_message, response.mutable_json_payload()); - if (!status.ok()) { - response.set_serialize_error(absl::StrCat( - "failed to serialize JSON output: ", status.message())); - } - break; - } - - case ::conformance::TEXT_FORMAT: { - TextFormat::Printer printer; - printer.SetHideUnknownFields(!request.print_unknown_fields()); - ABSL_CHECK(printer.PrintToString(*test_message, - response.mutable_text_payload())); - break; - } - - default: - return absl::InvalidArgumentError(absl::StrCat( - "unknown output format", request.requested_output_format())); - } - - return response; -} - -absl::StatusOr<bool> Harness::ServeConformanceRequest() { +// Reads one request from stdin, runs it through `harness`, and writes the +// response to stdout. Returns Ok(true) if we're done processing requests. +absl::StatusOr<bool> ServeConformanceRequest( + const CppConformanceHarness& harness) { uint32_t in_len; if (!ReadAll(stdin, reinterpret_cast<char*>(&in_len), sizeof(in_len)).ok()) { // EOF means we're done. return true; } - in_len = ::google::protobuf::internal::little_endian::ToHost(in_len); + in_len = google::protobuf::internal::little_endian::ToHost(in_len); std::string serialized_input; serialized_input.resize(in_len); @@ -244,14 +84,14 @@ ConformanceRequest request; ABSL_CHECK(request.ParseFromString(serialized_input)); - absl::StatusOr<ConformanceResponse> response = RunTest(request); + absl::StatusOr<ConformanceResponse> response = harness.RunTest(request); RETURN_IF_ERROR(response.status()); std::string serialized_output; // TODO: Remove this suppression. (void)response->SerializeToString(&serialized_output); - uint32_t out_len = ::google::protobuf::internal::little_endian::FromHost( + uint32_t out_len = google::protobuf::internal::little_endian::FromHost( static_cast<uint32_t>(serialized_output.size())); RETURN_IF_ERROR(WriteAll(stdout, &out_len, sizeof(out_len))); @@ -261,13 +101,14 @@ return absl::ErrnoToStatus(errno, "error flushing to test runner"); } - if (verbose_) { + if (kVerbose) { ABSL_LOG(INFO) << "conformance-cpp: request=" << google::protobuf::ShortFormat(request) << ", response=" << google::protobuf::ShortFormat(*response); } return false; } + } // namespace } // namespace conformance } // namespace protobuf @@ -278,10 +119,11 @@ _setmode(_fileno(stdin), _O_BINARY); _setmode(_fileno(stdout), _O_BINARY); #endif - google::protobuf::conformance::Harness harness; + google::protobuf::conformance::CppConformanceHarness harness; int total_runs = 0; while (true) { - auto is_done = harness.ServeConformanceRequest(); + absl::StatusOr<bool> is_done = + google::protobuf::conformance::ServeConformanceRequest(harness); if (!is_done.ok()) { ABSL_LOG(FATAL) << is_done.status(); } @@ -293,5 +135,3 @@ ABSL_LOG(INFO) << "conformance-cpp: received EOF from test runner after " << total_runs << " tests"; } - -#include "google/protobuf/port_undef.inc"
diff --git a/conformance/conformance_cpp_harness.cc b/conformance/conformance_cpp_harness.cc new file mode 100644 index 0000000..568d368 --- /dev/null +++ b/conformance/conformance_cpp_harness.cc
@@ -0,0 +1,174 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/conformance_cpp_harness.h" + +#include <memory> + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/api.pb.h" +#include "google/protobuf/duration.pb.h" +#include "google/protobuf/empty.pb.h" +#include "google/protobuf/field_mask.pb.h" +#include "google/protobuf/struct.pb.h" +#include "google/protobuf/timestamp.pb.h" +#include "google/protobuf/type.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "conformance/conformance.pb.h" +#include "conformance/test_protos/test_messages_edition2023.pb.h" +#include "conformance/test_protos/test_messages_edition2026.pb.h" +#include "conformance/test_protos/test_messages_edition_unstable.pb.h" +#include "google/protobuf/descriptor.h" +#include "editions/golden/test_messages_proto2_editions.pb.h" +#include "editions/golden/test_messages_proto3_editions.pb.h" +#include "google/protobuf/json/json.h" +#include "google/protobuf/message.h" +#include "google/protobuf/test_messages_proto2.pb.h" +#include "google/protobuf/test_messages_proto3.pb.h" +#include "google/protobuf/text_format.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::google::protobuf::json::JsonStringToMessage; +using ::google::protobuf::json::MessageToJsonString; +using ::google::protobuf::json::ParseOptions; +using ::protobuf_test_messages::edition_unstable::TestAllTypesEditionUnstable; +using ::protobuf_test_messages::editions::TestAllTypesEdition2023; +using ::protobuf_test_messages::editions::TestAllTypesEdition2026; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using TestAllTypesProto2Editions = + ::protobuf_test_messages::editions::proto2::TestAllTypesProto2; +using TestAllTypesProto3Editions = + ::protobuf_test_messages::editions::proto3::TestAllTypesProto3; + +} // namespace + +CppConformanceHarness::CppConformanceHarness() { + google::protobuf::LinkMessageReflection<TestAllTypesProto2>(); + google::protobuf::LinkMessageReflection<TestAllTypesProto3>(); + google::protobuf::LinkMessageReflection<TestAllTypesEdition2023>(); + google::protobuf::LinkMessageReflection<TestAllTypesEdition2026>(); + google::protobuf::LinkMessageReflection<TestAllTypesEditionUnstable>(); + google::protobuf::LinkMessageReflection<TestAllTypesProto2Editions>(); + google::protobuf::LinkMessageReflection<TestAllTypesProto3Editions>(); + + // Force link one wkt from each wkt file. + google::protobuf::LinkMessageReflection<google::protobuf::Any>(); + google::protobuf::LinkMessageReflection<google::protobuf::Api>(); + google::protobuf::LinkMessageReflection<google::protobuf::Duration>(); + google::protobuf::LinkMessageReflection<google::protobuf::Empty>(); + google::protobuf::LinkMessageReflection<google::protobuf::FieldMask>(); + google::protobuf::LinkMessageReflection<google::protobuf::Struct>(); + google::protobuf::LinkMessageReflection<google::protobuf::Timestamp>(); + google::protobuf::LinkMessageReflection<google::protobuf::Type>(); + google::protobuf::LinkMessageReflection<google::protobuf::DoubleValue>(); +} + +absl::StatusOr<ConformanceResponse> CppConformanceHarness::RunTest( + const ConformanceRequest& request) const { + const Descriptor* descriptor = + DescriptorPool::generated_pool()->FindMessageTypeByName( + request.message_type()); + if (descriptor == nullptr) { + return absl::NotFoundError( + absl::StrCat("No such message type: ", request.message_type())); + } + + std::unique_ptr<Message> test_message( + MessageFactory::generated_factory()->GetPrototype(descriptor)->New()); + ConformanceResponse response; + + switch (request.payload_case()) { + case ConformanceRequest::kProtobufPayload: { + if (!test_message->ParseFromString(request.protobuf_payload())) { + response.set_parse_error("parse error (no more details available)"); + return response; + } + break; + } + + case ConformanceRequest::kJsonPayload: { + ParseOptions options; + options.ignore_unknown_fields = + (request.test_category() == + ::conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + absl::Status status = JsonStringToMessage(request.json_payload(), + test_message.get(), options); + if (!status.ok()) { + response.set_parse_error( + absl::StrCat("parse error: ", status.message())); + return response; + } + break; + } + + case ConformanceRequest::kTextPayload: { + if (!TextFormat::ParseFromString(request.text_payload(), + test_message.get())) { + response.set_parse_error("parse error (no more details available)"); + return response; + } + break; + } + + case ConformanceRequest::PAYLOAD_NOT_SET: + return absl::InvalidArgumentError("request didn't have payload"); + + default: + return absl::InvalidArgumentError( + absl::StrCat("unknown payload type: ", request.payload_case())); + } + + switch (request.requested_output_format()) { + case ::conformance::UNSPECIFIED: + return absl::InvalidArgumentError("unspecified output format"); + + case ::conformance::PROTOBUF: { + ABSL_CHECK( + test_message->SerializeToString(response.mutable_protobuf_payload())); + break; + } + + case ::conformance::JSON: { + absl::Status status = + MessageToJsonString(*test_message, response.mutable_json_payload()); + if (!status.ok()) { + response.set_serialize_error(absl::StrCat( + "failed to serialize JSON output: ", status.message())); + } + break; + } + + case ::conformance::TEXT_FORMAT: { + TextFormat::Printer printer; + printer.SetHideUnknownFields(!request.print_unknown_fields()); + ABSL_CHECK(printer.PrintToString(*test_message, + response.mutable_text_payload())); + break; + } + + default: + return absl::InvalidArgumentError(absl::StrCat( + "unknown output format: ", request.requested_output_format())); + } + + return response; +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/conformance_cpp_harness.h b/conformance/conformance_cpp_harness.h new file mode 100644 index 0000000..e9633ff --- /dev/null +++ b/conformance/conformance_cpp_harness.h
@@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_CPP_HARNESS_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_CPP_HARNESS_H__ + +#include "absl/status/statusor.h" +#include "conformance/conformance.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// Implements the C++ conformance testee. Given a ConformanceRequest, it +// parses the payload into the requested test message and serializes it in +// the requested output format. +// +// Shared by the conformance_cpp binary (stdin/stdout protocol) and, via +// cpp_in_process_test_runner.h, by in-process conformance tests. +// +// Instances are stateless and cheap to construct. RunTest is const, so a +// single instance may be shared across threads. +class CppConformanceHarness { + public: + // Force-links the reflection data of every test message and well-known type + // into the binary so that they can be looked up by name in the generated + // descriptor pool. + CppConformanceHarness(); + + // Runs a single conformance test. + // + // Returns a response with `parse_error` or `serialize_error` set when the + // payload cannot be handled. Returns a non-OK status when the request + // itself is malformed: unknown message type, missing payload, or an + // unspecified output format. + absl::StatusOr<::conformance::ConformanceResponse> RunTest( + const ::conformance::ConformanceRequest& request) const; +}; + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_CPP_HARNESS_H__
diff --git a/conformance/conformance_cpp_harness_test.cc b/conformance/conformance_cpp_harness_test.cc new file mode 100644 index 0000000..83b8b0f --- /dev/null +++ b/conformance/conformance_cpp_harness_test.cc
@@ -0,0 +1,285 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/conformance_cpp_harness.h" + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_wireformat.h" +#include "conformance/conformance.pb.h" +#include "google/protobuf/test_messages_proto3.pb.h" +#include "google/protobuf/test_textproto.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using ::testing::HasSubstr; +using ::testing::Property; +using ::testing::StartsWith; +using ::testing::TestParamInfo; +using ::testing::TestWithParam; +using ::testing::Values; + +constexpr absl::string_view kProto3MessageType = + "protobuf_test_messages.proto3.TestAllTypesProto3"; + +// Returns a request for TestAllTypesProto3 with the given output format and no +// payload. Tests add whichever payload they exercise, or leave it unset to +// exercise the missing-payload path. +ConformanceRequest Proto3Request(::conformance::WireFormat output_format) { + ConformanceRequest request; + request.set_message_type(kProto3MessageType); + request.set_requested_output_format(output_format); + return request; +} + +// Serializes the TestAllTypesProto3 described by `textproto` to the wire +// format. +std::string SerializeProto3(absl::string_view textproto) { + TestAllTypesProto3 message = ParseTextOrDie(textproto); + return message.SerializeAsString(); +} + +// Serializes `optional_int32: 42` with an additional unknown varint field +// (number 9999, value 1) so that unknown-field handling can be observed. +std::string Proto3PayloadWithUnknownField() { + TestAllTypesProto3 message; + message.set_optional_int32(42); + message.GetReflection()->MutableUnknownFields(&message)->AddVarint(9999, 1); + return message.SerializeAsString(); +} + +TEST(CppConformanceHarnessTest, ProtobufRoundTrip) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + request.set_protobuf_payload( + SerializeProto3(R"pb(optional_int32: 42 optional_string: "hello")pb")); + + absl::StatusOr<ConformanceResponse> response = harness.RunTest(request); + + ASSERT_THAT(response, IsOk()); + ASSERT_EQ(response->result_case(), ConformanceResponse::kProtobufPayload); + TestAllTypesProto3 round_tripped; + ASSERT_TRUE(round_tripped.ParseFromString(response->protobuf_payload())); + EXPECT_THAT(round_tripped, EqualsProto(R"pb(optional_int32: 42 + optional_string: "hello")pb")); +} + +TEST(CppConformanceHarnessTest, ProtobufParseErrorOnMalformedBytes) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + // Field 1, length-delimited, claims 5 bytes but only 3 follow. + request.set_protobuf_payload( + Wire(Tag(1, WireType::kLengthPrefixed), Varint(5), "abc").str()); + + EXPECT_THAT(harness.RunTest(request), + IsOkAndHolds(EqualsProto( + R"pb(parse_error: "parse error (no more details available)" + )pb"))); +} + +TEST(CppConformanceHarnessTest, JsonOutput) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::JSON); + request.set_protobuf_payload(SerializeProto3(R"pb(optional_int32: 42)pb")); + + // Kept out of the macro arguments: MSVC's traditional preprocessor mis-scans + // raw string literals containing \" inside macro arguments. + constexpr absl::string_view kExpected = + R"pb(json_payload: "{\"optionalInt32\":42}")pb"; + EXPECT_THAT(harness.RunTest(request), IsOkAndHolds(EqualsProto(kExpected))); +} + +TEST(CppConformanceHarnessTest, JsonSerializeErrorOnUnresolvableAny) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::JSON); + // The payload parses from the wire format, but JSON printing has to expand + // the Any and cannot resolve its type URL. + request.set_protobuf_payload(SerializeProto3( + R"pb(optional_any { type_url: "type.googleapis.com/no.such.Message" } + )pb")); + + absl::StatusOr<ConformanceResponse> response = harness.RunTest(request); + + ASSERT_THAT(response, IsOk()); + ASSERT_EQ(response->result_case(), ConformanceResponse::kSerializeError); + EXPECT_THAT(response->serialize_error(), + StartsWith("failed to serialize JSON output: ")); +} + +TEST(CppConformanceHarnessTest, JsonUnknownFieldIsParseErrorByDefault) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + request.set_json_payload(R"json({"noSuchField": 1})json"); + request.set_test_category(::conformance::JSON_TEST); + + EXPECT_THAT(harness.RunTest(request), + IsOkAndHolds(Property(&ConformanceResponse::parse_error, + StartsWith("parse error: ")))); +} + +TEST(CppConformanceHarnessTest, JsonUnknownFieldIsIgnoredInIgnoreUnknownTest) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + request.set_json_payload( + R"json({"noSuchField": 1, "optionalInt32": 42})json"); + request.set_test_category(::conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + + absl::StatusOr<ConformanceResponse> response = harness.RunTest(request); + + ASSERT_THAT(response, IsOk()); + ASSERT_EQ(response->result_case(), ConformanceResponse::kProtobufPayload); + TestAllTypesProto3 parsed; + ASSERT_TRUE(parsed.ParseFromString(response->protobuf_payload())); + EXPECT_THAT(parsed, EqualsProto(R"pb(optional_int32: 42)pb")); +} + +TEST(CppConformanceHarnessTest, TextPayloadRoundTrip) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::TEXT_FORMAT); + request.set_text_payload( + R"pb(optional_int32: 42 optional_string: 'hello')pb"); + + constexpr absl::string_view kExpected = + R"pb(text_payload: "optional_int32: 42\n" + "optional_string: \"hello\"\n")pb"; + EXPECT_THAT(harness.RunTest(request), IsOkAndHolds(EqualsProto(kExpected))); +} + +TEST(CppConformanceHarnessTest, TextParseErrorOnMalformedInput) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::TEXT_FORMAT); + request.set_text_payload(R"pb(optional_int32: not_a_number)pb"); + + EXPECT_THAT(harness.RunTest(request), + IsOkAndHolds(EqualsProto( + R"pb(parse_error: "parse error (no more details available)" + )pb"))); +} + +TEST(CppConformanceHarnessTest, TextFormatOutputHidesUnknownFieldsByDefault) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::TEXT_FORMAT); + request.set_protobuf_payload(Proto3PayloadWithUnknownField()); + + EXPECT_THAT( + harness.RunTest(request), + IsOkAndHolds(EqualsProto(R"pb(text_payload: "optional_int32: 42\n")pb"))); +} + +TEST(CppConformanceHarnessTest, TextFormatOutputPrintsUnknownFieldsOnRequest) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::TEXT_FORMAT); + request.set_protobuf_payload(Proto3PayloadWithUnknownField()); + request.set_print_unknown_fields(true); + + EXPECT_THAT(harness.RunTest(request), + IsOkAndHolds(EqualsProto( + R"pb(text_payload: "optional_int32: 42\n9999: 1\n")pb"))); +} + +TEST(CppConformanceHarnessTest, UnknownMessageTypeIsNotFound) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + request.set_message_type("no.such.Message"); + request.set_protobuf_payload(""); + + EXPECT_THAT(harness.RunTest(request), StatusIs(absl::StatusCode::kNotFound, + HasSubstr("no.such.Message"))); +} + +TEST(CppConformanceHarnessTest, MissingPayloadIsInvalidArgument) { + CppConformanceHarness harness; + + EXPECT_THAT( + harness.RunTest(Proto3Request(::conformance::PROTOBUF)), + StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("payload"))); +} + +TEST(CppConformanceHarnessTest, JspbPayloadIsInvalidArgument) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::PROTOBUF); + request.set_jspb_payload("[]"); + + EXPECT_THAT(harness.RunTest(request), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("unknown payload type"))); +} + +TEST(CppConformanceHarnessTest, UnspecifiedOutputFormatIsInvalidArgument) { + CppConformanceHarness harness; + ConformanceRequest request = Proto3Request(::conformance::UNSPECIFIED); + request.set_protobuf_payload(""); + + EXPECT_THAT( + harness.RunTest(request), + StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("output format"))); +} + +TEST(CppConformanceHarnessTest, UnknownOutputFormatIsInvalidArgument) { + CppConformanceHarness harness; + ConformanceRequest request = + Proto3Request(static_cast<::conformance::WireFormat>(99)); + request.set_protobuf_payload(""); + + EXPECT_THAT( + harness.RunTest(request), + StatusIs(absl::StatusCode::kInvalidArgument, HasSubstr("output format"))); +} + +// Message types the conformance suites may name in a request, other than +// TestAllTypesProto3 (which this file references directly). The harness +// constructor force-links the reflection data for each of them. This test +// deliberately does not reference the generated C++ types, so it is intended +// to fail if the harness stops pulling one of them in; it is only a weak +// guarantee, though, since the linker may keep the descriptors alive for other +// reasons (e.g. alwayslink or dynamic linking). +class LinkedMessageTypeTest : public TestWithParam<absl::string_view> {}; + +TEST_P(LinkedMessageTypeTest, EmptyPayloadRoundTrips) { + CppConformanceHarness harness; + ConformanceRequest request; + request.set_message_type(GetParam()); + request.set_requested_output_format(::conformance::PROTOBUF); + request.set_protobuf_payload(""); + + EXPECT_THAT(harness.RunTest(request), + IsOkAndHolds(EqualsProto(R"pb(protobuf_payload: "")pb"))); +} + +INSTANTIATE_TEST_SUITE_P( + CppConformanceHarnessTest, LinkedMessageTypeTest, + Values("protobuf_test_messages.proto2.TestAllTypesProto2", + "protobuf_test_messages.editions.TestAllTypesEdition2023", + "protobuf_test_messages.edition_unstable." + "TestAllTypesEditionUnstable", + "protobuf_test_messages.editions.proto2.TestAllTypesProto2", + "protobuf_test_messages.editions.proto3.TestAllTypesProto3", + "google.protobuf.Any", "google.protobuf.Timestamp"), + [](const TestParamInfo<absl::string_view>& info) { + return absl::StrReplaceAll(info.param, {{".", "_"}}); + }); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc deleted file mode 100644 index 467bcfe..0000000 --- a/conformance/conformance_test.cc +++ /dev/null
@@ -1,798 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include "conformance/conformance_test.h" - -#include <stdarg.h> - -#include <algorithm> -#include <cstddef> -#include <cstdint> -#include <cstdio> -#include <fstream> -#include <memory> -#include <string> -#include <utility> - -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_set.h" -#include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "conformance/conformance.pb.h" -#include "conformance/failure_list_trie_node.h" -#include "google/protobuf/descriptor_legacy.h" -#include "google/protobuf/endian.h" -#include "google/protobuf/message.h" -#include "google/protobuf/text_format.h" -#include "google/protobuf/util/field_comparator.h" -#include "google/protobuf/util/message_differencer.h" - -using conformance::ConformanceRequest; -using conformance::ConformanceResponse; -using conformance::TestStatus; -using conformance::WireFormat; -using google::protobuf::util::DefaultFieldComparator; -using google::protobuf::util::MessageDifferencer; -using std::string; - -namespace { - -static void ReplaceAll(std::string& input, std::string replace_word, - std::string replace_by) { - size_t pos = input.find(replace_word); - while (pos != std::string::npos) { - input.replace(pos, replace_word.length(), replace_by); - pos = input.find(replace_word, pos + replace_by.length()); - } -} - -static std::string ToOctString(const std::string& binary_string) { - std::string oct_string; - for (size_t i = 0; i < binary_string.size(); i++) { - uint8_t c = binary_string.at(i); - uint8_t high = c / 64; - uint8_t mid = (c % 64) / 8; - uint8_t low = c % 8; - oct_string.push_back('\\'); - oct_string.push_back('0' + high); - oct_string.push_back('0' + mid); - oct_string.push_back('0' + low); - } - return oct_string; -} - -// Returns full filename path of written .txt file if successful -static std::string ProduceOctalSerialized(const std::string& request, - uint32_t len) { - char* len_split_bytes = static_cast<char*>(static_cast<void*>(&len)); - - std::string out; - - std::string hex_repr; - for (int i = 0; i < 4; i++) { - auto conversion = (unsigned int)static_cast<uint8_t>(len_split_bytes[i]); - std::string hex = absl::StrFormat("\\x%x", conversion); - absl::StrAppend(&hex_repr, hex); - } - - absl::StrAppend(&out, hex_repr); - - absl::StrAppend(&out, ToOctString(request)); - - return out; -} - -static std::string WriteToFile(const std::string& octal_serialized, - const std::string& output_dir, - const std::string& test_name) { - std::string test_name_txt = test_name; - ReplaceAll(test_name_txt, ".", "_"); - absl::StrAppend(&test_name_txt, ".txt"); - std::string full_filename; - if (!output_dir.empty()) { - full_filename = output_dir; - if (*output_dir.rbegin() != '/') { - full_filename.push_back('/'); - } - absl::StrAppend(&full_filename, test_name_txt); - } - std::ofstream os{std::string(full_filename)}; - if (os) { - os << octal_serialized; - return full_filename; - } else { - ABSL_LOG(INFO) << "Failed to open file for debugging: " << full_filename - << "\n"; - return ""; - } -} - -// Removes all newlines. -static void Normalize(std::string& input) { - input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); -} - -// Sets up a failure message properly for our failure lists. -static TestStatus FormatFailureMessage(const TestStatus& input) { - // Make copy just this once, as we need to modify it for our failure lists. - std::string formatted_failure_message = input.failure_message(); - // Remove newlines - Normalize(formatted_failure_message); - // Truncate failure message if needed - if (formatted_failure_message.length() > 128) { - formatted_failure_message = formatted_failure_message.substr(0, 128); - } - TestStatus properly_formatted; - properly_formatted.set_name(input.name()); - properly_formatted.set_failure_message(formatted_failure_message); - return properly_formatted; -} - -bool CheckSetEmpty(const absl::btree_map<std::string, TestStatus>& set_to_check, - absl::string_view write_to_file, absl::string_view msg, - absl::string_view output_dir, std::string* output) { - if (set_to_check.empty()) return true; - - absl::StrAppendFormat(output, "\n"); - absl::StrAppendFormat(output, "%s\n\n", msg); - for (const auto& pair : set_to_check) { - absl::StrAppendFormat(output, " %s # %s\n", pair.first, - pair.second.failure_message()); - } - absl::StrAppendFormat(output, "\n"); - - if (!write_to_file.empty()) { - std::string full_filename; - absl::string_view filename = write_to_file; - if (!output_dir.empty()) { - full_filename = std::string(output_dir); - absl::StrAppend(&full_filename, write_to_file); - filename = full_filename; - } - std::ofstream os{std::string(filename)}; - if (os) { - for (const auto& pair : set_to_check) { - // Additions will not have a 'matched_name' while removals will. - string potential_add_or_removal = pair.second.matched_name().empty() - ? pair.first - : pair.second.matched_name(); - os << potential_add_or_removal << " # " << pair.second.failure_message() - << "\n"; - } - } else { - absl::StrAppendFormat(output, - "Failed to open file: %s\n", - filename); - } - } - - return false; -} - -} // namespace - -namespace google { -namespace protobuf { -namespace conformance { - -constexpr int kMaximumWildcardExpansions = 20; - -ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( - ConformanceLevel level, ::conformance::WireFormat input_format, - ::conformance::WireFormat output_format, - ::conformance::TestCategory test_category, const Message& prototype_message, - const std::string& test_name, const std::string& input) - : level_(level), - input_format_(input_format), - output_format_(output_format), - prototype_message_(prototype_message), - prototype_message_for_compare_(prototype_message.New()), - test_name_(test_name) { - switch (input_format) { - case ::conformance::PROTOBUF: { - request_.set_protobuf_payload(input); - break; - } - - case ::conformance::JSON: { - request_.set_json_payload(input); - break; - } - - case ::conformance::JSPB: { - request_.set_jspb_payload(input); - break; - } - - case ::conformance::TEXT_FORMAT: { - request_.set_text_payload(input); - break; - } - - default: - ABSL_LOG(FATAL) << "Unspecified input format"; - } - - request_.set_test_category(test_category); - - request_.set_message_type(prototype_message.GetDescriptor()->full_name()); - request_.set_requested_output_format(output_format); -} - -std::unique_ptr<Message> -ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const { - return std::unique_ptr<Message>(prototype_message_for_compare_->New()); -} - -std::string -ConformanceTestSuite::ConformanceRequestSetting::GetSyntaxIdentifier() const { - switch (FileDescriptorLegacy(prototype_message_.GetDescriptor()->file()) - .edition()) { - case Edition::EDITION_PROTO3: - return "Proto3"; - case Edition::EDITION_PROTO2: - return "Proto2"; - case Edition::EDITION_UNSTABLE: - return "EditionUnstable"; - case Edition::EDITION_2026: - return "Edition2026"; - default: { - std::string id = "Editions"; - if (prototype_message_.GetDescriptor()->name() == "TestAllTypesProto2") { - absl::StrAppend(&id, "_Proto2"); - } else if (prototype_message_.GetDescriptor()->name() == - "TestAllTypesProto3") { - absl::StrAppend(&id, "_Proto3"); - } - return id; - } - } -} - -string ConformanceTestSuite::ConformanceRequestSetting::GetTestName() const { - return absl::StrCat(ConformanceLevelToString(level_), ".", - GetSyntaxIdentifier(), ".", - InputFormatString(input_format_), ".", test_name_, ".", - OutputFormatString(output_format_)); -} - -std::string -ConformanceTestSuite::ConformanceRequestSetting::ConformanceLevelToString( - ConformanceLevel level) const { - switch (level) { - case REQUIRED: - return "Required"; - case RECOMMENDED: - return "Recommended"; - } - ABSL_LOG(FATAL) << "Unknown value: " << level; - return ""; -} - -std::string ConformanceTestSuite::ConformanceRequestSetting::InputFormatString( - ::conformance::WireFormat format) const { - switch (format) { - case ::conformance::PROTOBUF: - return "ProtobufInput"; - case ::conformance::JSON: - return "JsonInput"; - case ::conformance::TEXT_FORMAT: - return "TextFormatInput"; - default: - ABSL_LOG(FATAL) << "Unspecified output format"; - } - return ""; -} - -std::string ConformanceTestSuite::ConformanceRequestSetting::OutputFormatString( - ::conformance::WireFormat format) const { - switch (format) { - case ::conformance::PROTOBUF: - return "ProtobufOutput"; - case ::conformance::JSON: - return "JsonOutput"; - case ::conformance::TEXT_FORMAT: - return "TextFormatOutput"; - default: - ABSL_LOG(FATAL) << "Unspecified output format"; - } - return ""; -} - -void ConformanceTestSuite::TruncateDebugPayload(string* payload) { - if (payload != nullptr && payload->size() > 200) { - payload->resize(200); - payload->append("...(truncated)"); - } -} - -ConformanceRequest ConformanceTestSuite::TruncateRequest( - const ConformanceRequest& request) { - ConformanceRequest debug_request(request); - switch (debug_request.payload_case()) { - case ConformanceRequest::kProtobufPayload: - TruncateDebugPayload(debug_request.mutable_protobuf_payload()); - break; - case ConformanceRequest::kJsonPayload: - TruncateDebugPayload(debug_request.mutable_json_payload()); - break; - case ConformanceRequest::kTextPayload: - TruncateDebugPayload(debug_request.mutable_text_payload()); - break; - case ConformanceRequest::kJspbPayload: - TruncateDebugPayload(debug_request.mutable_jspb_payload()); - break; - default: - // Do nothing. - break; - } - return debug_request; -} - -ConformanceResponse ConformanceTestSuite::TruncateResponse( - const ConformanceResponse& response) { - ConformanceResponse debug_response(response); - switch (debug_response.result_case()) { - case ConformanceResponse::kProtobufPayload: - TruncateDebugPayload(debug_response.mutable_protobuf_payload()); - break; - case ConformanceResponse::kJsonPayload: - TruncateDebugPayload(debug_response.mutable_json_payload()); - break; - case ConformanceResponse::kTextPayload: - TruncateDebugPayload(debug_response.mutable_text_payload()); - break; - case ConformanceResponse::kJspbPayload: - TruncateDebugPayload(debug_response.mutable_jspb_payload()); - break; - default: - // Do nothing. - break; - } - return debug_response; -} - -void ConformanceTestSuite::ReportSuccess(const TestStatus& test) { - if (expected_to_fail_.contains(test.name())) { - absl::StrAppendFormat(&output_, - "ERROR: test %s (matched to %s) is in the failure " - "list, but test succeeded. " - "Remove its match from the failure list.\n", - test.name(), - expected_to_fail_[test.name()].matched_name()); - unexpected_succeeding_tests_[test.name()] = expected_to_fail_[test.name()]; - } - expected_to_fail_.erase(test.name()); - successes_++; -} - -void ConformanceTestSuite::ReportFailure(TestStatus& test, - ConformanceLevel level, - const ConformanceRequest& request, - const ConformanceResponse& response) { - if (expected_to_fail_.contains(test.name())) { - // Make copy just this once, as we need to modify them for comparison. - // Failure message from the failure list. - string expected_failure_message = - expected_to_fail_[test.name()].failure_message(); - // Actual failure message from the test run. - std::string actual_failure_message = test.failure_message(); - - Normalize(actual_failure_message); - if (actual_failure_message.rfind(expected_failure_message, 0) == 0) { - // Our failure messages match. - expected_failures_++; - } else { - // We want to add the test to the failure list with its correct failure - // message. - unexpected_failure_messages_[test.name()] = FormatFailureMessage(test); - // We want to remove the test from the failure list. That means passing - // to it the same failure message that was in the list. - TestStatus incorrect_failure_message; - incorrect_failure_message.set_name(test.name()); - incorrect_failure_message.set_failure_message(expected_failure_message); - incorrect_failure_message.set_matched_name( - expected_to_fail_[test.name()].matched_name()); - - expected_failure_messages_[test.name()] = incorrect_failure_message; - } - expected_to_fail_.erase(test.name()); - if (!verbose_) return; - } else if (level == RECOMMENDED && !enforce_recommended_) { - absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test.name()); - } else { - absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test.name()); - - unexpected_failing_tests_[test.name()] = FormatFailureMessage(test); - } - - absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", - test.failure_message(), - TruncateRequest(request).ShortDebugString(), - TruncateResponse(response).ShortDebugString()); -} - -void ConformanceTestSuite::ReportSkip(const TestStatus& test, - const ConformanceRequest& request, - const ConformanceResponse& response) { - if (verbose_) { - absl::StrAppendFormat( - &output_, "SKIPPED, test=%s request=%s, response=%s\n", test.name(), - request.ShortDebugString(), response.ShortDebugString()); - } - skipped_[test.name()] = test; -} - -void ConformanceTestSuite::RunValidInputTest( - const ConformanceRequestSetting& setting, - const std::string& equivalent_text_format) { - std::unique_ptr<Message> reference_message(setting.NewTestMessage()); - ABSL_CHECK(TextFormat::ParseFromString(equivalent_text_format, - reference_message.get())) - << "Failed to parse data for test case: " << setting.GetTestName() - << ", data: " << equivalent_text_format; - const std::string equivalent_wire_format = - reference_message->SerializeAsString(); - RunValidBinaryInputTest(setting, equivalent_wire_format); -} - -void ConformanceTestSuite::RunValidBinaryInputTest( - const ConformanceRequestSetting& setting, - const std::string& equivalent_wire_format, bool require_same_wire_format) { - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - if (!RunTest(setting.GetTestName(), request, &response)) { - return; - } - - VerifyResponse(setting, equivalent_wire_format, response, true, - require_same_wire_format); -} - -void ConformanceTestSuite::VerifyResponse( - const ConformanceRequestSetting& setting, - const std::string& equivalent_wire_format, - const ConformanceResponse& response, bool need_report_success, - bool require_same_wire_format) { - std::unique_ptr<Message> test_message(setting.NewTestMessage()); - const ConformanceRequest& request = setting.GetRequest(); - const std::string& test_name = setting.GetTestName(); - ConformanceLevel level = setting.GetLevel(); - std::unique_ptr<Message> reference_message = setting.NewTestMessage(); - - ABSL_CHECK(reference_message->ParseFromString(equivalent_wire_format)) - << "Failed to parse wire data for test case: " << test_name; - - TestStatus test; - test.set_name(test_name); - - switch (response.result_case()) { - case ConformanceResponse::RESULT_NOT_SET: - test.set_failure_message( - "Response didn't have any field in the Response."); - ReportFailure(test, level, request, response); - return; - - case ConformanceResponse::kParseError: - case ConformanceResponse::kTimeoutError: - case ConformanceResponse::kRuntimeError: - case ConformanceResponse::kSerializeError: - test.set_failure_message("Failed to parse input or produce output."); - ReportFailure(test, level, request, response); - return; - - case ConformanceResponse::kSkipped: - ReportSkip(test, request, response); - return; - - default: - if (!ParseResponse(response, setting, test_message.get())) return; - } - - MessageDifferencer differencer; - DefaultFieldComparator field_comparator; - field_comparator.set_treat_nan_as_equal(true); - differencer.set_field_comparator(&field_comparator); - std::string differences; - differencer.ReportDifferencesToString(&differences); - - bool check = false; - - if (require_same_wire_format) { - ABSL_DCHECK_EQ(response.result_case(), - ConformanceResponse::kProtobufPayload); - const std::string& protobuf_payload = response.protobuf_payload(); - check = equivalent_wire_format == protobuf_payload; - differences = absl::StrCat("Expect: ", ToOctString(equivalent_wire_format), - ", but got: ", ToOctString(protobuf_payload)); - } else { - check = differencer.Compare(*reference_message, *test_message); - } - if (check) { - if (need_report_success) { - ReportSuccess(test); - } - } else { - test.set_failure_message(absl::StrCat( - "Output was not equivalent to reference message: ", differences)); - ReportFailure(test, level, request, response); - } -} - -bool ConformanceTestSuite::RunTest(const std::string& test_name, - const ConformanceRequest& request, - ConformanceResponse* response) { - if (test_names_ran_.insert(test_name).second == false) { - ABSL_LOG(FATAL) << "Duplicated test name: " << test_name; - } - - // In essence, find what wildcarded test names expand to or direct matches - // (without wildcards). - auto result = failure_list_root_.WalkDownMatch(test_name); - if (result.has_value()) { - string matched_equivalent = result.value(); - unmatched_.erase(matched_equivalent); - TestStatus expansion; - expansion.set_name(test_name); - expansion.set_matched_name(matched_equivalent); - expansion.set_failure_message(saved_failure_messages_[matched_equivalent]); - expected_to_fail_[test_name] = expansion; - - if (number_of_matches_.contains(matched_equivalent)) { - if (number_of_matches_[matched_equivalent] > kMaximumWildcardExpansions && - !exceeded_max_matches_.contains(matched_equivalent)) { - exceeded_max_matches_[matched_equivalent] = expansion; - } - number_of_matches_[matched_equivalent]++; - } else { - number_of_matches_[matched_equivalent] = 1; - } - } - - std::string serialized_request; - ABSL_CHECK(request.SerializeToString(&serialized_request)); - - uint32_t len = ::google::protobuf::internal::little_endian::FromHost( - static_cast<uint32_t>(serialized_request.size())); - - if (isolated_) { - if (names_to_test_.erase(test_name) == - 0) { // Tests were asked to be run in isolated mode, but this test was - // not asked to be run. - expected_to_fail_.erase(test_name); - return false; - } - if (debug_) { - std::string octal = ProduceOctalSerialized(serialized_request, len); - std::string full_filename = WriteToFile(octal, output_dir_, test_name); - if (!full_filename.empty()) { - absl::StrAppendFormat( - &output_, "Produced octal serialized request file for test %s\n", - test_name); - absl::StrAppendFormat( - &output_, - " To pipe the " - "serialized request directly to " - "the " - "testee run from the root of your workspace:\n printf $(" - "<\"%s\") | %s\n\n", - full_filename, testee_); - absl::StrAppendFormat( - &output_, - " To inspect the wire format of the serialized request with " - "protoscope run " - "(Disclaimer: This may not work properly on non-Linux " - "platforms):\n " - " " - "contents=$(<\"%s\"); sub=$(cut -d \\\\ -f 6- <<< " - "$contents) ; printf \"\\\\${sub}\" | protoscope \n\n\n", - full_filename); - } - } - } - - response->set_protobuf_payload(serialized_request); - - std::string serialized_response = - runner_->RunTest(test_name, serialized_request); - - if (!response->ParseFromString(serialized_response)) { - response->Clear(); - response->set_runtime_error("response proto could not be parsed."); - } - - if (verbose_) { - absl::StrAppendFormat( - &output_, "conformance test: name=%s, request=%s, response=%s\n", - test_name, TruncateRequest(request).ShortDebugString(), - TruncateResponse(*response).ShortDebugString()); - } - return true; -} - -std::string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) { - switch (wire_format) { - case ::conformance::PROTOBUF: - return "PROTOBUF"; - case ::conformance::JSON: - return "JSON"; - case ::conformance::JSPB: - return "JSPB"; - case ::conformance::TEXT_FORMAT: - return "TEXT_FORMAT"; - case ::conformance::UNSPECIFIED: - return "UNSPECIFIED"; - default: - ABSL_LOG(FATAL) << "unknown wire type: " << wire_format; - } - return ""; -} - -bool ConformanceTestSuite::AddExpectedFailedTest( - const TestStatus& expected_failure) { - absl::Status attempt = failure_list_root_.Insert(expected_failure.name()); - if (!attempt.ok()) { - absl::StrAppend(&output_, attempt.message(), "\n\n"); - return false; - } - unmatched_[expected_failure.name()] = expected_failure; - saved_failure_messages_[expected_failure.name()] = - expected_failure.failure_message(); - return true; -} - -bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, - std::string* output, - const std::string& filename, - ::conformance::FailureSet* failure_list) { - runner_ = runner; - failure_list_root_ = FailureListTrieNode("root"); - successes_ = 0; - expected_failures_ = 0; - skipped_.clear(); - test_names_ran_.clear(); - unexpected_failing_tests_.clear(); - unexpected_succeeding_tests_.clear(); - - std::string mode = debug_ ? "DEBUG" : "TEST"; - absl::StrAppendFormat( - &output_, "CONFORMANCE %s BEGIN ====================================\n\n", - mode); - - failure_list_filename_ = filename; - expected_to_fail_.clear(); - for (const TestStatus& expected_failure : failure_list->test()) { - if (!AddExpectedFailedTest(expected_failure)) { - output->assign(output_); - return false; - } - } - - RunSuiteImpl(); - - if (*output_dir_.rbegin() != '/') { - output_dir_.push_back('/'); - } - - bool ok = true; - if (!CheckSetEmpty( - unmatched_, "unmatched.txt", - absl::StrCat( - "These test names were listed in the failure list, but they " - "didn't match any actual test name. Remove them from the " - "failure list by running from the root of your workspace:\n" - " bazel run " - "//google/protobuf/conformance:update_failure_list -- ", - failure_list_filename_, " --remove ", output_dir_, - "unmatched.txt"), - output_dir_, &output_)) { - ok = false; - } - - if (!CheckSetEmpty( - expected_failure_messages_, "expected_failure_messages.txt", - absl::StrCat( - "These tests (either expanded from wildcard(s) or direct " - "matches) were listed in the failure list, but their " - "failure messages do not match. Remove their match from the " - "failure list by running from the root of your workspace:\n" - " bazel run ", - "//google/protobuf/conformance:update_failure_list -- ", - failure_list_filename_, " --remove ", output_dir_, - "expected_failure_messages.txt"), - output_dir_, &output_)) { - ok = false; - } - - if (!CheckSetEmpty( - unexpected_succeeding_tests_, "succeeding_tests.txt", - absl::StrCat( - "These tests succeeded, even though they were listed in " - "the failure list (expanded from wildcard(s) or direct matches). " - " Remove their match from the failure list by " - "running from the root of your workspace:\n" - " bazel run " - "//google/protobuf/conformance:update_failure_list -- ", - failure_list_filename_, " --remove ", output_dir_, - "succeeding_tests.txt"), - output_dir_, &output_)) { - ok = false; - } - - if (!CheckSetEmpty( - exceeded_max_matches_, "exceeded_max_matches.txt", - absl::StrFormat( - "These failure list entries served as matches to too many test " - "names exceeding the max amount of %d. " - "Remove them from the failure list by running from the root of " - "your workspace:\n" - " bazel run " - "//google/protobuf/conformance:update_failure_list -- %s " - "--remove %sexceeded_max_matches.txt", - kMaximumWildcardExpansions, failure_list_filename_, output_dir_), - output_dir_, &output_)) { - ok = false; - } - - if (!CheckSetEmpty( - unexpected_failure_messages_, "unexpected_failure_messages.txt", - absl::StrCat( - "These tests (expanded from wildcard(s) or direct matches from " - "the failure list) failed because their failure messages did " - "not match. If they can't be fixed right now, " - "you can add them to the failure list so the overall " - "suite can succeed. Add them to the failure list by " - "running from the root of your workspace:\n" - " bazel run " - "//google/protobuf/conformance:update_failure_list -- ", - failure_list_filename_, " --add ", output_dir_, - "unexpected_failure_messages.txt"), - output_dir_, &output_)) { - ok = false; - } - - if (!CheckSetEmpty( - unexpected_failing_tests_, "failing_tests.txt", - absl::StrCat( - "These tests failed. If they can't be fixed right now, " - "you can add them to the failure list so the overall " - "suite can succeed. Add them to the failure list by " - "running from the root of your workspace:\n" - " bazel run " - "//google/protobuf/conformance:update_failure_list -- ", - failure_list_filename_, " --add ", output_dir_, - "failing_tests.txt"), - output_dir_, &output_)) { - ok = false; - } - - if (verbose_) { - CheckSetEmpty(skipped_, "", - "These tests were skipped (probably because support for some " - "features is not implemented)", - output_dir_, &output_); - } - - absl::StrAppendFormat(&output_, - "CONFORMANCE SUITE %s: %d successes, %zu skipped, " - "%d expected failures, %zu unexpected failures.\n", - ok ? "PASSED" : "FAILED", successes_, skipped_.size(), - expected_failures_, unexpected_failing_tests_.size()); - absl::StrAppendFormat(&output_, "\n"); - - output->assign(output_); - - return ok; -} - -} // namespace conformance -} // namespace protobuf -} // namespace google
diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h deleted file mode 100644 index 991f9b0..0000000 --- a/conformance/conformance_test.h +++ /dev/null
@@ -1,322 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -// This file defines a protocol for running the conformance test suite -// in-process. In other words, the suite itself will run in the same process as -// the code under test. -// -// For pros and cons of this approach, please see conformance.proto. - -#ifndef GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_H__ -#define GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_H__ - -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "google/protobuf/descriptor.pb.h" -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_set.h" -#include "conformance/conformance.pb.h" -#include "conformance/failure_list_trie_node.h" -#include "conformance/test_runner.h" -#include "google/protobuf/descriptor.h" - -namespace conformance { -class ConformanceRequest; -class ConformanceResponse; -} // namespace conformance - -namespace protobuf_test_messages { -namespace proto3 { -class TestAllTypesProto3; -} // namespace proto3 -} // namespace protobuf_test_messages - -namespace google { -namespace protobuf { -namespace conformance { - -class ConformanceTestSuite; - -int RunConformanceTests(int argc, char* argv[], - const std::vector<ConformanceTestSuite*>& suites); - -// Class representing the test suite itself. To run it, implement your own -// class derived from ConformanceTestRunner, class derived from -// ConformanceTestSuite and then write code like: -// -// class MyConformanceTestSuite : public ConformanceTestSuite { -// public: -// void RunSuiteImpl() { -// // INSERT ACTUAL TESTS. -// } -// }; -// -// class MyConformanceTestRunner : public ConformanceTestRunner { -// public: -// static int Run(int argc, char *argv[], -// ConformanceTestSuite* suite); -// -// private: -// virtual void RunTest(...) { -// // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. -// } -// }; -// -// int main() { -// MyConformanceTestSuite suite; -// MyConformanceTestRunner::Run(argc, argv, &suite); -// } -// -class ConformanceTestSuite { - public: - ConformanceTestSuite() = default; - virtual ~ConformanceTestSuite() = default; - - void SetPerformance(bool performance) { performance_ = performance; } - void SetVerbose(bool verbose) { verbose_ = verbose; } - - // Whether to require the testee to pass RECOMMENDED tests. By default failing - // a RECOMMENDED test case will not fail the entire suite but will only - // generated a warning. If this flag is set to true, RECOMMENDED tests will - // be treated the same way as REQUIRED tests and failing a RECOMMENDED test - // case will cause the entire test suite to fail as well. An implementation - // can enable this if it wants to be strictly conforming to protobuf spec. - // See the comments about ConformanceLevel below to learn more about the - // difference between REQUIRED and RECOMMENDED test cases. - void SetEnforceRecommended(bool value) { enforce_recommended_ = value; } - - // Sets the maximum edition (inclusive) that should be tests for conformance. - void SetMaximumEdition(Edition edition) { maximum_edition_ = edition; } - - // Gets the flag name to the failure list file. - // By default, this would return --failure_list - std::string GetFailureListFlagName() { return failure_list_flag_name_; } - - void SetFailureListFlagName(const std::string& failure_list_flag_name) { - failure_list_flag_name_ = failure_list_flag_name; - } - - // Sets the path of the output directory. - void SetOutputDir(const std::string& output_dir) { output_dir_ = output_dir; } - - // Sets if we are running the test in debug mode. - void SetDebug(bool debug) { debug_ = debug; } - - // Sets if we are running ONLY the tests provided in the 'names_to_test_' set. - void SetIsolated(bool isolated) { isolated_ = isolated; } - - // Sets the file path of the testee. - void SetTestee(const std::string& testee) { testee_ = testee; } - - // Sets the names of tests to ONLY be run isolated from all the others. - void SetNamesToTest(absl::flat_hash_set<std::string> names_to_test) { - names_to_test_ = std::move(names_to_test); - } - - absl::flat_hash_set<std::string> GetExpectedTestsNotRun() { - return names_to_test_; - } - - // Run all the conformance tests against the given test runner. - // Test output will be stored in "output". - // - // Returns true if the set of failing tests was exactly the same as the - // failure list. - // The filename here is *only* used to create/format useful error messages for - // how to update the failure list. We do NOT read this file at all. - - bool RunSuite(ConformanceTestRunner* runner, std::string* output, - const std::string& filename, - ::conformance::FailureSet* failure_list); - - protected: - // Test cases are classified into a few categories: - // REQUIRED: the test case must be passed for an implementation to be - // interoperable with other implementations. For example, a - // parser implementation must accept both packed and unpacked - // form of repeated primitive fields. - // RECOMMENDED: the test case is not required for the implementation to - // be interoperable with other implementations, but is - // recommended for best performance and compatibility. For - // example, a proto3 serializer should serialize repeated - // primitive fields in packed form, but an implementation - // failing to do so will still be able to communicate with - // other implementations. - enum ConformanceLevel { - REQUIRED = 0, - RECOMMENDED = 1, - }; - - class ConformanceRequestSetting { - public: - ConformanceRequestSetting(ConformanceLevel level, - ::conformance::WireFormat input_format, - ::conformance::WireFormat output_format, - ::conformance::TestCategory test_category, - const Message& prototype_message, - const std::string& test_name, - const std::string& input); - virtual ~ConformanceRequestSetting() = default; - - std::unique_ptr<Message> NewTestMessage() const; - - std::string GetSyntaxIdentifier() const; - - std::string GetTestName() const; - - const ::conformance::ConformanceRequest& GetRequest() const { - return request_; - } - - ConformanceLevel GetLevel() const { return level_; } - - std::string ConformanceLevelToString(ConformanceLevel level) const; - - void SetPrintUnknownFields(bool print_unknown_fields) { - request_.set_print_unknown_fields(true); - } - - void SetPrototypeMessageForCompare(const Message& message) { - prototype_message_for_compare_.reset(message.New()); - } - - protected: - virtual std::string InputFormatString( - ::conformance::WireFormat format) const; - virtual std::string OutputFormatString( - ::conformance::WireFormat format) const; - ::conformance::ConformanceRequest request_; - - private: - ConformanceLevel level_; - ::conformance::WireFormat input_format_; - ::conformance::WireFormat output_format_; - const Message& prototype_message_; - std::unique_ptr<Message> prototype_message_for_compare_; - std::string test_name_; - }; - - std::string WireFormatToString(::conformance::WireFormat wire_format); - - // Parse payload in the response to the given message. Returns true on - // success. - virtual bool ParseResponse(const ::conformance::ConformanceResponse& response, - const ConformanceRequestSetting& setting, - Message* test_message) = 0; - - void VerifyResponse(const ConformanceRequestSetting& setting, - const std::string& equivalent_wire_format, - const ::conformance::ConformanceResponse& response, - bool need_report_success, bool require_same_wire_format); - - void TruncateDebugPayload(std::string* payload); - ::conformance::ConformanceRequest TruncateRequest( - const ::conformance::ConformanceRequest& request); - ::conformance::ConformanceResponse TruncateResponse( - const ::conformance::ConformanceResponse& response); - - void ReportSuccess(const ::conformance::TestStatus& test); - void ReportFailure(::conformance::TestStatus& test, ConformanceLevel level, - const ::conformance::ConformanceRequest& request, - const ::conformance::ConformanceResponse& response); - void ReportSkip(const ::conformance::TestStatus& test, - const ::conformance::ConformanceRequest& request, - const ::conformance::ConformanceResponse& response); - - void RunValidInputTest(const ConformanceRequestSetting& setting, - const std::string& equivalent_text_format); - void RunValidBinaryInputTest(const ConformanceRequestSetting& setting, - const std::string& equivalent_wire_format, - bool require_same_wire_format = false); - - // Returns true if our runner_ ran the test and false if it did not. - bool RunTest(const std::string& test_name, - const ::conformance::ConformanceRequest& request, - ::conformance::ConformanceResponse* response); - - // Will return false if an entry from the failure list was either a - // duplicate of an already added one to the trie or it contained invalid - // wildcards; otherwise, returns true. - bool AddExpectedFailedTest(const ::conformance::TestStatus& failure); - - virtual void RunSuiteImpl() = 0; - - ConformanceTestRunner* runner_; - FailureListTrieNode failure_list_root_; - std::string testee_; - int successes_; - int expected_failures_; - bool verbose_ = false; - bool performance_ = false; - bool enforce_recommended_ = false; - Edition maximum_edition_ = Edition::EDITION_PROTO3; - std::string output_; - std::string output_dir_; - std::string failure_list_flag_name_ = "--failure_list"; - std::string failure_list_filename_; - absl::flat_hash_set<std::string> names_to_test_; - bool debug_ = false; - // If names were given for names_to_test_, only those tests - // will be run and this bool will be set to true. - bool isolated_ = false; - - // The set of test names (expanded from wildcard(s) and non-expanded) that are - // expected to fail in this run, but haven't failed yet. - absl::btree_map<std::string, ::conformance::TestStatus> expected_to_fail_; - - // The set of tests that failed because their failure message did not match - // the actual failure message. These are failure messages that may need to be - // removed from our failure lists. - absl::btree_map<std::string, ::conformance::TestStatus> - expected_failure_messages_; - - // The set of test names that have been run. Used to ensure that there are no - // duplicate names in the suite. - absl::flat_hash_set<std::string> test_names_ran_; - - // The set of tests that failed, but weren't expected to: They weren't - // present in our failure lists. - absl::btree_map<std::string, ::conformance::TestStatus> - unexpected_failing_tests_; - - // The set of tests that succeeded, but weren't expected to: They were present - // in our failure lists, but managed to succeed. - absl::btree_map<std::string, ::conformance::TestStatus> - unexpected_succeeding_tests_; - - // The set of tests that failed because their failure message did not match - // the actual failure message. These are failure messages that may need to be - // added to our failure lists. - absl::btree_map<std::string, ::conformance::TestStatus> - unexpected_failure_messages_; - - // The set of test names (wildcarded or not) from the failure list that did - // not match any actual test name. - absl::btree_map<std::string, ::conformance::TestStatus> unmatched_; - - // The set of tests that the testee opted out of; - absl::btree_map<std::string, ::conformance::TestStatus> skipped_; - - // Allows us to remove from unmatched_. - absl::btree_map<std::string, std::string> saved_failure_messages_; - - // If a failure list entry served as a match for more than 'max_matches_', - // those will be added here for removal. - absl::btree_map<std::string, ::conformance::TestStatus> exceeded_max_matches_; - - // Keeps track of how many tests matched to each failure list entry. - absl::btree_map<std::string, int> number_of_matches_; -}; - -} // namespace conformance -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_H__
diff --git a/conformance/conformance_test_main.cc b/conformance/conformance_test_main.cc index 3e4dd84..cffd864 100644 --- a/conformance/conformance_test_main.cc +++ b/conformance/conformance_test_main.cc
@@ -5,13 +5,746 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -#include "conformance/binary_json_conformance_suite.h" -#include "conformance/conformance_test.h" -#include "conformance/text_format_conformance_suite.h" +// The main of conformance_test_runner. +// +// The runner runs the gtest-based conformance suites (see test_fixture.h) +// linked into this binary, the alwayslink libraries in conformance_test_main's +// BUILD deps (binary_conformance_tests, json_conformance_tests, +// performance_conformance_tests, text_conformance_tests), against the testee +// named on its command line. The command line and the report format are those +// of the legacy runner, which ran hand-written suites instead (see +// conformance_test_runner.h), so nothing changes for its users, the test +// targets and scripts that invoke it with their failure lists and --output_dir: +// +// * gtest flags given before the testee are accepted too. Both spellings +// work everywhere: --gtest_* (the open source one) and --gunit_* (the +// google3 one); a --gtest_ flag is rewritten to the prefix this binary's +// gtest was built with (see RewriteGtestFlagPrefix()), since google3's +// gtest rejects the other. +// * --failure_list and --text_format_failure_list (see kFailureListFlag and +// kTextFormatFailureListFlag in conformance_test_runner.h) are loaded into +// the gtest suites' one ResultLedger. An entry that matches no test is +// reported as unmatched, in the legacy format, and fails the run like it +// failed the legacy suites. +// * --test <name> isolates the named tests (see FilteringTestRunner), and +// names that don't exist are reported as before. +// * --performance selects the performance tests: the test suites of the +// performance conformance suite, which are the ones named *PerformanceTest +// (see test_fixture.h), are selected through the gtest filter, and +// deselected without the flag (see PerformanceGtestFilter()). +// * --record_requests writes every request into one file. +// * --output_dir receives the files update_failure_list consumes: the gtest +// suites write failing_tests.txt (unexpected failures), +// succeeding_tests.txt (unexpected successes), listed_skips.txt and +// unmatched.txt in the legacy format, see ReportTestStatusSet(). +// +// TODO: retire the legacy-compatible parts of this command line +// (--failure_list / --text_format_failure_list and conformance_test_wrapper.sh) +// once the remaining sh_test and script invocations have moved to the +// conformance_test() macro, which runs the gtest suites as ordinary test +// binaries (see test_environment_main.cc). +// +// Behavior notes, i.e. what does differ from the legacy runner: +// +// * gtest flags are accepted before the testee, in their `--flag=value` form +// only (the runner's own parser would take a separate value for the +// testee). Anything else starting with --gtest_ or --gunit_ is still +// rejected, but by whoever sees it first: in google3 InitGoogle() exits +// with its own "Unknown command line flag" error, in open source the +// runner prints its usage text as before. +// * The runner is a self-contained program: the environment variables +// through which bazel and the test bridge configure a gtest binary +// (kInheritedTestEnvironmentVariables: test filter, sharding, fail-fast, +// result and status files, ...) are meant for the enclosing test and are +// cleared before gtest initializes, like the legacy runner ignored them, +// and the gtest flags open source gtest has already defaulted from them by +// then are reset (see ResetInheritedGtestFlags()). Explicit gtest flags +// on the command line are the only way to filter, shard or get an XML/JSON +// report of the run. As a consequence, the testee doesn't see those +// variables either. +// * Unless a gtest flag was given, the run prints gtest's brief output +// (--gtest_brief=1): failures, skips and the summary, not every test. +// * --debug and --verbose are accepted but have no effect: they only ever +// affected the legacy suites (see ConformanceRunnerOptions). gtest's own +// flags take their place (--gtest_brief=0 lists every test's result). +// * testing::InitGoogleTest() runs. In google3 that also runs InitGoogle(): +// absl logging is initialized (ABSL_LOG(INFO) lines, e.g. the testee +// command line logged by ForkPipeRunner, go to the log files instead of +// stderr) and the failure signal handler is installed. +// * --gtest_repeat is not supported (see test_environment.h). +// * --gtest_filter, test sharding and --gtest_fail_fast make the run a +// partial one, which disables the unmatched-entry check: a failure list +// entry that no test matched may simply belong to a test that wasn't +// selected (or run). The same applies when the filter selects no test at +// all. The selection --performance (or its absence) makes is not partial: +// its baseline is the tests of the selected kind (see +// IsPartialGtestPhase()). An explicit --gtest_filter applies within that +// selection. +// * --gtest_output only takes effect when given on the command line itself, +// not through --gtest_flagfile (see GtestOutputRequested()). +// * If writing the --record_requests file fails *and* --test named tests +// that don't exist, the legacy "do not exist" report is printed first and +// the write error last. +// * The gtest suites only report the --output_dir categories above. A test +// that was expected to fail with a different message is an unexpected +// failure to them (the legacy suites reported it separately, in +// unexpected_failure_messages.txt and expected_failure_messages.txt). +// Since the gtest suites load every failure list into one ResultLedger, +// their report comes in one block per failure list flag, each with the +// hint naming that flag's last file: a result matched by a failure list +// entry goes to the flag whose file lists the entry, a test in no list to +// --text_format_failure_list if it is a text-format test (see +// IsTextFormatTest()) and to --failure_list otherwise. Like the legacy +// suites among themselves, a later block overwrites the --output_dir +// files of an earlier one. +// TODO: goes away with the --text_format_failure_list flag; +// conformance_test() passes one failure list per suite instead. -int main(int argc, char *argv[]) { - google::protobuf::conformance::BinaryAndJsonConformanceSuite binary_and_json_suite; - google::protobuf::conformance::TextFormatConformanceTestSuite text_format_suite; - return google::protobuf::conformance::RunConformanceTests( - argc, argv, {&binary_and_json_suite, &text_format_suite}); +#include <algorithm> +#include <cstddef> +#include <cstdio> // IWYU pragma: keep +#include <cstdlib> +#include <fstream> +#include <memory> +#include <ostream> +#include <string> +#include <utility> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include <gtest/gtest.h> +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" +#include "conformance/conformance_test_runner.h" +#include "conformance/filtering_test_runner.h" +#include "conformance/fork_pipe_runner.h" +#include "conformance/recording_test_runner.h" +#include "conformance/result_ledger.h" +#include "conformance/test_environment.h" +#include "conformance/test_runner.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +// The environment variables through which bazel and the test bridge configure +// the gtest binary they run. Under `bazel test` they describe the *enclosing* +// test (typically an sh_test wrapping this runner), so the gtest phase must not +// act on them: it would run only that test's shard or filter, stop at its +// first failure, or overwrite its result file. Cleared before +// testing::InitGoogleTest(); see the file comment and +// ResetInheritedGtestFlags(). Both the open source (GTEST_) and google3 +// (GUNIT_) spellings are listed. +constexpr const char* kInheritedTestEnvironmentVariables[] = { + // Test selection, sharding and shuffling. + "TESTBRIDGE_TEST_ONLY", + "TEST_TOTAL_SHARDS", + "TEST_SHARD_INDEX", + "TEST_SHARD_STATUS_FILE", + "GTEST_FILTER", + "GUNIT_FILTER", + "GTEST_FAIL_IF_NO_TEST_SELECTED", + "GUNIT_FAIL_IF_NO_TEST_SELECTED", + "GTEST_ALSO_RUN_DISABLED_TESTS", + "GUNIT_ALSO_RUN_DISABLED_TESTS", + "GTEST_SHUFFLE", + "GUNIT_SHUFFLE", + "GTEST_RANDOM_SEED", + "GUNIT_RANDOM_SEED", + // How far the run goes and how it is reported. + "TESTBRIDGE_TEST_RUNNER_FAIL_FAST", + "GTEST_FAIL_FAST", + "GUNIT_FAIL_FAST", + "GTEST_REPEAT", + "GUNIT_REPEAT", + "GTEST_BRIEF", + "GUNIT_BRIEF", + "GTEST_FLAGFILE", + "GUNIT_FLAGFILE", + // Result and status files. + "GTEST_OUTPUT", + "GUNIT_OUTPUT", + "XML_OUTPUT_FILE", + "TEST_PREMATURE_EXIT_FILE", + "TEST_WARNINGS_OUTPUT_FILE", +}; + +// Resets the gtest flags that kInheritedTestEnvironmentVariables would have +// defaulted to their built-in values, and picks the brief output unless +// `gtest_args` (the explicit gtest flags) configure gtest. Must be called +// after the variables are cleared and before testing::InitGoogleTest(). +// +// Clearing the variables is enough in google3, whose gtest flags are absl +// flags that evaluate their environment-derived default lazily, on first +// access; that happens after the variables are gone. Open source gtest +// evaluates them once, at static initialization, before main() runs: by now +// the enclosing test's filter, fail-fast, shuffle, ... settings are already +// baked into the flags, so they are reset here. Explicit gtest flags are +// parsed afterwards, by InitGoogleTest(), and therefore still win. +// +// Not reset here: sharding, whose flags don't exist in every gtest version +// and whose variables gtest only reads in RUN_ALL_TESTS() (after they are +// cleared; a shard that got through anyway makes the run partial, see +// IsPartialGtestPhase()), and the output flag, which is handled after +// InitGoogleTest() (see GtestOutputRequested()). +void ResetInheritedGtestFlags(absl::Span<const std::string> gtest_args) { + GTEST_FLAG_SET(filter, "*"); + GTEST_FLAG_SET(also_run_disabled_tests, false); + GTEST_FLAG_SET(shuffle, false); + GTEST_FLAG_SET(random_seed, 0); + GTEST_FLAG_SET(fail_fast, false); + GTEST_FLAG_SET(repeat, 1); + // Keep the gtest phase's output as short as the legacy reports, unless the + // caller configured gtest themselves. Like the others, this has to happen + // before InitGoogleTest(), which is where gtest picks its result printer + // (in PostFlagParsingInit()). + GTEST_FLAG_SET(brief, gtest_args.empty()); +} + +// The gtest output flag, spelled as this binary's gtest expects it, e.g. +// "--gunit_output=". gtest only accepts the `--flag=value` form for it (and so +// does this runner, see the file comment), so that is the only form matched. +constexpr absl::string_view kGtestOutputFlagPrefix = + "--" GTEST_FLAG_PREFIX_ "output="; + +// Returns true if gtest output was asked for explicitly on the command line. +// `gtest_args` must already have been through RewriteGtestFlagPrefix(). The +// environment variables gtest would otherwise default the flag from are +// deliberately not consulted: they belong to the enclosing test (see +// kInheritedTestEnvironmentVariables). Neither is --gtest_flagfile: an output +// flag given through it isn't seen here and so has no effect. +bool GtestOutputRequested(absl::Span<const std::string> gtest_args) { + for (const std::string& arg : gtest_args) { + if (absl::StartsWith(arg, kGtestOutputFlagPrefix)) return true; + } + return false; +} + +// The names of every gtest test suite registered in this binary. +std::vector<std::string> RegisteredTestSuiteNames() { + const testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + const int count = unit_test.total_test_suite_count(); + std::vector<std::string> names; + names.reserve(static_cast<size_t>(count)); + for (int i = 0; i < count; ++i) { + names.push_back(unit_test.GetTestSuite(i)->name()); + } + return names; +} + +// Narrows the gtest filter to the performance test suites if `performance`, +// and to all the others otherwise; see PerformanceGtestFilter(). Must be +// called after testing::InitGoogleTest() (which parses an explicit +// --gtest_filter) and before the tests are listed or run. +void SelectPerformanceTests(bool performance) { + GTEST_FLAG_SET(filter, + PerformanceGtestFilter(GTEST_FLAG_GET(filter), performance, + RegisteredTestSuiteNames())); +} + +// The number of tests a full run runs after SelectPerformanceTests(): +// the tests of the selected kind, minus the DISABLED_ ones (like the baseline +// of the environment's IsPartialRun(), which counts every registered test). +int SelectedTestCount(bool performance) { + const testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + int count = 0; + for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { + const testing::TestSuite& test_suite = *unit_test.GetTestSuite(i); + if (IsPerformanceFixture(test_suite.name()) != performance) continue; + count += test_suite.total_test_count() - test_suite.disabled_test_count(); + } + return count; +} + +// Whether only some of the `selected_test_count` tests of the selected kind +// ran, because of an explicit --gtest_filter, sharding or --gtest_fail_fast. +// The runner's counterpart of the environment's IsPartialRun() (see +// test_environment.cc), which would count the tests SelectPerformanceTests() +// deselected as not run, and so call every run partial. +bool IsPartialGtestPhase(int selected_test_count) { + const testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + if (GTEST_FLAG_GET(fail_fast) && unit_test.Failed()) return true; + return unit_test.test_to_run_count() < selected_test_count; +} + +internal::ConformanceEnvironmentOptions EnvironmentOptionsFor( + const ConformanceRunnerOptions& options, + ConformanceTestRunner* absl_nonnull runner) { + internal::ConformanceEnvironmentOptions env_options; + env_options.runner = runner; + for (const auto& [flag, file] : options.failure_list_files) { + env_options.failure_list_files.push_back(file); + } + env_options.enforcement_level = + options.enforce_recommended ? kLowestPriority : kP0; + // The legacy runner treats any maximum below PROTO3 (its EDITION_UNKNOWN + // default, or an explicit PROTO2) as "proto2 and proto3 only", which is the + // environment's EDITION_PROTO3. ConformanceEnvironment clamps this itself; + // the std::max here just keeps the intent visible. + env_options.maximum_edition = + std::max(options.maximum_edition, EDITION_PROTO3); + // --performance is applied through the gtest filter instead; see + // SelectPerformanceTests(). + // Unmatched entries are reported per failure list flag below instead. + env_options.check_unseen_expected_failures = false; + // --debug and --verbose have no effect (see the file comment). + return env_options; +} + +// Converts `results` for ReportTestStatusSet(). +std::vector<ReportedTestStatus> ToReportedTestStatuses( + absl::Span<const internal::UnexpectedResult> results) { + std::vector<ReportedTestStatus> statuses; + statuses.reserve(results.size()); + for (const internal::UnexpectedResult& result : results) { + ReportedTestStatus status; + status.test_name = result.test_name; + status.failure_message = result.failure_message; + status.matched_name = result.matched_entry.value_or(""); + statuses.push_back(std::move(status)); + } + return statuses; +} + +// Whether `test_name` is a text-format conformance test, i.e. one the legacy +// text-format suite used to run: a test with text-format input or output. +// Test names read "<Level>.<Edition>.<Format>Input.<name>[.<Format>Output]" +// (see Testee). +bool IsTextFormatTest(absl::string_view test_name) { + return absl::StrContains(test_name, ".TextFormatInput.") || + absl::EndsWith(test_name, ".TextFormatOutput"); +} + +// The failure list flag whose block reports a test that is in no failure +// list, with the hint to add it to that flag's last file (the flags are those +// of conformance_test_runner.h). +absl::string_view FailureListFlagFor(absl::string_view test_name) { + return IsTextFormatTest(test_name) ? kTextFormatFailureListFlag + : kFailureListFlag; +} + +// One block of the gtest suites' report: the results one failure list flag's +// files are responsible for, reported by ReportFailureListResults() with the +// hint naming the flag's last file (like each legacy suite names the last +// file of its flag). +struct FailureListReport { + std::string flag; + std::vector<std::string> files; + std::vector<internal::UnexpectedResult> successes; + std::vector<internal::UnexpectedResult> failures; + // (test name, matched entry); see ResultLedger::ListedSkips(). + std::vector<std::pair<std::string, std::string>> listed_skips; + // The entries no test matched, with their messages. + std::vector<ReportedTestStatus> unmatched; +}; + +// Where a failure list entry (as listed, wildcards included) comes from. +struct FailureListEntry { + size_t report = 0; // An index into the FailureListReports. + std::string message; +}; + +// The index of the report block for `flag`, appended if there is none yet +// (with no files: the caller appends those). +size_t ReportIndexFor(std::vector<FailureListReport>& reports, + absl::string_view flag) { + for (size_t i = 0; i < reports.size(); ++i) { + if (reports[i].flag == flag) return i; + } + reports.push_back({/*flag=*/std::string(flag)}); + return reports.size() - 1; +} + +// Groups the failure lists (`failure_list_files`, (flag, file) pairs in +// command line order) into one report block per flag, in order of the flags' +// first appearance, and indexes their entries in `entries`. The files are +// loaded into a scratch ResultLedger exactly as ConformanceEnvironment will, +// so that a bad failure list is reported once and clearly instead of failing +// every gtest test in SetUp(). The environment loads all the files into one +// ResultLedger, so unlike with the legacy suites (each of which only loaded its +// own flag's files) an entry may only appear in one of them: the same name in +// both --failure_list and --text_format_failure_list, or a wildcard in one +// covering an entry of the other, is an error. +// TODO: goes away with the --text_format_failure_list flag; +// conformance_test() passes one failure list per suite instead. +absl::Status LoadFailureListReports( + absl::Span<const std::pair<std::string, std::string>> failure_list_files, + std::vector<FailureListReport>& reports, + absl::flat_hash_map<std::string, FailureListEntry>& entries) { + internal::ResultLedger ledger; + absl::Status status; + std::vector<std::string> loaded; + for (const auto& [flag, file] : failure_list_files) { + status = ledger.LoadFailureList(file); + if (!status.ok()) { + if (absl::IsAlreadyExists(status)) { + status = absl::InvalidArgumentError(absl::StrCat( + "While loading failure list ", file, + loaded.empty() + ? "" + : absl::StrCat(" after ", absl::StrJoin(loaded, ", ")), + ": ", status.message(), + ". The gtest conformance suites load all the failure lists into " + "one set, so an entry (or a wildcard covering it) may only appear " + "in one of them.")); + } + break; + } + loaded.push_back(file); + const size_t index = ReportIndexFor(reports, flag); + reports[index].files.push_back(file); + // The entries this file added are the ones not indexed yet. (Nothing has + // been reported, so no entry has been matched.) + for (const std::string& entry : ledger.UnmatchedExpectedFailures()) { + if (entries.contains(entry)) continue; + entries.emplace( + entry, FailureListEntry{ + /*report=*/index, + /*message=*/*ledger.failure_list().ExpectedMessage(entry), + }); + } + } + // ResultLedger insists on being finalized before destruction. + ledger.Finalize().IgnoreError(); + return status; +} + +// Reports `report` (the gtest suites' unexpected successes, unexpected +// failures, listed skips, i.e. tests in the failure list that the testee +// skipped, and unmatched entries) to stderr and to succeeding_tests.txt / +// failing_tests.txt / listed_skips.txt / unmatched.txt in `output_dir` (which +// must end in '/' unless empty), exactly as each legacy suite reports its own; +// see ReportTestStatusSet(). The hints tell the user to update the report's +// last file. Returns false if there were unmatched entries (the only category +// the gtest assertions haven't already failed for). +bool ReportFailureListResults(const FailureListReport& report, + absl::string_view output_dir) { + // A block may have no file: a test in no list is routed to its flag's block + // (see FailureListFlagFor()) whether or not that flag was given. + const std::string failure_list = + report.files.empty() ? absl::StrCat("<the ", report.flag, " file>") + : report.files.back(); + std::string output; + ReportTestStatusSet( + ToReportedTestStatuses(report.successes), "succeeding_tests.txt", + absl::StrCat( + "These tests succeeded, even though they were listed in " + "the failure list (expanded from wildcard(s) or direct matches). " + " Remove their match from the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//third_party/protobuf/conformance:update_failure_list -- ", + failure_list, " --remove ", output_dir, "succeeding_tests.txt"), + output_dir, output); + ReportTestStatusSet( + ToReportedTestStatuses(report.failures), "failing_tests.txt", + absl::StrCat("These tests failed. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed. Add them to the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//third_party/protobuf/conformance:update_failure_list -- ", + failure_list, " --add ", output_dir, "failing_tests.txt"), + output_dir, output); + std::vector<ReportedTestStatus> listed_skips; + listed_skips.reserve(report.listed_skips.size()); + for (const auto& [test_name, matched_entry] : report.listed_skips) { + listed_skips.push_back( + {/*test_name=*/test_name, + /*failure_message=*/ + absl::StrCat("skipped by the testee (matched to ", matched_entry, ")"), + /*matched_name=*/matched_entry}); + } + ReportTestStatusSet( + listed_skips, "listed_skips.txt", + absl::StrCat("These tests were skipped by the testee, even though they " + "were listed in the failure list (expanded from wildcard(s) " + "or direct matches). A skipped test can't be an expected " + "failure. Remove their match from the failure list by " + "running from the root of your workspace:\n" + " bazel run " + "//third_party/protobuf/conformance:update_failure_list -- ", + failure_list, " --remove ", output_dir, "listed_skips.txt"), + output_dir, output); + const bool no_unmatched = ReportTestStatusSet( + report.unmatched, "unmatched.txt", + absl::StrCat("These test names were listed in the failure list, but they " + "didn't match any actual test name. Remove them from the " + "failure list by running from the root of your workspace:\n" + " bazel run " + "//third_party/protobuf/conformance:update_failure_list -- ", + failure_list, " --remove ", output_dir, "unmatched.txt"), + output_dir, output); + absl::FPrintF(stderr, "%s", output); + return no_unmatched; +} + +// What running the gtest suites leaves behind for Main(). +struct GtestPhaseResult { + // RUN_ALL_TESTS()'s result, or 1 if a failure list had unmatched entries. + int exit_code = 0; + // The tests --test named that the gtest suites ran. + std::vector<std::string> names_run; +}; + +// Runs the gtest suites linked into this binary against `options.testee` and +// prints their report: one block per failure list flag (see +// ReportFailureListResults()) and a summary line. A failure list entry no +// test matched fails the run, like it failed the legacy suites. Returns an +// error, before anything runs, if the failure lists can't be loaded into one +// ResultLedger (see LoadFailureListReports()). Must be called after +// testing::InitGoogleTest(), and at most once (it installs the +// ConformanceEnvironment and calls RUN_ALL_TESTS()). +absl::StatusOr<GtestPhaseResult> RunGtestPhase( + const ConformanceRunnerOptions& options, + std::ostream* absl_nullable record_requests) { + GtestPhaseResult result; + testing::UnitTest* unit_test = testing::UnitTest::GetInstance(); + + // The ForkPipeRunner (which only spawns the testee on first use) is wrapped + // for recording and for --test isolation. The runners only need to live for + // this call: the environment installed below outlives them, but it doesn't + // own the runner and drops its reference to it in TearDown() (run by gtest, + // or below if gtest selected no test); ~ConformanceEnvironment only + // finalizes its ResultLedger. + ForkPipeRunner fork_pipe_runner(options.testee, options.testee_args); + ConformanceTestRunner* gtest_runner = &fork_pipe_runner; + absl::optional<RecordingTestRunner> recording_runner; + if (record_requests != nullptr) { + recording_runner.emplace(gtest_runner, record_requests); + gtest_runner = &*recording_runner; + } + absl::optional<FilteringTestRunner> filtering_runner; + if (options.isolated) { + filtering_runner.emplace(gtest_runner, options.names_to_test); + gtest_runner = &*filtering_runner; + } + + internal::ConformanceEnvironmentOptions env_options = + EnvironmentOptionsFor(options, gtest_runner); + std::vector<FailureListReport> reports; + absl::flat_hash_map<std::string, FailureListEntry> entries; + if (absl::Status status = + LoadFailureListReports(options.failure_list_files, reports, entries); + !status.ok()) { + return status; + } + + internal::ConformanceEnvironment& environment = + internal::ConformanceEnvironment::Install(std::move(env_options)); + const int selected_test_count = SelectedTestCount(options.performance); + result.exit_code = RUN_ALL_TESTS(); + if (unit_test->test_to_run_count() == 0) { + // gtest only runs an environment's SetUp() and TearDown() if it selected + // some test to run (which is only known now, so the runners above can't + // be created conditionally). Run TearDown() ourselves, so that the + // environment (which outlives this call, see Install()) lets go of the + // runners destroyed at the end of it instead of keeping a dangling + // pointer. There is nothing else for it to do: SetUp() never ran, so no + // failure list was loaded, and no test ran either. + environment.TearDown(); + } + + const internal::ResultLedger& ledger = environment.ledger(); + // The report block a result belongs to: the one of the entry it matched, + // or, for a test in no list, the one FailureListFlagFor() says. + auto report_for_entry = [&](absl::string_view entry) -> FailureListReport& { + return reports[entries.at(entry).report]; + }; + // Only a full run can tell that an entry is stale: if only some of the gtest + // tests of the selected kind ran (--gtest_filter, sharding, + // --gtest_fail_fast), or none at all (in which case gtest didn't even set + // the environment up), an entry they didn't match may well belong to a test + // that wasn't selected, so nothing is reported rather than guessed. Nor is + // anything when the gtest suites have no test of the selected kind at all + // (e.g. --performance was given and no performance test is linked in): a + // full run of nothing, which loaded no list. + if (selected_test_count > 0 && !IsPartialGtestPhase(selected_test_count)) { + for (const std::string& entry : ledger.UnmatchedExpectedFailures()) { + report_for_entry(entry).unmatched.push_back( + {/*test_name=*/entry, /*failure_message=*/entries.at(entry).message}); + } + } + if (filtering_runner.has_value()) { + result.names_run = filtering_runner->NamesRun(); + } + + for (const internal::UnexpectedResult& success : + ledger.UnexpectedSuccesses()) { + report_for_entry(*success.matched_entry).successes.push_back(success); + } + // An unexpected failure has no matched entry (see UnexpectedResult), also + // when it was listed with a different message, so it is routed by name. + // A text-format test listed in a --failure_list file with a stale message + // therefore gets the --text_format_failure_list hint; the list it is in is + // unaffected either way, since SaveFailureList() replaces the entry in + // place. + for (const internal::UnexpectedResult& failure : + ledger.UnexpectedFailures()) { + reports[ReportIndexFor(reports, FailureListFlagFor(failure.test_name))] + .failures.push_back(failure); + } + for (const auto& listed_skip : ledger.ListedSkips()) { + report_for_entry(listed_skip.second).listed_skips.push_back(listed_skip); + } + + // One report block per failure list flag, like the legacy suites' own + // reports. + std::string output_dir = options.output_dir; + if (!output_dir.empty() && output_dir.back() != '/') { + output_dir.push_back('/'); + } + for (const FailureListReport& report : reports) { + if (!ReportFailureListResults(report, output_dir) && + result.exit_code == 0) { + result.exit_code = 1; + } + } + absl::FPrintF(stderr, + "CONFORMANCE GTEST SUITES %s: %d tests: %d successes, %d " + "skipped, %d expected failures, %d unexpected failures, " + "%d unexpected successes, %d listed skips\n\n", + result.exit_code == 0 ? "PASSED" : "FAILED", + unit_test->test_to_run_count(), ledger.expected_successes(), + ledger.skipped(), ledger.expected_failures(), + ledger.unexpected_failures(), ledger.unexpected_successes(), + ledger.listed_skips()); + return result; +} + +int Main(int argc, char* argv[]) { + // The runner's own parser knows which of its flags take a value, so it is + // the one that can tell the gtest flags (which it collects) from the + // testee's arguments (which may well start with --gtest_ too). + absl::StatusOr<ConformanceRunnerOptions> options = + ParseConformanceRunnerArgs(argc, argv); + if (!options.ok()) { + PrintConformanceRunnerUsage(options.status().message()); + return EXIT_FAILURE; + } + for (std::string& arg : options->gtest_args) { + arg = RewriteGtestFlagPrefix(arg, GTEST_FLAG_PREFIX_); + } + + // See the file comment: only the command line configures the gtest phase. + // (In google3, unsetenv() alone does the job because the flags' defaults are + // evaluated lazily; see ResetInheritedGtestFlags() for open source.) + for (const char* variable : kInheritedTestEnvironmentVariables) { +#ifdef _WIN32 + _putenv_s(variable, ""); // the MSVC CRT has no unsetenv() +#else + unsetenv(variable); +#endif + } + ResetInheritedGtestFlags(options->gtest_args); + + // In google3, InitGoogleTest() also runs InitGoogle(), which exits on any + // flag it doesn't know, so it only gets to see the gtest flags. (In open + // source it's plain gtest, which leaves unknown flags in place.) Whatever it + // leaves behind is an unknown option, as it was for the legacy runner. + std::vector<char*> gtest_argv = {argv[0]}; + gtest_argv.reserve(options->gtest_args.size() + 2); + for (std::string& arg : options->gtest_args) { + gtest_argv.push_back(arg.data()); + } + // Like a real argv, the vector must be null-terminated: InitGoogleTest() + // removes the flags it consumed by shifting the rest down and writing + // argv[argc] = nullptr. + gtest_argv.push_back(nullptr); + int gtest_argc = static_cast<int>(gtest_argv.size()) - 1; + testing::InitGoogleTest(>est_argc, gtest_argv.data()); + if (gtest_argc > 1) { + PrintConformanceRunnerUsage( + absl::StrCat("Unknown option: ", gtest_argv[1])); + return EXIT_FAILURE; + } + + SelectPerformanceTests(options->performance); + + if (GTEST_FLAG_GET(list_tests)) return RUN_ALL_TESTS(); + + // The environment variables gtest would default --gtest_output from are + // cleared above, so by now the flag is only set if the command line asked + // for a report. Should gtest have found a default some other way, don't let + // it write anywhere: InitGoogleTest() already installed the output listener + // for the default, so clearing the flag isn't enough, release the listener + // as well. Despite its name, default_xml_generator() is the listener for + // whichever format the flag selected, so this covers the JSON printer too. + if (!GtestOutputRequested(options->gtest_args)) { + GTEST_FLAG_SET(output, ""); + testing::TestEventListeners& listeners = + testing::UnitTest::GetInstance()->listeners(); + std::unique_ptr<testing::TestEventListener> released_listener( + listeners.Release(listeners.default_xml_generator())); + } + + std::ofstream record_requests_file; + if (!options->record_requests_file.empty()) { + record_requests_file.open(options->record_requests_file); + if (!record_requests_file.is_open()) { + absl::FPrintF(stderr, "Couldn't open record_requests file: %s\n", + options->record_requests_file); + return EXIT_FAILURE; + } + } + std::ostream* record_requests = + record_requests_file.is_open() ? &record_requests_file : nullptr; + + absl::StatusOr<GtestPhaseResult> gtest_result = + RunGtestPhase(*options, record_requests); + if (!gtest_result.ok()) { + absl::FPrintF(stderr, "%s\n", gtest_result.status().message()); + return EXIT_FAILURE; + } + + // Like the legacy runner, list the tests --test named that don't exist, + // without failing the run. + absl::flat_hash_set<std::string> names_not_run = options->names_to_test; + for (const std::string& name : gtest_result->names_run) { + names_not_run.erase(name); + } + if (!names_not_run.empty()) { + // Sorted: flat_hash_set iteration order varies from run to run. + std::vector<std::string> sorted(names_not_run.begin(), names_not_run.end()); + std::sort(sorted.begin(), sorted.end()); + absl::FPrintF(stderr, + "These tests were requested to be run isolated, but they do " + "not exist. Revise the test names:\n\n"); + for (const std::string& test_name : sorted) { + absl::FPrintF(stderr, " %s\n", test_name); + } + absl::FPrintF(stderr, "\n\n"); + } + + if (record_requests_file.is_open()) { + record_requests_file.close(); + if (!record_requests_file) { + absl::FPrintF(stderr, "Error writing record_requests file: %s\n", + options->record_requests_file); + return EXIT_FAILURE; + } + } + + return gtest_result->exit_code == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + return google::protobuf::conformance::Main(argc, argv); }
diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index f4ef9af..a328d4e 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc
@@ -5,99 +5,75 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -// This file contains a program for running the test suite in a separate -// process. The other alternative is to run the suite in-process. See -// conformance.proto for pros/cons of these two options. -// -// This program will fork the process under test and communicate with it over -// its stdin/stdout: -// -// +--------+ pipe +----------+ -// | tester | <------> | testee | -// | | | | -// | C++ | | any lang | -// +--------+ +----------+ -// -// The tester contains all of the test cases and their expected output. -// The testee is a simple program written in the target language that reads -// each test case and attempts to produce acceptable output for it. -// -// Every test consists of a ConformanceRequest/ConformanceResponse -// request/reply pair. The protocol on the pipe is simply: -// -// 1. tester sends 4-byte length N (little endian) -// 2. tester sends N bytes representing a ConformanceRequest proto -// 3. testee sends 4-byte length M (little endian) -// 4. testee sends M bytes representing a ConformanceResponse proto +// The command line parser and the report writer of conformance_test_runner; +// see conformance_test_runner.h and, for the runner itself, +// conformance_test_main.cc. -#include <stdio.h> +#include "conformance/conformance_test_runner.h" -#include <algorithm> -#include <cctype> #include <cstdio> -#include <cstdlib> #include <cstring> #include <fstream> #include <string> #include <vector> -#include "absl/container/flat_hash_set.h" -#include "absl/strings/ascii.h" +#include "google/protobuf/descriptor.pb.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "absl/types/optional.h" -#include "conformance/conformance.pb.h" -#include "conformance/conformance_test.h" -#include "conformance/fork_pipe_runner.h" -#include "conformance/recording_test_runner.h" -#include "conformance/test_runner.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" namespace google { namespace protobuf { namespace conformance { -namespace { -void ParseFailureList(const char* filename, - ::conformance::FailureSet* failure_list) { - std::ifstream infile(filename); - - if (!infile.is_open()) { - fprintf(stderr, "Couldn't open failure list file: %s\n", filename); - exit(1); - } - - for (std::string line; std::getline(infile, line);) { - // Remove comments. - std::string test_name = line.substr(0, line.find('#')); - - test_name.erase( - std::remove_if(test_name.begin(), test_name.end(), ::isspace), - test_name.end()); - - if (test_name.empty()) { // Skip empty lines. - continue; - } - - // If we remove whitespace from the beginning of a line, and what we have - // left at first is a '#', then we have a comment. - if (test_name[0] != '#') { - // Find our failure message if it exists. Will be set to an empty string - // if no message is found. Empty failure messages also pass our tests. - size_t check_message = line.find('#'); - std::string message; - if (check_message != std::string::npos) { - message = line.substr(check_message + 1); // +1 to skip the delimiter - // If we had only whitespace after the delimiter, we will have an empty - // failure message and the test will still pass. - message = std::string(absl::StripAsciiWhitespace(message)); - } - ::conformance::TestStatus* test = failure_list->add_test(); - test->set_name(test_name); - test->set_failure_message(message); - } - } +std::string RewriteGtestFlagPrefix(absl::string_view arg, + absl::string_view flag_prefix) { + constexpr absl::string_view kGtestPrefix = "--gtest_"; + if (!absl::StartsWith(arg, kGtestPrefix)) return std::string(arg); + return absl::StrCat("--", flag_prefix, arg.substr(kGtestPrefix.size())); } -void UsageError() { +bool IsPerformanceFixture(absl::string_view test_suite_name) { + for (absl::string_view component : absl::StrSplit(test_suite_name, '/')) { + if (absl::EndsWith(component, "PerformanceTest")) return true; + } + return false; +} + +std::string PerformanceGtestFilter( + absl::string_view filter, bool performance, + absl::Span<const std::string> test_suite_names) { + std::vector<std::string> excluded; + for (const std::string& test_suite_name : test_suite_names) { + if (IsPerformanceFixture(test_suite_name) != performance) { + excluded.push_back(absl::StrCat(test_suite_name, ".*")); + } + } + if (excluded.empty()) return std::string(filter); + + // As gtest reads a filter: everything before the first '-' is the positive + // patterns (none means "*"), everything after it the negative ones. + absl::string_view positive = filter; + absl::string_view negative; + if (size_t dash = filter.find('-'); dash != absl::string_view::npos) { + positive = filter.substr(0, dash); + negative = filter.substr(dash + 1); + } + if (positive.empty()) positive = "*"; + return absl::StrCat(positive, "-", negative, negative.empty() ? "" : ":", + absl::StrJoin(excluded, ":")); +} + +void PrintConformanceRunnerUsage(absl::string_view error_message) { + if (!error_message.empty()) { + absl::FPrintF(stderr, "%s\n", error_message); + } fprintf(stderr, "Usage: conformance-test-runner [options] <test-program>\n"); fprintf(stderr, "\n"); fprintf(stderr, "Options:\n"); @@ -115,7 +91,7 @@ " of tests that are expected to \n"); fprintf(stderr, " fail in the \n"); fprintf(stderr, - " text_format_conformance_suite. \n"); + " text-format conformance tests. \n"); fprintf(stderr, " File should contain one test name \n"); fprintf(stderr, @@ -144,14 +120,17 @@ " can be specified by repeating the \n" " flag.\n\n"); fprintf(stderr, - " --debug Enable debug mode\n" - " to produce octal serialized\n" - " ConformanceRequest for the tests\n" - " passed to --test (required)\n\n"); + " --debug Accepted for compatibility; has\n" + " no effect. Requires --test, as\n" + " it always did.\n\n"); fprintf(stderr, " --performance Boolean option\n"); fprintf(stderr, " for enabling run of\n"); fprintf(stderr, " performance tests.\n\n"); fprintf(stderr, + " --verbose Accepted for compatibility; has\n" + " no effect (use --gtest_brief=0 to\n" + " list every test's result).\n\n"); + fprintf(stderr, " --record_requests <file> Write one line per request sent\n" " to the testee, of the form\n" " '<test_name> <input_size> " @@ -159,158 +138,121 @@ " CRC32C is of the request's\n" " canonical rendering, for\n" " verifying that refactorings of\n" - " the suites preserve behavior.\n"); - exit(1); + " the suites preserve behavior.\n\n"); + fprintf(stderr, + " --gtest_<flag>=<value> Passed on to the gtest-based\n" + " suites (e.g. --gtest_filter,\n" + " --gtest_output); --gunit_ is\n" + " accepted too. Must precede the\n" + " testee and use the --flag=value\n" + " form.\n"); } -} // namespace - -int RunConformanceTests(int argc, char *argv[], - const std::vector<ConformanceTestSuite *> &suites) { - if (suites.empty()) { - fprintf(stderr, "No test suites found.\n"); - return EXIT_FAILURE; - } - - std::string program; - std::string testee; - std::vector<std::string> program_args; - bool performance = false; +absl::StatusOr<ConformanceRunnerOptions> ParseConformanceRunnerArgs( + int argc, char* argv[]) { + ConformanceRunnerOptions options; bool debug = false; - absl::flat_hash_set<std::string> names_to_test; - bool enforce_recommended = false; - Edition maximum_edition = EDITION_UNKNOWN; - std::string output_dir; - std::string record_requests_filename; - bool verbose = false; - bool isolated = false; + + // The error for a flag at `arg` that takes a value but is the last argument. + auto missing_value = [&](int arg) { + return absl::InvalidArgumentError( + absl::StrCat("Missing value for ", argv[arg])); + }; for (int arg = 1; arg < argc; ++arg) { if (strcmp(argv[arg], "--performance") == 0) { - performance = true; + options.performance = true; } else if (strcmp(argv[arg], "--debug") == 0) { + // No effect (see ConformanceRunnerOptions); only its "requires --test" + // check below survives. debug = true; } else if (strcmp(argv[arg], "--verbose") == 0) { - verbose = true; + // No effect (see ConformanceRunnerOptions). } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { - enforce_recommended = true; + options.enforce_recommended = true; } else if (strcmp(argv[arg], "--maximum_edition") == 0) { - if (++arg == argc) UsageError(); + if (arg + 1 == argc) return missing_value(arg); + ++arg; Edition edition = EDITION_UNKNOWN; if (!Edition_Parse(absl::StrCat("EDITION_", argv[arg]), &edition)) { - fprintf(stderr, "Unknown edition: %s\n", argv[arg]); - UsageError(); + return absl::InvalidArgumentError( + absl::StrCat("Unknown edition: ", argv[arg])); } - maximum_edition = edition; + options.maximum_edition = edition; } else if (strcmp(argv[arg], "--output_dir") == 0) { - if (++arg == argc) UsageError(); - output_dir = argv[arg]; + if (arg + 1 == argc) return missing_value(arg); + options.output_dir = argv[++arg]; } else if (strcmp(argv[arg], "--record_requests") == 0) { - if (++arg == argc) UsageError(); - record_requests_filename = argv[arg]; + if (arg + 1 == argc) return missing_value(arg); + options.record_requests_file = argv[++arg]; } else if (strcmp(argv[arg], "--test") == 0) { - if (++arg == argc) UsageError(); - names_to_test.insert(argv[arg]); + if (arg + 1 == argc) return missing_value(arg); + options.names_to_test.insert(argv[++arg]); + + } else if (argv[arg] == kFailureListFlag || + argv[arg] == kTextFormatFailureListFlag) { + if (arg + 1 == argc) return missing_value(arg); + options.failure_list_files.emplace_back(argv[arg], argv[arg + 1]); + ++arg; + + } else if (absl::StartsWith(argv[arg], "--gtest_") || + absl::StartsWith(argv[arg], "--gunit_")) { + options.gtest_args.push_back(argv[arg]); } else if (argv[arg][0] == '-') { - bool recognized_flag = false; - for (ConformanceTestSuite *suite : suites) { - if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { - if (++arg == argc) UsageError(); - recognized_flag = true; - } - } - if (!recognized_flag) { - fprintf(stderr, "Unknown option: %s\n", argv[arg]); - UsageError(); - } + return absl::InvalidArgumentError( + absl::StrCat("Unknown option: ", argv[arg])); } else { - program += argv[arg++]; + options.testee = argv[arg++]; while (arg < argc) { - program_args.push_back(argv[arg]); + options.testee_args.push_back(argv[arg]); arg++; } } } - if (debug && names_to_test.empty()) { - UsageError(); + if (debug && options.names_to_test.empty()) { + return absl::InvalidArgumentError("--debug requires --test"); } + options.isolated = !options.names_to_test.empty(); - if (!names_to_test.empty()) { - isolated = true; + return options; +} + +bool ReportTestStatusSet(absl::Span<const ReportedTestStatus> statuses, + absl::string_view file_name, absl::string_view message, + absl::string_view output_dir, std::string& output) { + if (statuses.empty()) return true; + + absl::StrAppend(&output, "\n", message, "\n\n"); + for (const ReportedTestStatus& status : statuses) { + absl::StrAppendFormat(&output, " %s # %s\n", status.test_name, + status.failure_message); } + absl::StrAppend(&output, "\n"); - // Opened once (truncating) so that requests from all suites are appended in - // order to a single file. - std::ofstream record_requests_file; - if (!record_requests_filename.empty()) { - record_requests_file.open(record_requests_filename); - if (!record_requests_file.is_open()) { - fprintf(stderr, "Couldn't open record_requests file: %s\n", - record_requests_filename.c_str()); - return EXIT_FAILURE; - } - } - - bool all_ok = true; - for (ConformanceTestSuite *suite : suites) { - std::string failure_list_filename; - ::conformance::FailureSet failure_list; - for (int arg = 1; arg < argc; ++arg) { - if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) { - if (++arg == argc) UsageError(); - failure_list_filename = argv[arg]; - ParseFailureList(argv[arg], &failure_list); + if (!file_name.empty()) { + const std::string filename = absl::StrCat(output_dir, file_name); + std::ofstream os(filename); + if (os) { + for (const ReportedTestStatus& status : statuses) { + // Additions will not have a 'matched_name' while removals will. + absl::string_view potential_add_or_removal = status.matched_name.empty() + ? status.test_name + : status.matched_name; + os << potential_add_or_removal << " # " << status.failure_message + << "\n"; } - } - suite->SetPerformance(performance); - suite->SetVerbose(verbose); - suite->SetEnforceRecommended(enforce_recommended); - suite->SetMaximumEdition(maximum_edition); - suite->SetOutputDir(output_dir); - suite->SetDebug(debug); - suite->SetNamesToTest(names_to_test); - suite->SetTestee(program); - suite->SetIsolated(isolated); - - ForkPipeRunner fork_pipe_runner(program, program_args); - ConformanceTestRunner* runner = &fork_pipe_runner; - absl::optional<RecordingTestRunner> recording_runner; - if (record_requests_file.is_open()) { - recording_runner.emplace(&fork_pipe_runner, &record_requests_file); - runner = &*recording_runner; - } - - std::string output; - all_ok = all_ok && suite->RunSuite(runner, &output, failure_list_filename, - &failure_list); - - names_to_test = suite->GetExpectedTestsNotRun(); - fwrite(output.c_str(), 1, output.size(), stderr); - } - - if (record_requests_file.is_open()) { - record_requests_file.close(); - if (!record_requests_file) { - fprintf(stderr, "Error writing record_requests file: %s\n", - record_requests_filename.c_str()); - return EXIT_FAILURE; + } else { + absl::StrAppendFormat(&output, + "Failed to open file: %s\n", + filename); } } - if (!names_to_test.empty()) { - fprintf(stderr, - "These tests were requested to be ran isolated, but they do " - "not exist. Revise the test names:\n\n"); - for (const std::string &test_name : names_to_test) { - fprintf(stderr, " %s\n", test_name.c_str()); - } - fprintf(stderr, "\n\n"); - } - return all_ok ? EXIT_SUCCESS : EXIT_FAILURE; + return false; } } // namespace conformance
diff --git a/conformance/conformance_test_runner.h b/conformance/conformance_test_runner.h new file mode 100644 index 0000000..f1ff657 --- /dev/null +++ b/conformance/conformance_test_runner.h
@@ -0,0 +1,160 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The command line of conformance_test_runner and the report format its users +// rely on. Those users are the conformance test targets, their +// update_failure_list tooling and the scripts around them. The runner itself +// is conformance_test_main.cc. It runs the gtest-based conformance suites +// (see test_fixture.h) against the testee named on the command line. See +// PrintConformanceRunnerUsage() for the options. +// +// The command line and the report format are those of the legacy +// conformance_test_runner, which ran hand-written ConformanceTestSuite classes +// instead of gtest suites. They are kept so that nothing changes for its +// users. +// TODO: retire the failure list flags below together with the +// per-flag report blocks of conformance_test_main.cc once the remaining sh_test +// and script invocations have moved to the conformance_test() macro, which runs +// the gtest suites as ordinary test binaries. + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_RUNNER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_RUNNER_H__ + +#include <string> +#include <utility> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// The failure list flags the runner accepts. Every file given for either is +// loaded into the gtest suites' one ResultLedger. The files are recorded under +// their flag (see ConformanceRunnerOptions::failure_list_files) so that the +// runner can tell which list an entry came from and report per list. +// --failure_list is the flag of the binary and JSON tests. +// --text_format_failure_list is the one of the text-format tests. The test +// targets and scripts pass both. +inline constexpr absl::string_view kFailureListFlag = "--failure_list"; +inline constexpr absl::string_view kTextFormatFailureListFlag = + "--text_format_failure_list"; + +// The options accepted on the conformance_test_runner command line. +struct ConformanceRunnerOptions { + // The first non-flag argument and everything after it. + std::string testee; + std::vector<std::string> testee_args; + + // Every failure list flag given, as (flag name, file) pairs in command line + // order, e.g. ("--failure_list", "failure_list_cpp.txt"). + std::vector<std::pair<std::string, std::string>> failure_list_files; + + bool performance = false; + bool enforce_recommended = false; + // EDITION_UNKNOWN (the default) runs the proto2 and proto3 tests only. + Edition maximum_edition = EDITION_UNKNOWN; + std::string output_dir; + // Where --record_requests should write, if given. + std::string record_requests_file; + // The tests named with --test, and whether to run only those (set by + // ParseConformanceRunnerArgs() when --test was given). + absl::flat_hash_set<std::string> names_to_test; + bool isolated = false; + + // Arguments starting with --gtest_ or --gunit_ that appeared before the + // testee, in order and as given. conformance_test_main.cc passes them on to + // gtest, after RewriteGtestFlagPrefix(). + std::vector<std::string> gtest_args; + + // --debug and --verbose are accepted too, for the scripts that still pass + // them, but have no effect. They only ever configured the legacy runner's + // suites. gtest's own flags take their place. For example, --gtest_brief=0 + // lists every test's result. +}; + +// Parses the conformance_test_runner command line. On error, returns +// InvalidArgumentError with the message the runner prints before its usage +// text; see PrintConformanceRunnerUsage(). +absl::StatusOr<ConformanceRunnerOptions> ParseConformanceRunnerArgs( + int argc, char* argv[]); + +// Returns `arg` with a leading "--gtest_" replaced by "--" + `flag_prefix`. +// Any other argument, including one already spelled with `flag_prefix`, is +// returned unchanged. conformance_test_main.cc applies this to +// ConformanceRunnerOptions::gtest_args with the prefix its gtest was built +// with, so that the documented --gtest_ spelling works everywhere. That +// prefix is GTEST_FLAG_PREFIX_ from gtest-port.h: "gtest_" in open source and +// "gunit_" in google3, whose gtest rejects the --gtest_ spelling. +// GTEST_FLAG_PREFIX_ is one of gtest's internal macros, but it is what gtest +// itself parses its flags by. merged_runner_test.sh checks end to end that a +// --gtest_ flag reaches gtest. +std::string RewriteGtestFlagPrefix(absl::string_view arg, + absl::string_view flag_prefix); + +// Returns whether the gtest test suite (fixture) named `test_suite_name` +// belongs to the performance conformance suite, that is, whether its fixture +// is named `*PerformanceTest` (the naming rule in test_fixture.h). gtest +// names an instantiated fixture `<instantiation>/<fixture>` for a value +// parameterized one, `<fixture>/<index or type name>` for a typed one, and +// `<instantiation>/<fixture>/<index or type name>` for a type parameterized +// one, so every '/'-separated component is checked. +bool IsPerformanceFixture(absl::string_view test_suite_name); + +// Returns the gtest filter that narrows `filter` to the test suites of the +// kind `performance` selects. The filter has the form +// `positive patterns[-negative patterns]`, each list ':'-separated, see +// gtest's --gtest_filter. `test_suite_names` are the names of every +// registered gtest test suite. With `performance`, the performance fixtures +// among them are selected (see IsPerformanceFixture()), otherwise all the +// others. The suites of the other kind are appended to the negative patterns +// as `<test suite name>.*`, so `filter` still applies within the selected +// kind. `filter` is typically an explicit --gtest_filter, or "*". This is +// how the runner (conformance_test_main.cc) implements --performance. +std::string PerformanceGtestFilter( + absl::string_view filter, bool performance, + absl::Span<const std::string> test_suite_names); + +// Prints `error_message` (if non-empty) followed by the usage text to stderr. +void PrintConformanceRunnerUsage(absl::string_view error_message); + +// One entry of a report written by ReportTestStatusSet(). +struct ReportedTestStatus { + // The full name of the test. + std::string test_name; + // The failure message shown next to the test: the test's own for a failure, + // the failure list entry's for an unexpected success. + std::string failure_message; + // The failure list entry (possibly a wildcard) the test matched, when the + // report is about removing that entry from the list. Empty when it is about + // adding `test_name` to the list. + std::string matched_name; +}; + +// Reports one category of test statuses in the runner's report format. Does +// nothing and returns true if `statuses` is empty. Otherwise appends +// `message` followed by one " <test name> # <failure message>" line per +// status to `output` and returns false. If `file_name` is non-empty, it also +// writes one "<name> # <failure message>" line per status to the file +// `output_dir` + `file_name`, in the form update_failure_list expects. There +// <name> is the status's matched_name if set (an entry to remove from the +// failure list) and its test name otherwise (an entry to add). A file that +// cannot be opened is reported in `output` too. +bool ReportTestStatusSet(absl::Span<const ReportedTestStatus> statuses, + absl::string_view file_name, absl::string_view message, + absl::string_view output_dir, std::string& output); + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_CONFORMANCE_TEST_RUNNER_H__
diff --git a/conformance/conformance_test_runner_test.cc b/conformance/conformance_test_runner_test.cc new file mode 100644 index 0000000..150b70f --- /dev/null +++ b/conformance/conformance_test_runner_test.cc
@@ -0,0 +1,516 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Tests of the conformance_test_runner command line parsing and of the report +// writer in conformance_test_runner.cc (see conformance_test_runner.h). + +#include "conformance/conformance_test_runner.h" + +#ifdef _WIN32 +#include <direct.h> // for _mkdir +#else +#include <sys/stat.h> // for mkdir +#endif + +#include <cerrno> +#include <fstream> +#include <iterator> +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::testing::ElementsAre; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Not; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Parses `args` (without argv[0], which is added). +absl::StatusOr<ConformanceRunnerOptions> Parse( + const std::vector<std::string>& args) { + std::vector<std::string> owned_args = {"conformance_test_runner"}; + owned_args.insert(owned_args.end(), args.begin(), args.end()); + std::vector<char*> argv; + for (std::string& arg : owned_args) argv.push_back(arg.data()); + argv.push_back(nullptr); + return ParseConformanceRunnerArgs(static_cast<int>(owned_args.size()), + argv.data()); +} + +// mkdir() and its MSVC CRT spelling. +int MakeDirectory(const std::string& dir) { +#ifdef _WIN32 + return _mkdir(dir.c_str()); +#else + return mkdir(dir.c_str(), 0755); +#endif +} + +// Returns a fresh directory for one test's output files, with a trailing '/'. +std::string NewOutputDir() { + const testing::TestInfo& test_info = + *testing::UnitTest::GetInstance()->current_test_info(); + std::string dir = + absl::StrCat(testing::TempDir(), test_info.test_suite_name(), "_", + test_info.name(), "/"); + ABSL_PCHECK(MakeDirectory(dir) == 0 || errno == EEXIST) << dir; + return dir; +} + +std::string ReadFile(absl::string_view path) { + std::ifstream in{std::string(path)}; + ABSL_CHECK(in.is_open()) << path; + return std::string(std::istreambuf_iterator<char>(in), {}); +} + +bool FileExists(absl::string_view path) { + return std::ifstream(std::string(path)).is_open(); +} + +TEST(ParseConformanceRunnerArgsTest, Defaults) { + absl::StatusOr<ConformanceRunnerOptions> options = Parse({"testee"}); + ASSERT_THAT(options, IsOk()); + + EXPECT_EQ(options->testee, "testee"); + EXPECT_THAT(options->testee_args, IsEmpty()); + EXPECT_THAT(options->failure_list_files, IsEmpty()); + EXPECT_FALSE(options->performance); + EXPECT_FALSE(options->enforce_recommended); + EXPECT_EQ(options->maximum_edition, EDITION_UNKNOWN); + EXPECT_THAT(options->output_dir, IsEmpty()); + EXPECT_THAT(options->record_requests_file, IsEmpty()); + EXPECT_THAT(options->names_to_test, IsEmpty()); + EXPECT_FALSE(options->isolated); + EXPECT_THAT(options->gtest_args, IsEmpty()); +} + +TEST(ParseConformanceRunnerArgsTest, NoTesteeIsAccepted) { + // The legacy runner never checked for one; the testee fails to spawn later. + absl::StatusOr<ConformanceRunnerOptions> options = Parse({}); + ASSERT_THAT(options, IsOk()); + EXPECT_THAT(options->testee, IsEmpty()); +} + +TEST(ParseConformanceRunnerArgsTest, ParsesEveryFlag) { + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--performance", "--debug", "--verbose", "--enforce_recommended", + "--maximum_edition", "2023", "--output_dir", "out", + "--record_requests", "requests.txt", "--test", "Some.Test", + "--failure_list", "list.txt", "testee"}); + ASSERT_THAT(options, IsOk()); + + EXPECT_TRUE(options->performance); + EXPECT_TRUE(options->enforce_recommended); + EXPECT_EQ(options->maximum_edition, EDITION_2023); + EXPECT_EQ(options->output_dir, "out"); + EXPECT_EQ(options->record_requests_file, "requests.txt"); + EXPECT_THAT(options->names_to_test, ElementsAre("Some.Test")); + EXPECT_THAT(options->failure_list_files, + ElementsAre(Pair("--failure_list", "list.txt"))); + EXPECT_EQ(options->testee, "testee"); +} + +TEST(ParseConformanceRunnerArgsTest, EverythingAfterTesteeGoesToTestee) { + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--verbose", "testee", "--verbose", "--failure_list", "x", + "--gtest_filter=*", "plain", "--"}); + ASSERT_THAT(options, IsOk()); + + EXPECT_EQ(options->testee, "testee"); + EXPECT_THAT(options->testee_args, + ElementsAre("--verbose", "--failure_list", "x", + "--gtest_filter=*", "plain", "--")); + EXPECT_THAT(options->gtest_args, IsEmpty()); + EXPECT_THAT(options->failure_list_files, IsEmpty()); +} + +TEST(ParseConformanceRunnerArgsTest, GtestFlagsBeforeTesteeAreCollected) { + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--gtest_filter=Foo.*", "--verbose", "--gunit_brief=1", + "--gtest_list_tests", "testee", "--gtest_after"}); + ASSERT_THAT(options, IsOk()); + + EXPECT_THAT(options->gtest_args, + ElementsAre("--gtest_filter=Foo.*", "--gunit_brief=1", + "--gtest_list_tests")); + EXPECT_THAT(options->testee_args, ElementsAre("--gtest_after")); +} + +// --debug and --verbose only ever configured the legacy suites; they are still +// accepted (conformance_test_wrapper.sh passes --debug on) and change nothing. +TEST(ParseConformanceRunnerArgsTest, NoEffectFlagsAreAccepted) { + absl::StatusOr<ConformanceRunnerOptions> with_flags = + Parse({"--verbose", "--debug", "--test", "Some.Test", "testee"}); + absl::StatusOr<ConformanceRunnerOptions> without_flags = + Parse({"--test", "Some.Test", "testee"}); + ASSERT_THAT(with_flags, IsOk()); + ASSERT_THAT(without_flags, IsOk()); + + EXPECT_EQ(with_flags->testee, without_flags->testee); + EXPECT_EQ(with_flags->names_to_test, without_flags->names_to_test); + EXPECT_EQ(with_flags->isolated, without_flags->isolated); + EXPECT_EQ(with_flags->performance, without_flags->performance); + EXPECT_EQ(with_flags->enforce_recommended, + without_flags->enforce_recommended); + EXPECT_EQ(with_flags->gtest_args, without_flags->gtest_args); +} + +TEST(ParseConformanceRunnerArgsTest, DoubleDashIsAnUnknownOption) { + // Pins the legacy behavior: there is no way to end the runner's own flags + // other than naming the testee. + EXPECT_THAT(Parse({"--", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown option: --"))); +} + +TEST(ParseConformanceRunnerArgsTest, UnknownOption) { + EXPECT_THAT(Parse({"--bogus", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown option: --bogus"))); + EXPECT_THAT(Parse({"-v", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown option: -v"))); +} + +class MissingValueTest : public testing::TestWithParam<std::string> {}; + +TEST_P(MissingValueTest, LastArgumentIsAValueFlag) { + EXPECT_THAT(Parse({GetParam()}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Missing value for " + GetParam()))) + << GetParam(); + EXPECT_THAT(Parse({"--verbose", GetParam()}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Missing value for " + GetParam()))) + << GetParam(); + // After the testee it is just an argument for the testee. + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"testee", GetParam()}); + ASSERT_THAT(options, IsOk()) << GetParam(); + EXPECT_THAT(options->testee_args, ElementsAre(GetParam())); +} + +INSTANTIATE_TEST_SUITE_P(ValueFlags, MissingValueTest, + testing::Values("--maximum_edition", "--output_dir", + "--record_requests", "--test", + "--failure_list", + "--text_format_failure_list")); + +TEST(ParseConformanceRunnerArgsTest, DebugRequiresTest) { + EXPECT_THAT(Parse({"--debug", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("--debug requires --test"))); + EXPECT_THAT(Parse({"--debug", "--test", "Some.Test", "testee"}), IsOk()); +} + +TEST(ParseConformanceRunnerArgsTest, MaximumEdition) { + EXPECT_THAT(Parse({"--maximum_edition", "bogus", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown edition: bogus"))); + // The legacy runner prepends "EDITION_", so only the short form works. + EXPECT_THAT(Parse({"--maximum_edition", "EDITION_2023", "testee"}), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Unknown edition: EDITION_2023"))); + + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--maximum_edition", "PROTO3", "testee"}); + ASSERT_THAT(options, IsOk()); + EXPECT_EQ(options->maximum_edition, EDITION_PROTO3); +} + +// Both failure list flags are recorded under their own name, in command line +// order. +TEST(ParseConformanceRunnerArgsTest, FailureListFlagsAreRecordedByName) { + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--text_format_failure_list", "t.txt", "--failure_list", "f.txt", + "testee"}); + ASSERT_THAT(options, IsOk()); + EXPECT_THAT(options->failure_list_files, + ElementsAre(Pair("--text_format_failure_list", "t.txt"), + Pair("--failure_list", "f.txt"))); +} + +TEST(ParseConformanceRunnerArgsTest, FailureListFlagsKeepCommandLineOrder) { + absl::StatusOr<ConformanceRunnerOptions> options = + Parse({"--text_format_failure_list", "t1.txt", "--failure_list", "b1.txt", + "--text_format_failure_list", "t2.txt", "--failure_list", "b2.txt", + "testee"}); + ASSERT_THAT(options, IsOk()); + + EXPECT_THAT(options->failure_list_files, + ElementsAre(Pair("--text_format_failure_list", "t1.txt"), + Pair("--failure_list", "b1.txt"), + Pair("--text_format_failure_list", "t2.txt"), + Pair("--failure_list", "b2.txt"))); +} + +TEST(ParseConformanceRunnerArgsTest, IsolatedFollowsFromTest) { + absl::StatusOr<ConformanceRunnerOptions> options = Parse( + {"--test", "Test.B", "--test", "Test.A", "--test", "Test.B", "testee"}); + ASSERT_THAT(options, IsOk()); + EXPECT_TRUE(options->isolated); + EXPECT_THAT(options->names_to_test, UnorderedElementsAre("Test.A", "Test.B")); + + options = Parse({"testee"}); + ASSERT_THAT(options, IsOk()); + EXPECT_FALSE(options->isolated); +} + +TEST(RewriteGtestFlagPrefixTest, RewritesLeadingGtestPrefixOnly) { + EXPECT_EQ(RewriteGtestFlagPrefix("--gtest_filter=Foo.*", "gunit_"), + "--gunit_filter=Foo.*"); + EXPECT_EQ(RewriteGtestFlagPrefix("--gtest_list_tests", "gunit_"), + "--gunit_list_tests"); + // A no-op when the prefix already is gtest's own, as in open source. + EXPECT_EQ(RewriteGtestFlagPrefix("--gtest_filter=Foo.*", "gtest_"), + "--gtest_filter=Foo.*"); + // Already spelled with the target prefix, or not a gtest flag at all. + EXPECT_EQ(RewriteGtestFlagPrefix("--gunit_filter=Foo.*", "gunit_"), + "--gunit_filter=Foo.*"); + EXPECT_EQ(RewriteGtestFlagPrefix("--verbose", "gunit_"), "--verbose"); + EXPECT_EQ(RewriteGtestFlagPrefix("gtest_filter=x", "gunit_"), + "gtest_filter=x"); + EXPECT_EQ(RewriteGtestFlagPrefix("--gtest_", "gunit_"), "--gunit_"); + EXPECT_EQ(RewriteGtestFlagPrefix("", "gunit_"), ""); +} + +TEST(IsPerformanceFixtureTest, PlainFixture) { + EXPECT_TRUE(IsPerformanceFixture("BinaryPerformanceTest")); + EXPECT_TRUE(IsPerformanceFixture("PerformanceTest")); + EXPECT_FALSE(IsPerformanceFixture("BinaryTest")); + EXPECT_FALSE(IsPerformanceFixture("PerformanceTestOfSomething")); + EXPECT_FALSE(IsPerformanceFixture("BinaryPerformance")); + EXPECT_FALSE(IsPerformanceFixture("")); +} + +TEST(IsPerformanceFixtureTest, InstantiatedFixtures) { + // Value-parameterized: <instantiation>/<fixture>. + EXPECT_TRUE(IsPerformanceFixture("Proto3/UnknownFieldsPerformanceTest")); + EXPECT_FALSE(IsPerformanceFixture("Proto3/UnknownFieldsTest")); + // Typed: <fixture>/<index or type name>. + EXPECT_TRUE(IsPerformanceFixture("DepthLimitPerformanceTest/0")); + EXPECT_TRUE(IsPerformanceFixture("DepthLimitPerformanceTest/TestAllTypes")); + EXPECT_FALSE(IsPerformanceFixture("DepthLimitTest/0")); + // Type-parameterized: <instantiation>/<fixture>/<index or type name>. + EXPECT_TRUE(IsPerformanceFixture("Editions/DepthLimitPerformanceTest/1")); + EXPECT_FALSE(IsPerformanceFixture("Editions/DepthLimitTest/1")); + // A two-component name does not tell whether it is <instantiation>/<fixture> + // or <fixture>/<type name>, so every component counts: an instantiation or + // type name ending in the suffix marks the suite too (nothing else is + // expected to be named that way), while the suffix inside a component or + // in the test name does not. + EXPECT_TRUE(IsPerformanceFixture("PerformanceTest/DepthLimitTest")); + EXPECT_TRUE(IsPerformanceFixture("Inst/DepthLimitTest/PerformanceTest")); + EXPECT_FALSE(IsPerformanceFixture("Inst/DepthLimitTest/PerformanceTestType")); +} + +// Fixtures of the performance kind in every form gtest registers a fixture +// in, so that the test below can check the names gtest reports for them (the +// input of IsPerformanceFixture() in the merged runner) against real gtest +// rather than against this file's reading of gtest's documentation. +class ProbePerformanceTest : public testing::Test {}; +TEST_F(ProbePerformanceTest, IsRegistered) {} + +class ProbeParameterizedPerformanceTest : public testing::TestWithParam<int> {}; +TEST_P(ProbeParameterizedPerformanceTest, IsRegistered) {} +INSTANTIATE_TEST_SUITE_P(Probe, ProbeParameterizedPerformanceTest, + testing::Values(0)); + +template <typename T> +class ProbeTypedPerformanceTest : public testing::Test {}; +TYPED_TEST_SUITE(ProbeTypedPerformanceTest, testing::Types<int>); +TYPED_TEST(ProbeTypedPerformanceTest, IsRegistered) {} + +template <typename T> +class ProbeTypeParameterizedPerformanceTest : public testing::Test {}; +TYPED_TEST_SUITE_P(ProbeTypeParameterizedPerformanceTest); +TYPED_TEST_P(ProbeTypeParameterizedPerformanceTest, IsRegistered) {} +REGISTER_TYPED_TEST_SUITE_P(ProbeTypeParameterizedPerformanceTest, + IsRegistered); +INSTANTIATE_TYPED_TEST_SUITE_P(Probe, ProbeTypeParameterizedPerformanceTest, + testing::Types<int>); + +TEST(IsPerformanceFixtureTest, RecognizesEveryFormGtestRegisters) { + const testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + std::vector<std::string> probes; + for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { + const std::string name = unit_test.GetTestSuite(i)->name(); + if (absl::StrContains(name, "Probe")) probes.push_back(name); + } + EXPECT_THAT(probes, UnorderedElementsAre( + "ProbePerformanceTest", + "Probe/ProbeParameterizedPerformanceTest", + "ProbeTypedPerformanceTest/0", + "Probe/ProbeTypeParameterizedPerformanceTest/0")); + for (const std::string& name : probes) { + EXPECT_TRUE(IsPerformanceFixture(name)) << name; + } + // The fixtures of this file's other tests are of the regular kind. + EXPECT_FALSE(IsPerformanceFixture("IsPerformanceFixtureTest")); + EXPECT_FALSE(IsPerformanceFixture("ValueFlags/MissingValueTest")); +} + +TEST(PerformanceGtestFilterTest, ExcludesTheOtherKind) { + const std::vector<std::string> suites = { + "BinaryTest", "Proto3/UnknownFieldsPerformanceTest", + "DepthLimitPerformanceTest/0", "Editions/DelimitedTest/1"}; + EXPECT_EQ(PerformanceGtestFilter("*", /*performance=*/true, suites), + "*-BinaryTest.*:Editions/DelimitedTest/1.*"); + EXPECT_EQ(PerformanceGtestFilter("*", /*performance=*/false, suites), + "*-Proto3/UnknownFieldsPerformanceTest.*:" + "DepthLimitPerformanceTest/0.*"); +} + +TEST(PerformanceGtestFilterTest, KeepsAnExplicitFilter) { + const std::vector<std::string> suites = {"BinaryTest", + "BinaryPerformanceTest"}; + // Positive patterns only. + EXPECT_EQ( + PerformanceGtestFilter("Binary*.Foo:*.Bar", /*performance=*/true, suites), + "Binary*.Foo:*.Bar-BinaryTest.*"); + // Positive and negative patterns. + EXPECT_EQ(PerformanceGtestFilter("*.Foo-*.Bar:*.Baz", /*performance=*/false, + suites), + "*.Foo-*.Bar:*.Baz:BinaryPerformanceTest.*"); + // Negative patterns only: gtest takes the positive part as "*". + EXPECT_EQ(PerformanceGtestFilter("-*.Bar", /*performance=*/true, suites), + "*-*.Bar:BinaryTest.*"); + EXPECT_EQ(PerformanceGtestFilter("", /*performance=*/true, suites), + "*-BinaryTest.*"); +} + +TEST(PerformanceGtestFilterTest, UnchangedWhenNothingToExclude) { + EXPECT_EQ(PerformanceGtestFilter("*", /*performance=*/true, {}), "*"); + EXPECT_EQ(PerformanceGtestFilter("Foo.*-Bar.*", /*performance=*/false, + {"BinaryTest"}), + "Foo.*-Bar.*"); + EXPECT_EQ(PerformanceGtestFilter("Foo.*", /*performance=*/true, + {"BinaryPerformanceTest"}), + "Foo.*"); +} + +TEST(PrintConformanceRunnerUsageTest, DocumentsEveryOption) { + testing::internal::CaptureStderr(); + PrintConformanceRunnerUsage("Some error"); + const std::string usage = testing::internal::GetCapturedStderr(); + + EXPECT_THAT(usage, HasSubstr("Some error\nUsage: conformance-test-runner")); + for (absl::string_view option : + {"--failure_list", "--text_format_failure_list", "--enforce_recommended", + "--maximum_edition", "--output_dir", "--test", "--debug", + "--performance", "--verbose", "--record_requests", + "--gtest_<flag>=<value>"}) { + EXPECT_THAT(usage, HasSubstr(absl::StrCat("\n ", option, " "))) << option; + } + EXPECT_THAT(usage, HasSubstr("--gunit_")); +} + +TEST(ReportTestStatusSetTest, EmptySetReportsNothing) { + const std::string output_dir = NewOutputDir(); + std::string output = "before"; + EXPECT_TRUE( + ReportTestStatusSet({}, "empty.txt", "Some message", output_dir, output)); + EXPECT_EQ(output, "before"); + EXPECT_FALSE(FileExists(absl::StrCat(output_dir, "empty.txt"))); +} + +TEST(ReportTestStatusSetTest, WritesReportAndFile) { + const std::string output_dir = NewOutputDir(); + ReportedTestStatus added; + added.test_name = "Test.Added"; + added.failure_message = "it failed"; + ReportedTestStatus removed; + removed.test_name = "Test.Removed"; + removed.failure_message = "it passed"; + removed.matched_name = "Test.*"; + const std::vector<ReportedTestStatus> statuses = {added, removed}; + + std::string output = "before\n"; + EXPECT_FALSE(ReportTestStatusSet(statuses, "report.txt", "Some message", + output_dir, output)); + // The report always names the test itself ... + EXPECT_EQ(output, + "before\n" + "\n" + "Some message\n" + "\n" + " Test.Added # it failed\n" + " Test.Removed # it passed\n" + "\n"); + // ... whereas the file for update_failure_list has the failure list entry + // to remove, if there is one, and the test name to add otherwise. + EXPECT_EQ(ReadFile(absl::StrCat(output_dir, "report.txt")), + "Test.Added # it failed\n" + "Test.* # it passed\n"); +} + +TEST(ReportTestStatusSetTest, WritesRelativeToCurrentDirWithoutOutputDir) { + // An empty output_dir means the file name is used as is. Point it into the + // temp dir to keep the test hermetic. + const std::string output_dir = NewOutputDir(); + ReportedTestStatus status; + status.test_name = "Test.A"; + const std::vector<ReportedTestStatus> statuses = {status}; + + std::string output; + EXPECT_FALSE(ReportTestStatusSet(statuses, + absl::StrCat(output_dir, "plain.txt"), + "Some message", "", output)); + EXPECT_EQ(ReadFile(absl::StrCat(output_dir, "plain.txt")), "Test.A # \n"); +} + +TEST(ReportTestStatusSetTest, NoFileNameMeansNoFile) { + const std::string output_dir = NewOutputDir(); + ReportedTestStatus status; + status.test_name = "Test.A"; + status.failure_message = "message"; + const std::vector<ReportedTestStatus> statuses = {status}; + + std::string output; + EXPECT_FALSE( + ReportTestStatusSet(statuses, "", "Some message", output_dir, output)); + EXPECT_THAT(output, HasSubstr(" Test.A # message\n")); + EXPECT_THAT(output, Not(HasSubstr("Failed to open file"))); +} + +TEST(ReportTestStatusSetTest, ReportsUnwritableFile) { + const std::string output_dir = + absl::StrCat(NewOutputDir(), "does_not_exist/"); + ReportedTestStatus status; + status.test_name = "Test.A"; + const std::vector<ReportedTestStatus> statuses = {status}; + + std::string output; + EXPECT_FALSE(ReportTestStatusSet(statuses, "report.txt", "Some message", + output_dir, output)); + EXPECT_THAT(output, HasSubstr(" Test.A # \n")); + EXPECT_THAT(output, HasSubstr(absl::StrCat( + "Failed to open file: ", output_dir, "report.txt"))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/cpp_in_process_test_runner.cc b/conformance/cpp_in_process_test_runner.cc new file mode 100644 index 0000000..e80c826 --- /dev/null +++ b/conformance/cpp_in_process_test_runner.cc
@@ -0,0 +1,31 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/cpp_in_process_test_runner.h" + +#include <memory> + +#include "conformance/conformance.pb.h" +#include "conformance/conformance_cpp_harness.h" +#include "conformance/in_process_test_runner.h" +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +std::unique_ptr<ConformanceTestRunner> NewCppInProcessTestRunner() { + return std::make_unique<InProcessTestRunner>( + [harness = CppConformanceHarness()]( + const ::conformance::ConformanceRequest& request) { + return harness.RunTest(request); + }); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/cpp_in_process_test_runner.h b/conformance/cpp_in_process_test_runner.h new file mode 100644 index 0000000..4adf801 --- /dev/null +++ b/conformance/cpp_in_process_test_runner.h
@@ -0,0 +1,32 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_CPP_IN_PROCESS_TEST_RUNNER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_CPP_IN_PROCESS_TEST_RUNNER_H__ + +#include <memory> + +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// Returns a ConformanceTestRunner that runs every request through a +// CppConformanceHarness owned by the runner, i.e. the C++ testee hosted in the +// test process instead of behind a fork/pipe. +// +// This is the glue between the testee side (conformance_cpp_harness) and the +// tester side (in_process_test_runner); it lives in its own library so that +// the conformance_cpp binary does not link any runner code. +std::unique_ptr<ConformanceTestRunner> NewCppInProcessTestRunner(); + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_CPP_IN_PROCESS_TEST_RUNNER_H__
diff --git a/conformance/cpp_in_process_test_runner_test.cc b/conformance/cpp_in_process_test_runner_test.cc new file mode 100644 index 0000000..bcf71af --- /dev/null +++ b/conformance/cpp_in_process_test_runner_test.cc
@@ -0,0 +1,118 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/cpp_in_process_test_runner.h" + +#include <memory> +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/status/status_matchers.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" +#include "conformance/conformance_cpp_harness.h" +#include "conformance/test_runner.h" +#include "google/protobuf/test_messages_proto3.pb.h" +#include "google/protobuf/test_textproto.h" +#include "google/protobuf/util/message_differencer.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using ::testing::HasSubstr; + +// Returns a request that exercises the harness end to end: a binary +// TestAllTypesProto3 payload to be re-serialized as JSON. +ConformanceRequest Proto3JsonRequest() { + TestAllTypesProto3 message = + ParseTextOrDie(R"pb(optional_int32: 42 optional_string: "hello")pb"); + + ConformanceRequest request; + request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3"); + request.set_requested_output_format(::conformance::JSON); + request.set_protobuf_payload(message.SerializeAsString()); + return request; +} + +// Parses the serialized ConformanceResponse returned by a runner, aborting the +// test if the runner produced bytes that are not a valid response. +ConformanceResponse ParseResponse(absl::string_view serialized) { + ConformanceResponse response; + ABSL_CHECK(response.ParseFromString(serialized)) + << "runner returned an unparseable ConformanceResponse"; + return response; +} + +// The response every Proto3JsonRequest() yields. Kept out of the macro +// arguments: MSVC's traditional preprocessor mis-scans raw string literals +// containing \" inside macro arguments. +constexpr absl::string_view kProto3JsonResponse = + R"pb(json_payload: "{\"optionalInt32\":42,\"optionalString\":\"hello\"}")pb"; + +TEST(CppInProcessTestRunnerTest, RunsHarness) { + std::unique_ptr<ConformanceTestRunner> runner = NewCppInProcessTestRunner(); + + ConformanceResponse response = ParseResponse( + runner->RunTest("Suite.Test", Proto3JsonRequest().SerializeAsString())); + + EXPECT_THAT(response, EqualsProto(kProto3JsonResponse)); +} + +TEST(CppInProcessTestRunnerTest, MatchesDirectHarnessResult) { + std::unique_ptr<ConformanceTestRunner> runner = NewCppInProcessTestRunner(); + ConformanceRequest request = Proto3JsonRequest(); + + absl::StatusOr<ConformanceResponse> direct = + CppConformanceHarness().RunTest(request); + ConformanceResponse via_runner = + ParseResponse(runner->RunTest("Suite.Test", request.SerializeAsString())); + + ASSERT_THAT(direct, IsOkAndHolds(EqualsProto(kProto3JsonResponse))); + EXPECT_TRUE(util::MessageDifferencer::Equals(via_runner, *direct)) + << via_runner.ShortDebugString(); +} + +TEST(CppInProcessTestRunnerTest, ReportsHarnessErrorAsRuntimeError) { + std::unique_ptr<ConformanceTestRunner> runner = NewCppInProcessTestRunner(); + ConformanceRequest request = Proto3JsonRequest(); + request.set_message_type("no.such.Message"); + + ConformanceResponse response = + ParseResponse(runner->RunTest("Suite.Test", request.SerializeAsString())); + + ASSERT_EQ(response.result_case(), ConformanceResponse::kRuntimeError); + EXPECT_THAT(response.runtime_error(), HasSubstr("no.such.Message")); + EXPECT_THAT(response.runtime_error(), HasSubstr("Suite.Test")); +} + +TEST(CppInProcessTestRunnerTest, RunnerIsReusableAcrossRequests) { + std::unique_ptr<ConformanceTestRunner> runner = NewCppInProcessTestRunner(); + std::string request = Proto3JsonRequest().SerializeAsString(); + + ConformanceResponse first = + ParseResponse(runner->RunTest("Suite.A", request)); + ConformanceResponse second = + ParseResponse(runner->RunTest("Suite.B", request)); + + EXPECT_TRUE(util::MessageDifferencer::Equals(first, second)) + << second.ShortDebugString(); + EXPECT_EQ(second.result_case(), ConformanceResponse::kJsonPayload); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/cpp_in_process_testee.cc b/conformance/cpp_in_process_testee.cc new file mode 100644 index 0000000..deb1d1c --- /dev/null +++ b/conformance/cpp_in_process_testee.cc
@@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The C++ runtime's in-process conformance testee: defines MakeTesteeRunner() +// (testee_runner.h) to host CppConformanceHarness in the test process. +// `conformance_test(testee = ":cpp_in_process_testee", in_process = True)` +// links it into the C++ suites' tests in place of :forked_testee, so they run +// the testee without forking conformance_cpp. Other runtimes that can be +// linked into a C++ binary (e.g. upb) provide their own definition in their +// own library. + +#include <memory> +#include <string> + +#include "absl/log/absl_check.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "conformance/cpp_in_process_test_runner.h" +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +std::unique_ptr<ConformanceTestRunner> MakeTesteeRunner( + absl::string_view testee_binary, + absl::Span<const std::string> testee_args) { + // The flags of a forked testee have no meaning here; refusing them keeps a + // stale command line from silently testing the linked testee instead. + ABSL_QCHECK(testee_binary.empty()) + << "--testee_binary=" << testee_binary + << " was given, but this test links the C++ testee in-process " + "(cpp_in_process_testee) and can't run another binary; drop " + "--testee_binary."; + ABSL_QCHECK(testee_args.empty()) + << "--testee_args / positional arguments [" + << absl::StrJoin(testee_args, " ") + << "] were given, but this test links the C++ testee in-process " + "(cpp_in_process_testee), which takes no arguments; drop them."; + return NewCppInProcessTestRunner(); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/cpp_in_process_testee_test.cc b/conformance/cpp_in_process_testee_test.cc new file mode 100644 index 0000000..5d70031 --- /dev/null +++ b/conformance/cpp_in_process_testee_test.cc
@@ -0,0 +1,60 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include <memory> +#include <string> +#include <vector> + +#include <gtest/gtest.h> +#include "conformance/conformance.pb.h" +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" +#include "google/protobuf/test_messages_proto3.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; + +TEST(CppInProcessTesteeTest, AnswersRequestsWithTheCppTestee) { + std::unique_ptr<ConformanceTestRunner> runner = MakeTesteeRunner("", {}); + ASSERT_NE(runner, nullptr); + + TestAllTypesProto3 message; + message.set_optional_int32(99); + ConformanceRequest request; + request.set_message_type(TestAllTypesProto3::descriptor()->full_name()); + request.set_protobuf_payload(message.SerializeAsString()); + request.set_requested_output_format(::conformance::TEXT_FORMAT); + + ConformanceResponse response; + ASSERT_TRUE(response.ParseFromString( + runner->RunTest("SomeTest", request.SerializeAsString()))); + EXPECT_EQ(response.text_payload(), "optional_int32: 99\n"); +} + +TEST(CppInProcessTesteeDeathTest, RejectsATesteeBinary) { + EXPECT_DEATH(MakeTesteeRunner("/some/testee", {}), + "--testee_binary=/some/testee was given, but this test links " + "the C\\+\\+ testee in-process"); +} + +TEST(CppInProcessTesteeDeathTest, RejectsTesteeArgs) { + const std::vector<std::string> args = {"--foo", "bar"}; + EXPECT_DEATH(MakeTesteeRunner("", args), + "positional arguments \\[--foo bar\\] were given, but this test " + "links the C\\+\\+ testee in-process"); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/failure_list.cc b/conformance/failure_list.cc new file mode 100644 index 0000000..ac97276 --- /dev/null +++ b/conformance/failure_list.cc
@@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/failure_list.h" + +#include <algorithm> +#include <string> + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +constexpr int kFailureMessageLengthLimit = 128; + +void Normalize(std::string& input) { + input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); +} + +} // namespace + +std::string FormatFailureMessage(absl::string_view message) { + std::string result(absl::StripAsciiWhitespace(message)); + // Remove newlines + Normalize(result); + // Truncate failure message if needed + if (result.length() > kFailureMessageLengthLimit) { + result = result.substr(0, kFailureMessageLengthLimit); + } + return result; +} + +absl::Status FailureList::Add(absl::string_view test_name, + absl::string_view failure_message) { + if (absl::Status status = entries_.Insert(test_name); !status.ok()) { + return status; + } + // Normalize the message the same way reported failures are normalized so + // that the two compare equal in VerdictOnFailure(). + ABSL_CHECK(expected_messages_ + .emplace(test_name, FormatFailureMessage(failure_message)) + .second); + return absl::OkStatus(); +} + +absl::Status FailureList::VerdictOnSuccess(absl::string_view test_name) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + return absl::OkStatus(); + } + return absl::FailedPreconditionError(absl::StrCat( + "test ", test_name, " (matched to ", *failure_match, + ") is in the failure list, but test succeeded. Remove its match from " + "the failure list.")); +} + +absl::Status FailureList::VerdictOnFailure( + absl::string_view test_name, TestPriority priority, + absl::string_view failure_message) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + if (priority > enforcement_level_) { + return absl::OkStatus(); // Tolerated. + } + return absl::FailedPreconditionError( + absl::StrCat("Unexpected failure for test: ", test_name)); + } + + // Like the legacy runner, only require the actual failure message to start + // with the expected one. + std::string formatted_failure_message = FormatFailureMessage(failure_message); + const std::string& expected_failure_message = + expected_messages_.at(*failure_match); + if (!absl::StartsWith(formatted_failure_message, expected_failure_message)) { + return absl::FailedPreconditionError(absl::StrCat( + "Unexpected failure message for test: ", test_name, " expected: ", + expected_failure_message, " actual: ", formatted_failure_message)); + } + return absl::OkStatus(); +} + +absl::Status FailureList::VerdictOnSkip(absl::string_view test_name, + absl::string_view skip_reason) const { + absl::optional<std::string> failure_match = MatchingEntry(test_name); + if (!failure_match.has_value()) { + return absl::OkStatus(); + } + return absl::FailedPreconditionError(absl::StrCat( + "test ", test_name, " (matched to ", *failure_match, + ") is in the failure list but was skipped by the testee: ", skip_reason, + ". Remove its match from the failure list.")); +} + +absl::optional<std::string> FailureList::MatchingEntry( + absl::string_view test_name) const { + return entries_.WalkDownMatch(test_name); +} + +absl::optional<std::string> FailureList::ExpectedMessage( + absl::string_view entry) const { + auto it = expected_messages_.find(entry); + if (it == expected_messages_.end()) return absl::nullopt; + return it->second; +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/failure_list.h b/conformance/failure_list.h new file mode 100644 index 0000000..bcbcc63 --- /dev/null +++ b/conformance/failure_list.h
@@ -0,0 +1,98 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__ + +#include <string> + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/failure_list_trie_node.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// The form a failure message takes in a failure list: surrounding whitespace +// and newlines removed, truncated to 128 characters. Expected messages are +// normalized when they are added and actual ones when they are judged, so +// that FailureList::VerdictOnFailure() compares like with like. +std::string FormatFailureMessage(absl::string_view message); + +// The expected failures of a conformance run, as the failure list files list +// them, plus the enforcement level: everything needed to judge one test's +// outcome, and nothing that changes once the tests run. The ResultLedger +// (result_ledger.h) builds it while loading the failure lists and tallies the +// outcomes against it. Yields() (matchers.h) reads the run's list through +// GetGlobalFailureList() (global_test_environment.h) and changes nothing. +class FailureList { + public: + FailureList() : entries_("root") {} + + // The lowest priority whose failures fail the suite (see TestPriority in + // testee.h). VerdictOnFailure() tolerates a failing test of a lower priority + // unless it is listed. Defaults to kLowestPriority, which enforces every + // priority. + void set_enforcement_level(TestPriority level) { enforcement_level_ = level; } + TestPriority enforcement_level() const { return enforcement_level_; } + + // Adds the entry `test_name`, a test name or a wildcard, as expected to fail + // with `failure_message` (see VerdictOnFailure()). Returns an error if the + // entry is already present or contains an invalid wildcard. + absl::Status Add(absl::string_view test_name, + absl::string_view failure_message); + + // The verdict on a test that succeeded: an error naming the entry it + // matched if the test was expected to fail. + absl::Status VerdictOnSuccess(absl::string_view test_name) const; + + // The verdict on a test that failed with `failure_message`. OK if the + // failure doesn't fail the suite. That is the case when the test is in the + // failure list with a matching message (an expected failure), or when it + // isn't listed and `priority` is above the enforcement level (a tolerated + // failure). Otherwise the failure is unexpected and the error says why. A + // listed test is checked whatever its priority, so that the failure list + // can't go stale unnoticed. The normalized actual message only needs to + // start with the expected message, so an empty expected message matches any + // failure. + absl::Status VerdictOnFailure(absl::string_view test_name, + TestPriority priority, + absl::string_view failure_message) const; + + // The verdict on a test the testee skipped for `skip_reason`: an error + // naming the entry it matched if the test is in the failure list, since a + // skipped test can't be an expected failure. Whether that error fails the + // test is the caller's policy; Yields() fails such a test. + absl::Status VerdictOnSkip(absl::string_view test_name, + absl::string_view skip_reason) const; + + // The failure list entry `test_name` matches, possibly a wildcard, if any. + absl::optional<std::string> MatchingEntry(absl::string_view test_name) const; + + // The expected failure message of the entry `entry`, normalized (see + // FormatFailureMessage()). `entry` is an entry exactly as listed, wildcards + // included, not a test name to match. Returns nullopt if there is no such + // entry. + absl::optional<std::string> ExpectedMessage(absl::string_view entry) const; + + private: + FailureListTrieNode entries_; + absl::flat_hash_map<std::string, std::string> expected_messages_; + TestPriority enforcement_level_ = kLowestPriority; +}; + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_FAILURE_LIST_H__
diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 3b8d0e9..70597f1 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt
@@ -9,9 +9,9 @@ Recommended.*.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. Recommended.*.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. -Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: Recommended.*.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. Recommended.*.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. Recommended.*.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't.
diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index 61872eb..4cfe7d9 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt
@@ -14,7 +14,7 @@ Recommended.*.JsonInput.DoubleFieldInfinityNotQuoted # Should have failed to parse, but didn't. Recommended.*.JsonInput.DoubleFieldNanNotQuoted # Should have failed to parse, but didn't. Recommended.*.JsonInput.DoubleFieldNegativeInfinityNotQuoted # Should have failed to parse, but didn't. -Recommended.*.JsonInput.FieldNameExtension.JsonOutput # Expected JSON payload but got type 1 +Recommended.*.JsonInput.FieldNameExtension.JsonOutput # Failed to parse input or produce output. Recommended.*.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. Recommended.*.JsonInput.FloatFieldInfinityNotQuoted # Should have failed to parse, but didn't. Recommended.*.JsonInput.FloatFieldNanNotQuoted # Should have failed to parse, but didn't.
diff --git a/conformance/failure_list_jruby_ffi.txt b/conformance/failure_list_jruby_ffi.txt index d1d4c33..9606e4d 100644 --- a/conformance/failure_list_jruby_ffi.txt +++ b/conformance/failure_list_jruby_ffi.txt
@@ -1,6 +1,6 @@ -Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: Required.*.JsonInput.Int32FieldQuotedExponentialValue.* # Failed to parse input or produce output. Required.*.JsonInput.AnyWithNoType.* # Failed to parse input or produce output. # TODO: Uncomment once conformance tests can express failures that are not expected to be fixed.
diff --git a/conformance/failure_list_php.txt b/conformance/failure_list_php.txt index 88cb0c4..31d9342 100644 --- a/conformance/failure_list_php.txt +++ b/conformance/failure_list_php.txt
@@ -6,10 +6,10 @@ Recommended.Editions_Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput Recommended.*.JsonInput.FieldMaskInvalidCharacter -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: Recommended.Editions_Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput Recommended.Proto3.ProtobufInput.ValidDataOneofBinary.MESSAGE.Merge.ProtobufOutput Recommended.*.ProtobufInput.ValueRejectInfNumberValue.JsonOutput # Should have failed to serialize, but didn't.
diff --git a/conformance/failure_list_php_c.txt b/conformance/failure_list_php_c.txt index adc306a..dff232c 100644 --- a/conformance/failure_list_php_c.txt +++ b/conformance/failure_list_php_c.txt
@@ -1,9 +1,9 @@ -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. -Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: Required.Editions_Proto3.JsonInput.Int32FieldQuotedExponentialValue.* # Failed to parse input or produce output. Required.Proto3.JsonInput.Int32FieldQuotedExponentialValue.* # Failed to parse input or produce output. Required.*.JsonInput.AnyWithNoType.*
diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt index 873421d..fe59385 100644 --- a/conformance/failure_list_python.txt +++ b/conformance/failure_list_python.txt
@@ -1,5 +1,5 @@ -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: Required.*.JsonInput.DoubleFieldFalseValue # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueArabicIndicDigits # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueFullwidthDigits # Should have failed to parse, but didn't.
diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt index f49eaba..c5af70d 100644 --- a/conformance/failure_list_python_cpp.txt +++ b/conformance/failure_list_python_cpp.txt
@@ -7,8 +7,8 @@ # TODO: insert links to corresponding bugs tracking the issue. # Should we use GitHub issues or the Google-internal bug tracker? -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: Required.*.JsonInput.DoubleFieldFalseValue # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueArabicIndicDigits # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueFullwidthDigits # Should have failed to parse, but didn't.
diff --git a/conformance/failure_list_python_upb.txt b/conformance/failure_list_python_upb.txt index 72fb8b8..9be5513 100644 --- a/conformance/failure_list_python_upb.txt +++ b/conformance/failure_list_python_upb.txt
@@ -1,5 +1,5 @@ -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: Required.*.JsonInput.DoubleFieldFalseValue # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueArabicIndicDigits # Should have failed to parse, but didn't. Required.*.JsonInput.DoubleFieldStringValueFullwidthDigits # Should have failed to parse, but didn't.
diff --git a/conformance/failure_list_ruby.txt b/conformance/failure_list_ruby.txt index 4a79ef7..8196d08 100644 --- a/conformance/failure_list_ruby.txt +++ b/conformance/failure_list_ruby.txt
@@ -1,6 +1,6 @@ -Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: Required.*.JsonInput.Int32FieldQuotedExponentialValue.* # Failed to parse input or produce output. Required.*.JsonInput.AnyWithNoType.* # TODO: Uncomment once conformance tests can express failures that are not expected to be fixed.
diff --git a/conformance/failure_lists/cpp_binary.txt b/conformance/failure_lists/cpp_binary.txt new file mode 100644 index 0000000..b02b49a --- /dev/null +++ b/conformance/failure_lists/cpp_binary.txt
@@ -0,0 +1,4 @@ +Required.Editions_Proto2.ProtobufInput.BadTag_FieldNumberSlightlyTooHigh # Should have failed to parse, but didn't. +Required.Editions_Proto3.ProtobufInput.BadTag_FieldNumberSlightlyTooHigh # Should have failed to parse, but didn't. +Required.Proto2.ProtobufInput.BadTag_FieldNumberSlightlyTooHigh # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.BadTag_FieldNumberSlightlyTooHigh # Should have failed to parse, but didn't.
diff --git a/conformance/failure_lists/cpp_json.txt b/conformance/failure_lists/cpp_json.txt new file mode 100644 index 0000000..c0d4486 --- /dev/null +++ b/conformance/failure_lists/cpp_json.txt
@@ -0,0 +1,122 @@ +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.ProtobufInput.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.ProtobufInput.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.ProtobufInput.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Editions_Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto2.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto2.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto2.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.ProtobufInput.FieldMaskNumbersDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.ProtobufInput.FieldMaskPathsDontRoundTrip.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.ProtobufInput.FieldMaskTooManyUnderscore.JsonOutput # Should have failed to serialize, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedFalse # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.BoolFieldDoubleQuotedTrue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldMaskInvalidCharacter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.Proto3.JsonInput.FieldNameNotQuoted # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.MapFieldValueIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldPrimitiveElementIsNull # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingComma # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.RepeatedFieldTrailingCommaWithSpaceCommaSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteBoth # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteKey # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldSingleQuoteValue # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.StringFieldUppercaseEscapeLetter # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObject # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithNewlines # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpace # Should have failed to parse, but didn't. +Recommended.Proto3.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.SingleValueForRepeatedFieldInt32 # Should have failed to parse, but didn't. +Required.Editions_Proto2.JsonInput.SingleValueForRepeatedFieldMessage # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.SingleValueForRepeatedFieldInt32 # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.SingleValueForRepeatedFieldMessage # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputDayTooLarge # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputHourTooLarge # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputHourTooLarge25 # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputInvalidOffsetHour # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputInvalidOffsetMinute # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputMinuteTooLarge # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputMonthTooLarge # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputNonLeapFeb29 # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputOffsetBoundaryOverflow # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputOffsetBoundaryUnderflow # Should have failed to parse, but didn't. +Required.Editions_Proto3.JsonInput.TimestampJsonInputSecondTooLarge # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.SingleValueForRepeatedFieldInt32 # Should have failed to parse, but didn't. +Required.Proto2.JsonInput.SingleValueForRepeatedFieldMessage # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.SingleValueForRepeatedFieldInt32 # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.SingleValueForRepeatedFieldMessage # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputDayTooLarge # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputHourTooLarge # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputHourTooLarge25 # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputInvalidOffsetHour # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputInvalidOffsetMinute # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputMinuteTooLarge # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputMonthTooLarge # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputNonLeapFeb29 # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputOffsetBoundaryOverflow # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputOffsetBoundaryUnderflow # Should have failed to parse, but didn't. +Required.Proto3.JsonInput.TimestampJsonInputSecondTooLarge # Should have failed to parse, but didn't.
diff --git a/conformance/failure_lists/cpp_performance.txt b/conformance/failure_lists/cpp_performance.txt new file mode 100644 index 0000000..72f63eb --- /dev/null +++ b/conformance/failure_lists/cpp_performance.txt
@@ -0,0 +1,16 @@ +# This is the list of conformance performance tests that are known to +# fail for the C++ implementation right now. + +# Expected: Non-repeated field "recursive_message" is specified multiple times. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldBool.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldBool.TextFormatOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldBytes.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldBytes.TextFormatOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldDouble.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldDouble.TextFormatOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt32.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt32.TextFormatOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt64.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt64.TextFormatOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldString.ProtobufOutput # Failed to parse input or produce output. +Recommended.*.TextFormatInput.TestTextFormatPerformanceMergeMessageWithRepeatedFieldString.TextFormatOutput # Failed to parse input or produce output.
diff --git a/conformance/failure_lists/cpp_text.txt b/conformance/failure_lists/cpp_text.txt new file mode 100644 index 0000000..69895e1 --- /dev/null +++ b/conformance/failure_lists/cpp_text.txt
@@ -0,0 +1,45 @@ +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Editions_Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes # Should have failed to parse, but didn't. +Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Editions_Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes # Should have failed to parse, but didn't. +Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString # Should have failed to parse, but didn't. + +# Hex literals with the high bit set are parsed as negative numbers instead of +# being rejected. (The same entries are in text_format_failure_list_cpp.txt, +# the list merged_runner_test and the open-source build pass to +# conformance_test_runner.)
diff --git a/conformance/filtering_test_runner.cc b/conformance/filtering_test_runner.cc new file mode 100644 index 0000000..d65df2f --- /dev/null +++ b/conformance/filtering_test_runner.cc
@@ -0,0 +1,53 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/filtering_test_runner.h" + +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +FilteringTestRunner::FilteringTestRunner( + ConformanceTestRunner* absl_nonnull delegate, + absl::flat_hash_set<std::string> names_to_run) + : delegate_(delegate), names_to_run_(std::move(names_to_run)) {} + +FilteringTestRunner::~FilteringTestRunner() = default; + +std::string FilteringTestRunner::RunTest(absl::string_view test_name, + absl::string_view input) { + if (!names_to_run_.contains(test_name)) { + // Fully qualified: inside google::protobuf::conformance, a bare `conformance::` + // would name this namespace rather than the protos' one. + ::conformance::ConformanceResponse response; + response.set_skipped(kNotSelectedSkipReason); + return response.SerializeAsString(); + } + names_run_.insert(std::string(test_name)); + return delegate_->RunTest(test_name, input); +} + +std::vector<std::string> FilteringTestRunner::NamesRun() const { + std::vector<std::string> names_run(names_run_.begin(), names_run_.end()); + absl::c_sort(names_run); + return names_run; +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/filtering_test_runner.h b/conformance/filtering_test_runner.h new file mode 100644 index 0000000..b7a4a1a --- /dev/null +++ b/conformance/filtering_test_runner.h
@@ -0,0 +1,68 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_FILTERING_TEST_RUNNER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_FILTERING_TEST_RUNNER_H__ + +#include <string> +#include <vector> + +#include "absl/base/nullability.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/string_view.h" +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// Decorates a ConformanceTestRunner so that only the tests named in a set are +// forwarded to the delegate. Any other test gets a ConformanceResponse with +// `skipped` set to kNotSelectedSkipReason, without contacting the delegate. +// So no testee process is ever started if none of the tests is selected. +// +// This implements the `--test <name>` isolation of conformance_test_runner +// for gtest-based conformance suites. The shared reason string is what lets +// the reporting side (Yields(), see matchers.h) tell such a test apart from +// one the testee itself opted out of. NamesRun() tells which of the +// requested names came up, so that the caller can tell which ones are still +// to be looked for elsewhere. +// +// The delegate is not owned and must outlive this object. +class FilteringTestRunner : public ConformanceTestRunner { + public: + // The `skipped` reason answered for every test that was not selected. It + // is not a testee skip: Yields() (matchers.h) passes such a response + // silently and only records that the test exists. A testee must never + // answer with it, or its skip would go unnoticed. + static constexpr absl::string_view kNotSelectedSkipReason = + "Not selected by --test."; + + FilteringTestRunner(ConformanceTestRunner* absl_nonnull delegate, + absl::flat_hash_set<std::string> names_to_run); + ~FilteringTestRunner() override; + + FilteringTestRunner(const FilteringTestRunner&) = delete; + FilteringTestRunner& operator=(const FilteringTestRunner&) = delete; + + std::string RunTest(absl::string_view test_name, + absl::string_view input) override; + + // The names forwarded to the delegate so far, in sorted order. + std::vector<std::string> NamesRun() const; + + private: + ConformanceTestRunner* absl_nonnull delegate_; + absl::flat_hash_set<std::string> names_to_run_; + absl::flat_hash_set<std::string> names_run_; +}; + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_FILTERING_TEST_RUNNER_H__
diff --git a/conformance/filtering_test_runner_test.cc b/conformance/filtering_test_runner_test.cc new file mode 100644 index 0000000..b698898 --- /dev/null +++ b/conformance/filtering_test_runner_test.cc
@@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/filtering_test_runner.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/conformance.pb.h" +#include "conformance/mock_test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ElementsAre; +using ::testing::InSequence; +using ::testing::IsEmpty; +using ::testing::Return; +using ::testing::StrictMock; + +// The delegate is a strict mock throughout: a call the filter should not +// have forwarded fails the test. +TEST(FilteringTestRunnerTest, ForwardsSelectedTests) { + StrictMock<MockTestRunner> delegate; + FilteringTestRunner runner(&delegate, {"Test.One", "Test.Two"}); + + InSequence in_sequence; + EXPECT_CALL(delegate, RunTest("Test.One", "abc")).WillOnce(Return("one")); + EXPECT_CALL(delegate, RunTest("Test.Two", "")).WillOnce(Return("two")); + EXPECT_EQ(runner.RunTest("Test.One", "abc"), "one"); + EXPECT_EQ(runner.RunTest("Test.Two", ""), "two"); +} + +TEST(FilteringTestRunnerTest, SkipsUnselectedTestsWithoutDelegating) { + StrictMock<MockTestRunner> delegate; + FilteringTestRunner runner(&delegate, {"Test.One"}); + + ::conformance::ConformanceResponse response; + ASSERT_TRUE(response.ParseFromString(runner.RunTest("Test.Other", "abc"))); + EXPECT_EQ(response.result_case(), + ::conformance::ConformanceResponse::kSkipped); + // The reason Yields() (matchers.h) recognizes. + EXPECT_EQ(response.skipped(), "Not selected by --test."); +} + +TEST(FilteringTestRunnerTest, EmptySelectionSkipsEverything) { + StrictMock<MockTestRunner> delegate; + FilteringTestRunner runner(&delegate, {}); + + ::conformance::ConformanceResponse response; + ASSERT_TRUE(response.ParseFromString(runner.RunTest("Test.One", "abc"))); + EXPECT_EQ(response.result_case(), + ::conformance::ConformanceResponse::kSkipped); + EXPECT_THAT(runner.NamesRun(), IsEmpty()); +} + +TEST(FilteringTestRunnerTest, TracksNamesRunInSortedOrder) { + StrictMock<MockTestRunner> delegate; + FilteringTestRunner runner(&delegate, {"Test.C", "Test.A", "Test.B"}); + + EXPECT_THAT(runner.NamesRun(), IsEmpty()); + + InSequence in_sequence; + EXPECT_CALL(delegate, RunTest("Test.C", "")); + EXPECT_CALL(delegate, RunTest("Test.B", "")); + EXPECT_CALL(delegate, RunTest("Test.C", "")); + runner.RunTest("Test.C", ""); + runner.RunTest("Test.Other", ""); + runner.RunTest("Test.B", ""); + // Running a test twice is fine and counts once. + runner.RunTest("Test.C", ""); + + EXPECT_THAT(runner.NamesRun(), ElementsAre("Test.B", "Test.C")); +} + +TEST(FilteringTestRunnerTest, MatchesExactNamesOnly) { + StrictMock<MockTestRunner> delegate; + FilteringTestRunner runner(&delegate, {"Test.One"}); + + runner.RunTest("Test.On", ""); + runner.RunTest("Test.One.More", ""); + runner.RunTest("test.one", ""); + + EXPECT_THAT(runner.NamesRun(), IsEmpty()); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/forked_testee.cc b/conformance/forked_testee.cc new file mode 100644 index 0000000..30a864e --- /dev/null +++ b/conformance/forked_testee.cc
@@ -0,0 +1,35 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The default testee of a conformance test binary: --testee_binary, run as a +// child process (see testee_runner.h). + +#include <memory> +#include <string> + +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "conformance/fork_pipe_runner.h" +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +std::unique_ptr<ConformanceTestRunner> MakeTesteeRunner( + absl::string_view testee_binary, + absl::Span<const std::string> testee_args) { + ABSL_QCHECK(!testee_binary.empty()) << "--testee_binary is required."; + // Spawns the testee lazily, on the first request. + return std::make_unique<ForkPipeRunner>(testee_binary, testee_args); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/forked_testee_test.cc b/conformance/forked_testee_test.cc new file mode 100644 index 0000000..cb870e7 --- /dev/null +++ b/conformance/forked_testee_test.cc
@@ -0,0 +1,50 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include <memory> +#include <string> +#include <vector> + +#include <gtest/gtest.h> +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +// TODO: cover Windows too, with a portable stand-in testee. +#ifndef _WIN32 +TEST(ForkedTesteeTest, RunsTheTesteeBinaryWithItsArgs) { + // `cat` echoes the length-prefixed request straight back, so a round trip + // shows that --testee_binary and --testee_args reached a child process. + const std::vector<std::string> args = {"-c", "exec cat"}; + std::unique_ptr<ConformanceTestRunner> runner = + MakeTesteeRunner("/bin/sh", args); + ASSERT_NE(runner, nullptr); + EXPECT_EQ(runner->RunTest("SomeTest", "request bytes"), "request bytes"); +} +#endif // !_WIN32 + +TEST(ForkedTesteeTest, AcceptsANonexistentBinary) { + // MakeTesteeRunner() only builds the runner; ForkPipeRunner spawns the + // testee on the first request, so the binary need not even exist until a + // test runs (--gtest_list_tests and an all-skipped run never do). + std::unique_ptr<ConformanceTestRunner> runner = + MakeTesteeRunner("/no/such/testee", {}); + EXPECT_NE(runner, nullptr); +} + +TEST(ForkedTesteeDeathTest, RequiresATesteeBinary) { + EXPECT_DEATH(MakeTesteeRunner("", {}), "--testee_binary is required"); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/global_test_environment.h b/conformance/global_test_environment.h index 8742960..45e658b 100644 --- a/conformance/global_test_environment.h +++ b/conformance/global_test_environment.h
@@ -5,7 +5,7 @@ // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd -// Declares the process-wide state that the Yields() matcher depends on (see +// Declares the process-wide state that the Yields() matcher reads (see // matchers.h). The definition is deliberately not provided here. It is the // link-time seam between the matchers and the process that hosts them. The // conformance test binary provides it from its global test environment, @@ -15,18 +15,20 @@ #ifndef GOOGLE_PROTOBUF_CONFORMANCE_GLOBAL_TEST_ENVIRONMENT_H__ #define GOOGLE_PROTOBUF_CONFORMANCE_GLOBAL_TEST_ENVIRONMENT_H__ -#include "conformance/test_manager.h" +#include "conformance/failure_list.h" namespace google { namespace protobuf { namespace conformance { namespace internal { -// Returns the process-wide TestManager that records test results against the -// expected-failure list. Yields() reports the outcome of every test here. -// That lets the failure list be validated and regenerated, and the enforcement -// level be applied (see TestManager::set_enforcement_level()). -TestManager& GetGlobalTestManager(); +// Returns the process-wide expected-failure list. Yields() asks it for the +// verdict on every test, which applies the failure list and the enforcement +// level (see FailureList::set_enforcement_level()). The test environment, +// which owns the list as part of its ResultLedger, is what tallies the +// outcomes (see ConformanceEnvironment in test_environment.h); the matchers +// never change anything. +const FailureList& GetGlobalFailureList(); } // namespace internal } // namespace conformance
diff --git a/conformance/in_process_test_runner.cc b/conformance/in_process_test_runner.cc new file mode 100644 index 0000000..342b7d3 --- /dev/null +++ b/conformance/in_process_test_runner.cc
@@ -0,0 +1,57 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/in_process_test_runner.h" + +#include <string> +#include <utility> + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +// Returns a serialized ConformanceResponse whose `runtime_error` is `message`. +std::string RuntimeError(absl::string_view message) { + ::conformance::ConformanceResponse response; + response.set_runtime_error(message); + return response.SerializeAsString(); +} + +} // namespace + +InProcessTestRunner::InProcessTestRunner(RequestHandler handler) + : handler_(std::move(handler)) {} + +InProcessTestRunner::~InProcessTestRunner() = default; + +std::string InProcessTestRunner::RunTest(absl::string_view test_name, + absl::string_view input) { + ::conformance::ConformanceRequest request; + if (!request.ParseFromString(input)) { + return RuntimeError(absl::StrCat( + "failed to parse ConformanceRequest for test ", test_name)); + } + + absl::StatusOr<::conformance::ConformanceResponse> response = + handler_(request); + if (!response.ok()) { + return RuntimeError(absl::StrCat("request handler failed for test ", + test_name, ": ", + response.status().ToString())); + } + return response->SerializeAsString(); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/in_process_test_runner.h b/conformance/in_process_test_runner.h new file mode 100644 index 0000000..0a7c084 --- /dev/null +++ b/conformance/in_process_test_runner.h
@@ -0,0 +1,56 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_IN_PROCESS_TEST_RUNNER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_IN_PROCESS_TEST_RUNNER_H__ + +#include <string> + +#include "absl/functional/any_invocable.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "conformance/conformance.pb.h" +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// A ConformanceTestRunner that dispatches each request to an in-process +// callback instead of a testee subprocess. Parses the serialized request, +// invokes `handler`, and serializes the response. +// +// If the request cannot be parsed, or `handler` returns a non-OK status, the +// returned response has `runtime_error` set describing the failure. +// +// Any state referenced by `handler` (e.g. a captured harness) must outlive the +// runner. Like every other ConformanceTestRunner, this class is meant for +// single-threaded use: RunTest is not thread-safe if `handler` mutates state. +class InProcessTestRunner : public ConformanceTestRunner { + public: + using RequestHandler = + absl::AnyInvocable<absl::StatusOr<::conformance::ConformanceResponse>( + const ::conformance::ConformanceRequest&)>; + + explicit InProcessTestRunner(RequestHandler handler); + ~InProcessTestRunner() override; + + InProcessTestRunner(const InProcessTestRunner&) = delete; + InProcessTestRunner& operator=(const InProcessTestRunner&) = delete; + + std::string RunTest(absl::string_view test_name, + absl::string_view input) override; + + private: + RequestHandler handler_; +}; + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_IN_PROCESS_TEST_RUNNER_H__
diff --git a/conformance/in_process_test_runner_test.cc b/conformance/in_process_test_runner_test.cc new file mode 100644 index 0000000..60e6498 --- /dev/null +++ b/conformance/in_process_test_runner_test.cc
@@ -0,0 +1,132 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/in_process_test_runner.h" + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_wireformat.h" +#include "conformance/conformance.pb.h" +#include "google/protobuf/test_textproto.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::testing::AllOf; +using ::testing::HasSubstr; + +// Parses the serialized ConformanceResponse returned by a runner, aborting the +// test if the runner produced bytes that are not a valid response. +ConformanceResponse ParseResponse(absl::string_view serialized) { + ConformanceResponse response; + ABSL_CHECK(response.ParseFromString(serialized)) + << "runner returned an unparseable ConformanceResponse"; + return response; +} + +TEST(InProcessTestRunnerTest, HandlerReceivesParsedRequest) { + ConformanceRequest received; + InProcessTestRunner runner([&received](const ConformanceRequest& request) + -> absl::StatusOr<ConformanceResponse> { + received = request; + return ConformanceResponse(); + }); + ConformanceRequest request = ParseTextOrDie(R"pb( + message_type: "some.Message" + requested_output_format: TEXT_FORMAT + protobuf_payload: "payload" + )pb"); + + runner.RunTest("Suite.Test", request.SerializeAsString()); + + EXPECT_THAT(received, EqualsProto(R"pb( + message_type: "some.Message" + requested_output_format: TEXT_FORMAT + protobuf_payload: "payload" + )pb")); +} + +TEST(InProcessTestRunnerTest, HandlerResponseIsSerialized) { + InProcessTestRunner runner( + [](const ConformanceRequest&) -> absl::StatusOr<ConformanceResponse> { + ConformanceResponse response; + response.set_text_payload("optional_int32: 1\n"); + return response; + }); + + std::string serialized = + runner.RunTest("Suite.Test", ConformanceRequest().SerializeAsString()); + + EXPECT_THAT(ParseResponse(serialized), + EqualsProto(R"pb(text_payload: "optional_int32: 1\n")pb")); +} + +TEST(InProcessTestRunnerTest, HandlerErrorBecomesRuntimeError) { + InProcessTestRunner runner( + [](const ConformanceRequest&) -> absl::StatusOr<ConformanceResponse> { + return absl::NotFoundError("no such message type"); + }); + + ConformanceResponse response = ParseResponse( + runner.RunTest("Suite.Test", ConformanceRequest().SerializeAsString())); + + ASSERT_EQ(response.result_case(), ConformanceResponse::kRuntimeError); + EXPECT_THAT( + response.runtime_error(), + AllOf(HasSubstr("Suite.Test"), HasSubstr("no such message type"))); +} + +TEST(InProcessTestRunnerTest, UnparseableRequestBecomesRuntimeError) { + bool handler_called = false; + InProcessTestRunner runner([&handler_called](const ConformanceRequest&) + -> absl::StatusOr<ConformanceResponse> { + handler_called = true; + return ConformanceResponse(); + }); + + // Field 1, length-delimited, claims 5 bytes but only 3 follow. + ConformanceResponse response = ParseResponse(runner.RunTest( + "Suite.Test", + Wire(Tag(1, WireType::kLengthPrefixed), Varint(5), "abc").str())); + + EXPECT_FALSE(handler_called); + ASSERT_EQ(response.result_case(), ConformanceResponse::kRuntimeError); + EXPECT_THAT(response.runtime_error(), + AllOf(HasSubstr("Suite.Test"), + HasSubstr("failed to parse ConformanceRequest"))); +} + +TEST(InProcessTestRunnerTest, HandlerMayBeStateful) { + InProcessTestRunner runner([calls = 0](const ConformanceRequest&) mutable + -> absl::StatusOr<ConformanceResponse> { + ConformanceResponse response; + response.set_text_payload(std::to_string(++calls)); + return response; + }); + std::string request = ConformanceRequest().SerializeAsString(); + + runner.RunTest("Suite.First", request); + ConformanceResponse response = + ParseResponse(runner.RunTest("Suite.Second", request)); + + EXPECT_THAT(response, EqualsProto(R"pb(text_payload: "2")pb")); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/in_process_testee_test.cc b/conformance/in_process_testee_test.cc new file mode 100644 index 0000000..70ce1d7 --- /dev/null +++ b/conformance/in_process_testee_test.cc
@@ -0,0 +1,111 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A conformance test binary with an in-process testee: this file defines +// MakeTesteeRunner() (see testee_runner.h) and links :test_environment_main +// without :forked_testee. It is run without --testee_binary (see BUILD), +// which a forked testee would refuse, and checks that the main hands the +// suites the linked testee. + +#include <memory> +#include <string> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "conformance/binary_wireformat.h" +#include "conformance/conformance.pb.h" +#include "conformance/matchers.h" +#include "conformance/test_fixture.h" +#include "conformance/test_runner.h" +#include "conformance/testee_runner.h" +#include "google/protobuf/test_messages_proto3.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using ::testing::IsEmpty; + +// What test_environment_main passed to MakeTesteeRunner(), and how many +// requests the testee it got back has answered. +struct LinkedTesteeState { + bool created = false; + std::string testee_binary; + std::vector<std::string> testee_args; + int requests_answered = 0; +}; + +LinkedTesteeState& State() { + static auto* const state = new LinkedTesteeState; + return *state; +} + +// An in-process testee that "parses" a binary payload by echoing it back, so +// the suites see a conforming response without any real parsing. +class EchoTestee : public ConformanceTestRunner { + public: + std::string RunTest(absl::string_view test_name, + absl::string_view serialized_request) override { + ConformanceRequest request; + ABSL_CHECK(request.ParseFromString(serialized_request)) << test_name; + ConformanceResponse response; + if (request.has_protobuf_payload() && + request.requested_output_format() == ::conformance::PROTOBUF) { + response.set_protobuf_payload(request.protobuf_payload()); + } else { + response.set_skipped("EchoTestee only echoes binary payloads."); + } + ++State().requests_answered; + return response.SerializeAsString(); + } +}; + +} // namespace + +std::unique_ptr<ConformanceTestRunner> MakeTesteeRunner( + absl::string_view testee_binary, + absl::Span<const std::string> testee_args) { + State().created = true; + State().testee_binary = std::string(testee_binary); + State().testee_args.assign(testee_args.begin(), testee_args.end()); + return std::make_unique<EchoTestee>(); +} + +namespace { + +using InProcessTesteeTest = ConformanceTest; + +TEST_F(InProcessTesteeTest, MainCreatesTheLinkedTesteeFromEmptyFlags) { + // Run without --testee_binary or --testee_args, which is how a test with an + // in-process testee is invoked (see conformance.bzl). + EXPECT_TRUE(State().created); + EXPECT_THAT(State().testee_binary, IsEmpty()); + EXPECT_THAT(State().testee_args, IsEmpty()); +} + +TEST_F(InProcessTesteeTest, SuitesTalkToTheLinkedTestee) { + const int requests_before = State().requests_answered; + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); + EXPECT_EQ(State().requests_answered, requests_before + 1); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_any_test.cc b/conformance/json_any_test.cc new file mode 100644 index 0000000..68616af --- /dev/null +++ b/conformance/json_any_test.cc
@@ -0,0 +1,374 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.Any fields: the "@type" member +// (in any position, or nested in another Any), the well-known types whose +// JSON form is a "value" member, the type URLs that must be rejected, an Any +// without a type, `null`, and an empty packed message. This replaces the +// legacy BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForAny(), which ran +// for the proto3-style message types only; the test names and the requests +// sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. The inputs are the legacy raw +// string literals verbatim; the ones packing the test message itself get its +// type URL substituted per message type, as in the legacy suite. + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonAnyTest = MessageTypeConformanceTest; + +TEST_P(JsonAnyTest, Any) { + const std::string type_url = + absl::StrCat("type.googleapis.com/", message()->full_name()); + const std::string input = absl::Substitute(R"({ + "optionalAny": { + "@type": "$0", + "optionalInt32": 12345 + } + })", + type_url); + const std::string expected = absl::Substitute(R"( + optional_any: { + [$0] { + optional_int32: 12345 + } + } + )", + type_url); + EXPECT_THAT(Testee("Any").ParseJson(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee("Any").ParseJson(message(), input).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(JsonAnyTest, AnyNested) { + const std::string type_url = + absl::StrCat("type.googleapis.com/", message()->full_name()); + const std::string input = absl::Substitute(R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Any", + "value": { + "@type": "$0", + "optionalInt32": 12345 + } + } + })", + type_url); + const std::string expected = absl::Substitute(R"( + optional_any: { + [type.googleapis.com/google.protobuf.Any] { + [$0] { + optional_int32: 12345 + } + } + } + )", + type_url); + EXPECT_THAT(Testee("AnyNested").ParseJson(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee("AnyNested").ParseJson(message(), input).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +// The special "@type" tag is not required to appear first. +TEST_P(JsonAnyTest, AnyUnorderedTypeTag) { + const std::string type_url = + absl::StrCat("type.googleapis.com/", message()->full_name()); + const std::string input = absl::Substitute(R"({ + "optionalAny": { + "optionalInt32": 12345, + "@type": "$0" + } + })", + type_url); + const std::string expected = absl::Substitute(R"( + optional_any: { + [$0] { + optional_int32: 12345 + } + } + )", + type_url); + EXPECT_THAT(Testee("AnyUnorderedTypeTag") + .ParseJson(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT( + Testee("AnyUnorderedTypeTag").ParseJson(message(), input).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +// Well-known types in Any. +TEST_P(JsonAnyTest, AnyWithInt32ValueWrapper) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Int32Value", + "value": 12345 + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Int32Value] { value: 12345 } + } + )pb"; + EXPECT_THAT(Testee("AnyWithInt32ValueWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("AnyWithInt32ValueWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithDuration) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Duration", + "value": "1.5s" + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Duration] { + seconds: 1 + nanos: 500000000 + } + } + )pb"; + EXPECT_THAT( + Testee("AnyWithDuration").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("AnyWithDuration").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithTimestamp) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Timestamp", + "value": "1970-01-01T00:00:00Z" + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Timestamp] { seconds: 0 nanos: 0 } + } + )pb"; + EXPECT_THAT( + Testee("AnyWithTimestamp").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("AnyWithTimestamp").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithFieldMask) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.FieldMask", + "value": "foo,barBaz" + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.FieldMask] { + paths: [ "foo", "bar_baz" ] + } + } + )pb"; + EXPECT_THAT( + Testee("AnyWithFieldMask").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("AnyWithFieldMask").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithStruct) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Struct", + "value": { + "foo": 1 + } + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Struct] { + fields: { + key: "foo" + value: { number_value: 1 } + } + } + } + )pb"; + EXPECT_THAT( + Testee("AnyWithStruct").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("AnyWithStruct").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithValueForJsonObject) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Value", + "value": { + "foo": 1 + } + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Value] { + struct_value: { + fields: { + key: "foo" + value: { number_value: 1 } + } + } + } + } + )pb"; + EXPECT_THAT(Testee("AnyWithValueForJsonObject") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("AnyWithValueForJsonObject") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonAnyTest, AnyWithValueForInteger) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Value", + "value": 1 + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Value] { number_value: 1 } + } + )pb"; + EXPECT_THAT(Testee("AnyWithValueForInteger") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("AnyWithValueForInteger") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// When the Any is in WKT form (with "@type"), the type_url must be present +// and URL shaped, otherwise it should be a parse error (because it can't be +// parsed into the Any schema). +TEST_P(JsonAnyTest, AnyWktRepresentationWithEmptyTypeAndValue) { + EXPECT_THAT(Testee("AnyWktRepresentationWithEmptyTypeAndValue") + .ParseJson(message(), R"json({ + "optionalAny": { + "@type": "", + "value": "" + } + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonAnyTest, AnyWktRepresentationWithBadType) { + EXPECT_THAT(Testee("AnyWktRepresentationWithBadType") + .ParseJson(message(), R"json({ + "optionalAny": { + "@type": "not_a_url", + "value": "" + } + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +// When the Any can be parsed as non-WKT form, the type_url could be missing +// or invalid, since that can still be parsed into the Any schema. +TEST_P(JsonAnyTest, AnyWithNoType) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": {} + })json"; + constexpr absl::string_view kExpected = R"pb(optional_any: {})pb"; + EXPECT_THAT( + Testee("AnyWithNoType").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("AnyWithNoType").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// `null` where an Any exists should just result in the field being unset. +TEST_P(JsonAnyTest, AnyNull) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": null + })json"; + constexpr absl::string_view kExpected = ""; + EXPECT_THAT(Testee("AnyNull").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("AnyNull").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// google.protobuf.Empty packed into an Any, implementations must accept it +// without the "value" field set. This also confirms that what they round trip +// does not have `"value":{}` set on it, since the test harness uses the C++ +// JSON parser which will reject it. +TEST_P(JsonAnyTest, AnyEmpty) { + constexpr absl::string_view kInput = R"json({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Empty" + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_any: { + [type.googleapis.com/google.protobuf.Empty] {} + } + )pb"; + EXPECT_THAT(Testee("AnyEmpty").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("AnyEmpty").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonAnyTest, ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_bool_test.cc b/conformance/json_bool_test.cc new file mode 100644 index 0000000..a8b1afc --- /dev/null +++ b/conformance/json_bool_test.cc
@@ -0,0 +1,131 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for bool fields: `true` / `false` round-trip, and +// every other spelling (0, 1, True, TRUE, "true", ...) is rejected. This +// replaces the "Bool fields" block of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(); the +// test names and the requests sent to the testee are identical to the legacy +// ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +class JsonBoolTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +TEST_P(JsonBoolTest, BoolFieldTrue) { + constexpr absl::string_view kInput = R"json({"optionalBool":true})json"; + constexpr absl::string_view kExpected = R"pb(optional_bool: true)pb"; + EXPECT_THAT(Testee(kP0, "BoolFieldTrue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee(kP0, "BoolFieldTrue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonBoolTest, BoolFieldFalse) { + constexpr absl::string_view kInput = R"json({"optionalBool":false})json"; + constexpr absl::string_view kExpected = R"pb(optional_bool: false)pb"; + EXPECT_THAT(Testee(kP0, "BoolFieldFalse") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee(kP0, "BoolFieldFalse") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// --------------------------------------------------------------------------- +// Other forms are not allowed. +// --------------------------------------------------------------------------- + +TEST_P(JsonBoolTest, BoolFieldIntegerZero) { + EXPECT_THAT(Testee("BoolFieldIntegerZero") + .ParseJson(message(), R"json({"optionalBool":0})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldIntegerOne) { + EXPECT_THAT(Testee("BoolFieldIntegerOne") + .ParseJson(message(), R"json({"optionalBool":1})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldCamelCaseTrue) { + EXPECT_THAT(Testee("BoolFieldCamelCaseTrue") + .ParseJson(message(), R"json({"optionalBool":True})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldCamelCaseFalse) { + EXPECT_THAT(Testee("BoolFieldCamelCaseFalse") + .ParseJson(message(), R"json({"optionalBool":False})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldAllCapitalTrue) { + EXPECT_THAT(Testee("BoolFieldAllCapitalTrue") + .ParseJson(message(), R"json({"optionalBool":TRUE})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldAllCapitalFalse) { + EXPECT_THAT(Testee("BoolFieldAllCapitalFalse") + .ParseJson(message(), R"json({"optionalBool":FALSE})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldDoubleQuotedTrue) { + EXPECT_THAT(Testee("BoolFieldDoubleQuotedTrue") + .ParseJson(message(), R"json({"optionalBool":"true"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonBoolTest, BoolFieldDoubleQuotedFalse) { + EXPECT_THAT(Testee("BoolFieldDoubleQuotedFalse") + .ParseJson(message(), R"json({"optionalBool":"false"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonBoolTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_duration_test.cc b/conformance/json_duration_test.cc new file mode 100644 index 0000000..ae9db0a --- /dev/null +++ b/conformance/json_duration_test.cc
@@ -0,0 +1,276 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.Duration: its "<seconds>s" +// string form with up to nine fractional digits, its range, the rejection of +// out-of-range or malformed values in either direction, and the number of +// fractional digits a serializer emits. This replaces the Duration part of +// the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForWrapperTypes(), which +// ran for the proto3-style message types only; the test names and the requests +// sent to the testee are identical to the legacy ones, except that the +// binary-input tests below are BINARY_TEST requests (see below). +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix; ExpectSerializeFailureForJson() sent +// a binary message (built here from its wire format) to be serialized as +// JSON ("ProtobufInput.<name>.JsonOutput"), categorized as JSON_TEST; here +// ParseBinary() categorizes them as BINARY_TEST, which no testee tells apart +// from JSON_TEST on a binary payload (b/563657722). And the +// legacy "validators" (RunValidJsonTestWithValidator()) looked at the +// serialized JSON text itself. + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonDurationTest = MessageTypeConformanceTest; + +// optional_duration is field 301 of the test message types; Duration's +// seconds is field 1 and nanos field 2. +constexpr uint32_t kOptionalDuration = 301; +constexpr uint32_t kSeconds = 1; +constexpr uint32_t kNanos = 2; + +TEST_P(JsonDurationTest, DurationMinValue) { + constexpr absl::string_view kInput = + R"json({"optionalDuration": "-315576000000.999999999s"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_duration: { seconds: -315576000000 nanos: -999999999 })pb"; + EXPECT_THAT( + Testee("DurationMinValue").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("DurationMinValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDurationTest, DurationMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalDuration": "315576000000.999999999s"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_duration: { seconds: 315576000000 nanos: 999999999 })pb"; + EXPECT_THAT( + Testee("DurationMaxValue").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("DurationMaxValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDurationTest, DurationRepeatedValue) { + constexpr absl::string_view kInput = + R"json({"repeatedDuration": ["1.5s", "-1.5s"]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_duration: { seconds: 1 nanos: 500000000 } + repeated_duration: { seconds: -1 nanos: -500000000 } + )pb"; + EXPECT_THAT(Testee("DurationRepeatedValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DurationRepeatedValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDurationTest, DurationNull) { + constexpr absl::string_view kInput = R"json({"optionalDuration": null})json"; + constexpr absl::string_view kExpected = ""; + EXPECT_THAT( + Testee("DurationNull").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("DurationNull").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDurationTest, DurationNegativeSeconds) { + constexpr absl::string_view kInput = R"json({"optionalDuration": "-5s"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_duration: { seconds: -5 nanos: 0 })pb"; + EXPECT_THAT(Testee("DurationNegativeSeconds") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DurationNegativeSeconds") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDurationTest, DurationNegativeNanos) { + constexpr absl::string_view kInput = + R"json({"optionalDuration": "-0.5s"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_duration: { seconds: 0 nanos: -500000000 })pb"; + EXPECT_THAT(Testee("DurationNegativeNanos") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DurationNegativeNanos") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Malformed and out-of-range JSON values are rejected. +TEST_P(JsonDurationTest, DurationMissingS) { + EXPECT_THAT(Testee("DurationMissingS") + .ParseJson(message(), R"json({"optionalDuration": "1"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDurationTest, DurationJsonInputTooSmall) { + EXPECT_THAT( + Testee("DurationJsonInputTooSmall") + .ParseJson( + message(), + R"json({"optionalDuration": "-315576000001.000000000s"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDurationTest, DurationJsonInputTooLarge) { + EXPECT_THAT( + Testee("DurationJsonInputTooLarge") + .ParseJson( + message(), + R"json({"optionalDuration": "315576000001.000000000s"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Out-of-range or inconsistent messages can't be serialized. The inputs +// are the legacy text-format messages on the wire (zero fields omitted). +TEST_P(JsonDurationTest, DurationProtoInputTooSmall) { + EXPECT_THAT( + Testee("DurationProtoInputTooSmall") + .ParseBinary(message(), LengthPrefixedField( + kOptionalDuration, + VarintField(kSeconds, -315576000001))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonDurationTest, DurationProtoInputTooLarge) { + EXPECT_THAT( + Testee("DurationProtoInputTooLarge") + .ParseBinary(message(), + LengthPrefixedField(kOptionalDuration, + VarintField(kSeconds, 315576000001))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonDurationTest, DurationProtoNanosWrongSign) { + EXPECT_THAT(Testee("DurationProtoNanosWrongSign") + .ParseBinary(message(), LengthPrefixedField( + kOptionalDuration, + Wire(VarintField(kSeconds, 1), + VarintField(kNanos, -1)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonDurationTest, DurationProtoNanosWrongSignNegativeSecs) { + EXPECT_THAT(Testee("DurationProtoNanosWrongSignNegativeSecs") + .ParseBinary(message(), LengthPrefixedField( + kOptionalDuration, + Wire(VarintField(kSeconds, -1), + VarintField(kNanos, 1)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonDurationTest, DurationProtoNanosTooSmall) { + EXPECT_THAT( + Testee("DurationProtoNanosTooSmall") + .ParseBinary(message(), LengthPrefixedField( + kOptionalDuration, + Wire(VarintField(kSeconds, -1), + VarintField(kNanos, -1000000000)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonDurationTest, DurationProtoNanosTooLarge) { + EXPECT_THAT( + Testee("DurationProtoNanosTooLarge") + .ParseBinary(message(), LengthPrefixedField( + kOptionalDuration, + Wire(VarintField(kSeconds, 1), + VarintField(kNanos, 1000000000)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +// Serializers emit 0, 3, 6 or 9 fractional digits, whichever is the fewest +// that represent the value exactly. +TEST_P(JsonDurationTest, DurationHasZeroFractionalDigit) { + EXPECT_THAT(Testee(kP1, "DurationHasZeroFractionalDigit") + .ParseJson(message(), + R"json({"optionalDuration": "1.000000000s"})json") + .SerializeJson(), + Yields(JsonPayload( + HasJsonMemberThat("optionalDuration", IsJsonString("1s"))))); +} + +TEST_P(JsonDurationTest, DurationHas3FractionalDigits) { + EXPECT_THAT(Testee(kP1, "DurationHas3FractionalDigits") + .ParseJson(message(), + R"json({"optionalDuration": "1.010000000s"})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat("optionalDuration", + IsJsonString("1.010s"))))); +} + +TEST_P(JsonDurationTest, DurationHas6FractionalDigits) { + EXPECT_THAT(Testee(kP1, "DurationHas6FractionalDigits") + .ParseJson(message(), + R"json({"optionalDuration": "1.000010000s"})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalDuration", IsJsonString("1.000010s"))))); +} + +TEST_P(JsonDurationTest, DurationHas9FractionalDigits) { + EXPECT_THAT(Testee(kP1, "DurationHas9FractionalDigits") + .ParseJson(message(), + R"json({"optionalDuration": "1.000000010s"})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalDuration", IsJsonString("1.000000010s"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonDurationTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_enum_test.cc b/conformance/json_enum_test.cc new file mode 100644 index 0000000..519fdc8 --- /dev/null +++ b/conformance/json_enum_test.cc
@@ -0,0 +1,211 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for enum fields: values by name and by number, +// rejected unquoted names, arrays and booleans, and, for proto3 message +// types only, like the legacy suite, enum aliases and unknown values being +// serialized as numbers. This replaces the "Enum fields" block of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(); the +// test names and the requests sent to the testee are identical to the legacy +// ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix, and the legacy "validators" +// (RunValidJsonTestWithValidator()) looked at the serialized JSON text itself. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// --------------------------------------------------------------------------- +// Every message type. +// --------------------------------------------------------------------------- + +using JsonEnumTest = MessageTypeConformanceTest; + +TEST_P(JsonEnumTest, EnumField) { + constexpr absl::string_view kInput = + R"json({"optionalNestedEnum": "FOO"})json"; + constexpr absl::string_view kExpected = R"pb(optional_nested_enum: FOO)pb"; + EXPECT_THAT( + Testee("EnumField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Enum values must be represented as strings. +TEST_P(JsonEnumTest, EnumFieldNotQuoted) { + EXPECT_THAT( + Testee("EnumFieldNotQuoted") + .ParseJson(message(), R"json({"optionalNestedEnum": FOO})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Numeric values are allowed. +TEST_P(JsonEnumTest, EnumFieldNumericValueZero) { + constexpr absl::string_view kInput = R"json({"optionalNestedEnum": 0})json"; + constexpr absl::string_view kExpected = R"pb(optional_nested_enum: FOO)pb"; + EXPECT_THAT(Testee("EnumFieldNumericValueZero") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumFieldNumericValueZero") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonEnumTest, EnumFieldNumericValueNonZero) { + constexpr absl::string_view kInput = R"json({"optionalNestedEnum": 1})json"; + constexpr absl::string_view kExpected = R"pb(optional_nested_enum: BAR)pb"; + EXPECT_THAT(Testee("EnumFieldNumericValueNonZero") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumFieldNumericValueNonZero") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Arrays are not allowed for non-repeated enum fields. +TEST_P(JsonEnumTest, EnumFieldSingleElementArrayEnumName) { + EXPECT_THAT( + Testee("EnumFieldSingleElementArrayEnumName") + .ParseJson(message(), R"json({"optionalNestedEnum": ["FOO"]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonEnumTest, EnumFieldSingleElementArrayNumericValue) { + EXPECT_THAT( + Testee("EnumFieldSingleElementArrayNumericValue") + .ParseJson(message(), R"json({"optionalNestedEnum": [2]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Booleans are not allowed for enum fields. +TEST_P(JsonEnumTest, EnumFieldTrueValue) { + EXPECT_THAT( + Testee("EnumFieldTrueValue") + .ParseJson(message(), R"json({"optionalNestedEnum": true})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonEnumTest, EnumFieldFalseValue) { + EXPECT_THAT( + Testee("EnumFieldFalseValue") + .ParseJson(message(), R"json({"optionalNestedEnum": false})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonEnumTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Proto3 message types only: enum aliases and unknown (open enum) values. +// --------------------------------------------------------------------------- + +using JsonProto3EnumTest = MessageTypeConformanceTest; + +// Enum fields with alias. +TEST_P(JsonProto3EnumTest, EnumFieldWithAlias) { + constexpr absl::string_view kInput = + R"json({"optionalAliasedEnum": "ALIAS_BAZ"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_aliased_enum: ALIAS_BAZ)pb"; + EXPECT_THAT(Testee("EnumFieldWithAlias") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("EnumFieldWithAlias").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonProto3EnumTest, EnumFieldWithAliasUseAlias) { + constexpr absl::string_view kInput = + R"json({"optionalAliasedEnum": "MOO"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_aliased_enum: ALIAS_BAZ)pb"; + EXPECT_THAT(Testee("EnumFieldWithAliasUseAlias") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumFieldWithAliasUseAlias") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonProto3EnumTest, EnumFieldWithAliasLowerCase) { + constexpr absl::string_view kInput = + R"json({"optionalAliasedEnum": "moo"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_aliased_enum: ALIAS_BAZ)pb"; + EXPECT_THAT(Testee("EnumFieldWithAliasLowerCase") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumFieldWithAliasLowerCase") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonProto3EnumTest, EnumFieldWithAliasDifferentCase) { + constexpr absl::string_view kInput = + R"json({"optionalAliasedEnum": "bAz"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_aliased_enum: ALIAS_BAZ)pb"; + EXPECT_THAT(Testee("EnumFieldWithAliasDifferentCase") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("EnumFieldWithAliasDifferentCase") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Unknown enum values are represented as numeric values. +TEST_P(JsonProto3EnumTest, EnumFieldUnknownValue) { + EXPECT_THAT( + Testee("EnumFieldUnknownValue") + .ParseJson(message(), R"json({"optionalNestedEnum": 123})json") + .SerializeJson(), + Yields(JsonPayload( + HasJsonMemberThat("optionalNestedEnum", IsJsonInt(123))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonProto3EnumTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_field_mask_test.cc b/conformance/json_field_mask_test.cc new file mode 100644 index 0000000..be19ce6 --- /dev/null +++ b/conformance/json_field_mask_test.cc
@@ -0,0 +1,127 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.FieldMask fields: the +// comma-separated lowerCamelCase form, the empty mask, a path character the +// JSON form cannot express, and the paths that don't round-trip through +// lowerCamelCase and so must be rejected by the serializer. This replaces the +// legacy BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForFieldMask(), +// which ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones, except that +// the binary-input tests below are BINARY_TEST requests (see below). +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. ExpectSerializeFailureForJson() sent +// the serialized text-format message as binary input, categorized as a JSON +// test, under "ProtobufInput.<name>.JsonOutput"; the same bytes are built here +// with the wire-format builders, and ParseBinary() categorizes them as +// BINARY_TEST, which no testee tells apart from JSON_TEST on a binary payload +// (b/563657722). + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonFieldMaskTest = MessageTypeConformanceTest; + +// Field numbers of the binary inputs of the serialize-failure tests: the +// test message's optional_field_mask and google.protobuf.FieldMask.paths. +constexpr uint32_t kOptionalFieldMask = 303; +constexpr uint32_t kFieldMaskPaths = 1; + +TEST_P(JsonFieldMaskTest, FieldMask) { + constexpr absl::string_view kInput = + R"json({"optionalFieldMask": "foo,barBaz"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_field_mask: { paths: "foo" paths: "bar_baz" })pb"; + EXPECT_THAT( + Testee("FieldMask").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FieldMask").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFieldMaskTest, EmptyFieldMask) { + constexpr absl::string_view kInput = R"json({"optionalFieldMask": ""})json"; + constexpr absl::string_view kExpected = R"pb(optional_field_mask: {})pb"; + EXPECT_THAT( + Testee("EmptyFieldMask").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("EmptyFieldMask").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFieldMaskTest, FieldMaskInvalidCharacter) { + EXPECT_THAT(Testee(kP1, "FieldMaskInvalidCharacter") + .ParseJson(message(), + R"json({"optionalFieldMask": "foo,bar_bar"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// The binary form of the legacy input +// `optional_field_mask: {paths: "fooBar"}`. +TEST_P(JsonFieldMaskTest, FieldMaskPathsDontRoundTrip) { + EXPECT_THAT(Testee(kP1, "FieldMaskPathsDontRoundTrip") + .ParseBinary(message(), LengthPrefixedField( + kOptionalFieldMask, + LengthPrefixedField( + kFieldMaskPaths, "fooBar"))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +// The binary form of the legacy input `optional_field_mask: {paths: +// "foo_3_bar"}`. +TEST_P(JsonFieldMaskTest, FieldMaskNumbersDontRoundTrip) { + EXPECT_THAT( + Testee(kP1, "FieldMaskNumbersDontRoundTrip") + .ParseBinary(message(), + LengthPrefixedField( + kOptionalFieldMask, + LengthPrefixedField(kFieldMaskPaths, "foo_3_bar"))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +// The binary form of the legacy input `optional_field_mask: {paths: +// "foo__bar"}`. +TEST_P(JsonFieldMaskTest, FieldMaskTooManyUnderscore) { + EXPECT_THAT(Testee(kP1, "FieldMaskTooManyUnderscore") + .ParseBinary(message(), LengthPrefixedField( + kOptionalFieldMask, + LengthPrefixedField( + kFieldMaskPaths, "foo__bar"))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonFieldMaskTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_field_name_test.cc b/conformance/json_field_name_test.cc new file mode 100644 index 0000000..83dfc5a --- /dev/null +++ b/conformance/json_field_name_test.cc
@@ -0,0 +1,534 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests about field names and object syntax: the +// lowerCamelCase / original-name conventions, escaped and unquoted names, +// trailing commas, comments, whitespace, duplicate members, and whether +// default values of proto2 and proto3 fields are serialized. This replaces +// the legacy BinaryAndJsonConformanceSuiteImpl<M>:: +// RunJsonTestsForFieldNameConvention() and +// RunJsonTestsForStoresDefaultPrimitive(); the test names and the requests +// sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). The legacy "validators" +// (RunValidJsonTestWithValidator()) looked at the serialized JSON text itself +// and are the tests that only serialize as JSON (FieldNameInLowerCamelCase, +// FieldNameWith*Validator, StoresDefaultPrimitive, ...). +// RunValidJsonTestOrParseFailure() asked for binary output under a name +// without an output suffix. + +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "conformance/binary_test_util.h" +#include "conformance/conformance.pb.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::AllOf; +using ::testing::AnyOf; +using ::testing::Not; +using ::testing::ValuesIn; + +// --------------------------------------------------------------------------- +// Parsing field names in either convention. +// --------------------------------------------------------------------------- + +using JsonFieldNameTest = MessageTypeConformanceTest; + +TEST_P(JsonFieldNameTest, FieldNameInSnakeCase) { + constexpr absl::string_view kInput = R"json({ + "fieldname1": 1, + "fieldName2": 2, + "FieldName3": 3, + "fieldName4": 4 + })json"; + constexpr absl::string_view kExpected = + R"pb(fieldname1: 1 field_name2: 2 _field_name3: 3 field__name4_: 4)pb"; + EXPECT_THAT(Testee("FieldNameInSnakeCase") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FieldNameInSnakeCase") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFieldNameTest, FieldNameWithNumbers) { + constexpr absl::string_view kInput = R"json({ + "field0name5": 5, + "field0Name6": 6 + })json"; + constexpr absl::string_view kExpected = + R"pb(field0name5: 5 field_0_name6: 6)pb"; + EXPECT_THAT(Testee("FieldNameWithNumbers") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FieldNameWithNumbers") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFieldNameTest, FieldNameWithMixedCases) { + constexpr absl::string_view kInput = R"json({ + "fieldName7": 7, + "FieldName8": 8, + "fieldName9": 9, + "FieldName10": 10, + "FIELDNAME11": 11, + "FIELDName12": 12 + })json"; + constexpr absl::string_view kExpected = R"pb( + fieldName7: 7 + FieldName8: 8 + field_Name9: 9 + Field_Name10: 10 + FIELD_NAME11: 11 + FIELD_name12: 12 + )pb"; + EXPECT_THAT(Testee("FieldNameWithMixedCases") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FieldNameWithMixedCases") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFieldNameTest, FieldNameWithDoubleUnderscores) { + constexpr absl::string_view kInput = R"json({ + "FieldName13": 13, + "FieldName14": 14, + "fieldName15": 15, + "fieldName16": 16, + "fieldName17": 17, + "FieldName18": 18 + })json"; + constexpr absl::string_view kExpected = R"pb( + __field_name13: 13 + __Field_name14: 14 + field__name15: 15 + field__Name16: 16 + field_name17__: 17 + Field_name18__: 18 + )pb"; + EXPECT_THAT(Testee(kP1, "FieldNameWithDoubleUnderscores") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee(kP1, "FieldNameWithDoubleUnderscores") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Using the original proto field name in JSON is also allowed. +TEST_P(JsonFieldNameTest, OriginalProtoFieldName) { + constexpr absl::string_view kInput = R"json({ + "fieldname1": 1, + "field_name2": 2, + "_field_name3": 3, + "field__name4_": 4, + "field0name5": 5, + "field_0_name6": 6, + "fieldName7": 7, + "FieldName8": 8, + "field_Name9": 9, + "Field_Name10": 10, + "FIELD_NAME11": 11, + "FIELD_name12": 12, + "__field_name13": 13, + "__Field_name14": 14, + "field__name15": 15, + "field__Name16": 16, + "field_name17__": 17, + "Field_name18__": 18 + })json"; + constexpr absl::string_view kExpected = R"pb( + fieldname1: 1 + field_name2: 2 + _field_name3: 3 + field__name4_: 4 + field0name5: 5 + field_0_name6: 6 + fieldName7: 7 + FieldName8: 8 + field_Name9: 9 + Field_Name10: 10 + FIELD_NAME11: 11 + FIELD_name12: 12 + __field_name13: 13 + __Field_name14: 14 + field__name15: 15 + field__Name16: 16 + field_name17__: 17 + Field_name18__: 18 + )pb"; + EXPECT_THAT(Testee("OriginalProtoFieldName") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OriginalProtoFieldName") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Field names can be escaped. +TEST_P(JsonFieldNameTest, FieldNameEscaped) { + constexpr absl::string_view kInput = R"json({"fieldn\u0061me1": 1})json"; + EXPECT_THAT( + Testee("FieldNameEscaped").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(fieldname1: 1)pb")))); + EXPECT_THAT( + Testee("FieldNameEscaped").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(fieldname1: 1)pb")))); +} + +// String ends with escape character. +TEST_P(JsonFieldNameTest, StringEndsWithEscapeChar) { + EXPECT_THAT(Testee(kP1, "StringEndsWithEscapeChar") + .ParseJson(message(), "{\"optionalString\": \"abc\\") + .ParseOnly(), + Yields(IsParseError())); +} + +// Field names must be quoted (or it's not valid JSON). +TEST_P(JsonFieldNameTest, FieldNameNotQuoted) { + EXPECT_THAT(Testee(kP1, "FieldNameNotQuoted") + .ParseJson(message(), "{fieldname1: 1}") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonFieldNameTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Object syntax: trailing commas, comments, whitespace, missing commas. +// --------------------------------------------------------------------------- + +class JsonObjectSyntaxTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Trailing comma is not allowed (not valid JSON). +TEST_P(JsonObjectSyntaxTest, TrailingCommaInAnObject) { + EXPECT_THAT(Testee("TrailingCommaInAnObject") + .ParseJson(message(), R"json({"fieldname1":1,})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonObjectSyntaxTest, TrailingCommaInAnObjectWithSpace) { + EXPECT_THAT(Testee("TrailingCommaInAnObjectWithSpace") + .ParseJson(message(), R"json({"fieldname1":1 ,})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonObjectSyntaxTest, TrailingCommaInAnObjectWithSpaceCommaSpace) { + EXPECT_THAT(Testee("TrailingCommaInAnObjectWithSpaceCommaSpace") + .ParseJson(message(), R"json({"fieldname1":1 , })json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonObjectSyntaxTest, TrailingCommaInAnObjectWithNewlines) { + EXPECT_THAT(Testee("TrailingCommaInAnObjectWithNewlines") + .ParseJson(message(), R"json({ + "fieldname1":1, + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +// JSON doesn't support comments. +TEST_P(JsonObjectSyntaxTest, JsonWithComments) { + EXPECT_THAT(Testee("JsonWithComments") + .ParseJson(message(), R"json({ + // This is a comment. + "fieldname1": 1 + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +// JSON spec says whitespace doesn't matter, so try a few spacings to be sure. +constexpr absl::string_view kTwoInt32sExpected = R"pb( + optional_int32: 1 + optional_int64: 2 +)pb"; + +TEST_P(JsonObjectSyntaxTest, OneLineNoSpaces) { + constexpr absl::string_view kInput = + R"json({"optionalInt32":1,"optionalInt64":2})json"; + EXPECT_THAT( + Testee("OneLineNoSpaces").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); + EXPECT_THAT( + Testee("OneLineNoSpaces").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); +} + +TEST_P(JsonObjectSyntaxTest, OneLineWithSpaces) { + constexpr absl::string_view kInput = + R"json({ "optionalInt32" : 1 , "optionalInt64" : 2 })json"; + EXPECT_THAT(Testee("OneLineWithSpaces") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); + EXPECT_THAT( + Testee("OneLineWithSpaces").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); +} + +TEST_P(JsonObjectSyntaxTest, MultilineNoSpaces) { + constexpr absl::string_view kInput = + "{\n\"optionalInt32\"\n:\n1\n,\n\"optionalInt64\"\n:\n2\n}"; + EXPECT_THAT(Testee("MultilineNoSpaces") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); + EXPECT_THAT( + Testee("MultilineNoSpaces").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); +} + +TEST_P(JsonObjectSyntaxTest, MultilineWithSpaces) { + constexpr absl::string_view kInput = + "{\n \"optionalInt32\" : 1\n ,\n \"optionalInt64\" : 2\n}\n"; + EXPECT_THAT(Testee("MultilineWithSpaces") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); + EXPECT_THAT(Testee("MultilineWithSpaces") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kTwoInt32sExpected)))); +} + +// Missing comma between key/value pairs. +TEST_P(JsonObjectSyntaxTest, MissingCommaOneLine) { + EXPECT_THAT( + Testee("MissingCommaOneLine") + .ParseJson(message(), + R"json({ "optionalInt32": 1 "optionalInt64": 2 })json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonObjectSyntaxTest, MissingCommaMultiline) { + EXPECT_THAT( + Testee("MissingCommaMultiline") + .ParseJson(message(), + "{\n \"optionalInt32\": 1\n \"optionalInt64\": 2\n}") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonObjectSyntaxTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Duplicated field names have either last-wins or parse failure. Like the +// legacy RunValidJsonTestOrParseFailure(), binary output is requested under a +// name without an output suffix. +// --------------------------------------------------------------------------- + +class JsonDuplicateFieldNameTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +TEST_P(JsonDuplicateFieldNameTest, FieldNameDuplicate) { + EXPECT_THAT(Testee("FieldNameDuplicate") + .ParseJson(message(), R"json({ + "optionalNestedMessage": {"a": 1}, + "optionalNestedMessage": {} + })json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsTextProto(R"pb( + optional_nested_message: {} + )pb"))))); +} + +TEST_P(JsonDuplicateFieldNameTest, FieldNameDuplicateDifferentCasing1) { + EXPECT_THAT(Testee("FieldNameDuplicateDifferentCasing1") + .ParseJson(message(), R"json({ + "optional_nested_message": {"a": 1}, + "optionalNestedMessage": {} + })json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsTextProto(R"pb( + optional_nested_message: {} + )pb"))))); +} + +TEST_P(JsonDuplicateFieldNameTest, FieldNameDuplicateDifferentCasing2) { + EXPECT_THAT(Testee("FieldNameDuplicateDifferentCasing2") + .ParseJson(message(), R"json({ + "optionalNestedMessage": {"a": 1}, + "optional_nested_message": {} + })json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsTextProto(R"pb( + optional_nested_message: {} + )pb"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonDuplicateFieldNameTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Serializers should use lowerCamelCase by default. These look at the JSON +// text the testee produces, not at the message it decodes to. +// --------------------------------------------------------------------------- + +using JsonFieldNameSerializationTest = MessageTypeConformanceTest; + +TEST_P(JsonFieldNameSerializationTest, FieldNameInLowerCamelCase) { + EXPECT_THAT(Testee("FieldNameInLowerCamelCase") + .ParseJson(message(), R"json({ + "fieldname1": 1, + "fieldName2": 2, + "FieldName3": 3, + "fieldName4": 4 + })json") + .SerializeJson(), + Yields(JsonPayload(AllOf( + HasJsonMember("fieldname1"), HasJsonMember("fieldName2"), + HasJsonMember("FieldName3"), HasJsonMember("fieldName4"))))); +} + +TEST_P(JsonFieldNameSerializationTest, FieldNameWithNumbersValidator) { + EXPECT_THAT(Testee("FieldNameWithNumbersValidator") + .ParseJson(message(), R"json({ + "field0name5": 5, + "field0Name6": 6 + })json") + .SerializeJson(), + Yields(JsonPayload(AllOf(HasJsonMember("field0name5"), + HasJsonMember("field0Name6"))))); +} + +TEST_P(JsonFieldNameSerializationTest, FieldNameWithMixedCasesValidator) { + EXPECT_THAT( + Testee("FieldNameWithMixedCasesValidator") + .ParseJson(message(), R"json({ + "fieldName7": 7, + "FieldName8": 8, + "fieldName9": 9, + "FieldName10": 10, + "FIELDNAME11": 11, + "FIELDName12": 12 + })json") + .SerializeJson(), + Yields(JsonPayload( + AllOf(HasJsonMember("fieldName7"), HasJsonMember("FieldName8"), + HasJsonMember("fieldName9"), HasJsonMember("FieldName10"), + HasJsonMember("FIELDNAME11"), HasJsonMember("FIELDName12"))))); +} + +TEST_P(JsonFieldNameSerializationTest, + FieldNameWithDoubleUnderscoresValidator) { + EXPECT_THAT( + Testee(kP1, "FieldNameWithDoubleUnderscoresValidator") + .ParseJson(message(), R"json({ + "FieldName13": 13, + "FieldName14": 14, + "fieldName15": 15, + "fieldName16": 16, + "fieldName17": 17, + "FieldName18": 18 + })json") + .SerializeJson(), + Yields(JsonPayload( + AllOf(HasJsonMember("FieldName13"), HasJsonMember("FieldName14"), + HasJsonMember("fieldName15"), HasJsonMember("fieldName16"), + HasJsonMember("fieldName17"), HasJsonMember("FieldName18"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonFieldNameSerializationTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Default values: proto3 (implicit presence) skips them, proto2 (explicit +// presence) stores them. Extensions are serialized under their bracketed +// full name. +// --------------------------------------------------------------------------- + +using JsonSkipsDefaultTest = MessageTypeConformanceTest; + +TEST_P(JsonSkipsDefaultTest, SkipsDefaultPrimitive) { + EXPECT_THAT(Testee("SkipsDefaultPrimitive") + .ParseJson(message(), R"json({"FieldName13": 0})json") + .SerializeJson(), + Yields(JsonPayload(Not(HasJsonMember("FieldName13"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonSkipsDefaultTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +// Only run on proto2 message types for now; proto3 fields with explicit +// presence would be a candidate as well. +using JsonStoresDefaultTest = MessageTypeConformanceTest; + +TEST_P(JsonStoresDefaultTest, StoresDefaultPrimitive) { + EXPECT_THAT(Testee("StoresDefaultPrimitive") + .ParseJson(message(), R"json({ + "FieldName13": 0 + })json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMember("FieldName13")))); +} + +TEST_P(JsonStoresDefaultTest, FieldNameExtension) { + // Like the legacy suite, the first extension of the message type in its + // pool is used, whichever it is. + std::vector<const FieldDescriptor*> extensions; + message()->file()->pool()->FindAllExtensions(message(), &extensions); + ASSERT_FALSE(extensions.empty()); + const absl::string_view extension_name = extensions[0]->full_name(); + EXPECT_THAT(Testee(kP1, "FieldNameExtension") + .ParseJson(message(), absl::Substitute(R"({ + "[$0]": 1 + })", + extension_name)) + .SerializeJson(), + Yields(JsonPayload( + HasJsonMember(absl::StrCat("[", extension_name, "]"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonStoresDefaultTest, + ValuesIn(Proto2TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_float_test.cc b/conformance/json_float_test.cc new file mode 100644 index 0000000..8b70281 --- /dev/null +++ b/conformance/json_float_test.cc
@@ -0,0 +1,594 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for float and double fields: the value ranges, +// quoted values, the special values NaN / Infinity / -Infinity (which must be +// quoted), the normalization of non-canonical NaN bit patterns when +// serializing to JSON, and rejected out-of-range, empty, non-numeric and +// boolean values. This replaces the "Float fields" and "Double fields" blocks +// of the legacy BinaryAndJsonConformanceSuiteImpl<M>:: +// RunJsonTestsForNonRepeatedTypes(); the test names and the requests sent to +// the testee are identical to the legacy ones, except that the NaN +// normalization tests are BINARY_TEST requests (see below). +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. The NaN normalization tests +// (RunValidJsonTestWithProtobufInput()) send a *binary* payload and ask for +// JSON output ("<S>.<E>.ProtobufInput.<name>.JsonOutput"); the legacy suite +// filed them under the JSON_TEST category, here ParseBinary() categorizes them +// as BINARY_TEST, which no testee tells apart from JSON_TEST on a binary +// payload (b/563657722). + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// The field numbers of optional_float and optional_double, the same in every +// test message type. +constexpr uint32_t kOptionalFloatFieldNumber = 11; +constexpr uint32_t kOptionalDoubleFieldNumber = 12; + +// --------------------------------------------------------------------------- +// Float fields. +// --------------------------------------------------------------------------- + +using JsonFloatTest = MessageTypeConformanceTest; + +TEST_P(JsonFloatTest, FloatFieldMinPositiveValue) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": 1.175494e-38})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: 1.175494e-38)pb"; + EXPECT_THAT(Testee("FloatFieldMinPositiveValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldMinPositiveValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFloatTest, FloatFieldMaxNegativeValue) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": -1.175494e-38})json"; + constexpr absl::string_view kExpected = + R"pb(optional_float: -1.175494e-38)pb"; + EXPECT_THAT(Testee("FloatFieldMaxNegativeValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldMaxNegativeValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFloatTest, FloatFieldMaxPositiveValue) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": 3.402823e+38})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: 3.402823e+38)pb"; + EXPECT_THAT(Testee("FloatFieldMaxPositiveValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldMaxPositiveValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Sends the same (positive) value as FloatFieldMaxPositiveValue, like the +// legacy suite did. +TEST_P(JsonFloatTest, FloatFieldMinNegativeValue) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": 3.402823e+38})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: 3.402823e+38)pb"; + EXPECT_THAT(Testee("FloatFieldMinNegativeValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldMinNegativeValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Values can be quoted. +TEST_P(JsonFloatTest, FloatFieldQuotedValue) { + constexpr absl::string_view kInput = R"json({"optionalFloat": "1"})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: 1)pb"; + EXPECT_THAT(Testee("FloatFieldQuotedValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldQuotedValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFloatTest, FloatFieldQuotedExponentialValue) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": "1.175494e-38"})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: 1.175494e-38)pb"; + EXPECT_THAT(Testee("FloatFieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Special values. +TEST_P(JsonFloatTest, FloatFieldNan) { + constexpr absl::string_view kInput = R"json({"optionalFloat": "NaN"})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: nan)pb"; + EXPECT_THAT( + Testee("FloatFieldNan").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("FloatFieldNan").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFloatTest, FloatFieldInfinity) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": "Infinity"})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: inf)pb"; + EXPECT_THAT(Testee("FloatFieldInfinity") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("FloatFieldInfinity").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonFloatTest, FloatFieldNegativeInfinity) { + constexpr absl::string_view kInput = + R"json({"optionalFloat": "-Infinity"})json"; + constexpr absl::string_view kExpected = R"pb(optional_float: -inf)pb"; + EXPECT_THAT(Testee("FloatFieldNegativeInfinity") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("FloatFieldNegativeInfinity") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Non-canonical NaNs are normalized: the binary payload carries a NaN with +// a non-canonical payload (0x7FA12345; the legacy names notwithstanding, +// bit 22 is clear in both patterns, so both are signaling NaNs), ... +TEST_P(JsonFloatTest, FloatFieldNormalizeQuietNan) { + EXPECT_THAT( + Testee("FloatFieldNormalizeQuietNan") + .ParseBinary(message(), + Fixed32Field(kOptionalFloatFieldNumber, 0x7FA12345)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: nan)pb")))); +} + +// ... or a NaN with the sign bit set and another payload (0xFFB54321). +TEST_P(JsonFloatTest, FloatFieldNormalizeSignalingNan) { + EXPECT_THAT( + Testee("FloatFieldNormalizeSignalingNan") + .ParseBinary(message(), + Fixed32Field(kOptionalFloatFieldNumber, 0xFFB54321)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: nan)pb")))); +} + +// Special values must be quoted. +TEST_P(JsonFloatTest, FloatFieldNanNotQuoted) { + EXPECT_THAT(Testee(kP1, "FloatFieldNanNotQuoted") + .ParseJson(message(), R"json({"optionalFloat": NaN})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldInfinityNotQuoted) { + EXPECT_THAT( + Testee(kP1, "FloatFieldInfinityNotQuoted") + .ParseJson(message(), R"json({"optionalFloat": Infinity})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldNegativeInfinityNotQuoted) { + EXPECT_THAT( + Testee(kP1, "FloatFieldNegativeInfinityNotQuoted") + .ParseJson(message(), R"json({"optionalFloat": -Infinity})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers should reject out-of-bound values. +TEST_P(JsonFloatTest, FloatFieldTooSmall) { + EXPECT_THAT( + Testee("FloatFieldTooSmall") + .ParseJson(message(), R"json({"optionalFloat": -3.502823e+38})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldTooLarge) { + EXPECT_THAT( + Testee("FloatFieldTooLarge") + .ParseJson(message(), R"json({"optionalFloat": 3.502823e+38})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers should reject empty string values. +TEST_P(JsonFloatTest, FloatFieldEmptyString) { + EXPECT_THAT(Testee("FloatFieldEmptyString") + .ParseJson(message(), R"json({"optionalFloat": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers reject non-numeric string values. +TEST_P(JsonFloatTest, FloatFieldStringValuePartiallyNumeric) { + EXPECT_THAT(Testee("FloatFieldStringValuePartiallyNumeric") + .ParseJson(message(), R"json({"optionalFloat": "12abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldStringValueNonNumeric) { + EXPECT_THAT(Testee("FloatFieldStringValueNonNumeric") + .ParseJson(message(), R"json({"optionalFloat": "abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldStringValuePartiallyNumericSpace) { + EXPECT_THAT(Testee("FloatFieldStringValuePartiallyNumericSpace") + .ParseJson(message(), R"json({"optionalFloat": "12 34"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldStringValuePartiallyNumericComma) { + EXPECT_THAT(Testee("FloatFieldStringValuePartiallyNumericComma") + .ParseJson(message(), R"json({"optionalFloat": "12,34"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldStringValuePartiallyNumericUnicode) { + // Not a raw string literal: the payload holds the UTF-8 bytes of U+8C37 + // U+6B4C between the digits. + EXPECT_THAT(Testee("FloatFieldStringValuePartiallyNumericUnicode") + .ParseJson(message(), + "{\"optionalFloat\": \"12\xE8\xB0\xB7\xE6\xAD\x8C" + "34\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +// Only ASCII digits are accepted in quoted numbers. Not raw string +// literals: the payloads hold the UTF-8 bytes of the Arabic-Indic digits +// U+0661 U+0662 U+0663 and of the fullwidth digits U+FF11 U+FF12 U+FF13. +TEST_P(JsonFloatTest, FloatFieldStringValueArabicIndicDigits) { + constexpr absl::string_view kInput = + "{\"optionalFloat\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"; + EXPECT_THAT(Testee("FloatFieldStringValueArabicIndicDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldStringValueFullwidthDigits) { + constexpr absl::string_view kInput = + "{\"optionalFloat\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"; + EXPECT_THAT(Testee("FloatFieldStringValueFullwidthDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers reject boolean values for float fields. +TEST_P(JsonFloatTest, FloatFieldTrueValue) { + EXPECT_THAT(Testee("FloatFieldTrueValue") + .ParseJson(message(), R"json({"optionalFloat": true})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonFloatTest, FloatFieldFalseValue) { + EXPECT_THAT(Testee("FloatFieldFalseValue") + .ParseJson(message(), R"json({"optionalFloat": false})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonFloatTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Double fields. +// --------------------------------------------------------------------------- + +using JsonDoubleTest = MessageTypeConformanceTest; + +TEST_P(JsonDoubleTest, DoubleFieldMinPositiveValue) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": 2.22507e-308})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double: 2.22507e-308)pb"; + EXPECT_THAT(Testee("DoubleFieldMinPositiveValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldMinPositiveValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldMaxNegativeValue) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": -2.22507e-308})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double: -2.22507e-308)pb"; + EXPECT_THAT(Testee("DoubleFieldMaxNegativeValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldMaxNegativeValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldMaxPositiveValue) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": 1.79769e+308})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double: 1.79769e+308)pb"; + EXPECT_THAT(Testee("DoubleFieldMaxPositiveValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldMaxPositiveValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldMinNegativeValue) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": -1.79769e+308})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double: -1.79769e+308)pb"; + EXPECT_THAT(Testee("DoubleFieldMinNegativeValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldMinNegativeValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Values can be quoted. +TEST_P(JsonDoubleTest, DoubleFieldQuotedValue) { + constexpr absl::string_view kInput = R"json({"optionalDouble": "1"})json"; + constexpr absl::string_view kExpected = R"pb(optional_double: 1)pb"; + EXPECT_THAT(Testee("DoubleFieldQuotedValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldQuotedValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldQuotedExponentialValue) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": "2.22507e-308"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double: 2.22507e-308)pb"; + EXPECT_THAT(Testee("DoubleFieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Special values. +TEST_P(JsonDoubleTest, DoubleFieldNan) { + constexpr absl::string_view kInput = R"json({"optionalDouble": "NaN"})json"; + constexpr absl::string_view kExpected = R"pb(optional_double: nan)pb"; + EXPECT_THAT( + Testee("DoubleFieldNan").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("DoubleFieldNan").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldInfinity) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": "Infinity"})json"; + constexpr absl::string_view kExpected = R"pb(optional_double: inf)pb"; + EXPECT_THAT(Testee("DoubleFieldInfinity") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldInfinity") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonDoubleTest, DoubleFieldNegativeInfinity) { + constexpr absl::string_view kInput = + R"json({"optionalDouble": "-Infinity"})json"; + constexpr absl::string_view kExpected = R"pb(optional_double: -inf)pb"; + EXPECT_THAT(Testee("DoubleFieldNegativeInfinity") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("DoubleFieldNegativeInfinity") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Non-canonical NaNs are normalized, as for floats above. +TEST_P(JsonDoubleTest, DoubleFieldNormalizeQuietNan) { + EXPECT_THAT( + Testee("DoubleFieldNormalizeQuietNan") + .ParseBinary(message(), Fixed64Field(kOptionalDoubleFieldNumber, + 0x7FFA123456789ABC)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: nan)pb")))); +} + +TEST_P(JsonDoubleTest, DoubleFieldNormalizeSignalingNan) { + EXPECT_THAT( + Testee("DoubleFieldNormalizeSignalingNan") + .ParseBinary(message(), Fixed64Field(kOptionalDoubleFieldNumber, + 0xFFFBCBA987654321)) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: nan)pb")))); +} + +// Special values must be quoted. +TEST_P(JsonDoubleTest, DoubleFieldNanNotQuoted) { + EXPECT_THAT(Testee(kP1, "DoubleFieldNanNotQuoted") + .ParseJson(message(), R"json({"optionalDouble": NaN})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldInfinityNotQuoted) { + EXPECT_THAT( + Testee(kP1, "DoubleFieldInfinityNotQuoted") + .ParseJson(message(), R"json({"optionalDouble": Infinity})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldNegativeInfinityNotQuoted) { + EXPECT_THAT( + Testee(kP1, "DoubleFieldNegativeInfinityNotQuoted") + .ParseJson(message(), R"json({"optionalDouble": -Infinity})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers should reject out-of-bound values. +TEST_P(JsonDoubleTest, DoubleFieldTooSmall) { + EXPECT_THAT( + Testee("DoubleFieldTooSmall") + .ParseJson(message(), R"json({"optionalDouble": -1.89769e+308})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldTooLarge) { + EXPECT_THAT( + Testee("DoubleFieldTooLarge") + .ParseJson(message(), R"json({"optionalDouble": 1.89769e+308})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers should reject empty string values. +TEST_P(JsonDoubleTest, DoubleFieldEmptyString) { + EXPECT_THAT(Testee("DoubleFieldEmptyString") + .ParseJson(message(), R"json({"optionalDouble": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers reject non-numeric string values. +TEST_P(JsonDoubleTest, DoubleFieldStringValuePartiallyNumeric) { + EXPECT_THAT( + Testee("DoubleFieldStringValuePartiallyNumeric") + .ParseJson(message(), R"json({"optionalDouble": "12abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldStringValueNonNumeric) { + EXPECT_THAT(Testee("DoubleFieldStringValueNonNumeric") + .ParseJson(message(), R"json({"optionalDouble": "abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Only ASCII digits are accepted in quoted numbers. Not raw string +// literals: the payloads hold the UTF-8 bytes of the Arabic-Indic digits +// U+0661 U+0662 U+0663 and of the fullwidth digits U+FF11 U+FF12 U+FF13. +TEST_P(JsonDoubleTest, DoubleFieldStringValueArabicIndicDigits) { + constexpr absl::string_view kInput = + "{\"optionalDouble\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"; + EXPECT_THAT(Testee("DoubleFieldStringValueArabicIndicDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldStringValueFullwidthDigits) { + constexpr absl::string_view kInput = + "{\"optionalDouble\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"; + EXPECT_THAT(Testee("DoubleFieldStringValueFullwidthDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +// Parsers reject boolean values for double fields. +TEST_P(JsonDoubleTest, DoubleFieldTrueValue) { + EXPECT_THAT(Testee("DoubleFieldTrueValue") + .ParseJson(message(), R"json({"optionalDouble": true})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonDoubleTest, DoubleFieldFalseValue) { + EXPECT_THAT(Testee("DoubleFieldFalseValue") + .ParseJson(message(), R"json({"optionalDouble": false})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonDoubleTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_hello_world_test.cc b/conformance/json_hello_world_test.cc new file mode 100644 index 0000000..4554f77 --- /dev/null +++ b/conformance/json_hello_world_test.cc
@@ -0,0 +1,112 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The smoke tests of the JSON conformance suite: a single string field +// round-trips through the testee, unknown members of every JSON kind are +// ignored when asked to, and a top-level `null` is rejected. This replaces +// the tests the legacy BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTests() +// ran directly (rather than through one of its RunJsonTestsFor*() groups), +// for every test message type; the test names and the requests sent to the +// testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). The legacy RunValidJsonIgnoreUnknownTest() only had +// the binary leg, and ExpectParseFailureForJson() asked for JSON output under +// a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonHelloWorldTest = MessageTypeConformanceTest; + +TEST_P(JsonHelloWorldTest, HelloWorld) { + constexpr absl::string_view kInput = + R"json({"optionalString":"Hello, World!"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: 'Hello, World!')pb"; + EXPECT_THAT( + Testee("HelloWorld").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("HelloWorld").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Unknown members are ignored whatever kind of JSON value they hold. +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonNumber) { + EXPECT_THAT(Testee("IgnoreUnknownJsonNumber") + .ParseJson(message(), R"json({"unknown": 1})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonString) { + EXPECT_THAT(Testee("IgnoreUnknownJsonString") + .ParseJson(message(), R"json({"unknown": "a"})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonTrue) { + EXPECT_THAT(Testee("IgnoreUnknownJsonTrue") + .ParseJson(message(), R"json({"unknown": true})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonFalse) { + EXPECT_THAT(Testee("IgnoreUnknownJsonFalse") + .ParseJson(message(), R"json({"unknown": false})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonNull) { + EXPECT_THAT(Testee("IgnoreUnknownJsonNull") + .ParseJson(message(), R"json({"unknown": null})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, IgnoreUnknownJsonObject) { + EXPECT_THAT(Testee("IgnoreUnknownJsonObject") + .ParseJson(message(), R"json({"unknown": {"a": 1}})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonHelloWorldTest, RejectTopLevelNull) { + EXPECT_THAT( + Testee("RejectTopLevelNull").ParseJson(message(), "null").ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonHelloWorldTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_integer_test.cc b/conformance/json_integer_test.cc new file mode 100644 index 0000000..151a0ba --- /dev/null +++ b/conformance/json_integer_test.cc
@@ -0,0 +1,613 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for integer fields: the value ranges of the 32- and +// 64-bit types, integers as JSON strings and as float literals, rejected +// out-of-range, non-integer, boolean, non-numeric and malformed values, and +// the 64-bit types being serialized as strings. This replaces the "Integer +// fields" block of the legacy BinaryAndJsonConformanceSuiteImpl<M>:: +// RunJsonTestsForNonRepeatedTypes(); the test names and the requests sent to +// the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix, and the legacy "validators" +// (RunValidJsonTestWithValidator()) looked at the serialized JSON text itself. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonIntegerTest = MessageTypeConformanceTest; + +// --------------------------------------------------------------------------- +// Value ranges. 64-bit values are quoted in JSON. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": 2147483647})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 2147483647)pb"; + EXPECT_THAT(Testee("Int32FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int32FieldMaxValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldMinValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": -2147483648})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: -2147483648)pb"; + EXPECT_THAT(Testee("Int32FieldMinValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int32FieldMinValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Uint32FieldMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalUint32": 4294967295})json"; + constexpr absl::string_view kExpected = R"pb(optional_uint32: 4294967295)pb"; + EXPECT_THAT(Testee("Uint32FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Uint32FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int64FieldMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt64": "9223372036854775807"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int64: 9223372036854775807)pb"; + EXPECT_THAT(Testee("Int64FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int64FieldMaxValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int64FieldMinValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt64": "-9223372036854775808"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int64: -9223372036854775808)pb"; + EXPECT_THAT(Testee("Int64FieldMinValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int64FieldMinValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Uint64FieldMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalUint64": "18446744073709551615"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_uint64: 18446744073709551615)pb"; + EXPECT_THAT(Testee("Uint64FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Uint64FieldMaxValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// While not the largest Int64, this is the largest Int64 which can be exactly +// represented within an IEEE-754 64-bit float, which is the expected level of +// interoperability guarantee. Larger values may work in some implementations, +// but should not be relied upon. +TEST_P(JsonIntegerTest, Int64FieldMaxValueNotQuoted) { + constexpr absl::string_view kInput = + R"json({"optionalInt64": 9223372036854774784})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int64: 9223372036854774784)pb"; + EXPECT_THAT(Testee("Int64FieldMaxValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int64FieldMaxValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int64FieldMinValueNotQuoted) { + constexpr absl::string_view kInput = + R"json({"optionalInt64": -9223372036854775808})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int64: -9223372036854775808)pb"; + EXPECT_THAT(Testee("Int64FieldMinValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int64FieldMinValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Largest interoperable Uint64; see the comment above for +// Int64FieldMaxValueNotQuoted. +TEST_P(JsonIntegerTest, Uint64FieldMaxValueNotQuoted) { + constexpr absl::string_view kInput = + R"json({"optionalUint64": 18446744073709549568})json"; + constexpr absl::string_view kExpected = + R"pb(optional_uint64: 18446744073709549568)pb"; + EXPECT_THAT(Testee("Uint64FieldMaxValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Uint64FieldMaxValueNotQuoted") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Values can be represented as JSON strings. +TEST_P(JsonIntegerTest, Int32FieldStringValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": "2147483647"})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 2147483647)pb"; + EXPECT_THAT(Testee("Int32FieldStringValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldStringValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValueEscaped) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": "2\u003147483647"})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 2147483647)pb"; + EXPECT_THAT(Testee("Int32FieldStringValueEscaped") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldStringValueEscaped") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValueZero) { + constexpr absl::string_view kInput = R"json({"optionalInt32": "0"})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 0)pb"; + EXPECT_THAT(Testee("Int32FieldStringValueZero") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldStringValueZero") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldQuotedExponentialValue) { + constexpr absl::string_view kInput = R"json({"optionalInt32": "1e5"})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 100000)pb"; + EXPECT_THAT(Testee("Int32FieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldQuotedExponentialValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// --------------------------------------------------------------------------- +// Parsers reject out-of-bound integer values. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldTooLarge) { + EXPECT_THAT( + Testee("Int32FieldTooLarge") + .ParseJson(message(), R"json({"optionalInt32": 2147483648})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldTooSmall) { + EXPECT_THAT( + Testee("Int32FieldTooSmall") + .ParseJson(message(), R"json({"optionalInt32": -2147483649})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint32FieldTooLarge) { + EXPECT_THAT( + Testee("Uint32FieldTooLarge") + .ParseJson(message(), R"json({"optionalUint32": 4294967296})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldTooLarge) { + EXPECT_THAT( + Testee("Int64FieldTooLarge") + .ParseJson(message(), + R"json({"optionalInt64": "9223372036854775808"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldTooSmall) { + EXPECT_THAT( + Testee("Int64FieldTooSmall") + .ParseJson(message(), + R"json({"optionalInt64": "-9223372036854775809"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint64FieldTooLarge) { + EXPECT_THAT( + Testee("Uint64FieldTooLarge") + .ParseJson(message(), + R"json({"optionalUint64": "18446744073709551616"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint64QuotedExponentFieldTooLarge) { + EXPECT_THAT( + Testee("Uint64QuotedExponentFieldTooLarge") + .ParseJson(message(), R"json({"optionalUint64": "1e536870000"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// Parsers reject non-integer numeric values. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldNotInteger) { + EXPECT_THAT(Testee("Int32FieldNotInteger") + .ParseJson(message(), R"json({"optionalInt32": 0.5})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint32FieldNotInteger) { + EXPECT_THAT(Testee("Uint32FieldNotInteger") + .ParseJson(message(), R"json({"optionalUint32": 0.5})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldNotInteger) { + EXPECT_THAT(Testee("Int64FieldNotInteger") + .ParseJson(message(), R"json({"optionalInt64": "0.5"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint64FieldNotInteger) { + EXPECT_THAT(Testee("Uint64FieldNotInteger") + .ParseJson(message(), R"json({"optionalUint64": "0.5"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// Parsers reject boolean values for integer fields. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldTrueValue) { + EXPECT_THAT(Testee("Int32FieldTrueValue") + .ParseJson(message(), R"json({"optionalInt32": true})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldFalseValue) { + EXPECT_THAT(Testee("Int32FieldFalseValue") + .ParseJson(message(), R"json({"optionalInt32": false})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// Parsers reject non-numeric string values. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldStringValuePartiallyNumeric) { + EXPECT_THAT(Testee("Int32FieldStringValuePartiallyNumeric") + .ParseJson(message(), R"json({"optionalInt32": "12abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValuePartiallyNumericSpace) { + EXPECT_THAT(Testee("Int32FieldStringValuePartiallyNumericSpace") + .ParseJson(message(), R"json({"optionalInt32": "12 34"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValuePartiallyNumericComma) { + EXPECT_THAT(Testee("Int32FieldStringValuePartiallyNumericComma") + .ParseJson(message(), R"json({"optionalInt32": "12,34"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValuePartiallyNumericUnicode) { + // Not a raw string literal: the payload holds the UTF-8 bytes of U+8C37 + // U+6B4C between the digits. + EXPECT_THAT(Testee("Int32FieldStringValuePartiallyNumericUnicode") + .ParseJson(message(), + "{\"optionalInt32\": \"12\xE8\xB0\xB7\xE6\xAD\x8C" + "34\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldStringValueNonNumeric) { + EXPECT_THAT(Testee("Int32FieldStringValueNonNumeric") + .ParseJson(message(), R"json({"optionalInt32": "abc"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Only ASCII digits are accepted in quoted integers. Other Unicode +// decimal digits, such as Arabic-Indic or fullwidth digits, are rejected. +// Not raw string literals: the payloads hold the UTF-8 bytes of U+0661 +// U+0662 U+0663 and of U+FF11 U+FF12 U+FF13. +TEST_P(JsonIntegerTest, Int32FieldStringValueArabicIndicDigits) { + constexpr absl::string_view kInput = + "{\"optionalInt32\": \"\xD9\xA1\xD9\xA2\xD9\xA3\"}"; + EXPECT_THAT(Testee("Int32FieldStringValueArabicIndicDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldStringValueFullwidthDigits) { + constexpr absl::string_view kInput = + "{\"optionalInt64\": \"\xEF\xBC\x91\xEF\xBC\x92\xEF\xBC\x93\"}"; + EXPECT_THAT(Testee("Int64FieldStringValueFullwidthDigits") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// Parsers reject empty string values. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldEmptyString) { + EXPECT_THAT(Testee("Int32FieldEmptyString") + .ParseJson(message(), R"json({"optionalInt32": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint32FieldEmptyString) { + EXPECT_THAT(Testee("Uint32FieldEmptyString") + .ParseJson(message(), R"json({"optionalUint32": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldEmptyString) { + EXPECT_THAT(Testee("Int64FieldEmptyString") + .ParseJson(message(), R"json({"optionalInt64": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint64FieldEmptyString) { + EXPECT_THAT(Testee("Uint64FieldEmptyString") + .ParseJson(message(), R"json({"optionalUint64": ""})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// Integers represented as float values are accepted. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldFloatTrailingZero) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": 100000.000})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 100000)pb"; + EXPECT_THAT(Testee("Int32FieldFloatTrailingZero") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldFloatTrailingZero") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldExponentialFormat) { + constexpr absl::string_view kInput = R"json({"optionalInt32": 1e5})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 100000)pb"; + EXPECT_THAT(Testee("Int32FieldExponentialFormat") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldExponentialFormat") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldMaxFloatValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": 2.147483647e9})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: 2147483647)pb"; + EXPECT_THAT(Testee("Int32FieldMaxFloatValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldMaxFloatValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Int32FieldMinFloatValue) { + constexpr absl::string_view kInput = + R"json({"optionalInt32": -2.147483648e9})json"; + constexpr absl::string_view kExpected = R"pb(optional_int32: -2147483648)pb"; + EXPECT_THAT(Testee("Int32FieldMinFloatValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Int32FieldMinFloatValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonIntegerTest, Uint32FieldMaxFloatValue) { + constexpr absl::string_view kInput = + R"json({"optionalUint32": 4.294967295e9})json"; + constexpr absl::string_view kExpected = R"pb(optional_uint32: 4294967295)pb"; + EXPECT_THAT(Testee("Uint32FieldMaxFloatValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Uint32FieldMaxFloatValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// --------------------------------------------------------------------------- +// Parsers reject non-numeric and malformed numeric values. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int32FieldNotNumber) { + EXPECT_THAT(Testee("Int32FieldNotNumber") + .ParseJson(message(), R"json({"optionalInt32": "3x3"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint32FieldNotNumber) { + EXPECT_THAT(Testee("Uint32FieldNotNumber") + .ParseJson(message(), R"json({"optionalUint32": "3x3"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int64FieldNotNumber) { + EXPECT_THAT(Testee("Int64FieldNotNumber") + .ParseJson(message(), R"json({"optionalInt64": "3x3"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Uint64FieldNotNumber) { + EXPECT_THAT(Testee("Uint64FieldNotNumber") + .ParseJson(message(), R"json({"optionalUint64": "3x3"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// JSON does not allow "+" on numeric values. +TEST_P(JsonIntegerTest, Int32FieldPlusSign) { + EXPECT_THAT(Testee("Int32FieldPlusSign") + .ParseJson(message(), R"json({"optionalInt32": +1})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// JSON doesn't allow leading 0s. +TEST_P(JsonIntegerTest, Int32FieldLeadingZero) { + EXPECT_THAT(Testee("Int32FieldLeadingZero") + .ParseJson(message(), R"json({"optionalInt32": 01})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldNegativeWithLeadingZero) { + EXPECT_THAT(Testee("Int32FieldNegativeWithLeadingZero") + .ParseJson(message(), R"json({"optionalInt32": -01})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// String values must follow the same syntax rule. Specifically leading or +// trailing spaces are not allowed. +TEST_P(JsonIntegerTest, Int32FieldLeadingSpace) { + EXPECT_THAT(Testee("Int32FieldLeadingSpace") + .ParseJson(message(), R"json({"optionalInt32": " 1"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonIntegerTest, Int32FieldTrailingSpace) { + EXPECT_THAT(Testee("Int32FieldTrailingSpace") + .ParseJson(message(), R"json({"optionalInt32": "1 "})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// 64-bit values are serialized as strings. +// --------------------------------------------------------------------------- + +TEST_P(JsonIntegerTest, Int64FieldBeString) { + EXPECT_THAT(Testee(kP1, "Int64FieldBeString") + .ParseJson(message(), R"json({"optionalInt64": 1})json") + .SerializeJson(), + Yields(JsonPayload( + HasJsonMemberThat("optionalInt64", IsJsonString("1"))))); +} + +TEST_P(JsonIntegerTest, Uint64FieldBeString) { + EXPECT_THAT(Testee(kP1, "Uint64FieldBeString") + .ParseJson(message(), R"json({"optionalUint64": 1})json") + .SerializeJson(), + Yields(JsonPayload( + HasJsonMemberThat("optionalUint64", IsJsonString("1"))))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonIntegerTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_map_test.cc b/conformance/json_map_test.cc new file mode 100644 index 0000000..f92eb75 --- /dev/null +++ b/conformance/json_map_test.cc
@@ -0,0 +1,229 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for map fields: keys are always JSON strings (so +// unquoted integer and bool keys are rejected and escapes in keys are +// honored) and message values are nested objects. This replaces the "Map +// fields" block of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(), +// for every test message type; the test names and the requests sent to the +// testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), the valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). The legacy ExpectParseFailureForJson() asked for +// JSON output under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonMapTest = MessageTypeConformanceTest; + +// Map keys are always JSON strings, even for integer and bool key types. +TEST_P(JsonMapTest, Int32MapField) { + constexpr absl::string_view kInput = + R"json({"mapInt32Int32": {"1": 2, "3": 4}})json"; + constexpr absl::string_view kExpected = R"pb( + map_int32_int32: { key: 1 value: 2 } + map_int32_int32: { key: 3 value: 4 } + )pb"; + EXPECT_THAT( + Testee("Int32MapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int32MapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, Int32MapFieldKeyNotQuoted) { + EXPECT_THAT( + Testee(kP1, "Int32MapFieldKeyNotQuoted") + .ParseJson(message(), R"json({"mapInt32Int32": {1: 2, 3: 4}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonMapTest, Uint32MapField) { + constexpr absl::string_view kInput = + R"json({"mapUint32Uint32": {"1": 2, "3": 4}})json"; + constexpr absl::string_view kExpected = R"pb( + map_uint32_uint32: { key: 1 value: 2 } + map_uint32_uint32: { key: 3 value: 4 } + )pb"; + EXPECT_THAT( + Testee("Uint32MapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Uint32MapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, Uint32MapFieldKeyNotQuoted) { + EXPECT_THAT( + Testee(kP1, "Uint32MapFieldKeyNotQuoted") + .ParseJson(message(), R"json({"mapUint32Uint32": {1: 2, 3: 4}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonMapTest, Int64MapField) { + constexpr absl::string_view kInput = + R"json({"mapInt64Int64": {"1": 2, "3": 4}})json"; + constexpr absl::string_view kExpected = R"pb( + map_int64_int64: { key: 1 value: 2 } + map_int64_int64: { key: 3 value: 4 } + )pb"; + EXPECT_THAT( + Testee("Int64MapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int64MapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, Int64MapFieldKeyNotQuoted) { + EXPECT_THAT( + Testee(kP1, "Int64MapFieldKeyNotQuoted") + .ParseJson(message(), R"json({"mapInt64Int64": {1: 2, 3: 4}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonMapTest, Uint64MapField) { + constexpr absl::string_view kInput = + R"json({"mapUint64Uint64": {"1": 2, "3": 4}})json"; + constexpr absl::string_view kExpected = R"pb( + map_uint64_uint64: { key: 1 value: 2 } + map_uint64_uint64: { key: 3 value: 4 } + )pb"; + EXPECT_THAT( + Testee("Uint64MapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Uint64MapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, Uint64MapFieldKeyNotQuoted) { + EXPECT_THAT( + Testee(kP1, "Uint64MapFieldKeyNotQuoted") + .ParseJson(message(), R"json({"mapUint64Uint64": {1: 2, 3: 4}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonMapTest, BoolMapField) { + constexpr absl::string_view kInput = + R"json({"mapBoolBool": {"true": true, "false": false}})json"; + constexpr absl::string_view kExpected = R"pb( + map_bool_bool: { key: true value: true } + map_bool_bool: { key: false value: false } + )pb"; + EXPECT_THAT( + Testee("BoolMapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("BoolMapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, BoolMapFieldKeyNotQuoted) { + EXPECT_THAT( + Testee(kP1, "BoolMapFieldKeyNotQuoted") + .ParseJson(message(), + R"json({"mapBoolBool": {true: true, false: false}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonMapTest, MessageMapField) { + constexpr absl::string_view kInput = R"json({ + "mapStringNestedMessage": { + "hello": {"a": 1234}, + "world": {"a": 5678} + } + })json"; + constexpr absl::string_view kExpected = R"pb( + map_string_nested_message: { + key: "hello" + value: { a: 1234 } + } + map_string_nested_message: { + key: "world" + value: { a: 5678 } + } + )pb"; + EXPECT_THAT( + Testee("MessageMapField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("MessageMapField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Since Map keys are represented as JSON strings, escaping should be allowed. +TEST_P(JsonMapTest, Int32MapEscapedKey) { + constexpr absl::string_view kInput = + R"json({"mapInt32Int32": {"\u0031": 2}})json"; + constexpr absl::string_view kExpected = + R"pb(map_int32_int32: { key: 1 value: 2 })pb"; + EXPECT_THAT(Testee("Int32MapEscapedKey") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int32MapEscapedKey").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, Int64MapEscapedKey) { + constexpr absl::string_view kInput = + R"json({"mapInt64Int64": {"\u0031": 2}})json"; + constexpr absl::string_view kExpected = + R"pb(map_int64_int64: { key: 1 value: 2 })pb"; + EXPECT_THAT(Testee("Int64MapEscapedKey") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("Int64MapEscapedKey").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonMapTest, BoolMapEscapedKey) { + constexpr absl::string_view kInput = + R"json({"mapBoolBool": {"tr\u0075e": true}})json"; + constexpr absl::string_view kExpected = + R"pb(map_bool_bool: { key: true value: true })pb"; + EXPECT_THAT(Testee("BoolMapEscapedKey") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("BoolMapEscapedKey").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonMapTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_message_test.cc b/conformance/json_message_test.cc new file mode 100644 index 0000000..92907ed --- /dev/null +++ b/conformance/json_message_test.cc
@@ -0,0 +1,55 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for (singular) message fields: a nested message +// given as a JSON object. This replaces the "Message fields" block of the +// legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(); the +// test name and the requests sent to the testee are identical to the legacy +// ones. +// +// Like the legacy RunValidJsonTest(), the valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonMessageTest = MessageTypeConformanceTest; + +TEST_P(JsonMessageTest, MessageField) { + constexpr absl::string_view kInput = + R"json({"optionalNestedMessage": {"a": 1234}})json"; + constexpr absl::string_view kExpected = + R"pb(optional_nested_message: { a: 1234 })pb"; + EXPECT_THAT( + Testee("MessageField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("MessageField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonMessageTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_null_test.cc b/conformance/json_null_test.cc new file mode 100644 index 0000000..403dfe0 --- /dev/null +++ b/conformance/json_null_test.cc
@@ -0,0 +1,134 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for `null`: it is accepted (and means "unset") as +// the value of a field of any type, but not as an element of a repeated +// field or as a map key or value. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNullTypes(), for +// every test message type; the test names and the requests sent to the +// testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), the valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). The legacy ExpectParseFailureForJson() asked for +// JSON output under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +class JsonNullTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// "null" is accepted for all field types, and leaves the field unset. +TEST_P(JsonNullTest, AllFieldAcceptNull) { + constexpr absl::string_view kInput = R"json({ + "optionalInt32": null, + "optionalInt64": null, + "optionalUint32": null, + "optionalUint64": null, + "optionalSint32": null, + "optionalSint64": null, + "optionalFixed32": null, + "optionalFixed64": null, + "optionalSfixed32": null, + "optionalSfixed64": null, + "optionalFloat": null, + "optionalDouble": null, + "optionalBool": null, + "optionalString": null, + "optionalBytes": null, + "optionalNestedEnum": null, + "optionalNestedMessage": null, + "repeatedInt32": null, + "repeatedInt64": null, + "repeatedUint32": null, + "repeatedUint64": null, + "repeatedSint32": null, + "repeatedSint64": null, + "repeatedFixed32": null, + "repeatedFixed64": null, + "repeatedSfixed32": null, + "repeatedSfixed64": null, + "repeatedFloat": null, + "repeatedDouble": null, + "repeatedBool": null, + "repeatedString": null, + "repeatedBytes": null, + "repeatedNestedEnum": null, + "repeatedNestedMessage": null, + "mapInt32Int32": null, + "mapBoolBool": null, + "mapStringNestedMessage": null + })json"; + EXPECT_THAT(Testee(kP0, "AllFieldAcceptNull") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); + EXPECT_THAT(Testee(kP0, "AllFieldAcceptNull") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +// Repeated field elements cannot be null. +TEST_P(JsonNullTest, RepeatedFieldPrimitiveElementIsNull) { + EXPECT_THAT( + Testee("RepeatedFieldPrimitiveElementIsNull") + .ParseJson(message(), R"json({"repeatedInt32": [1, null, 2]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonNullTest, RepeatedFieldMessageElementIsNull) { + EXPECT_THAT( + Testee("RepeatedFieldMessageElementIsNull") + .ParseJson( + message(), + R"json({"repeatedNestedMessage": [{"a":1}, null, {"a":2}]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Map field keys cannot be null. +TEST_P(JsonNullTest, MapFieldKeyIsNull) { + EXPECT_THAT( + Testee("MapFieldKeyIsNull") + .ParseJson(message(), R"json({"mapInt32Int32": {null: 1}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Map field values cannot be null. +TEST_P(JsonNullTest, MapFieldValueIsNull) { + EXPECT_THAT( + Testee("MapFieldValueIsNull") + .ParseJson(message(), R"json({"mapInt32Int32": {"0": null}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonNullTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_oneof_test.cc b/conformance/json_oneof_test.cc new file mode 100644 index 0000000..540cf12 --- /dev/null +++ b/conformance/json_oneof_test.cc
@@ -0,0 +1,201 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for oneof fields: setting more than one member of a +// oneof in the same object, `null` members, and explicitly zero/empty +// members (which must still mark the oneof as set). This replaces the +// "Oneof fields" block of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(), +// for every test message type; the test names and the requests sent to the +// testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), the valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). The legacy RunValidJsonTestOrParseFailure() asked +// for binary output under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/conformance.pb.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::AnyOf; +using ::testing::ValuesIn; + +class JsonOneofTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Setting two members of the same oneof may either be rejected or resolved +// in favor of the last one. +TEST_P(JsonOneofTest, OneofFieldDuplicate) { + EXPECT_THAT( + Testee(kP0, "OneofFieldDuplicate") + .ParseJson(message(), + R"json({"oneofUint32": 1, "oneofString": "test"})json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}), + Yields(AnyOf(IsParseError(), WhenParsed(EqualsTextProto( + R"pb(oneof_string: "test")pb"))))); +} + +TEST_P(JsonOneofTest, OneofFieldDuplicate2) { + EXPECT_THAT( + Testee(kP0, "OneofFieldDuplicate2") + .ParseJson(message(), + R"json({"oneofString": "test", "oneofUint32": 1})json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}), + Yields(AnyOf(IsParseError(), + WhenParsed(EqualsTextProto(R"pb(oneof_uint32: 1)pb"))))); +} + +// A null oneof member does not count as setting the oneof, so a second, +// non-null member is not a duplicate. +TEST_P(JsonOneofTest, OneofFieldNullFirst) { + constexpr absl::string_view kInput = + R"json({"oneofUint32": null, "oneofString": "test"})json"; + constexpr absl::string_view kExpected = R"pb(oneof_string: "test")pb"; + EXPECT_THAT(Testee(kP0, "OneofFieldNullFirst") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee(kP0, "OneofFieldNullFirst") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofFieldNullSecond) { + constexpr absl::string_view kInput = + R"json({"oneofString": "test", "oneofUint32": null})json"; + constexpr absl::string_view kExpected = R"pb(oneof_string: "test")pb"; + EXPECT_THAT(Testee(kP0, "OneofFieldNullSecond") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee(kP0, "OneofFieldNullSecond") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// A oneof member explicitly set to its default value is still set. +TEST_P(JsonOneofTest, OneofZeroUint32) { + constexpr absl::string_view kInput = R"json({"oneofUint32": 0})json"; + constexpr absl::string_view kExpected = R"pb(oneof_uint32: 0)pb"; + EXPECT_THAT( + Testee("OneofZeroUint32").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroUint32").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroMessage) { + constexpr absl::string_view kInput = R"json({"oneofNestedMessage": {}})json"; + constexpr absl::string_view kExpected = R"pb(oneof_nested_message: {})pb"; + EXPECT_THAT( + Testee("OneofZeroMessage").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroMessage").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroString) { + constexpr absl::string_view kInput = R"json({"oneofString": ""})json"; + constexpr absl::string_view kExpected = R"pb(oneof_string: "")pb"; + EXPECT_THAT( + Testee("OneofZeroString").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroString").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroBytes) { + constexpr absl::string_view kInput = R"json({"oneofBytes": ""})json"; + constexpr absl::string_view kExpected = R"pb(oneof_bytes: "")pb"; + EXPECT_THAT( + Testee("OneofZeroBytes").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroBytes").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroBool) { + constexpr absl::string_view kInput = R"json({"oneofBool": false})json"; + constexpr absl::string_view kExpected = R"pb(oneof_bool: false)pb"; + EXPECT_THAT( + Testee("OneofZeroBool").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroBool").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroUint64) { + constexpr absl::string_view kInput = R"json({"oneofUint64": 0})json"; + constexpr absl::string_view kExpected = R"pb(oneof_uint64: 0)pb"; + EXPECT_THAT( + Testee("OneofZeroUint64").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroUint64").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroFloat) { + constexpr absl::string_view kInput = R"json({"oneofFloat": 0.0})json"; + constexpr absl::string_view kExpected = R"pb(oneof_float: 0)pb"; + EXPECT_THAT( + Testee("OneofZeroFloat").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroFloat").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroDouble) { + constexpr absl::string_view kInput = R"json({"oneofDouble": 0.0})json"; + constexpr absl::string_view kExpected = R"pb(oneof_double: 0)pb"; + EXPECT_THAT( + Testee("OneofZeroDouble").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroDouble").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonOneofTest, OneofZeroEnum) { + constexpr absl::string_view kInput = R"json({"oneofEnum":"FOO"})json"; + constexpr absl::string_view kExpected = R"pb(oneof_enum: FOO)pb"; + EXPECT_THAT( + Testee("OneofZeroEnum").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("OneofZeroEnum").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonOneofTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_repeated_test.cc b/conformance/json_repeated_test.cc new file mode 100644 index 0000000..bc94712 --- /dev/null +++ b/conformance/json_repeated_test.cc
@@ -0,0 +1,269 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for repeated fields: JSON arrays of primitives, +// enums, strings, bytes and messages round-trip, array elements of the wrong +// type are rejected, a single value where an array was expected is rejected, +// and so is a trailing comma in an array. This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForRepeatedTypes(); the +// test names and the requests sent to the testee are identical to the legacy +// ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonRepeatedTest = MessageTypeConformanceTest; + +// --------------------------------------------------------------------------- +// Repeated fields of each kind round-trip. +// --------------------------------------------------------------------------- + +TEST_P(JsonRepeatedTest, PrimitiveRepeatedField) { + constexpr absl::string_view kInput = + R"json({"repeatedInt32": [1, 2, 3, 4]})json"; + constexpr absl::string_view kExpected = + R"pb(repeated_int32: [ 1, 2, 3, 4 ])pb"; + EXPECT_THAT(Testee("PrimitiveRepeatedField") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("PrimitiveRepeatedField") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonRepeatedTest, EnumRepeatedField) { + constexpr absl::string_view kInput = + R"json({"repeatedNestedEnum": ["FOO", "BAR", "BAZ"]})json"; + constexpr absl::string_view kExpected = + R"pb(repeated_nested_enum: [ FOO, BAR, BAZ ])pb"; + EXPECT_THAT(Testee("EnumRepeatedField") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("EnumRepeatedField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonRepeatedTest, StringRepeatedField) { + constexpr absl::string_view kInput = + R"json({"repeatedString": ["Hello", "world"]})json"; + constexpr absl::string_view kExpected = + R"pb(repeated_string: [ "Hello", "world" ])pb"; + EXPECT_THAT(Testee("StringRepeatedField") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("StringRepeatedField") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonRepeatedTest, BytesRepeatedField) { + constexpr absl::string_view kInput = + R"json({"repeatedBytes": ["AAEC", "AQI="]})json"; + constexpr absl::string_view kExpected = + R"pb(repeated_bytes: [ "\x00\x01\x02", "\x01\x02" ])pb"; + EXPECT_THAT(Testee("BytesRepeatedField") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("BytesRepeatedField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonRepeatedTest, MessageRepeatedField) { + constexpr absl::string_view kInput = + R"json({"repeatedNestedMessage": [{"a": 1234}, {"a": 5678}]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_nested_message: { a: 1234 } + repeated_nested_message: { a: 5678 } + )pb"; + EXPECT_THAT(Testee("MessageRepeatedField") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("MessageRepeatedField") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// --------------------------------------------------------------------------- +// Repeated field elements of the wrong type are rejected. +// --------------------------------------------------------------------------- + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingIntegersGotBool) { + EXPECT_THAT(Testee("RepeatedFieldWrongElementTypeExpectingIntegersGotBool") + .ParseJson(message(), + R"json({"repeatedInt32": [1, false, 3, 4]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingIntegersGotString) { + EXPECT_THAT(Testee("RepeatedFieldWrongElementTypeExpectingIntegersGotString") + .ParseJson(message(), + R"json({"repeatedInt32": [1, 2, "name", 4]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingIntegersGotMessage) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingIntegersGotMessage") + .ParseJson(message(), + R"json({"repeatedInt32": [1, 2, 3, {"a": 4}]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldWrongElementTypeExpectingStringsGotInt) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingStringsGotInt") + .ParseJson(message(), + R"json({"repeatedString": ["1", 2, "3", "4"]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldWrongElementTypeExpectingStringsGotBool) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingStringsGotBool") + .ParseJson(message(), + R"json({"repeatedString": ["1", "2", false, "4"]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingStringsGotMessage) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingStringsGotMessage") + .ParseJson(message(), + R"json({"repeatedString": ["1", 2, "3", {"a": 4}]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldWrongElementTypeExpectingMessagesGotInt) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingMessagesGotInt") + .ParseJson(message(), + R"json({"repeatedNestedMessage": [{"a": 1}, 2]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingMessagesGotBool) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingMessagesGotBool") + .ParseJson(message(), + R"json({"repeatedNestedMessage": [{"a": 1}, false]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, + RepeatedFieldWrongElementTypeExpectingMessagesGotString) { + EXPECT_THAT( + Testee("RepeatedFieldWrongElementTypeExpectingMessagesGotString") + .ParseJson(message(), + R"json({"repeatedNestedMessage": [{"a": 1}, "2"]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// A single value where a repeated field was expected is rejected, even if it +// is of the right type. +// --------------------------------------------------------------------------- + +TEST_P(JsonRepeatedTest, SingleValueForRepeatedFieldInt32) { + EXPECT_THAT(Testee("SingleValueForRepeatedFieldInt32") + .ParseJson(message(), R"json({"repeatedInt32": 1})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, SingleValueForRepeatedFieldMessage) { + EXPECT_THAT(Testee("SingleValueForRepeatedFieldMessage") + .ParseJson(message(), + R"json({"repeatedNestedMessage": {"a": 1}})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// --------------------------------------------------------------------------- +// A trailing comma in a repeated field is rejected. +// --------------------------------------------------------------------------- + +TEST_P(JsonRepeatedTest, RepeatedFieldTrailingComma) { + EXPECT_THAT( + Testee(kP1, "RepeatedFieldTrailingComma") + .ParseJson(message(), R"json({"repeatedInt32": [1, 2, 3, 4,]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldTrailingCommaWithSpace) { + EXPECT_THAT( + Testee(kP1, "RepeatedFieldTrailingCommaWithSpace") + .ParseJson(message(), R"json({"repeatedInt32": [1, 2, 3, 4 ,]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldTrailingCommaWithSpaceCommaSpace) { + EXPECT_THAT( + Testee(kP1, "RepeatedFieldTrailingCommaWithSpaceCommaSpace") + .ParseJson(message(), R"json({"repeatedInt32": [1, 2, 3, 4 , ]})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonRepeatedTest, RepeatedFieldTrailingCommaWithNewlines) { + EXPECT_THAT( + Testee(kP1, "RepeatedFieldTrailingCommaWithNewlines") + .ParseJson(message(), + "{\"repeatedInt32\": [\n 1,\n 2,\n 3,\n 4,\n]}") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonRepeatedTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_string_test.cc b/conformance/json_string_test.cc new file mode 100644 index 0000000..a9bde6f --- /dev/null +++ b/conformance/json_string_test.cc
@@ -0,0 +1,258 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for string and bytes fields: Unicode text, JSON +// escapes (including \u escapes, surrogate pairs and embedded NUL), rejected +// malformed escapes, unpaired surrogates, non-string values and single-quoted +// strings, and base64 / base64url bytes. This replaces the "String fields" +// and "Bytes fields" blocks (and the single-quote tests at the end) of the +// legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForNonRepeatedTypes(); the +// test names and the requests sent to the testee are identical to the legacy +// ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// --------------------------------------------------------------------------- +// String fields. +// --------------------------------------------------------------------------- + +using JsonStringTest = MessageTypeConformanceTest; + +TEST_P(JsonStringTest, StringField) { + constexpr absl::string_view kInput = + R"json({"optionalString": "Hello world!"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "Hello world!")pb"; + EXPECT_THAT( + Testee("StringField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("StringField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Non-ASCII characters in the JSON text itself, as opposed to the \uXXXX +// escapes tested below. Note that kInput is deliberately not a raw string +// literal: the compiler resolves the \x escapes, so the payload holds the +// UTF-8 bytes of U+8C37 U+6B4C ("Google" in Chinese). In the raw string +// literals below, the \u escapes reach the testee's JSON parser verbatim. +TEST_P(JsonStringTest, StringFieldUnicode) { + constexpr absl::string_view kInput = + "{\"optionalString\": \"\xE8\xB0\xB7\xE6\xAD\x8C\"}"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "\xE8\xB0\xB7\xE6\xAD\x8C")pb"; + EXPECT_THAT(Testee("StringFieldUnicode") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("StringFieldUnicode").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonStringTest, StringFieldEscape) { + constexpr absl::string_view kInput = + R"json({"optionalString": "\"\\\/\b\f\n\r\t"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "\"\\/\b\f\n\r\t")pb"; + EXPECT_THAT(Testee("StringFieldEscape") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("StringFieldEscape").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonStringTest, StringFieldUnicodeEscape) { + constexpr absl::string_view kInput = + R"json({"optionalString": "\u8C37\u6B4C"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "\xE8\xB0\xB7\xE6\xAD\x8C")pb"; + EXPECT_THAT(Testee("StringFieldUnicodeEscape") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("StringFieldUnicodeEscape") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// U+1F601 (grinning face with smiling eyes). +TEST_P(JsonStringTest, StringFieldSurrogatePair) { + constexpr absl::string_view kInput = + R"json({"optionalString": "\uD83D\uDE01"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "\xF0\x9F\x98\x81")pb"; + EXPECT_THAT(Testee("StringFieldSurrogatePair") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("StringFieldSurrogatePair") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonStringTest, StringFieldEmbeddedNull) { + constexpr absl::string_view kInput = + R"json({"optionalString": "Hello\u0000world!"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string: "Hello\000world!")pb"; + EXPECT_THAT(Testee("StringFieldEmbeddedNull") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("StringFieldEmbeddedNull") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Unicode escapes must start with "\u" (lowercase u). +TEST_P(JsonStringTest, StringFieldUppercaseEscapeLetter) { + EXPECT_THAT( + Testee(kP1, "StringFieldUppercaseEscapeLetter") + .ParseJson(message(), "{\"optionalString\": \"\\U8C37\\U6b4C\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldInvalidEscape) { + EXPECT_THAT( + Testee(kP1, "StringFieldInvalidEscape") + .ParseJson(message(), "{\"optionalString\": \"\\uXXXX\\u6B4C\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldUnterminatedEscape) { + EXPECT_THAT( + Testee(kP1, "StringFieldUnterminatedEscape") + .ParseJson(message(), R"json({"optionalString": "\u8C3"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// The unpaired-surrogate inputs below are ordinary (not raw) string literals: +// MSVC rejects a lone surrogate written as \uXXXX even inside a raw string +// literal (error C3850), so the backslash is escaped to keep the \u escape +// out of the compiler's hands and deliver it to the testee's JSON parser. +TEST_P(JsonStringTest, StringFieldUnpairedHighSurrogate) { + EXPECT_THAT(Testee(kP1, "StringFieldUnpairedHighSurrogate") + .ParseJson(message(), "{\"optionalString\": \"\\uD800\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldUnpairedLowSurrogate) { + EXPECT_THAT(Testee(kP1, "StringFieldUnpairedLowSurrogate") + .ParseJson(message(), "{\"optionalString\": \"\\uDC00\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldSurrogateInWrongOrder) { + EXPECT_THAT( + Testee(kP1, "StringFieldSurrogateInWrongOrder") + .ParseJson(message(), "{\"optionalString\": \"\\uDE01\\uD83D\"}") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldNotAString) { + EXPECT_THAT(Testee("StringFieldNotAString") + .ParseJson(message(), R"json({"optionalString": 12345})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// http://www.rfc-editor.org/rfc/rfc7159.txt says strings have to use double +// quotes. +TEST_P(JsonStringTest, StringFieldSingleQuoteKey) { + EXPECT_THAT( + Testee(kP1, "StringFieldSingleQuoteKey") + .ParseJson(message(), R"json({'optionalString': "Hello world!"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldSingleQuoteValue) { + EXPECT_THAT( + Testee(kP1, "StringFieldSingleQuoteValue") + .ParseJson(message(), R"json({"optionalString": 'Hello world!'})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonStringTest, StringFieldSingleQuoteBoth) { + EXPECT_THAT( + Testee(kP1, "StringFieldSingleQuoteBoth") + .ParseJson(message(), R"json({'optionalString': 'Hello world!'})json") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonStringTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +// --------------------------------------------------------------------------- +// Bytes fields. +// --------------------------------------------------------------------------- + +using JsonBytesTest = MessageTypeConformanceTest; + +TEST_P(JsonBytesTest, BytesField) { + constexpr absl::string_view kInput = R"json({"optionalBytes": "AQI="})json"; + constexpr absl::string_view kExpected = R"pb(optional_bytes: "\x01\x02")pb"; + EXPECT_THAT( + Testee("BytesField").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("BytesField").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonBytesTest, BytesFieldBase64Url) { + constexpr absl::string_view kInput = R"json({"optionalBytes": "-_"})json"; + constexpr absl::string_view kExpected = R"pb(optional_bytes: "\xfb")pb"; + EXPECT_THAT(Testee(kP1, "BytesFieldBase64Url") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee(kP1, "BytesFieldBase64Url") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonBytesTest, ValuesIn(AllTestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_struct_test.cc b/conformance/json_struct_test.cc new file mode 100644 index 0000000..6d2fd03 --- /dev/null +++ b/conformance/json_struct_test.cc
@@ -0,0 +1,176 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.Struct fields: a Struct holding +// every kind of Value, an empty ListValue member, and Structs nested 25 deep +// (must be accepted) and 200 deep (must be rejected). This replaces the +// legacy BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForStruct(), which +// ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix. + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonStructTest = MessageTypeConformanceTest; + +TEST_P(JsonStructTest, Struct) { + constexpr absl::string_view kInput = R"json({ + "optionalStruct": { + "nullValue": null, + "intValue": 1234, + "boolValue": true, + "doubleValue": 1234.5678, + "stringValue": "Hello world!", + "listValue": [1234, "5678"], + "objectValue": { + "value": 0 + } + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_struct: { + fields: { + key: "nullValue" + value: { null_value: NULL_VALUE } + } + fields: { + key: "intValue" + value: { number_value: 1234 } + } + fields: { + key: "boolValue" + value: { bool_value: true } + } + fields: { + key: "doubleValue" + value: { number_value: 1234.5678 } + } + fields: { + key: "stringValue" + value: { string_value: "Hello world!" } + } + fields: { + key: "listValue" + value: { + list_value: { + values: { number_value: 1234 } + values: { string_value: "5678" } + } + } + } + fields: { + key: "objectValue" + value: { + struct_value: { + fields: { + key: "value" + value: { number_value: 0 } + } + } + } + } + } + )pb"; + EXPECT_THAT(Testee("Struct").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("Struct").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonStructTest, StructWithEmptyListValue) { + constexpr absl::string_view kInput = R"json({ + "optionalStruct": { + "listValue": [] + } + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_struct: { + fields: { + key: "listValue" + value: { list_value: {} } + } + } + )pb"; + EXPECT_THAT(Testee("StructWithEmptyListValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("StructWithEmptyListValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonStructTest, StructDeepNesting25) { + const std::string input = + absl::StrCat(R"({"optionalStruct": {)", Repeat(R"("n": {)", 25), + R"("value": 1)", std::string(25, '}'), "}}"); + const std::string expected = + absl::StrCat("optional_struct: {\n", + Repeat(" fields: {\n" + " key: \"n\"\n" + " value: {\n" + " struct_value: {\n", + 25), + " fields: {\n" + " key: \"value\"\n" + " value: { number_value: 1 }\n" + " }\n", + Repeat(" }\n" + " }\n" + " }\n", + 25), + "}\n"); + EXPECT_THAT(Testee(kP1, "StructDeepNesting25") + .ParseJson(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kP1, "StructDeepNesting25") + .ParseJson(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(JsonStructTest, StructDeepNesting200) { + const std::string input = + absl::StrCat(R"({"optionalStruct": {)", Repeat(R"("n": {)", 200), + R"("value": 1)", std::string(200, '}'), "}}"); + EXPECT_THAT(Testee(kP1, "StructDeepNesting200") + .ParseJson(message(), input) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonStructTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_test_util.h b/conformance/json_test_util.h new file mode 100644 index 0000000..fc18fa9 --- /dev/null +++ b/conformance/json_test_util.h
@@ -0,0 +1,133 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_JSON_TEST_UTIL_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_JSON_TEST_UTIL_H__ + +#include <string> + +#include <gmock/gmock.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "json/value.h" + +// Helpers shared by the gtest-based JSON conformance tests (json_*_test.cc). +// +// Like binary_test_util.h, this file only holds matchers and pure values. +// Every EXPECT_THAT lives in the tests themselves, so that a failure points +// at the test and not at a helper; the tests use the framework directly: +// +// EXPECT_THAT( +// Testee("HelloWorld") +// .ParseJson(message(), R"json({"optionalString": "Hi"})json") +// .SerializeBinary(), +// Yields(WhenParsed(EqualsTextProto(R"pb(optional_string: 'Hi')pb")))); +// +// The message type sets the tests are parameterized over are in +// binary_test_util.h: AllTestMessageTypes(), Proto3TestMessageTypes() and +// Proto2TestMessageTypes(). + +namespace google { +namespace protobuf { +namespace conformance { + +// Returns `s` repeated `count` times, for building the deeply nested JSON +// inputs of the recursion-limit tests. +inline std::string Repeat(absl::string_view s, int count) { + std::string result; + for (int i = 0; i < count; ++i) { + absl::StrAppend(&result, s); + } + return result; +} + +// Matches a JSON object that has a member named `name`. Meant for use inside +// JsonPayload(): +// +// Yields(JsonPayload(AllOf(HasJsonMember("x"), HasJsonMember("y")))) +// Yields(JsonPayload(Not(HasJsonMember("x")))) +// +// A value that isn't a JSON object does not match. The failure message is +// "Expect: JSON payload has member "x", but got: ...". +MATCHER_P(HasJsonMember, name, + absl::StrCat(negation ? "doesn't have" : "has", " member \"", name, + "\"")) { + if (!arg.isObject()) { + *result_listener << "which is not a JSON object"; + return false; + } + return arg.isMember(name); +} + +// Matches a JSON object that has a member named `name` whose value matches +// `member_matcher`, a matcher on a Json::Value. For example, this checks +// that an int64 field is serialized as a string: +// +// Yields(JsonPayload(HasJsonMemberThat("optionalInt64", IsJsonString("1")))) +// +// A value that isn't a JSON object does not match. +MATCHER_P2(HasJsonMemberThat, name, member_matcher, + absl::StrCat(negation ? "doesn't have" : "has", " member \"", name, + "\" that ", + ::testing::DescribeMatcher<const Json::Value&>( + member_matcher))) { + if (!arg.isObject()) { + *result_listener << "which is not a JSON object"; + return false; + } + return arg.isMember(name) && ::testing::ExplainMatchResult( + member_matcher, arg[name], result_listener); +} + +// Matches a JSON string equal to `expected`. A number or any other JSON kind +// doesn't match, whatever its textual form. +// +// Yields(JsonPayload(HasJsonMemberThat("optionalInt64", IsJsonString("1")))) +MATCHER_P(IsJsonString, expected, + absl::StrCat(negation ? "isn't" : "is", " the JSON string \"", + expected, "\"")) { + return arg.isString() && arg.asString() == expected; +} + +// Matches a JSON number that jsoncpp parsed as a signed integer +// (Json::intValue) equal to `expected`. A string "123" or a real 123.0 +// doesn't match. +// +// Yields(JsonPayload(HasJsonMemberThat("optionalInt32", IsJsonInt(123)))) +MATCHER_P(IsJsonInt, expected, + absl::StrCat(negation ? "isn't" : "is", " the JSON integer ", + expected)) { + return arg.type() == Json::intValue && arg.asInt() == expected; +} + +// Matches the JSON value `null`. +// +// Yields(JsonPayload(HasJsonMemberThat("oneofNullValue", IsJsonNull()))) +MATCHER(IsJsonNull, absl::StrCat(negation ? "isn't" : "is", " JSON null")) { + return arg.isNull(); +} + +// Matches a JSON object with no members, that is `{}`. This is the +// serialization of a message none of whose fields is emitted: +// +// Yields(JsonPayload(IsEmptyJsonObject())) +// +// Unlike Json::Value::empty(), neither `null` nor `[]` matches. +MATCHER(IsEmptyJsonObject, + absl::StrCat(negation ? "isn't" : "is", " an empty JSON object")) { + if (!arg.isObject()) { + *result_listener << "which is not a JSON object"; + return false; + } + return arg.empty(); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_JSON_TEST_UTIL_H__
diff --git a/conformance/json_test_util_test.cc b/conformance/json_test_util_test.cc new file mode 100644 index 0000000..24ccb50 --- /dev/null +++ b/conformance/json_test_util_test.cc
@@ -0,0 +1,163 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/json_test_util.h" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "json/config.h" +#include "json/value.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::DescribeMatcher; +using ::testing::ExplainMatchResult; +using ::testing::Not; +using ::testing::StringMatchResultListener; + +TEST(HasJsonMemberTest, Matches) { + Json::Value value; + value["foo"] = 1; + EXPECT_THAT(value, HasJsonMember("foo")); + EXPECT_THAT(value, Not(HasJsonMember("bar"))); + // Only objects have members. + EXPECT_THAT(Json::Value(Json::nullValue), Not(HasJsonMember("foo"))); + EXPECT_THAT(Json::Value(Json::arrayValue), Not(HasJsonMember("foo"))); +} + +TEST(HasJsonMemberTest, ExplainsNonObject) { + StringMatchResultListener listener; + EXPECT_FALSE(ExplainMatchResult(HasJsonMember("foo"), + Json::Value(Json::arrayValue), &listener)); + EXPECT_EQ(listener.str(), "which is not a JSON object"); + + // Objects are matched by membership alone, with no further explanation. + Json::Value value; + value["foo"] = 1; + StringMatchResultListener object_listener; + EXPECT_TRUE( + ExplainMatchResult(HasJsonMember("foo"), value, &object_listener)); + EXPECT_EQ(object_listener.str(), ""); +} + +TEST(HasJsonMemberTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>(HasJsonMember("foo")), + R"(has member "foo")"); + EXPECT_EQ(DescribeMatcher<const Json::Value&>(HasJsonMember("foo"), + /*negation=*/true), + R"(doesn't have member "foo")"); +} + +TEST(HasJsonMemberThatTest, Matches) { + Json::Value value; + value["str"] = "1"; + value["num"] = 1; + EXPECT_THAT(value, HasJsonMemberThat("str", IsJsonString("1"))); + EXPECT_THAT(value, Not(HasJsonMemberThat("str", IsJsonString("2")))); + EXPECT_THAT(value, Not(HasJsonMemberThat("num", IsJsonString("1")))); + EXPECT_THAT(value, Not(HasJsonMemberThat("missing", IsJsonString("1")))); + // Only objects have members. + EXPECT_THAT(Json::Value(Json::nullValue), + Not(HasJsonMemberThat("str", IsJsonString("1")))); + EXPECT_THAT(Json::Value(Json::arrayValue), + Not(HasJsonMemberThat("str", IsJsonString("1")))); +} + +TEST(HasJsonMemberThatTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>( + HasJsonMemberThat("foo", IsJsonString("1"))), + R"(has member "foo" that is the JSON string "1")"); + EXPECT_EQ(DescribeMatcher<const Json::Value&>( + HasJsonMemberThat("foo", IsJsonString("1")), + /*negation=*/true), + R"(doesn't have member "foo" that is the JSON string "1")"); +} + +TEST(IsJsonStringTest, Matches) { + EXPECT_THAT(Json::Value("1"), IsJsonString("1")); + EXPECT_THAT(Json::Value("2"), Not(IsJsonString("1"))); + EXPECT_THAT(Json::Value(1), Not(IsJsonString("1"))); + EXPECT_THAT(Json::Value(Json::nullValue), Not(IsJsonString("1"))); +} + +TEST(IsJsonStringTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsJsonString("1")), + R"(is the JSON string "1")"); + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsJsonString("1"), + /*negation=*/true), + R"(isn't the JSON string "1")"); +} + +TEST(IsJsonIntTest, Matches) { + EXPECT_THAT(Json::Value(123), IsJsonInt(123)); + EXPECT_THAT(Json::Value(-1), IsJsonInt(-1)); + EXPECT_THAT(Json::Value(124), Not(IsJsonInt(123))); + // Only signed integers (Json::intValue) match, like the legacy validators. + EXPECT_THAT(Json::Value("123"), Not(IsJsonInt(123))); + EXPECT_THAT(Json::Value(123.0), Not(IsJsonInt(123))); + EXPECT_THAT(Json::Value(Json::UInt(123)), Not(IsJsonInt(123))); + EXPECT_THAT(Json::Value(Json::nullValue), Not(IsJsonInt(123))); +} + +TEST(IsJsonIntTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsJsonInt(123)), + "is the JSON integer 123"); + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsJsonInt(123), + /*negation=*/true), + "isn't the JSON integer 123"); +} + +TEST(IsJsonNullTest, Matches) { + EXPECT_THAT(Json::Value(Json::nullValue), IsJsonNull()); + EXPECT_THAT(Json::Value(), IsJsonNull()); + EXPECT_THAT(Json::Value(0), Not(IsJsonNull())); + EXPECT_THAT(Json::Value(""), Not(IsJsonNull())); + EXPECT_THAT(Json::Value("null"), Not(IsJsonNull())); + EXPECT_THAT(Json::Value(Json::objectValue), Not(IsJsonNull())); +} + +TEST(IsJsonNullTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsJsonNull()), "is JSON null"); + EXPECT_EQ( + DescribeMatcher<const Json::Value&>(IsJsonNull(), /*negation=*/true), + "isn't JSON null"); +} + +TEST(IsEmptyJsonObjectTest, Matches) { + EXPECT_THAT(Json::Value(Json::objectValue), IsEmptyJsonObject()); + Json::Value value; + value["foo"] = 1; + EXPECT_THAT(value, Not(IsEmptyJsonObject())); + // Only objects match, even though Json::Value::empty() is also true for + // null and for an empty array. + EXPECT_THAT(Json::Value(Json::nullValue), Not(IsEmptyJsonObject())); + EXPECT_THAT(Json::Value(Json::arrayValue), Not(IsEmptyJsonObject())); + EXPECT_THAT(Json::Value(""), Not(IsEmptyJsonObject())); +} + +TEST(IsEmptyJsonObjectTest, ExplainsNonObject) { + StringMatchResultListener listener; + EXPECT_FALSE(ExplainMatchResult(IsEmptyJsonObject(), + Json::Value(Json::arrayValue), &listener)); + EXPECT_EQ(listener.str(), "which is not a JSON object"); +} + +TEST(IsEmptyJsonObjectTest, Description) { + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsEmptyJsonObject()), + "is an empty JSON object"); + EXPECT_EQ(DescribeMatcher<const Json::Value&>(IsEmptyJsonObject(), + /*negation=*/true), + "isn't an empty JSON object"); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_timestamp_test.cc b/conformance/json_timestamp_test.cc new file mode 100644 index 0000000..23e1fd6 --- /dev/null +++ b/conformance/json_timestamp_test.cc
@@ -0,0 +1,690 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.Timestamp: its RFC 3339 string +// form, its range, UTC offsets, the rejection of out-of-range, malformed or +// invalid date-time components in either direction, and the number of +// fractional digits a serializer emits. This replaces the Timestamp part of +// the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForWrapperTypes(), which +// ran for the proto3-style message types only; the test names and the requests +// sent to the testee are identical to the legacy ones, except that the +// binary-input tests below are BINARY_TEST requests (see below). +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix; ExpectSerializeFailureForJson() sent +// a binary message (built here from its wire format) to be serialized as +// JSON ("ProtobufInput.<name>.JsonOutput"), categorized as JSON_TEST; here +// ParseBinary() categorizes them as BINARY_TEST, which no testee tells apart +// from JSON_TEST on a binary payload (b/563657722). And the +// legacy "validators" (RunValidJsonTestWithValidator()) looked at the +// serialized JSON text itself. + +#include <cstdint> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonTimestampTest = MessageTypeConformanceTest; + +// optional_timestamp is field 302 of the test message types; Timestamp's +// seconds is field 1 and nanos field 2. +constexpr uint32_t kOptionalTimestamp = 302; +constexpr uint32_t kSeconds = 1; +constexpr uint32_t kNanos = 2; + +TEST_P(JsonTimestampTest, TimestampMinValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "0001-01-01T00:00:00Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: -62135596800 })pb"; + EXPECT_THAT(Testee("TimestampMinValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("TimestampMinValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampMaxValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "9999-12-31T23:59:59.999999999Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 253402300799 nanos: 999999999 })pb"; + EXPECT_THAT(Testee("TimestampMaxValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("TimestampMaxValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampRepeatedValue) { + constexpr absl::string_view kInput = R"json({ + "repeatedTimestamp": [ + "0001-01-01T00:00:00Z", + "9999-12-31T23:59:59.999999999Z" + ] + })json"; + constexpr absl::string_view kExpected = R"pb( + repeated_timestamp: { seconds: -62135596800 } + repeated_timestamp: { seconds: 253402300799 nanos: 999999999 } + )pb"; + EXPECT_THAT(Testee("TimestampRepeatedValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampRepeatedValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.000Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 0 })pb"; + EXPECT_THAT(Testee("TimestampEpochValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampEpochValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampNanoAfterEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.000000001Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 0 nanos: 1 })pb"; + EXPECT_THAT(Testee("TimestampNanoAfterEpochlValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampNanoAfterEpochlValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampNanoBeforeEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1969-12-31T23:59:59.999999999Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: -1 nanos: 999999999 })pb"; + EXPECT_THAT(Testee("TimestampNanoBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampNanoBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampLittleAfterEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:01.000000001Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 1 nanos: 1 })pb"; + EXPECT_THAT(Testee("TimestampLittleAfterEpochlValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampLittleAfterEpochlValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampLittleBeforeEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1969-12-31T23:59:58.999999999Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: -2 nanos: 999999999 })pb"; + EXPECT_THAT(Testee("TimestampLittleBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampLittleBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampTenAndHalfSecondsAfterEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:10.500Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 10 nanos: 500000000 })pb"; + EXPECT_THAT(Testee("TimestampTenAndHalfSecondsAfterEpochValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampTenAndHalfSecondsAfterEpochValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampTenAndHalfSecondsBeforeEpochValue) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1969-12-31T23:59:49.500Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: -11 nanos: 500000000 })pb"; + EXPECT_THAT(Testee("TimestampTenAndHalfSecondsBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampTenAndHalfSecondsBeforeEpochValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampLeap) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1993-02-10T00:00:00.000Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 729302400 })pb"; + EXPECT_THAT( + Testee("TimestampLeap").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("TimestampLeap").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampWithPositiveOffset) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T08:00:01+08:00"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 1 })pb"; + EXPECT_THAT(Testee("TimestampWithPositiveOffset") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithPositiveOffset") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampWithNegativeOffset) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1969-12-31T16:00:01-08:00"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 1 })pb"; + EXPECT_THAT(Testee("TimestampWithNegativeOffset") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithNegativeOffset") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampNull) { + constexpr absl::string_view kInput = R"json({"optionalTimestamp": null})json"; + constexpr absl::string_view kExpected = ""; + EXPECT_THAT( + Testee("TimestampNull").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("TimestampNull").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Out-of-range and malformed JSON values are rejected. +TEST_P(JsonTimestampTest, TimestampJsonInputTooSmall) { + EXPECT_THAT( + Testee("TimestampJsonInputTooSmall") + .ParseJson(message(), + R"json({"optionalTimestamp": "0000-01-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputTooLarge") + .ParseJson( + message(), + R"json({"optionalTimestamp": "10000-01-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputMissingZ) { + EXPECT_THAT( + Testee("TimestampJsonInputMissingZ") + .ParseJson(message(), + R"json({"optionalTimestamp": "0001-01-01T00:00:00"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputMissingT) { + EXPECT_THAT( + Testee("TimestampJsonInputMissingT") + .ParseJson(message(), + R"json({"optionalTimestamp": "0001-01-01 00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputLowercaseZ) { + EXPECT_THAT( + Testee("TimestampJsonInputLowercaseZ") + .ParseJson(message(), + R"json({"optionalTimestamp": "0001-01-01T00:00:00z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputLowercaseT) { + EXPECT_THAT( + Testee("TimestampJsonInputLowercaseT") + .ParseJson(message(), + R"json({"optionalTimestamp": "0001-01-01t00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampWithMissingColonInOffset) { + EXPECT_THAT( + Testee("TimestampWithMissingColonInOffset") + .ParseJson( + message(), + R"json({"optionalTimestamp": "1970-01-01T08:00:01+0800"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +// Out-of-range or inconsistent messages can't be serialized. The inputs +// are the legacy text-format messages on the wire (zero fields omitted). +TEST_P(JsonTimestampTest, TimestampProtoInputTooSmall) { + EXPECT_THAT( + Testee("TimestampProtoInputTooSmall") + .ParseBinary(message(), + LengthPrefixedField(kOptionalTimestamp, + VarintField(kSeconds, -62135596801))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonTimestampTest, TimestampProtoInputTooLarge) { + EXPECT_THAT( + Testee("TimestampProtoInputTooLarge") + .ParseBinary(message(), + LengthPrefixedField(kOptionalTimestamp, + VarintField(kSeconds, 253402300800))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonTimestampTest, TimestampProtoNegativeNanos) { + EXPECT_THAT(Testee("TimestampProtoNegativeNanos") + .ParseBinary(message(), LengthPrefixedField( + kOptionalTimestamp, + Wire(VarintField(kSeconds, 5000), + VarintField(kNanos, -1)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonTimestampTest, TimestampProtoNanoTooLarge) { + EXPECT_THAT( + Testee("TimestampProtoNanoTooLarge") + .ParseBinary(message(), LengthPrefixedField( + kOptionalTimestamp, + Wire(VarintField(kSeconds, 5000), + VarintField(kNanos, 1000000000)))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +// Serializers normalize to UTC and emit 0, 3, 6 or 9 fractional digits, +// whichever is the fewest that represent the value exactly. +TEST_P(JsonTimestampTest, TimestampZeroNormalized) { + EXPECT_THAT( + Testee(kP1, "TimestampZeroNormalized") + .ParseJson( + message(), + R"json({"optionalTimestamp": "1969-12-31T16:00:00-08:00"})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalTimestamp", IsJsonString("1970-01-01T00:00:00Z"))))); +} + +TEST_P(JsonTimestampTest, TimestampHasZeroFractionalDigit) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.000000000Z"})json"; + EXPECT_THAT(Testee(kP1, "TimestampHasZeroFractionalDigit") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalTimestamp", IsJsonString("1970-01-01T00:00:00Z"))))); +} + +TEST_P(JsonTimestampTest, TimestampHas3FractionalDigits) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.010000000Z"})json"; + EXPECT_THAT( + Testee(kP1, "TimestampHas3FractionalDigits") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalTimestamp", IsJsonString("1970-01-01T00:00:00.010Z"))))); +} + +TEST_P(JsonTimestampTest, TimestampHas6FractionalDigits) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.000010000Z"})json"; + EXPECT_THAT( + Testee(kP1, "TimestampHas6FractionalDigits") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalTimestamp", IsJsonString("1970-01-01T00:00:00.000010Z"))))); +} + +TEST_P(JsonTimestampTest, TimestampHas9FractionalDigits) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.000000010Z"})json"; + EXPECT_THAT(Testee(kP1, "TimestampHas9FractionalDigits") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat( + "optionalTimestamp", + IsJsonString("1970-01-01T00:00:00.000000010Z"))))); +} + +// Out of bounds / invalid components should JSON parse fail. +TEST_P(JsonTimestampTest, TimestampJsonInputMonthTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputMonthTooLarge") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-13-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputMonthZero) { + EXPECT_THAT( + Testee("TimestampJsonInputMonthZero") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-00-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputDayTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputDayTooLarge") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-32T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputDayZero) { + EXPECT_THAT( + Testee("TimestampJsonInputDayZero") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-00T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputHourTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputHourTooLarge") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-01T24:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputHourTooLarge25) { + EXPECT_THAT( + Testee("TimestampJsonInputHourTooLarge25") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-01T25:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputMinuteTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputMinuteTooLarge") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-01T00:60:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputSecondTooLarge) { + EXPECT_THAT( + Testee("TimestampJsonInputSecondTooLarge") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-01T00:00:60Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputInvalidOffsetHour) { + EXPECT_THAT( + Testee("TimestampJsonInputInvalidOffsetHour") + .ParseJson( + message(), + R"json({"optionalTimestamp": "1970-01-01T00:00:00+24:00"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputInvalidOffsetMinute) { + EXPECT_THAT( + Testee("TimestampJsonInputInvalidOffsetMinute") + .ParseJson( + message(), + R"json({"optionalTimestamp": "1970-01-01T00:00:00+00:60"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputOffsetBoundaryUnderflow) { + EXPECT_THAT( + Testee("TimestampJsonInputOffsetBoundaryUnderflow") + .ParseJson( + message(), + R"json({"optionalTimestamp": "0001-01-01T00:00:00+00:01"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputOffsetBoundaryOverflow) { + EXPECT_THAT( + Testee("TimestampJsonInputOffsetBoundaryOverflow") + .ParseJson( + message(), + R"json({"optionalTimestamp": "9999-12-31T23:59:59-00:01"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputInvalidNanos) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00.1234567890Z"})json"; + EXPECT_THAT(Testee("TimestampJsonInputInvalidNanos") + .ParseJson(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputInvalidCharsInNanos) { + EXPECT_THAT( + Testee("TimestampJsonInputInvalidCharsInNanos") + .ParseJson( + message(), + R"json({"optionalTimestamp": "1970-01-01T00:00:00.123aZ"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputYearTooShort) { + EXPECT_THAT( + Testee("TimestampJsonInputYearTooShort") + .ParseJson(message(), + R"json({"optionalTimestamp": "999-01-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputYearTooLong) { + EXPECT_THAT( + Testee("TimestampJsonInputYearTooLong") + .ParseJson( + message(), + R"json({"optionalTimestamp": "00001-01-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputMonthTooShort) { + EXPECT_THAT( + Testee("TimestampJsonInputMonthTooShort") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-1-01T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputDayTooShort) { + EXPECT_THAT( + Testee("TimestampJsonInputDayTooShort") + .ParseJson(message(), + R"json({"optionalTimestamp": "1970-01-1T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputNonLeapFeb29) { + EXPECT_THAT( + Testee("TimestampJsonInputNonLeapFeb29") + .ParseJson(message(), + R"json({"optionalTimestamp": "2001-02-29T00:00:00Z"})json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonTimestampTest, TimestampJsonInputLeapFeb29) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "2000-02-29T00:00:00Z"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 951782400 })pb"; + EXPECT_THAT(Testee("TimestampJsonInputLeapFeb29") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampJsonInputLeapFeb29") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Honoring time zones correctly. +TEST_P(JsonTimestampTest, TimestampWithOffsetShiftsDay) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-02T01:00:00+02:00"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 82800 })pb"; + EXPECT_THAT(Testee("TimestampWithOffsetShiftsDay") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithOffsetShiftsDay") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampWithOffsetBoundaryInBoundsMin) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "0001-01-01T00:00:00-00:01"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: -62135596740 })pb"; + EXPECT_THAT(Testee("TimestampWithOffsetBoundaryInBoundsMin") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithOffsetBoundaryInBoundsMin") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampWithOffsetBoundaryInBoundsMax) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "9999-12-31T23:59:59+00:01"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 253402300739 })pb"; + EXPECT_THAT(Testee("TimestampWithOffsetBoundaryInBoundsMax") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithOffsetBoundaryInBoundsMax") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonTimestampTest, TimestampWithComplexOffset) { + constexpr absl::string_view kInput = + R"json({"optionalTimestamp": "1970-01-01T00:00:00-11:30"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_timestamp: { seconds: 41400 })pb"; + EXPECT_THAT(Testee("TimestampWithComplexOffset") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("TimestampWithComplexOffset") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonTimestampTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_unknown_enum_test.cc b/conformance/json_unknown_enum_test.cc new file mode 100644 index 0000000..b02b0c4 --- /dev/null +++ b/conformance/json_unknown_enum_test.cc
@@ -0,0 +1,146 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for unknown enum values given as string labels, in +// a singular field, a repeated field and a map value: a parser must reject +// them unless it is asked to ignore unknown fields, in which case it drops +// the unknown value (and, in a list or map mixing known and unknown labels, +// keeps the known ones). This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>:: +// RunJsonTestsForUnknownEnumStringValues(), +// which ran for every test message type; the test names and the requests sent +// to the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonIgnoreUnknownTest(), the ignore-unknown tests +// only have the binary-output leg ("<name>.ProtobufOutput", in the +// JSON_IGNORE_UNKNOWN_PARSING_TEST category), and like +// ExpectParseFailureForJson() the rejections ask for JSON output under a name +// without an output suffix. The inputs are the legacy raw string literals +// verbatim, including their snake_case member names and whitespace. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +class JsonUnknownEnumTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Unknown enum string values are a parse failure when not ignoring unknown +// fields. +TEST_P(JsonUnknownEnumTest, RejectUnknownEnumStringValueInOptionalField) { + EXPECT_THAT(Testee("RejectUnknownEnumStringValueInOptionalField") + .ParseJson(message(), R"json({ + "optional_nested_enum": "UNKNOWN_ENUM_VALUE" + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonUnknownEnumTest, RejectUnknownEnumStringValueInRepeatedField) { + EXPECT_THAT(Testee("RejectUnknownEnumStringValueInRepeatedField") + .ParseJson(message(), R"json({ + "repeated_nested_enum": ["UNKNOWN_ENUM_VALUE"] + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonUnknownEnumTest, RejectUnknownEnumStringValueInMapValue) { + EXPECT_THAT(Testee("RejectUnknownEnumStringValueInMapValue") + .ParseJson(message(), R"json({ + "map_string_nested_enum": {"key": "UNKNOWN_ENUM_VALUE"} + })json") + .ParseOnly(), + Yields(IsParseError())); +} + +// ... and are ignored when ignoring unknown fields. +TEST_P(JsonUnknownEnumTest, IgnoreUnknownEnumStringValueInOptionalField) { + EXPECT_THAT(Testee("IgnoreUnknownEnumStringValueInOptionalField") + .ParseJson(message(), R"json({ + "optional_nested_enum": "UNKNOWN_ENUM_VALUE" + })json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonUnknownEnumTest, IgnoreUnknownEnumStringValueInRepeatedField) { + EXPECT_THAT(Testee("IgnoreUnknownEnumStringValueInRepeatedField") + .ParseJson(message(), R"json({ + "repeated_nested_enum": ["UNKNOWN_ENUM_VALUE"] + })json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(JsonUnknownEnumTest, IgnoreUnknownEnumStringValueInMapValue) { + EXPECT_THAT(Testee("IgnoreUnknownEnumStringValueInMapValue") + .ParseJson(message(), R"json({ + "map_string_nested_enum": {"key": "UNKNOWN_ENUM_VALUE"} + })json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +// Like IgnoreUnknownEnumStringValueInRepeatedField, but highlights the +// potentially unexpected behavior in an array with mixed known and unknown +// enum string values. +TEST_P(JsonUnknownEnumTest, IgnoreUnknownEnumStringValueInRepeatedPart) { + EXPECT_THAT(Testee("IgnoreUnknownEnumStringValueInRepeatedPart") + .ParseJson(message(), R"json({ + "repeated_nested_enum": [ + "FOO", + "UNKNOWN_ENUM_VALUE", + "FOO" + ]})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb( + repeated_nested_enum: FOO + repeated_nested_enum: FOO + )pb")))); +} + +// Like IgnoreUnknownEnumStringValueInMapValue, with a mixture of known and +// unknown enum string values in the map. +TEST_P(JsonUnknownEnumTest, IgnoreUnknownEnumStringValueInMapPart) { + EXPECT_THAT(Testee("IgnoreUnknownEnumStringValueInMapPart") + .ParseJson(message(), R"json({ + "map_string_nested_enum": { + "key1": "FOO", + "key2": "UNKNOWN_ENUM_VALUE" + }})json", + {/*ignore_unknown_fields=*/true}) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(map_string_nested_enum: { + key: "key1" + value: FOO + })pb")))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonUnknownEnumTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_value_test.cc b/conformance/json_value_test.cc new file mode 100644 index 0000000..95f49e9 --- /dev/null +++ b/conformance/json_value_test.cc
@@ -0,0 +1,336 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for google.protobuf.Value, ListValue and NullValue +// fields: every JSON kind a Value accepts, repeated Values and ListValues, how +// a NullValue is serialized inside and outside a oneof, the non-finite numbers +// a Value cannot express, and lists and Structs nested 25 deep (must be +// accepted) and 200 deep (must be rejected). This replaces the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForValue(), which ran for +// the proto3-style message types only; the test names and the requests sent to +// the testee are identical to the legacy ones, except that the binary-input +// tests below are BINARY_TEST requests (see below). +// +// Like the legacy RunValidJsonTest(), each valid input is sent twice: once to +// be serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). ExpectParseFailureForJson() asked for JSON output +// under a name without an output suffix; RunValidJsonTestWithValidator() +// asked for JSON output and inspected the JSON text. +// ExpectSerializeFailureForJson() sent the serialized text-format message as +// binary input, categorized as a JSON test, under +// "ProtobufInput.<name>.JsonOutput"; the same bytes are built here with the +// wire-format builders, and ParseBinary() categorizes them as BINARY_TEST, +// which no testee tells apart from JSON_TEST on a binary payload +// (b/563657722). + +#include <cstdint> +#include <limits> +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonValueTest = MessageTypeConformanceTest; + +// Field numbers of the binary inputs of the serialize-failure tests: the +// test message's optional_value and google.protobuf.Value.number_value. +constexpr uint32_t kOptionalValue = 306; +constexpr uint32_t kValueNumberValue = 2; + +TEST_P(JsonValueTest, ValueAcceptInteger) { + constexpr absl::string_view kInput = R"json({"optionalValue": 1})json"; + constexpr absl::string_view kExpected = + R"pb(optional_value: { number_value: 1 })pb"; + EXPECT_THAT(Testee("ValueAcceptInteger") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptInteger").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptFloat) { + constexpr absl::string_view kInput = R"json({"optionalValue": 1.5})json"; + constexpr absl::string_view kExpected = + R"pb(optional_value: { number_value: 1.5 })pb"; + EXPECT_THAT( + Testee("ValueAcceptFloat").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptFloat").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptBool) { + constexpr absl::string_view kInput = R"json({"optionalValue": false})json"; + constexpr absl::string_view kExpected = + R"pb(optional_value: { bool_value: false })pb"; + EXPECT_THAT( + Testee("ValueAcceptBool").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptBool").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptNull) { + constexpr absl::string_view kInput = R"json({"optionalValue": null})json"; + constexpr absl::string_view kExpected = + R"pb(optional_value: { null_value: NULL_VALUE })pb"; + EXPECT_THAT( + Testee("ValueAcceptNull").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptNull").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptString) { + constexpr absl::string_view kInput = R"json({"optionalValue": "hello"})json"; + constexpr absl::string_view kExpected = + R"pb(optional_value: { string_value: "hello" })pb"; + EXPECT_THAT(Testee("ValueAcceptString") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptString").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptList) { + constexpr absl::string_view kInput = + R"json({"optionalValue": [0, "hello"]})json"; + constexpr absl::string_view kExpected = R"pb( + optional_value: { + list_value: { + values: { number_value: 0 } + values: { string_value: "hello" } + } + } + )pb"; + EXPECT_THAT( + Testee("ValueAcceptList").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptList").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, ValueAcceptObject) { + constexpr absl::string_view kInput = + R"json({"optionalValue": {"value": 1}})json"; + constexpr absl::string_view kExpected = R"pb( + optional_value: { + struct_value: { + fields: { + key: "value" + value: { number_value: 1 } + } + } + } + )pb"; + EXPECT_THAT(Testee("ValueAcceptObject") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("ValueAcceptObject").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, RepeatedValue) { + constexpr absl::string_view kInput = R"json({ + "repeatedValue": [["a"]] + })json"; + constexpr absl::string_view kExpected = R"pb( + repeated_value: + [ { + list_value: { + values: + [ { string_value: "a" }] + } + }] + )pb"; + EXPECT_THAT( + Testee("RepeatedValue").ParseJson(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("RepeatedValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonValueTest, RepeatedListValue) { + constexpr absl::string_view kInput = R"json({ + "repeatedListValue": [["a"]] + })json"; + constexpr absl::string_view kExpected = R"pb( + repeated_list_value: + [ { + values: + [ { string_value: "a" }] + }] + )pb"; + EXPECT_THAT(Testee("RepeatedListValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT( + Testee("RepeatedListValue").ParseJson(message(), kInput).SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// A NullValue set in a oneof is emitted as `null`, whether it was given as +// the enum name or as `null`. +TEST_P(JsonValueTest, NullValueInOtherOneofOldFormat) { + EXPECT_THAT( + Testee(kP1, "NullValueInOtherOneofOldFormat") + .ParseJson(message(), R"json({"oneofNullValue": "NULL_VALUE"})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat("oneofNullValue", IsJsonNull())))); +} + +TEST_P(JsonValueTest, NullValueInOtherOneofNewFormat) { + EXPECT_THAT( + Testee(kP1, "NullValueInOtherOneofNewFormat") + .ParseJson(message(), R"json({"oneofNullValue": null})json") + .SerializeJson(), + Yields(JsonPayload(HasJsonMemberThat("oneofNullValue", IsJsonNull())))); +} + +// A NullValue field outside a oneof has no presence, so it is omitted. +TEST_P(JsonValueTest, NullValueInNormalMessage) { + EXPECT_THAT( + Testee(kP1, "NullValueInNormalMessage") + .ParseJson(message(), R"json({"optionalNullValue": null})json") + .SerializeJson(), + Yields(JsonPayload(IsEmptyJsonObject()))); +} + +// The binary form of the legacy input `optional_value: { number_value: nan}`. +TEST_P(JsonValueTest, ValueRejectNanNumberValue) { + EXPECT_THAT( + Testee(kP1, "ValueRejectNanNumberValue") + .ParseBinary( + message(), + LengthPrefixedField( + kOptionalValue, + DoubleField(kValueNumberValue, + std::numeric_limits<double>::quiet_NaN()))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +// The binary form of the legacy input `optional_value: { number_value: inf}`. +TEST_P(JsonValueTest, ValueRejectInfNumberValue) { + EXPECT_THAT(Testee(kP1, "ValueRejectInfNumberValue") + .ParseBinary( + message(), + LengthPrefixedField( + kOptionalValue, + DoubleField(kValueNumberValue, + std::numeric_limits<double>::infinity()))) + .SerializeJson(), + Yields(IsSerializeError())); +} + +TEST_P(JsonValueTest, ListValueDeepNesting25) { + const std::string input = + absl::StrCat(R"({"optionalValue": )", std::string(25, '['), "1", + std::string(25, ']'), "}"); + const std::string expected = absl::StrCat("optional_value: {\n", + Repeat(" list_value: {\n" + " values: {\n", + 25), + " number_value: 1\n", + Repeat(" }\n" + " }\n", + 25), + "}\n"); + EXPECT_THAT(Testee(kP1, "ListValueDeepNesting25") + .ParseJson(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kP1, "ListValueDeepNesting25") + .ParseJson(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(JsonValueTest, ListValueDeepNesting200) { + const std::string input = + absl::StrCat(R"({"optionalValue": )", std::string(200, '['), "1", + std::string(200, ']'), "}"); + EXPECT_THAT(Testee(kP1, "ListValueDeepNesting200") + .ParseJson(message(), input) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(JsonValueTest, ValueDeepNesting25) { + const std::string input = + absl::StrCat(R"({"optionalValue": {)", Repeat(R"("n": {)", 25), + R"("value": 1)", std::string(25, '}'), "}}"); + const std::string expected = + absl::StrCat("optional_value: {\n", " struct_value: {\n", + Repeat(" fields: {\n" + " key: \"n\"\n" + " value: {\n" + " struct_value: {\n", + 25), + " fields: {\n" + " key: \"value\"\n" + " value: { number_value: 1 }\n" + " }\n", + Repeat(" }\n" + " }\n" + " }\n", + 25), + " }\n", "}\n"); + EXPECT_THAT(Testee(kP1, "ValueDeepNesting25") + .ParseJson(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kP1, "ValueDeepNesting25") + .ParseJson(message(), input) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(JsonValueTest, ValueDeepNesting200) { + const std::string input = + absl::StrCat(R"({"optionalValue": {)", Repeat(R"("n": {)", 200), + R"("value": 1)", std::string(200, '}'), "}}"); + EXPECT_THAT(Testee(kP1, "ValueDeepNesting200") + .ParseJson(message(), input) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonValueTest, ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/json_wrapper_test.cc b/conformance/json_wrapper_test.cc new file mode 100644 index 0000000..f86c729 --- /dev/null +++ b/conformance/json_wrapper_test.cc
@@ -0,0 +1,398 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// JSON conformance tests for the well-known wrapper types +// (google.protobuf.BoolValue, Int32Value, ...): a wrapper is represented by +// its value alone, is present even when that value is the default, and is +// unset by `null`. This replaces the wrapper type part of the legacy +// BinaryAndJsonConformanceSuiteImpl<M>::RunJsonTestsForWrapperTypes(), which +// ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidJsonTest(), each input is sent twice: once to be +// serialized as binary ("<name>.ProtobufOutput") and once as JSON +// ("<name>.JsonOutput"). + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using JsonWrapperTest = MessageTypeConformanceTest; + +// Singular wrappers holding their default value are present (a wrapper +// message is set even when its value is the default). +TEST_P(JsonWrapperTest, OptionalBoolWrapper) { + constexpr absl::string_view kInput = + R"json({"optionalBoolWrapper": false})json"; + constexpr absl::string_view kExpected = + R"pb(optional_bool_wrapper: { value: false })pb"; + EXPECT_THAT(Testee("OptionalBoolWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalBoolWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalInt32Wrapper) { + constexpr absl::string_view kInput = R"json({"optionalInt32Wrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int32_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalInt32Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalInt32Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalUint32Wrapper) { + constexpr absl::string_view kInput = + R"json({"optionalUint32Wrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_uint32_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalUint32Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalUint32Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalInt64Wrapper) { + constexpr absl::string_view kInput = R"json({"optionalInt64Wrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_int64_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalInt64Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalInt64Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalUint64Wrapper) { + constexpr absl::string_view kInput = + R"json({"optionalUint64Wrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_uint64_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalUint64Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalUint64Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalFloatWrapper) { + constexpr absl::string_view kInput = R"json({"optionalFloatWrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_float_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalFloatWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalFloatWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalDoubleWrapper) { + constexpr absl::string_view kInput = + R"json({"optionalDoubleWrapper": 0})json"; + constexpr absl::string_view kExpected = + R"pb(optional_double_wrapper: { value: 0 })pb"; + EXPECT_THAT(Testee("OptionalDoubleWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalDoubleWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalStringWrapper) { + constexpr absl::string_view kInput = + R"json({"optionalStringWrapper": ""})json"; + constexpr absl::string_view kExpected = + R"pb(optional_string_wrapper: { value: "" })pb"; + EXPECT_THAT(Testee("OptionalStringWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalStringWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalBytesWrapper) { + constexpr absl::string_view kInput = + R"json({"optionalBytesWrapper": ""})json"; + constexpr absl::string_view kExpected = + R"pb(optional_bytes_wrapper: { value: "" })pb"; + EXPECT_THAT(Testee("OptionalBytesWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalBytesWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, OptionalWrapperTypesWithNonDefaultValue) { + constexpr absl::string_view kInput = R"json({ + "optionalBoolWrapper": true, + "optionalInt32Wrapper": 1, + "optionalUint32Wrapper": 1, + "optionalInt64Wrapper": "1", + "optionalUint64Wrapper": "1", + "optionalFloatWrapper": 1, + "optionalDoubleWrapper": 1, + "optionalStringWrapper": "1", + "optionalBytesWrapper": "AQI=" + })json"; + constexpr absl::string_view kExpected = R"pb( + optional_bool_wrapper: { value: true } + optional_int32_wrapper: { value: 1 } + optional_uint32_wrapper: { value: 1 } + optional_int64_wrapper: { value: 1 } + optional_uint64_wrapper: { value: 1 } + optional_float_wrapper: { value: 1 } + optional_double_wrapper: { value: 1 } + optional_string_wrapper: { value: "1" } + optional_bytes_wrapper: { value: "\x01\x02" } + )pb"; + EXPECT_THAT(Testee("OptionalWrapperTypesWithNonDefaultValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("OptionalWrapperTypesWithNonDefaultValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// Repeated wrappers. +TEST_P(JsonWrapperTest, RepeatedBoolWrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedBoolWrapper": [true, false]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_bool_wrapper: { value: true } + repeated_bool_wrapper: { value: false } + )pb"; + EXPECT_THAT(Testee("RepeatedBoolWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedBoolWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedInt32Wrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedInt32Wrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_int32_wrapper: { value: 0 } + repeated_int32_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedInt32Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedInt32Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedUint32Wrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedUint32Wrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_uint32_wrapper: { value: 0 } + repeated_uint32_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedUint32Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedUint32Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedInt64Wrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedInt64Wrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_int64_wrapper: { value: 0 } + repeated_int64_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedInt64Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedInt64Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedUint64Wrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedUint64Wrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_uint64_wrapper: { value: 0 } + repeated_uint64_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedUint64Wrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedUint64Wrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedFloatWrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedFloatWrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_float_wrapper: { value: 0 } + repeated_float_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedFloatWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedFloatWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedDoubleWrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedDoubleWrapper": [0, 1]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_double_wrapper: { value: 0 } + repeated_double_wrapper: { value: 1 } + )pb"; + EXPECT_THAT(Testee("RepeatedDoubleWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedDoubleWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedStringWrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedStringWrapper": ["", "AQI="]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_string_wrapper: { value: "" } + repeated_string_wrapper: { value: "AQI=" } + )pb"; + EXPECT_THAT(Testee("RepeatedStringWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedStringWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +TEST_P(JsonWrapperTest, RepeatedBytesWrapper) { + constexpr absl::string_view kInput = + R"json({"repeatedBytesWrapper": ["", "AQI="]})json"; + constexpr absl::string_view kExpected = R"pb( + repeated_bytes_wrapper: { value: "" } + repeated_bytes_wrapper: { value: "\x01\x02" } + )pb"; + EXPECT_THAT(Testee("RepeatedBytesWrapper") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("RepeatedBytesWrapper") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +// `null` unsets a wrapper (unlike a Value, for which it is a null value). +TEST_P(JsonWrapperTest, WrapperTypesWithNullValue) { + constexpr absl::string_view kInput = R"json({ + "optionalBoolWrapper": null, + "optionalInt32Wrapper": null, + "optionalUint32Wrapper": null, + "optionalInt64Wrapper": null, + "optionalUint64Wrapper": null, + "optionalFloatWrapper": null, + "optionalDoubleWrapper": null, + "optionalStringWrapper": null, + "optionalBytesWrapper": null, + "repeatedBoolWrapper": null, + "repeatedInt32Wrapper": null, + "repeatedUint32Wrapper": null, + "repeatedInt64Wrapper": null, + "repeatedUint64Wrapper": null, + "repeatedFloatWrapper": null, + "repeatedDoubleWrapper": null, + "repeatedStringWrapper": null, + "repeatedBytesWrapper": null + })json"; + constexpr absl::string_view kExpected = ""; + EXPECT_THAT(Testee("WrapperTypesWithNullValue") + .ParseJson(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); + EXPECT_THAT(Testee("WrapperTypesWithNullValue") + .ParseJson(message(), kInput) + .SerializeJson(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, JsonWrapperTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/matchers.cc b/conformance/matchers.cc index 7ea51a4..cdc516d 100644 --- a/conformance/matchers.cc +++ b/conformance/matchers.cc
@@ -16,27 +16,32 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> #include "absl/base/nullability.h" -#include "absl/base/thread_annotations.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" -#include "absl/synchronization/mutex.h" #include "absl/types/optional.h" +#include "json/reader.h" +#include "json/value.h" #include "conformance/binary_wireformat.h" #include "conformance/conformance.pb.h" +#include "conformance/failure_list.h" +#include "conformance/filtering_test_runner.h" #include "conformance/global_test_environment.h" -#include "conformance/test_manager.h" +#include "conformance/result_record.h" #include "conformance/testee.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/json/json.h" #include "google/protobuf/message.h" #include "google/protobuf/port.h" #include "google/protobuf/text_format.h" +#include "google/protobuf/unknown_field_set.h" #include "google/protobuf/util/field_comparator.h" #include "google/protobuf/util/message_differencer.h" @@ -47,9 +52,15 @@ using ::conformance::ConformanceResponse; using ::conformance::WireFormat; -using ::google::protobuf::conformance::internal::GetGlobalTestManager; -using ::google::protobuf::conformance::internal::TestManager; +using ::google::protobuf::conformance::internal::CurrentGtestResult; +using ::google::protobuf::conformance::internal::FailureList; +using ::google::protobuf::conformance::internal::GetGlobalFailureList; +using ::google::protobuf::conformance::internal::ParseResultRecordMessage; +using ::google::protobuf::conformance::internal::RecordedResult; +using ::google::protobuf::conformance::internal::ResultRecord; +using ::google::protobuf::conformance::internal::ResultRecordMessage; using ::google::protobuf::conformance::internal::TestResult; +using ::google::protobuf::conformance::internal::YieldedResult; // Implements EqualsTextProto() and EqualsBinaryProto(): matches a message // equivalent to the one obtained by decoding `expected` (in `format`) as the @@ -127,8 +138,9 @@ failure_message_(failure_message), runtime_error_failure_message_(runtime_error_failure_message) {} - bool MatchAndExplain(const TestResult& result, + bool MatchAndExplain(const YieldedResult& yielded, testing::MatchResultListener* listener) const { + const TestResult& result = yielded.result(); ConformanceResponse::ResultCase actual = result.response().result_case(); if (actual == expected_result_) { return true; @@ -299,20 +311,32 @@ } return message; } - case ::conformance::JSON: + case ::conformance::JSON: { + // Decoded with the default json::ParseOptions, which is effectively what + // the legacy runner did. + // TODO: Set allow_legacy_nonconformant_behavior = false + // once the failures that surfaces have been triaged across languages. + absl::Status status = json::JsonStringToMessage( + result.response().json_payload(), message.get()); + if (!status.ok()) { + ABSL_LOG(INFO) << "JSON output of " << result.name() + << " is unparseable: " << status; + failure_message = UnparseableMessage(result.format()); + return nullptr; + } + return message; + } case ::conformance::JSPB: case ::conformance::UNSPECIFIED: default: - // TODO: b/410122158 - Support JSON once the JSON suite is migrated. - failure_message = absl::StrCat( - "WhenParsed is not supported for ", WireFormat_Name(result.format()), - " output; use RawPayload() to match the raw JSON text until JSON " - "matching is migrated to gtest (b/410122158)."); - return nullptr; + // PayloadMatcher only calls ParsePayload() once the response carries + // the payload the test asked for, and tests can only ask for PROTOBUF, + // TEXT_FORMAT or JSON output. + google::protobuf::internal::Unreachable(); } } -// Formats binary data the same way the legacy runner does in failure messages. +// Formats binary data the same way the legacy runner did in failure messages. std::string ToOctString(absl::string_view binary_string) { std::string oct_string; for (char ch : binary_string) { @@ -331,7 +355,7 @@ virtual ~PayloadMatcher() = default; - bool MatchAndExplain(const TestResult& result, + bool MatchAndExplain(const YieldedResult& yielded, testing::MatchResultListener* listener) const; void DescribeTo(std::ostream* os) const { DescribeInnerTo(os, /*negation=*/false); @@ -357,7 +381,9 @@ }; bool PayloadMatcher::MatchAndExplain( - const TestResult& result, testing::MatchResultListener* listener) const { + const YieldedResult& yielded, + testing::MatchResultListener* listener) const { + const TestResult& result = yielded.result(); const ConformanceResponse& response = result.response(); switch (response.result_case()) { case ConformanceResponse::RESULT_NOT_SET: @@ -461,8 +487,8 @@ bool RawPayloadMatcher::MatchPayload( const TestResult& result, testing::MatchResultListener* listener) const { - // Like the legacy runner's `require_same_wire_format`, binary output must at - // least be parseable as the test's message type. + // Like the legacy runner's byte-exact mode, binary output must at least be + // parseable as the test's message type. if (result.format() == ::conformance::PROTOBUF && !NewMessage(result.type()) ->ParseFromString(result.response().protobuf_payload())) { @@ -481,51 +507,160 @@ return false; } -// Implements Yields(). This is the single matcher that talks to the global -// TestManager; see matchers.h for the policy it applies. +// Implements JsonPayload(). +class JsonPayloadMatcher : public PayloadMatcher { + public: + explicit JsonPayloadMatcher(testing::Matcher<const Json::Value&> matcher) + : matcher_(std::move(matcher)) {} + + private: + bool MatchPayload(const TestResult& result, + testing::MatchResultListener* listener) const override; + void DescribeInnerTo(std::ostream* os, bool negation) const override; + + testing::Matcher<const Json::Value&> matcher_; +}; + +bool JsonPayloadMatcher::MatchPayload( + const TestResult& result, testing::MatchResultListener* listener) const { + // PayloadMatcher only checked that the payload is in the requested format; + // a test that asked for anything but JSON can't be matched this way. This + // is an authoring bug, so it is deliberately checked only after the base + // class has reported the "real" conformance failures (errors, skips, wrong + // payload kind) that also apply to correctly written tests. + if (result.format() != ::conformance::JSON) { + *listener << "JsonPayload() needs JSON output, but the test asked for " + << WireFormat_Name(result.format()) << " output."; + return false; + } + + // Parsed with jsoncpp's defaults, like the legacy runner's validators. + const std::string& payload = result.response().json_payload(); + Json::CharReaderBuilder builder; + const std::unique_ptr<Json::CharReader> reader(builder.newCharReader()); + Json::Value value; + std::string error; + if (!reader->parse(payload.data(), payload.data() + payload.size(), &value, + &error)) { + *listener << "JSON payload cannot be parsed as valid JSON: " << error; + return false; + } + + testing::StringMatchResultListener inner_listener; + if (matcher_.MatchAndExplain(value, &inner_listener)) { + *listener << inner_listener.str(); + return true; + } + if (inner_listener.str().empty()) { + // JSON is readable, so it is quoted and escaped rather than shown in octal. + *listener << "Expect: JSON payload " + << testing::DescribeMatcher<const Json::Value&>(matcher_) + << ", but got: \"" << absl::CEscape(payload) << "\""; + } else { + *listener << inner_listener.str(); + } + return false; +} + +void JsonPayloadMatcher::DescribeInnerTo(std::ostream* os, + bool negation) const { + *os << "JSON payload " + << testing::DescribeMatcher<const Json::Value&>(matcher_, negation); +} + +// What an earlier check of the conformance test `name` recorded in the gtest +// result Yields() records to, if anything (see ResultRecordMessage()), as +// ResultRecord::ToString(). +absl::optional<std::string> RecordedValue(absl::string_view name) { + const testing::TestResult& gtest_result = CurrentGtestResult(); + for (int i = 0; i < gtest_result.total_part_count(); ++i) { + const testing::TestPartResult& part = gtest_result.GetTestPartResult(i); + if (part.type() != testing::TestPartResult::kSuccess) continue; + absl::StatusOr<RecordedResult> recorded = + ParseResultRecordMessage(part.message()); + if (recorded.ok() && recorded->test_name == name) { + return recorded->record.ToString(); + } + } + return absl::nullopt; +} + +// The status to record for a result that is a failure: kCrash for a testee +// that raised an error or timed out, kFail otherwise (including a response +// with no result at all). +ResultRecord::Status FailureStatus(const ConformanceResponse& response) { + switch (response.result_case()) { + case ConformanceResponse::kRuntimeError: + case ConformanceResponse::kTimeoutError: + return ResultRecord::Status::kCrash; + default: + return ResultRecord::Status::kFail; + } +} + +// Logs what became of a result the first time it is checked, as the legacy +// runner did: a testee skip at INFO, and a failure that doesn't fail the test +// at INFO if it is listed (an expected failure) or as a WARNING if it isn't (a +// tolerated one). +void LogOutcome(const TestResult& result, const ResultRecord& record, + const absl::Status& verdict) { + absl::string_view message = + absl::StripTrailingAsciiWhitespace(record.message); + switch (record.status) { + case ResultRecord::Status::kSkip: + ABSL_LOG(INFO) << "Skipping test " << result.name() << ": " << message; + break; + case ResultRecord::Status::kFail: + case ResultRecord::Status::kCrash: + if (!verdict.ok()) break; + if (GetGlobalFailureList().MatchingEntry(result.name()).has_value()) { + ABSL_LOG(INFO) << "Ignoring expected failure for test " << result.name() + << ": " << message; + } else { + ABSL_LOG(WARNING) << "WARNING, test=" << result.name() << ": " + << message; + } + break; + default: + break; + } +} + +// Implements Yields(). This is the single matcher that reads the global +// failure list; see matchers.h for the policy it applies. // -// The TestManager must only hear about each test once. It remembers the -// names it was told (TestManager::WasReported()), and a result it already -// knows is not evaluated again. gtest evaluates a failing matcher a second -// time, through the same matcher object, to explain the failure. The matcher -// keeps its last failed verdict for that rerun. Anything else that checks a -// reported result is a test bug and fails. -// -// Matchers are nominally immutable. The cache only affects how a failure -// that has already been decided is explained, never a verdict, and copies of -// a Matcher share one impl, so the matcher behaves the same through every -// copy. +// The matcher changes nothing in the process but the gtest result it runs +// in: what it learns about a result goes into a gtest success whose message +// carries a ResultRecord (see ResultRecordMessage()), which the test +// environment tallies as gtest reports it. The success also tells the +// matcher whether the test was checked before. gtest evaluates a failing +// matcher a second time to explain the failure; that evaluation finds its own +// record and only repeats the verdict. A check that reaches a different +// outcome than the recorded one is a test bug and fails. class YieldsMatcherImpl : public testing::MatcherInterface<const TestResult&> { public: - explicit YieldsMatcherImpl(testing::Matcher<const TestResult&> inner) + explicit YieldsMatcherImpl(testing::Matcher<const YieldedResult&> inner) : inner_(std::move(inner)) {} bool MatchAndExplain(const TestResult& result, testing::MatchResultListener* listener) const override { - { - absl::MutexLock lock(&mutex_); - if (last_failure_.has_value() && last_failure_->result == &result && - last_failure_->name == result.name()) { - *listener << last_failure_->explanation; - return false; - } - } - if (GetGlobalTestManager().WasReported(result.name())) { + ResultRecord record; + absl::Status verdict = Evaluate(result, record); + const std::string value = record.ToString(); + absl::optional<std::string> recorded = RecordedValue(result.name()); + if (!recorded.has_value()) { + SUCCEED() << ResultRecordMessage(result.name(), record); + LogOutcome(result, record, verdict); + } else if (*recorded != value) { *listener << "TestResult for " << result.name() - << " was already checked; each result may be checked exactly " - "once"; + << " was already checked, with a different outcome: the first " + "check recorded \"" + << *recorded << "\", this one would record \"" << value + << "\"; each result may be checked once"; return false; } - absl::Status status = Evaluate(result); - *listener << status.message(); - absl::MutexLock lock(&mutex_); - if (status.ok()) { - last_failure_.reset(); - } else { - last_failure_ = LastFailure{&result, std::string(result.name()), - std::string(status.message())}; - } - return status.ok(); + *listener << verdict.message(); + return verdict.ok(); } void DescribeTo(std::ostream* os) const override { @@ -541,60 +676,135 @@ } private: - // Reports `result` to the global TestManager and returns the gtest verdict: - // OK if the test passes, otherwise an error whose message explains why not. - absl::Status Evaluate(const TestResult& result) const { - TestManager& manager = GetGlobalTestManager(); + // Decides `result`: fills in `record` with what to record about it and + // returns the gtest verdict, OK if the test passes, otherwise an error whose + // message explains why not. Reads the global failure list and changes + // nothing. + absl::Status Evaluate(const TestResult& result, ResultRecord& record) const { + const FailureList& failure_list = GetGlobalFailureList(); const std::string name(result.name()); const ConformanceResponse& response = result.response(); + record.priority = result.priority(); - // A skip is decided here; the inner matcher never sees it. The manager - // marks a listed entry as seen and returns an error for it, so this is the - // only place that reports a listed test the testee skipped. + // A test the runner filtered out with --test never ran: it is neither a + // testee skip nor, if listed, a "listed but skipped" failure. The record + // only says that the test exists, so that its failure list entry isn't + // reported as matching no test. + if (response.result_case() == ConformanceResponse::kSkipped && + response.skipped() == FilteringTestRunner::kNotSelectedSkipReason) { + record.status = ResultRecord::Status::kNotSelected; + return absl::OkStatus(); + } + + // A skip is decided here; the inner matcher never sees it. A listed test + // the testee skipped fails (see FailureList::VerdictOnSkip()). if (response.result_case() == ConformanceResponse::kSkipped) { - absl::Status status = manager.ReportSkip(name, response.skipped()); - ABSL_LOG(INFO) << "Skipping test " << name << ": " << response.skipped(); - return status; + record.status = ResultRecord::Status::kSkip; + record.message = response.skipped(); + return failure_list.VerdictOnSkip(name, response.skipped()); } testing::StringMatchResultListener inner_listener; - bool matched = inner_.MatchAndExplain(result, &inner_listener); + bool matched = + inner_.MatchAndExplain(YieldedResult(result), &inner_listener); std::string message = inner_listener.str(); if (matched) { - return manager.ReportSuccess(name); + record.status = ResultRecord::Status::kPass; + return failure_list.VerdictOnSuccess(name); } if (message.empty()) { message = absl::StrCat( "which doesn't match (", - testing::DescribeMatcher<const TestResult&>(inner_), ")"); + testing::DescribeMatcher<const YieldedResult&>(inner_), ")"); } + record.status = FailureStatus(response); + record.message = message; - // The manager decides whether the failure is expected (listed), tolerated - // (above the enforcement level) or unexpected, and logs the first two. - absl::Status status = - manager.ReportFailure(name, result.priority(), message); - if (status.ok()) { - return status; + // The failure list decides whether the failure is expected (listed), + // tolerated (above the enforcement level) or unexpected. + absl::Status verdict = + failure_list.VerdictOnFailure(name, result.priority(), message); + if (verdict.ok()) { + return verdict; } - return absl::Status(status.code(), - absl::StrCat(message, "\n", status.message())); + return absl::Status(verdict.code(), + absl::StrCat(message, "\n", verdict.message())); } - // The failure this matcher explained last, identified by the result's - // address and name, so that gtest's rerun gets the same explanation. A - // destroyed result's address may be reused, but not its name: Testee::Run() - // rejects duplicate test names. - struct LastFailure { - const TestResult* result; - std::string name; - std::string explanation; - }; + testing::Matcher<const YieldedResult&> inner_; +}; - testing::Matcher<const TestResult&> inner_; - mutable absl::Mutex mutex_; - mutable absl::optional<LastFailure> last_failure_ ABSL_GUARDED_BY(mutex_); +// Formats `fields` for descriptions, e.g. `[666: "abc", 666: 123]`. +std::string DescribeUnknownFields(const UnknownFieldSet& fields) { + std::string out = "["; + for (int i = 0; i < fields.field_count(); ++i) { + const UnknownField& field = fields.field(i); + if (i > 0) absl::StrAppend(&out, ", "); + absl::StrAppend(&out, field.number(), ": "); + switch (field.type()) { + case UnknownField::TYPE_VARINT: + absl::StrAppend(&out, field.varint()); + break; + case UnknownField::TYPE_FIXED32: + absl::StrAppend(&out, "fixed32(", field.fixed32(), ")"); + break; + case UnknownField::TYPE_FIXED64: + absl::StrAppend(&out, "fixed64(", field.fixed64(), ")"); + break; + case UnknownField::TYPE_LENGTH_DELIMITED: + absl::StrAppend(&out, "\"", absl::CEscape(field.length_delimited()), + "\""); + break; + case UnknownField::TYPE_GROUP: + absl::StrAppend(&out, DescribeUnknownFields(field.group())); + break; + } + } + return absl::StrCat(out, "]"); +} + +// Implements HasUnknownFieldsInOrder(): matches a message whose unknown field +// set holds exactly the fields of `expected`, in the same order. +// +// UnknownFieldSet serializes its fields in order with a canonical encoding for +// each, so two sets hold equal fields in the same order exactly when their +// serializations are equal. Comparing those also sidesteps copying the +// expected set, which UnknownFieldSet doesn't support. +class UnknownFieldsInOrderMatcher { + public: + using is_gtest_matcher = void; + + explicit UnknownFieldsInOrderMatcher(const UnknownFieldSet& expected) + : description_(DescribeUnknownFields(expected)) { + ABSL_CHECK(expected.SerializeToString(&expected_)) + << "Failed to serialize the expected unknown fields " << description_; + } + + bool MatchAndExplain(const Message& actual, + testing::MatchResultListener* listener) const { + std::string serialized; + if (actual.GetReflection()->GetUnknownFields(actual).SerializeToString( + &serialized) && + serialized == expected_) { + return true; + } + *listener << "Unknown field mismatch"; + return false; + } + void DescribeTo(std::ostream* os) const { + *os << "has exactly the unknown fields " << description_ + << ", in that order"; + } + void DescribeNegationTo(std::ostream* os) const { + *os << "doesn't have exactly the unknown fields " << description_ + << " in that order"; + } + + private: + std::string description_; // `expected`, formatted for descriptions + std::string expected_; // `expected`'s serialization }; } // namespace @@ -617,20 +827,36 @@ } } -testing::Matcher<const TestResult&> MakeWhenParsedMatcher( +testing::Matcher<const YieldedResult&> MakeWhenParsedMatcher( testing::Matcher<const Message&> m, const Descriptor* absl_nullable type_override) { return WhenParsedMatcher(std::move(m), type_override); } +testing::Matcher<const YieldedResult&> MakeJsonPayloadMatcher( + testing::Matcher<const Json::Value&> m) { + return JsonPayloadMatcher(std::move(m)); +} + testing::Matcher<const TestResult&> MakeYieldsMatcher( - testing::Matcher<const TestResult&> inner) { + testing::Matcher<const YieldedResult&> inner) { return testing::MakeMatcher(new YieldsMatcherImpl(std::move(inner))); } +const testing::TestResult& CurrentGtestResult() { + const testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + if (const testing::TestInfo* test_info = unit_test.current_test_info()) { + return *test_info->result(); + } + if (const testing::TestSuite* suite = unit_test.current_test_suite()) { + return suite->ad_hoc_test_result(); + } + return unit_test.ad_hoc_test_result(); +} + } // namespace internal -testing::Matcher<const internal::TestResult&> RawPayload(Wire bytes) { +testing::Matcher<const internal::YieldedResult&> RawPayload(Wire bytes) { return RawPayloadMatcher(std::move(bytes).str()); } @@ -644,14 +870,19 @@ std::move(bytes).str()); } -testing::Matcher<const internal::TestResult&> IsParseError() { +testing::Matcher<const Message&> HasUnknownFieldsInOrder( + const UnknownFieldSet& expected) { + return UnknownFieldsInOrderMatcher(expected); +} + +testing::Matcher<const internal::YieldedResult&> IsParseError() { return FailureMatcher( "parse error", ConformanceResponse::kParseError, "Should have failed to parse, but didn't.", "Should have failed to parse, but raised an error instead."); } -testing::Matcher<const internal::TestResult&> IsSerializeError() { +testing::Matcher<const internal::YieldedResult&> IsSerializeError() { return FailureMatcher( "serialize error", ConformanceResponse::kSerializeError, "Should have failed to serialize, but didn't.",
diff --git a/conformance/matchers.h b/conformance/matchers.h index 080d4bf..52218b1 100644 --- a/conformance/matchers.h +++ b/conformance/matchers.h
@@ -16,28 +16,38 @@ // .SerializeBinary(), // Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 1)pb")))); // -// The leaf matchers (WhenParsed, RawPayload, IsParseError, ...) only look at -// the TestResult, so they compose with other gMock matchers: -// Yields(AnyOf(IsParseError(), WhenParsed(m))) works as expected. -// EqualsTextProto() and EqualsBinaryProto() match a `const Message&` and are -// meant to be used inside WhenParsed(). +// The leaf matchers (WhenParsed, RawPayload, JsonPayload, IsParseError, ...) +// only look at the result, so they compose with other gMock matchers: +// Yields(AnyOf(IsParseError(), WhenParsed(m))) works as expected. They match +// the YieldedResult that Yields() hands them rather than the TestResult, so a +// leaf matcher applied to a TestResult without Yields() does not compile. +// EqualsTextProto(), EqualsBinaryProto() and HasUnknownFieldsInOrder() match +// a `const Message&` and are meant to be used inside WhenParsed(). // -// Yields() is the one matcher that talks to the global TestManager. It -// records the outcome of the test and turns the failure list into the gtest -// verdict: +// Yields() is the one matcher that reads the global failure list (see +// GetGlobalFailureList() in global_test_environment.h). It turns the failure +// list into the gtest verdict: // // - A failure that is in the failure list passes. A listed test that // succeeds fails. // - A failure of a test above the enforcement level is tolerated unless // the test is listed. kP0 failures always count (see TestPriority). -// - A test the testee skipped passes unless it is listed. The TestManager -// counts it in ListedSkips(). +// - A test the testee skipped passes unless it is listed. The test +// environment counts it as a listed skip. +// - A test the runner filtered out with --test is not counted at all. +// +// Yields() changes nothing in the process. It records the outcome of the +// test as a gtest success in the running test, whose message carries an +// internal::ResultRecord (see ResultRecordMessage() in result_record.h). The +// test environment picks the records up as gtest reports them, tallies them in +// its ResultLedger and reports each test's outcome as a property of the gtest +// test (see test_environment.h). // // The failure messages of the leaf matchers end up in failure lists, so each // matcher documents its message and keeps it stable. Messages that come from // an inner matcher use gMock's wording, which can change between versions. // Failure list entries for those should only use a prefix of the message (see -// TestManager::ReportFailure()). +// ResultLedger::VerdictOnFailure()). // // This file also defines PrintTo() for TestResult, which gtest uses to print a // result when a matcher on it fails. @@ -52,16 +62,38 @@ #include <gtest/gtest.h> #include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "json/value.h" #include "conformance/binary_wireformat.h" #include "conformance/testee.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" +#include "google/protobuf/unknown_field_set.h" namespace google { namespace protobuf { namespace conformance { namespace internal { +// The view of a TestResult that Yields() hands to its inner matcher. The +// leaf matchers match this rather than the TestResult itself, which is what +// makes a forgotten Yields() a compile error: +// +// EXPECT_THAT(Testee().ParseBinary(type, input).SerializeBinary(), +// IsParseError()); // error: TestResult isn't a YieldedResult +// +// It is a view: the result must outlive it. Only Yields() and the tests of +// the leaf matchers construct one. +class YieldedResult { + public: + explicit YieldedResult(const TestResult& result) : result_(&result) {} + YieldedResult(TestResult&&) = delete; + + const TestResult& result() const { return *result_; } + + private: + const TestResult* absl_nonnull result_; +}; + // Prints a TestResult in gtest failure output. The output has the test's // priority and name and a short form of the response. Long payloads are // truncated. Binary payloads are also decoded as the test's message type. @@ -73,19 +105,31 @@ // Implements WhenParsed() and WhenParsedAs() below. The payload is // decoded as `type_override` if it is non-null, and as the test's message type // otherwise. -testing::Matcher<const TestResult&> MakeWhenParsedMatcher( +testing::Matcher<const YieldedResult&> MakeWhenParsedMatcher( testing::Matcher<const Message&> m, const Descriptor* absl_nullable type_override = nullptr); +// Likewise for JsonPayload(m). +testing::Matcher<const YieldedResult&> MakeJsonPayloadMatcher( + testing::Matcher<const Json::Value&> m); + // Implements Yields(): see the function below for the semantics. testing::Matcher<const TestResult&> MakeYieldsMatcher( - testing::Matcher<const TestResult&> inner); + testing::Matcher<const YieldedResult&> inner); + +// The gtest result that a test part result reported now goes to: the running +// test's, else (in SetUpTestSuite() or TearDownTestSuite()) the current test +// suite's ad hoc result, else the whole run's. Yields() records what it +// learns about each conformance test as a success there, and finds there what +// an earlier check of the same test recorded. +const testing::TestResult& CurrentGtestResult(); } // namespace internal // Matches a result whose payload, decoded as the test's message type, matches -// `m`. The payload is decoded from binary or text output. `m` is any matcher -// on `const Message&`, usually EqualsTextProto() or EqualsBinaryProto(). +// `m`. The payload is decoded from whichever output format the test asked +// for. `m` is any matcher on `const Message&`, usually EqualsTextProto() or +// EqualsBinaryProto(). // // EXPECT_THAT( // Testee().ParseBinary(type, input).SerializeBinary(), @@ -93,10 +137,8 @@ // // A payload that can't be decoded fails with "<format> output we received // from test was unparseable." Otherwise the failure message comes from `m`. -// JSON output can't be decoded yet (b/410122158) and fails with a message -// saying so. Use RawPayload() for it. template <typename M> -testing::Matcher<const internal::TestResult&> WhenParsed(M m) { +testing::Matcher<const internal::YieldedResult&> WhenParsed(M m) { return internal::MakeWhenParsedMatcher( testing::SafeMatcherCast<const Message&>(std::move(m))); } @@ -112,7 +154,7 @@ // // Failure messages are the same as WhenParsed()'s. template <typename T, typename M> -testing::Matcher<const internal::TestResult&> WhenParsedAs(M m) { +testing::Matcher<const internal::YieldedResult&> WhenParsedAs(M m) { return internal::MakeWhenParsedMatcher( testing::SafeMatcherCast<const Message&>(std::move(m)), T::descriptor()); } @@ -129,7 +171,25 @@ // result, an error, a skipped test and the wrong output format. A PROTOBUF // payload must also be parseable as the test's message type. Otherwise it // fails with "Protobuf output we received from test was unparseable." -testing::Matcher<const internal::TestResult&> RawPayload(Wire bytes); +testing::Matcher<const internal::YieldedResult&> RawPayload(Wire bytes); + +// Matches a result whose JSON payload, parsed with jsoncpp, matches `m`. `m` +// is any matcher on `const Json::Value&`. The test must have asked for JSON +// output. +// +// EXPECT_THAT(Testee().ParseJson(type, input).SerializeJson(), +// Yields(JsonPayload(HasJsonMember("optionalInt32")))); +// +// Use this instead of WhenParsed() when the JSON text itself matters, for +// example to check that an int64 field is serialized as a string. +// +// A payload that isn't valid JSON fails with "JSON payload cannot be parsed +// as valid JSON: <error>". Otherwise the failure message comes from `m`. +template <typename M> +testing::Matcher<const internal::YieldedResult&> JsonPayload(M m) { + return internal::MakeJsonPayloadMatcher( + testing::SafeMatcherCast<const Json::Value&>(std::move(m))); +} // Matches a message equivalent to `text`, parsed as the actual message's type. // Messages are compared with MessageDifferencer, with NaN equal to NaN. @@ -147,7 +207,18 @@ // Yields(WhenParsed(EqualsBinaryProto(VarintField(1, 1))))); testing::Matcher<const Message&> EqualsBinaryProto(Wire bytes); -// TODO: b/410122158 - Add JSON matchers once JSON support is migrated. +// Matches a message whose unknown field set holds exactly the fields of +// `expected`, in the same order. Known fields are ignored. Meant for use +// inside WhenParsed() or WhenParsedAs(): +// +// UnknownFieldSet expected; +// expected.AddVarint(1, 1); +// EXPECT_THAT(Testee().ParseBinary(type, input).SerializeBinary(), +// Yields(WhenParsed(HasUnknownFieldsInOrder(expected)))); +// +// The failure message is "Unknown field mismatch". +testing::Matcher<const Message&> HasUnknownFieldsInOrder( + const UnknownFieldSet& expected); // Matches a response that reports a parse error. // @@ -157,7 +228,7 @@ // Any other response fails with "Should have failed to parse, but didn't." A // runtime error fails with "Should have failed to parse, but raised an error // instead." -testing::Matcher<const internal::TestResult&> IsParseError(); +testing::Matcher<const internal::YieldedResult&> IsParseError(); // Matches a response that reports a serialize error. // @@ -166,28 +237,33 @@ // // Any other response fails like it does for IsParseError(), with the // corresponding "Should have failed to serialize, ..." messages. -testing::Matcher<const internal::TestResult&> IsSerializeError(); +testing::Matcher<const internal::YieldedResult&> IsSerializeError(); // Wraps the matcher of every conformance test's EXPECT_THAT. // // EXPECT_THAT(Testee().ParseBinary(type, input).SerializeBinary(), // Yields(WhenParsed(EqualsBinaryProto(input)))); // -// Yields() evaluates `inner` against the TestResult, reports the outcome to -// the global TestManager and applies the failure list and priority rules -// described at the top of this file. Its verdict is not simply `inner`'s: an -// expected failure passes and an unexpected success fails. Do not wrap -// Yields() in Not() or other combinators. Compose `inner` instead. +// Yields() evaluates `inner` against the result, wrapped in an +// internal::YieldedResult (which is what the leaf matchers match), records +// the outcome in the running gtest test and applies the failure list and +// priority rules described at the top of this file. Its verdict is not +// simply `inner`'s: an expected failure passes and an unexpected success +// fails. Do not wrap Yields() in Not() or other combinators. Compose `inner` +// instead. // -// Each test may be checked exactly once. The first evaluation reports the -// result to the TestManager, which remembers the test's name. gtest -// evaluates a failing matcher a second time to explain the failure; Yields() -// replays its own failed verdict for that. Any other check of a result the -// TestManager has already heard about fails with "already checked". +// Each result is meant to be checked once. gtest evaluates a failing matcher +// a second time to explain the failure, so a second check is allowed as long +// as it reaches the same outcome, which it then only repeats. A check that +// reaches a different outcome than the one recorded is a test bug and fails +// with "already checked". A gtest test that runs a conformance test but +// never checks its result fails too (see ConformanceEnvironment in +// test_environment.h). template <typename M> testing::Matcher<const internal::TestResult&> Yields(M inner) { return internal::MakeYieldsMatcher( - testing::SafeMatcherCast<const internal::TestResult&>(std::move(inner))); + testing::SafeMatcherCast<const internal::YieldedResult&>( + std::move(inner))); } } // namespace conformance
diff --git a/conformance/matchers_test.cc b/conformance/matchers_test.cc index a8e3e19..83c2d7e 100644 --- a/conformance/matchers_test.cc +++ b/conformance/matchers_test.cc
@@ -12,10 +12,11 @@ #include <iterator> #include <ostream> #include <string> +#include <type_traits> #include <utility> +#include <vector> #include <gmock/gmock.h> -#include <gtest/gtest-spi.h> #include <gtest/gtest.h> #include "absl/base/log_severity.h" #include "absl/log/absl_check.h" @@ -26,16 +27,22 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "json/value.h" #include "conformance/binary_wireformat.h" #include "conformance/conformance.pb.h" +#include "conformance/failure_list.h" #include "conformance/global_test_environment.h" +#include "conformance/json_test_util.h" +#include "conformance/matchers_testing.h" #include "conformance/mock_test_runner.h" -#include "conformance/test_manager.h" +#include "conformance/result_ledger.h" +#include "conformance/result_record.h" #include "conformance/testee.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/message.h" #include "google/protobuf/test_messages_proto2.pb.h" #include "google/protobuf/text_format.h" +#include "google/protobuf/unknown_field_set.h" namespace google { namespace protobuf { @@ -46,13 +53,13 @@ // Backing state for the mocked-out global environment below. In the real // conformance binary this would be populated from CLI flags and the failure // list on disk, but we don't want to drag that in for a unit test. -TestManager* global_test_manager = nullptr; +const FailureList* global_failure_list = nullptr; } // namespace -TestManager& GetGlobalTestManager() { - ABSL_CHECK(global_test_manager != nullptr); - return *global_test_manager; +const FailureList& GetGlobalFailureList() { + ABSL_CHECK(global_failure_list != nullptr); + return *global_failure_list; } } // namespace internal @@ -63,8 +70,10 @@ using ::conformance::ConformanceResponse; using ::conformance::WireFormat; using ::google::protobuf::conformance::TestPriority; -using ::google::protobuf::conformance::internal::TestManager; +using ::google::protobuf::conformance::internal::ResultLedger; +using ::google::protobuf::conformance::internal::ResultRecord; using ::google::protobuf::conformance::internal::TestResult; +using ::google::protobuf::conformance::internal::YieldedResult; using ::protobuf_test_messages::proto2::TestAllTypesProto2; using ::protobuf_test_messages::proto2::UnknownToTestAllTypes; using ::testing::_; @@ -82,6 +91,8 @@ using ::testing::Pair; using ::testing::PrintToString; using ::testing::Return; +using ::testing::SizeIs; +using ::testing::StartsWith; using ::testing::TestParamInfo; using ::testing::Truly; using ::testing::Value; @@ -93,9 +104,9 @@ // `response`. The matchers never look at the input payload itself, only at // its format, so the input is empty. // -// The leaf matchers never touch the TestManager; the tests make sure of it by -// not installing one at all while they run. Only YieldsTest below installs -// one. +// The leaf matchers never touch the failure list; the tests make sure of it +// by not installing one at all while they run. Only YieldsTest below +// installs one. TestResult CreateResult(absl::string_view test_name, WireFormat input_format, WireFormat output_format, ConformanceResponse response) { @@ -149,14 +160,14 @@ return response; } -template <class M> -std::string Describe(const M& m) { - return testing::DescribeMatcher<const TestResult&>(m, /*negation=*/false); +template <class T> +std::string Describe(const testing::Matcher<T>& m) { + return testing::DescribeMatcher<T>(m, /*negation=*/false); } -template <class M> -std::string DescribeNegation(const M& m) { - return testing::DescribeMatcher<const TestResult&>(m, /*negation=*/true); +template <class T> +std::string DescribeNegation(const testing::Matcher<T>& m) { + return testing::DescribeMatcher<T>(m, /*negation=*/true); } // Runs `m` on `arg` once. The returned AssertionResult succeeds iff `m` @@ -170,6 +181,13 @@ << listener.str(); } +// The leaf matchers match a YieldedResult; this overload wraps `result` in +// one, so that their tests can pass the TestResult. +testing::AssertionResult Explain( + const testing::Matcher<const YieldedResult&>& m, const TestResult& result) { + return Explain(m, YieldedResult(result)); +} + // Matches an Explain() result whose matcher accepted the value and whose // explanation matches `explanation` (a string or a matcher on std::string). MATCHER_P(Accepts, explanation, @@ -282,6 +300,15 @@ } // --------------------------------------------------------------------------- +// YieldedResult +// --------------------------------------------------------------------------- + +// What makes a forgotten Yields() a compile error: the leaf matchers match a +// YieldedResult, and a TestResult does not convert to one. +static_assert(!std::is_convertible_v<const TestResult&, YieldedResult>); +static_assert(!std::is_constructible_v<YieldedResult, TestResult>); + +// --------------------------------------------------------------------------- // Descriptions // --------------------------------------------------------------------------- @@ -328,6 +355,36 @@ "doesn't equal binary proto \"\\010\\t\""); } +TEST(MatcherDescriptionTest, HasUnknownFieldsInOrder) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + expected.AddVarint(666, 123); + EXPECT_EQ(testing::DescribeMatcher<const Message&>( + HasUnknownFieldsInOrder(expected)), + "has exactly the unknown fields [666: \"abc\", 666: 123], in that " + "order"); + EXPECT_EQ(testing::DescribeMatcher<const Message&>( + HasUnknownFieldsInOrder(expected), /*negation=*/true), + "doesn't have exactly the unknown fields [666: \"abc\", 666: 123] " + "in that order"); +} + +TEST(MatcherDescriptionTest, HasUnknownFieldsInOrderCoversEveryFieldKind) { + UnknownFieldSet expected; + expected.AddVarint(1, 2); + expected.AddFixed32(3, 4); + expected.AddFixed64(5, 6); + expected.AddLengthDelimited(7, "\001\n"); + expected.AddGroup(8)->AddVarint(9, 10); + EXPECT_EQ(testing::DescribeMatcher<const Message&>( + HasUnknownFieldsInOrder(expected)), + "has exactly the unknown fields [1: 2, 3: fixed32(4), 5: " + "fixed64(6), 7: \"\\001\\n\", 8: [9: 10]], in that order"); + EXPECT_EQ(testing::DescribeMatcher<const Message&>( + HasUnknownFieldsInOrder(UnknownFieldSet())), + "has exactly the unknown fields [], in that order"); +} + TEST(MatcherDescriptionTest, FailureMatchers) { EXPECT_EQ(Describe(IsParseError()), "is a parse error"); EXPECT_EQ(DescribeNegation(IsParseError()), "is not a parse error"); @@ -484,16 +541,47 @@ Rejects("Protobuf output we received from test was unparseable.")); } -TEST(WhenParsedTest, JsonOutputIsNotSupported) { - TestResult result = - CreateResult("foo", ::conformance::JSON, - ResponseFromText(R"pb(json_payload: "{}")pb")); +TEST(WhenParsedTest, JsonOutput) { + TestResult result = CreateResult( + "foo", ::conformance::JSON, + ResponseFromText(R"pb(json_payload: "{\"optionalInt32\": 9}")pb")); EXPECT_THAT( - Explain(WhenParsed(_), result), - Rejects("WhenParsed is not supported for JSON output; use " - "RawPayload() to match the raw JSON text until JSON matching is " - "migrated to gtest (b/410122158).")); + Explain(WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")), result), + Accepts(IsEmpty())); + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsBinaryProto(VarintField(1, 9))))); + EXPECT_THAT( + Explain(WhenParsed(EqualsTextProto(R"pb(optional_int32: 1)pb")), result), + Rejects("Output was not equivalent to reference message: " + "modified: optional_int32: 1 -> 9\n")); +} + +TEST(WhenParsedTest, UnparseableJsonPayload) { + TestResult result = CreateResult( + "foo", ::conformance::JSON, + ResponseFromText(R"pb(json_payload: "{\"nonsense\": 1}")pb")); + + EXPECT_THAT(Explain(WhenParsed(_), result), + Rejects("JSON output we received from test was unparseable.")); + EXPECT_THAT( + Explain(WhenParsed(_), + CreateResult("bar", ::conformance::JSON, + ResponseFromText(R"pb(json_payload: "{")pb"))), + Rejects("JSON output we received from test was unparseable.")); +} + +TEST(WhenParsedTest, JsonOutputIsDecodedWithLegacyLeniency) { + // The legacy runner decoded JSON output with the default ParseOptions, + // which tolerate some non-conformant output; so does WhenParsed() until + // b/563658359 tightens it. Single-quoted strings are one such case; this + // test documents the current behaviour rather than endorsing it. + TestResult result = CreateResult( + "foo", ::conformance::JSON, + ResponseFromText(R"pb(json_payload: "{'optionalInt32': 9}")pb")); + + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); } TEST(WhenParsedTest, TextOutput) { @@ -501,8 +589,8 @@ "foo", ::conformance::TEXT_FORMAT, ResponseFromText(R"pb(text_payload: "optional_int32: 9")pb")); - EXPECT_TRUE( - Value(result, WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); } TEST(WhenParsedTest, TextOutputFieldNumbersAccepted) { @@ -512,8 +600,8 @@ CreateResult("foo", ::conformance::TEXT_FORMAT, ResponseFromText(R"pb(text_payload: "1: 9")pb")); - EXPECT_TRUE( - Value(result, WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); } TEST(WhenParsedTest, EqualsTextProtoMismatch) { @@ -534,7 +622,7 @@ CreateResult("foo", ::conformance::PROTOBUF, ProtobufPayload(Wire(message.SerializeAsString()))); - EXPECT_TRUE(Value(result, WhenParsed(EqualsTextProto(R"pb( + EXPECT_TRUE(Value(YieldedResult(result), WhenParsed(EqualsTextProto(R"pb( optional_float: nan optional_double: nan )pb")))); @@ -554,8 +642,8 @@ ProtobufPayload(VarintField(1, 2))); // Equivalent, but not byte-identical (over-long varint encoding). - EXPECT_TRUE( - Value(result, WhenParsed(EqualsBinaryProto(LongVarintField(1, 2, 1))))); + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsBinaryProto(LongVarintField(1, 2, 1))))); } TEST(WhenParsedTest, EqualsBinaryProtoWithTextOutput) { @@ -563,7 +651,8 @@ "foo", ::conformance::TEXT_FORMAT, ResponseFromText(R"pb(text_payload: "optional_int32: 9")pb")); - EXPECT_TRUE(Value(result, WhenParsed(EqualsBinaryProto(VarintField(1, 9))))); + EXPECT_TRUE(Value(YieldedResult(result), + WhenParsed(EqualsBinaryProto(VarintField(1, 9))))); } TEST(WhenParsedTest, EqualsBinaryProtoMismatch) { @@ -588,9 +677,9 @@ TestResult result = CreateResult("foo", ::conformance::PROTOBUF, ProtobufPayload(VarintField(1, 2))); - EXPECT_TRUE(Value(result, WhenParsed(_))); + EXPECT_TRUE(Value(YieldedResult(result), WhenParsed(_))); EXPECT_TRUE(Value( - result, + YieldedResult(result), WhenParsed(AllOf(EqualsTextProto(R"pb(optional_int32: 2)pb"), Not(EqualsTextProto(R"pb(optional_int32: 3)pb")))))); } @@ -612,11 +701,12 @@ TestResult result = CreateResult("foo", ::conformance::PROTOBUF, ProtobufPayload(VarintField(1, 2))); - EXPECT_TRUE(Value(result, Not(WhenParsed(EqualsTextProto(""))))); + YieldedResult yielded(result); + EXPECT_TRUE(Value(yielded, Not(WhenParsed(EqualsTextProto(""))))); EXPECT_TRUE(Value( - result, AnyOf(IsParseError(), - WhenParsed(EqualsTextProto(R"pb(optional_int32: 2)pb"))))); - EXPECT_TRUE(Value(result, AllOf(Not(IsParseError()), WhenParsed(_)))); + yielded, AnyOf(IsParseError(), + WhenParsed(EqualsTextProto(R"pb(optional_int32: 2)pb"))))); + EXPECT_TRUE(Value(yielded, AllOf(Not(IsParseError()), WhenParsed(_)))); } // --------------------------------------------------------------------------- @@ -719,7 +809,7 @@ TestResult result = CreateResult("foo", ::conformance::PROTOBUF, ProtobufPayload(VarintField(1, 2))); - EXPECT_TRUE(Value(result, RawPayload(VarintField(1, 2)))); + EXPECT_TRUE(Value(YieldedResult(result), RawPayload(VarintField(1, 2)))); EXPECT_THAT(Explain(RawPayload(VarintField(1, 2)), result), Accepts(IsEmpty())); } @@ -736,8 +826,8 @@ } TEST(RawPayloadTest, UnparseableProtobufOutputFailsEvenForTheExactBytes) { - // Like the legacy runner's require_same_wire_format, binary output must - // decode as the test's message type. + // Like the legacy runner's byte-exact mode, binary output must decode as + // the test's message type. TestResult result = CreateResult("foo", ::conformance::PROTOBUF, ProtobufPayload(Wire("\001"))); @@ -751,12 +841,13 @@ "foo", ::conformance::TEXT_FORMAT, ResponseFromText(R"pb(text_payload: "optional_int32: 9")pb")); - EXPECT_TRUE(Value(result, RawPayload(Wire("optional_int32: 9")))); + EXPECT_TRUE( + Value(YieldedResult(result), RawPayload(Wire("optional_int32: 9")))); // Text output isn't required to be parseable. - EXPECT_TRUE(Value( + TestResult nonsense = CreateResult("bar", ::conformance::TEXT_FORMAT, - ResponseFromText(R"pb(text_payload: "nonsense: 1")pb")), - RawPayload(Wire("nonsense: 1")))); + ResponseFromText(R"pb(text_payload: "nonsense: 1")pb")); + EXPECT_TRUE(Value(YieldedResult(nonsense), RawPayload(Wire("nonsense: 1")))); } TEST(RawPayloadTest, JsonOutput) { @@ -765,7 +856,7 @@ CreateResult("foo", ::conformance::JSON, ResponseFromText(R"pb(json_payload: "{}")pb")); - EXPECT_TRUE(Value(result, RawPayload(Wire("{}")))); + EXPECT_TRUE(Value(YieldedResult(result), RawPayload(Wire("{}")))); } // --------------------------------------------------------------------------- @@ -832,6 +923,172 @@ } // --------------------------------------------------------------------------- +// HasUnknownFieldsInOrder() +// --------------------------------------------------------------------------- + +// Field 666 is defined by none of the test messages, so it always stays +// unknown. + +TEST(HasUnknownFieldsInOrderTest, SameFieldsInSameOrder) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddLengthDelimited(666, "abc"); + message.mutable_unknown_fields()->AddVarint(666, 123); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Accepts(IsEmpty())); +} + +TEST(HasUnknownFieldsInOrderTest, DifferentOrder) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddVarint(666, 123); + message.mutable_unknown_fields()->AddLengthDelimited(666, "abc"); + + auto matcher = HasUnknownFieldsInOrder(expected); + EXPECT_THAT(Explain(matcher, message), Rejects("Unknown field mismatch")); + EXPECT_THAT(message, Not(matcher)); +} + +TEST(HasUnknownFieldsInOrderTest, MissingField) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddLengthDelimited(666, "abc"); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, ExtraField) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddLengthDelimited(666, "abc"); + message.mutable_unknown_fields()->AddVarint(666, 123); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, DifferentValue) { + UnknownFieldSet expected; + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddVarint(666, 124); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, DifferentFieldNumber) { + UnknownFieldSet expected; + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddVarint(667, 123); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, DifferentWireType) { + UnknownFieldSet expected; + expected.AddVarint(666, 1); + TestAllTypesProto2 message; + message.mutable_unknown_fields()->AddFixed32(666, 1); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, GroupsAreComparedByContent) { + UnknownFieldSet expected; + expected.AddGroup(666)->AddVarint(1, 2); + TestAllTypesProto2 same; + same.mutable_unknown_fields()->AddGroup(666)->AddVarint(1, 2); + TestAllTypesProto2 different; + different.mutable_unknown_fields()->AddGroup(666)->AddVarint(1, 3); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), same), + Accepts(IsEmpty())); + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), different), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, EmptyExpectedMatchesOnlyNoUnknownFields) { + TestAllTypesProto2 without; + TestAllTypesProto2 with; + with.mutable_unknown_fields()->AddVarint(666, 1); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(UnknownFieldSet()), without), + Accepts(IsEmpty())); + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(UnknownFieldSet()), with), + Rejects("Unknown field mismatch")); +} + +TEST(HasUnknownFieldsInOrderTest, IgnoresKnownFields) { + UnknownFieldSet expected; + expected.AddVarint(666, 123); + TestAllTypesProto2 message; + message.set_optional_int32(9); + message.set_optional_string("known"); + message.mutable_unknown_fields()->AddVarint(666, 123); + + EXPECT_THAT(Explain(HasUnknownFieldsInOrder(expected), message), + Accepts(IsEmpty())); +} + +// The end-to-end case: the mismatch text is what ends up in failure lists. +TEST(WhenParsedTest, HasUnknownFieldsInOrderKeepsTheLegacyMessage) { + UnknownFieldSet expected; + expected.AddLengthDelimited(666, "abc"); + expected.AddVarint(666, 123); + TestResult in_order = + CreateResult("foo", ::conformance::PROTOBUF, + ProtobufPayload(Wire(LengthPrefixedField(666, "abc"), + VarintField(666, 123)))); + TestResult reordered = + CreateResult("foo", ::conformance::PROTOBUF, + ProtobufPayload(Wire(VarintField(666, 123), + LengthPrefixedField(666, "abc")))); + + EXPECT_THAT(Explain(WhenParsed(HasUnknownFieldsInOrder(expected)), in_order), + Accepts(IsEmpty())); + EXPECT_THAT(Explain(WhenParsed(HasUnknownFieldsInOrder(expected)), reordered), + Rejects("Unknown field mismatch")); +} + +TEST(WhenParsedAsTest, HasUnknownFieldsInOrderSeesTheGivenTypesUnknowns) { + // Field 1001 is unknown to the test's type (TestAllTypesProto2) but is + // optional_int32 of the shadow type UnknownToTestAllTypes; 666 is unknown + // to both. + TestResult result = CreateResult( + "foo", ::conformance::PROTOBUF, + ProtobufPayload(Wire(VarintField(1001, 7), VarintField(666, 1)))); + UnknownFieldSet both; + both.AddVarint(1001, 7); + both.AddVarint(666, 1); + UnknownFieldSet only_666; + only_666.AddVarint(666, 1); + + EXPECT_THAT(Explain(WhenParsed(HasUnknownFieldsInOrder(both)), result), + Accepts(IsEmpty())); + EXPECT_THAT(Explain(WhenParsedAs<UnknownToTestAllTypes>( + HasUnknownFieldsInOrder(only_666)), + result), + Accepts(IsEmpty())); + EXPECT_THAT(Explain(WhenParsedAs<UnknownToTestAllTypes>( + HasUnknownFieldsInOrder(both)), + result), + Rejects("Unknown field mismatch")); +} + +// --------------------------------------------------------------------------- // Failure matchers // --------------------------------------------------------------------------- @@ -841,7 +1098,7 @@ ResponseFromText(R"pb(parse_error: "failed to parse")pb")); EXPECT_THAT(Explain(IsParseError(), result), Accepts(IsEmpty())); - EXPECT_FALSE(Value(result, IsSerializeError())); + EXPECT_FALSE(Value(YieldedResult(result), IsSerializeError())); } TEST(FailureMatcherTest, ParseErrorWithPayload) { @@ -895,7 +1152,7 @@ ResponseFromText(R"pb(serialize_error: "failed")pb")); EXPECT_THAT(Explain(IsSerializeError(), result), Accepts(IsEmpty())); - EXPECT_FALSE(Value(result, IsParseError())); + EXPECT_FALSE(Value(YieldedResult(result), IsParseError())); } TEST(FailureMatcherTest, SerializeErrorWithPayload) { @@ -921,16 +1178,18 @@ CreateResult("foo", ::conformance::PROTOBUF, ResponseFromText(R"pb(parse_error: "failed to parse")pb")); - EXPECT_TRUE(Value(result, Not(IsSerializeError()))); - EXPECT_TRUE(Value(result, AnyOf(IsSerializeError(), IsParseError()))); - EXPECT_FALSE(Value(result, AllOf(IsSerializeError(), IsParseError()))); + EXPECT_TRUE(Value(YieldedResult(result), Not(IsSerializeError()))); + EXPECT_TRUE( + Value(YieldedResult(result), AnyOf(IsSerializeError(), IsParseError()))); + EXPECT_FALSE( + Value(YieldedResult(result), AllOf(IsSerializeError(), IsParseError()))); } // --------------------------------------------------------------------------- // Yields() // --------------------------------------------------------------------------- -// A snapshot of the TestManager counters, for concise assertions. +// A snapshot of the ResultLedger counters, for concise assertions. struct Counts { int expected_successes = 0; int expected_failures = 0; @@ -960,16 +1219,16 @@ } }; -Counts GetCounts(const TestManager& manager) { +Counts GetCounts(const ResultLedger& ledger) { return Counts{ - /*expected_successes=*/manager.expected_successes(), - /*expected_failures=*/manager.expected_failures(), - /*unexpected_successes=*/manager.unexpected_successes(), - /*unexpected_failures=*/manager.unexpected_failures(), - /*skipped=*/manager.skipped(), - /*listed_skips=*/manager.listed_skips(), + /*expected_successes=*/ledger.expected_successes(), + /*expected_failures=*/ledger.expected_failures(), + /*unexpected_successes=*/ledger.unexpected_successes(), + /*unexpected_failures=*/ledger.unexpected_failures(), + /*skipped=*/ledger.skipped(), + /*listed_skips=*/ledger.listed_skips(), /*tolerated_failures=*/ - manager.tolerated_failures(), + ledger.tolerated_failures(), }; } @@ -985,6 +1244,7 @@ constexpr absl::string_view kSerializeError = R"pb(serialize_error: "can't serialize")pb"; constexpr absl::string_view kRuntimeError = R"pb(runtime_error: "crashed")pb"; +constexpr absl::string_view kTimeoutError = R"pb(timeout_error: "too slow")pb"; constexpr absl::string_view kSkipped = R"pb(skipped: "not supported")pb"; // The legacy failure messages the canned responses lead to. @@ -1015,10 +1275,10 @@ class YieldsTest : public testing::Test { protected: - YieldsTest() { internal::global_test_manager = &test_manager_; } + YieldsTest() { internal::global_failure_list = &ledger_.failure_list(); } ~YieldsTest() override { - test_manager_.Finalize().IgnoreError(); - internal::global_test_manager = nullptr; + ledger_.Finalize().IgnoreError(); + internal::global_failure_list = nullptr; } // Makes the testee answer every test with `response`. @@ -1052,13 +1312,27 @@ // failure list file would. void AddToFailureList(absl::string_view test_name, absl::string_view message) { - ASSERT_THAT(test_manager_.LoadFailureList(WriteTempFile( + ASSERT_THAT(ledger_.LoadFailureList(WriteTempFile( absl::StrCat("failure_list_", ++failure_lists_, ".txt"), absl::StrCat(test_name, " # ", message, "\n"))), IsOk()); } - TestManager test_manager_; + // Replays the outcomes Yields() recorded on the running gtest test into + // ledger_, as the test environment's listener does as they arrive, + // and returns the counters. Tests the ledger has heard about are skipped, + // so this can be called repeatedly. + Counts ReplayedCounts() { + for (const auto& [name, value] : + internal::RecordedResults(internal::CurrentGtestResult())) { + if (!ledger_.WasReported(name)) { + ledger_.Report(name, *ResultRecord::Parse(value)); + } + } + return GetCounts(ledger_); + } + + ResultLedger ledger_; NiceMock<MockTestRunner> runner_; internal::Testee testee_{&runner_}; int failure_lists_ = 0; @@ -1070,8 +1344,20 @@ EXPECT_THAT(Explain(matcher, Run(kPayload2)), Accepts(IsEmpty())); EXPECT_THAT(Run(kPayload2, TestPriority::kP0, "other"), matcher); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/2})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); +} + +TEST_F(YieldsTest, InnerMatcherSeesTheResultAsAYieldedResult) { + const TestResult* seen = nullptr; + auto matcher = Yields(Truly([&](const YieldedResult& yielded) { + seen = &yielded.result(); + return true; + })); + TestResult result = Run(kPayload2); + + EXPECT_THAT(Explain(matcher, result), Accepts(IsEmpty())); + EXPECT_EQ(seen, &result); } TEST_F(YieldsTest, UnlistedFailureFails) { @@ -1082,14 +1368,14 @@ EXPECT_THAT(Explain(matcher, result), Rejects(absl::StrCat(kMismatch1, "\nUnexpected failure for test: ", kP0Foo))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "modified: optional_int32: 1 -> 2"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "modified: optional_int32: 1 -> 2"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); // The recorded message is the legacy one, as the failure list would get it. - EXPECT_THAT(test_manager_.UnexpectedFailures(), + EXPECT_THAT(ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP0Foo, "Output was not equivalent to reference " "message: modified: optional_int32: 1 -> 2", @@ -1098,47 +1384,27 @@ TEST_F(YieldsTest, UnlistedFailureIsCountedOnceDespiteGtestRetrying) { // gtest evaluates a matcher a second time to explain a failed assertion. - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(Run(kPayload2), Yields(IsParseError())), - "Should have failed to parse, but didn't."); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), + EXPECT_YIELDS_FAILURE(EXPECT_THAT(Run(kPayload2), Yields(IsParseError())), + "Should have failed to parse, but didn't."); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP0Foo, kNotAParseError, absl::nullopt))); } -TEST_F(YieldsTest, StaleVerdictIsNeverReplayedForAnotherResult) { - // A long-lived matcher that checked a result which has since been destroyed - // must not replay that verdict for a different result, even one that reuses - // the same address: the replay is keyed to the result's name as well. - auto first_matcher = Yields(IsParseError()); - { - TestResult first = Run(kPayload2, TestPriority::kP0, "first"); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(first, first_matcher), - "Should have failed to parse, but didn't."); - } - TestResult second = Run(kParseError, TestPriority::kP0, "second"); - EXPECT_THAT(second, Yields(IsParseError())); - - EXPECT_THAT(Explain(first_matcher, second), - Rejects(HasSubstr("was already checked"))); - EXPECT_EQ(GetCounts(test_manager_), - (Counts{/*expected_successes=*/1, /*expected_failures=*/0, - /*unexpected_successes=*/0, /*unexpected_failures=*/1})); -} - TEST_F(YieldsTest, PayloadBytesMismatchRecordsTheLegacyOctalMessage) { - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kPayload2), Yields(RawPayload(VarintField(1, 1)))), "Output was not equivalent to reference message: " "Expect: \\010\\001, but got: \\010\\002"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); EXPECT_THAT( - test_manager_.UnexpectedFailures(), + ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP0Foo, "Output was not equivalent to reference message: " "Expect: \\010\\001, but got: \\010\\002", @@ -1153,23 +1419,23 @@ Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2)pb")))); EXPECT_THAT(RunText(kText2, TestPriority::kP0, "raw"), Yields(RawPayload(Wire("optional_int32: 2")))); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/2})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2})); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT( RunText(kText2, TestPriority::kP0, "mismatch"), Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 1)pb")))), "modified: optional_int32: 1 -> 2"); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(RunText(R"pb(text_payload: "nonsense: 1")pb", TestPriority::kP0, "unparseable"), Yields(WhenParsed(_))), "TEXT_FORMAT output we received from test was unparseable."); - EXPECT_EQ(GetCounts(test_manager_), + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2, /*expected_failures=*/0, /*unexpected_successes=*/0, /*unexpected_failures=*/2})); EXPECT_THAT( - test_manager_.UnexpectedFailures(), + ledger_.UnexpectedFailures(), ElementsAre( FieldsAre("Required.Proto2.ProtobufInput.mismatch.TextFormatOutput", "Output was not equivalent to reference message: " @@ -1199,10 +1465,10 @@ TestResult result = Run(kPayload2); EXPECT_THAT(Explain(matcher, result), Accepts(IsEmpty())); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, ListedFailureWithMessagePrefixPasses) { @@ -1210,9 +1476,9 @@ EXPECT_THAT(Run(kPayload2), Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 1)pb")))); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/1})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/1})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, ListedFailureViaWildcardPasses) { @@ -1220,9 +1486,9 @@ kNotAParseError); EXPECT_THAT(Run(kPayload2), Yields(IsParseError())); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/1})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/1})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, ListedFailureWithDifferentMessageFails) { @@ -1235,13 +1501,13 @@ Rejects(absl::StrCat( kNotAParseError, "\nUnexpected failure message for test: ", kP0Foo, " expected: Some other message actual: ", kNotAParseError))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "Unexpected failure message for test"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "Unexpected failure message for test"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP0Foo, kNotAParseError, absl::nullopt))); } @@ -1257,31 +1523,31 @@ Rejects(absl::StrCat("test ", kP0Foo, " (matched to ", kP0Foo, ") is in the failure list, but test succeeded. " "Remove its match from the failure list."))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "is in the failure list, but test succeeded"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/1})); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "is in the failure list, but test succeeded"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/1})); EXPECT_THAT( - test_manager_.UnexpectedSuccesses(), + ledger_.UnexpectedSuccesses(), ElementsAre(FieldsAre(kP0Foo, kNotAParseError, Optional(kP0Foo)))); } TEST_F(YieldsTest, ListedSuccessViaWildcardNamesTheWildcard) { AddToFailureList("Required.*.ProtobufInput.foo.ProtobufOutput", ""); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kParseError), Yields(IsParseError())), "test Required.Proto2.ProtobufInput.foo.ProtobufOutput (matched to " "Required.*.ProtobufInput.foo.ProtobufOutput) is in the failure list, " "but test succeeded. Remove its match from the failure list."); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/1})); } TEST_F(YieldsTest, P1FailureIsToleratedWhenNotEnforced) { - test_manager_.set_enforcement_level(kP0); + ledger_.set_enforcement_level(kP0); absl::ScopedMockLog log; EXPECT_CALL(log, Log).Times(AnyNumber()); EXPECT_CALL( @@ -1296,60 +1562,60 @@ EXPECT_THAT(Explain(matcher, result), Accepts(IsEmpty())); // It's neither a skip nor a failure. - EXPECT_EQ(GetCounts(test_manager_), + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, /*expected_failures=*/0, /*unexpected_successes=*/0, /*unexpected_failures=*/0, /*skipped=*/0, /*listed_skips=*/0, /*tolerated_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_THAT(ledger_.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, P1FailureFailsWhenEnforced) { - test_manager_.set_enforcement_level(kP1); + ledger_.set_enforcement_level(kP1); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kPayload2, TestPriority::kP1), Yields(IsParseError())), "Should have failed to parse, but didn't."); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP1Foo, kNotAParseError, absl::nullopt))); } TEST_F(YieldsTest, ListedP1FailureIsTrackedEvenWhenNotEnforced) { // A kP1 test that's already in the failure list keeps being tracked // there, so that the list can't go stale unnoticed. - test_manager_.set_enforcement_level(kP0); + ledger_.set_enforcement_level(kP0); AddToFailureList(kP1Foo, kNotAParseError); EXPECT_THAT(Run(kPayload2, TestPriority::kP1), Yields(IsParseError())); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/1})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/1})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, ListedP1FailureWithDifferentMessageFailsEvenWhenNotEnforced) { - test_manager_.set_enforcement_level(kP0); + ledger_.set_enforcement_level(kP0); AddToFailureList(kP1Foo, "Some other message"); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kPayload2, TestPriority::kP1), Yields(IsParseError())), "Unexpected failure message for test"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); } TEST_F(YieldsTest, P1SuccessPasses) { - test_manager_.set_enforcement_level(kP0); + ledger_.set_enforcement_level(kP0); EXPECT_THAT(Run(kParseError, TestPriority::kP1), Yields(IsParseError())); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/1})); } TEST_F(YieldsTest, UnlistedSkipPasses) { @@ -1363,7 +1629,7 @@ // The inner matcher never gets to see a skipped result. bool inner_called = false; - auto matcher = Yields(Truly([&](const TestResult&) { + auto matcher = Yields(Truly([&](const YieldedResult&) { inner_called = true; return false; })); @@ -1371,20 +1637,50 @@ EXPECT_THAT(Explain(matcher, result), Accepts(IsEmpty())); EXPECT_FALSE(inner_called); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/0, - /*skipped=*/1})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/0, + /*skipped=*/1})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); +} + +TEST_F(YieldsTest, NotSelectedByRunnerPassesSilently) { + // A filtering runner answers tests that weren't selected with this exact + // skip reason; that isn't a testee skip and must not be logged or counted. + AddToFailureList(kP0Foo, ""); + absl::ScopedMockLog log; + EXPECT_CALL(log, Log(_, _, HasSubstr(kP0Foo))).Times(0); + log.StartCapturingLogs(); + + bool inner_called = false; + auto matcher = Yields(Truly([&](const YieldedResult&) { + inner_called = true; + return false; + })); + const std::string response = R"pb(skipped: "Not selected by --test.")pb"; + TestResult result = Run(response); + + EXPECT_THAT(Explain(matcher, result), Accepts(IsEmpty())); + EXPECT_FALSE(inner_called); + EXPECT_EQ(ReplayedCounts(), Counts{}); + // Even a listed test is simply unseen, not "listed but skipped". Its entry + // does count as matched, though: the test exists, it just didn't run. + EXPECT_THAT(ledger_.Finalize().message(), + HasSubstr(absl::StrCat("were not seen: ", kP0Foo))); + EXPECT_THAT(ledger_.UnmatchedExpectedFailures(), IsEmpty()); + + // An unlisted one likewise. + EXPECT_THAT(Run(response, TestPriority::kP0, "other"), matcher); + EXPECT_EQ(ReplayedCounts(), Counts{}); } TEST_F(YieldsTest, ListedSkipFailsOnce) { // Deliberately stricter than the legacy runner, which passed this case: a // listed test the testee skips fails, but only here. The entry counts as - // seen (a skip says nothing about whether it is still needed), so Finalize() - // doesn't report it a second time and --fix keeps it. - ASSERT_THAT(test_manager_.LoadFailureList(WriteTempFile( + // seen, so Finalize() doesn't report it a second time, and --fix drops it + // like that of a test that passed. + ASSERT_THAT(ledger_.LoadFailureList(WriteTempFile( "failure_list.txt", absl::StrCat(kP0Foo, " # abc\n"))), IsOk()); auto matcher = Yields(IsParseError()); @@ -1397,30 +1693,34 @@ "test ", kP0Foo, " (matched to ", kP0Foo, ") is in the failure list but was skipped by the testee: " "not supported. Remove its match from the failure list."))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "is in the failure list but was skipped"); - // The manager records it as a listed skip, not as an unexpected failure. - EXPECT_EQ(GetCounts(test_manager_), + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "is in the failure list but was skipped"); + // The ledger records it as a listed skip, not as an unexpected failure. + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, /*expected_failures=*/0, /*unexpected_successes=*/0, /*unexpected_failures=*/0, /*skipped=*/1, /*listed_skips=*/1})); - EXPECT_THAT(test_manager_.ListedSkips(), ElementsAre(Pair(kP0Foo, kP0Foo))); - EXPECT_THAT(test_manager_.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_THAT(ledger_.ListedSkips(), ElementsAre(Pair(kP0Foo, kP0Foo))); + EXPECT_THAT(ledger_.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger_.Finalize(), IsOk()); const std::string fixed = WriteTempFile("fixed.txt", ""); - ASSERT_THAT(test_manager_.SaveFailureList(fixed), IsOk()); - EXPECT_EQ(ReadFile(fixed), absl::StrCat(kP0Foo, " # abc\n")); + ASSERT_THAT(ledger_.SaveFailureList(fixed), IsOk()); + EXPECT_EQ(ReadFile(fixed), ""); } TEST_F(YieldsTest, RuntimeErrorKeepsTheInnerMatchersFailureMessage) { // The legacy runner reports a runtime error where a parse error was expected // with a dedicated message, which the failure lists rely on. - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kRuntimeError), Yields(IsParseError())), "Should have failed to parse, but raised an error instead."); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); EXPECT_THAT( - test_manager_.UnexpectedFailures(), + ledger_.UnexpectedFailures(), ElementsAre(FieldsAre( kP0Foo, "Should have failed to parse, but raised an error instead.", absl::nullopt))); @@ -1431,9 +1731,9 @@ "Should have failed to parse, but raised an error instead."); EXPECT_THAT(Run(kRuntimeError), Yields(IsParseError())); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/1})); - EXPECT_THAT(test_manager_.Finalize(), IsOk()); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/1})); + EXPECT_THAT(ledger_.Finalize(), IsOk()); } TEST_F(YieldsTest, AnyOfMatchesEitherLeg) { @@ -1442,7 +1742,7 @@ EXPECT_THAT(Run(kParseError, TestPriority::kP0, "first"), matcher); EXPECT_THAT(Run(kPayload2, TestPriority::kP0, "second"), matcher); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/2})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2})); } TEST_F(YieldsTest, AnyOfFailsWhenNoLegMatches) { @@ -1456,17 +1756,17 @@ HasSubstr("Should have failed to parse, but didn't."), HasSubstr("Failed to parse input or produce output."), HasSubstr(absl::StrCat("Unexpected failure for test: ", kP0Foo))))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), "Unexpected failure"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), "Unexpected failure"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); } TEST_F(YieldsTest, NotComposes) { EXPECT_THAT(Run(kPayload2, TestPriority::kP0, "first"), Yields(Not(IsParseError()))); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/1})); auto matcher = Yields(Not(IsParseError())); TestResult result = Run(kParseError, TestPriority::kP0, "second"); @@ -1476,83 +1776,435 @@ "which doesn't match (is not a parse error)", "\nUnexpected failure for test: ", "Required.Proto2.ProtobufInput.second.ProtobufOutput"))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "which doesn't match (is not a parse error)"); - EXPECT_EQ(GetCounts(test_manager_), + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "which doesn't match (is not a parse error)"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/1, /*expected_failures=*/0, /*unexpected_successes=*/0, /*unexpected_failures=*/1})); } -TEST_F(YieldsTest, CheckingAPassedResultAgainFails) { - // Whatever the second matcher is, including the very same object. +TEST_F(YieldsTest, CheckingAPassedResultAgainWithTheSameOutcomePasses) { + // Whatever the second matcher is, as long as it reaches the same outcome; + // the test is recorded and counted once. auto matcher = Yields(IsParseError()); TestResult result = Run(kParseError); EXPECT_THAT(result, matcher); - EXPECT_NONFATAL_FAILURE( - EXPECT_THAT(result, matcher), - "TestResult for Required.Proto2.ProtobufInput.foo.ProtobufOutput was " - "already checked; each result may be checked exactly once"); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, Yields(IsParseError())), - "was already checked"); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, Yields(Not(IsParseError()))), - "was already checked"); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/1})); + EXPECT_THAT(result, matcher); + EXPECT_THAT(result, Yields(IsParseError())); + EXPECT_THAT(result, Yields(Not(WhenParsed(_)))); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/1})); } -TEST_F(YieldsTest, CheckingAFailedResultAgainReplaysTheFailure) { +TEST_F(YieldsTest, CheckingAResultAgainWithADifferentOutcomeFails) { + TestResult result = Run(kParseError); + EXPECT_THAT(result, Yields(IsParseError())); + + // The first outcome stays recorded; the ledger never hears of the second. + EXPECT_YIELDS_FAILURE( + EXPECT_THAT(result, Yields(Not(IsParseError()))), + "TestResult for Required.Proto2.ProtobufInput.foo.ProtobufOutput was " + "already checked, with a different outcome: the first check recorded " + "\"P0 PASS\", this one would record \"P0 FAIL: which doesn't match (is " + "not a parse error)\"; each result may be checked once"); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), IsEmpty()); +} + +TEST_F(YieldsTest, CheckingAFailedResultAgainRepeatsTheFailure) { // gtest re-evaluates a failing matcher, through the same matcher object, to - // explain the failure. The matcher replays its own failed verdict, so the - // manager hears about the test only once. Any other matcher finds the - // result already checked. + // explain the failure. That evaluation reaches the same outcome, so it + // only repeats the verdict, and the ledger hears about the test once. So + // does any other matcher that reaches the same outcome; one that doesn't + // finds the result already checked. auto matcher = Yields(IsParseError()); TestResult result = Run(kPayload2); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(result, matcher), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "Should have failed to parse, but didn't."); EXPECT_THAT(Explain(matcher, result), Rejects(absl::StrCat(kNotAParseError, "\nUnexpected failure for test: ", kP0Foo))); EXPECT_THAT(Explain(Yields(IsParseError()), result), - Rejects(HasSubstr("was already checked"))); + Rejects(absl::StrCat(kNotAParseError, + "\nUnexpected failure for test: ", kP0Foo))); EXPECT_THAT(Explain(Yields(WhenParsed(_)), result), - Rejects(HasSubstr("was already checked"))); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); - EXPECT_THAT(test_manager_.UnexpectedFailures(), + Rejects(HasSubstr("was already checked, with a different " + "outcome: the first check recorded \"P0 FAIL: " + "Should have failed to parse, but didn't.\", " + "this one would record \"P0 PASS\""))); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); + EXPECT_THAT(ledger_.UnexpectedFailures(), ElementsAre(FieldsAre(kP0Foo, kNotAParseError, absl::nullopt))); } TEST_F(YieldsTest, ACopyOfACheckedResultIsAlreadyChecked) { TestResult original = Run(kPayload2); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(original, Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(original, Yields(IsParseError())), + "Should have failed to parse, but didn't."); - // The manager knows the test by name, so a copy of the result is checked - // too and the manager hears nothing more. + // The record is keyed by the test's name, so a copy of the result counts as + // checked too: the same outcome is repeated, a different one is refused, + // and the ledger hears nothing more. TestResult copy = original; EXPECT_THAT(Explain(Yields(IsParseError()), copy), + Rejects(HasSubstr("Unexpected failure for test"))); + EXPECT_THAT(Explain(Yields(WhenParsed(_)), copy), Rejects(HasSubstr("was already checked"))); - EXPECT_EQ(GetCounts(test_manager_), (Counts{/*expected_successes=*/0, - /*expected_failures=*/0, - /*unexpected_successes=*/0, - /*unexpected_failures=*/1})); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/0, + /*expected_failures=*/0, + /*unexpected_successes=*/0, + /*unexpected_failures=*/1})); } TEST_F(YieldsTest, SameMatcherCanCheckSeveralResults) { auto matcher = Yields(IsParseError()); EXPECT_THAT(Run(kParseError, TestPriority::kP0, "first"), matcher); EXPECT_THAT(Run(kParseError, TestPriority::kP0, "second"), matcher); - EXPECT_NONFATAL_FAILURE( + EXPECT_YIELDS_FAILURE( EXPECT_THAT(Run(kPayload2, TestPriority::kP0, "third"), matcher), "Should have failed to parse, but didn't."); - EXPECT_EQ(GetCounts(test_manager_), + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2, /*expected_failures=*/0, /*unexpected_successes=*/0, /*unexpected_failures=*/1})); } +// --------------------------------------------------------------------------- +// The results Yields() records +// --------------------------------------------------------------------------- + +// The outcomes recorded so far on the running gtest test, in order, as +// (test name, ResultRecord::ToString()) pairs. +std::vector<std::pair<std::string, std::string>> RecordedResults() { + return internal::RecordedResults(internal::CurrentGtestResult()); +} + +TEST_F(YieldsTest, RecordsAPass) { + EXPECT_THAT(Run(kPayload2), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2)pb")))); + EXPECT_THAT(RecordedResults(), ElementsAre(Pair(kP0Foo, "P0 PASS"))); +} + +TEST_F(YieldsTest, RecordsAFailWithTheLegacyMessage) { + TestResult result = Run(kPayload2); + EXPECT_YIELDS_FAILURE( + EXPECT_THAT(result, Yields(WhenParsed(EqualsTextProto(R"pb( + optional_int32: 1 + )pb")))), + "modified: optional_int32: 1 -> 2"); + EXPECT_THAT( + RecordedResults(), + ElementsAre(Pair(kP0Foo, + "P0 FAIL: Output was not equivalent to reference " + "message: modified: optional_int32: 1 -> 2"))); +} + +TEST_F(YieldsTest, RecordsAnExpectedFailureAsAFail) { + // The record says what the testee did, not whether it was expected. + AddToFailureList(kP0Foo, kNotAParseError); + EXPECT_THAT(Run(kPayload2), Yields(IsParseError())); + EXPECT_THAT( + RecordedResults(), + ElementsAre(Pair(kP0Foo, absl::StrCat("P0 FAIL: ", kNotAParseError)))); +} + +TEST_F(YieldsTest, RecordsAToleratedP1FailureAsAFail) { + ledger_.set_enforcement_level(kP0); + EXPECT_THAT(Run(kPayload2, TestPriority::kP1), Yields(IsParseError())); + EXPECT_THAT( + RecordedResults(), + ElementsAre(Pair(kP1Foo, absl::StrCat("P1 FAIL: ", kNotAParseError)))); +} + +TEST_F(YieldsTest, RecordsARuntimeErrorAsACrash) { + EXPECT_YIELDS_FAILURE(EXPECT_THAT(Run(kRuntimeError), Yields(WhenParsed(_))), + "Failed to parse input or produce output."); + EXPECT_THAT(RecordedResults(), + ElementsAre(Pair(kP0Foo, + "P0 CRASH: Failed to parse input or produce " + "output."))); +} + +TEST_F(YieldsTest, RecordsATimeoutAsACrash) { + EXPECT_YIELDS_FAILURE(EXPECT_THAT(Run(kTimeoutError), Yields(WhenParsed(_))), + "Failed to parse input or produce output."); + EXPECT_THAT(RecordedResults(), + ElementsAre(Pair(kP0Foo, + "P0 CRASH: Failed to parse input or produce " + "output."))); +} + +TEST_F(YieldsTest, RecordsAnEmptyResponseAsAFail) { + EXPECT_YIELDS_FAILURE(EXPECT_THAT(Run(""), Yields(WhenParsed(_))), + "Response didn't have any field in the Response."); + EXPECT_THAT(RecordedResults(), + ElementsAre(Pair(kP0Foo, + "P0 FAIL: Response didn't have any field in " + "the Response."))); +} + +TEST_F(YieldsTest, RecordsASkipWithTheReason) { + EXPECT_THAT(Run(kSkipped), Yields(_)); + EXPECT_THAT(RecordedResults(), + ElementsAre(Pair(kP0Foo, "P0 SKIP: not supported"))); +} + +TEST_F(YieldsTest, RecordsAListedSkipAsASkip) { + AddToFailureList(kP0Foo, ""); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(Run(kSkipped), Yields(_)), + "is in the failure list but was skipped"); + EXPECT_THAT(RecordedResults(), + ElementsAre(Pair(kP0Foo, "P0 SKIP: not supported"))); +} + +TEST_F(YieldsTest, RecordsATestTheRunnerDidNotSelectAsNotSelected) { + const std::string response = R"pb(skipped: "Not selected by --test.")pb"; + EXPECT_THAT(Run(response), Yields(_)); + EXPECT_THAT(RecordedResults(), ElementsAre(Pair(kP0Foo, "P0 NOT_SELECTED"))); +} + +TEST_F(YieldsTest, RecordsOnceDespiteGtestRetrying) { + // gtest evaluates a failing matcher a second time to explain the failure, + // and a later check that reaches the same outcome repeats it. None of that + // records a second time. + auto matcher = Yields(IsParseError()); + TestResult result = Run(kPayload2); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, matcher), + "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(result, Yields(IsParseError())), + "Should have failed to parse, but didn't."); + EXPECT_THAT( + RecordedResults(), + ElementsAre(Pair(kP0Foo, absl::StrCat("P0 FAIL: ", kNotAParseError)))); +} + +TEST_F(YieldsTest, RecordsEveryResult) { + auto matcher = + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2)pb"))); + EXPECT_THAT(Run(kPayload2, TestPriority::kP0, "first"), matcher); + EXPECT_THAT(Run(kSkipped, TestPriority::kP1, "second"), matcher); + EXPECT_THAT( + RecordedResults(), + ElementsAre( + Pair("Required.Proto2.ProtobufInput.first.ProtobufOutput", "P0 PASS"), + Pair("Recommended.Proto2.ProtobufInput.second.ProtobufOutput", + "P1 SKIP: not supported"))); +} + +// --------------------------------------------------------------------------- +// JsonPayload() +// --------------------------------------------------------------------------- + +// A JSON-input, JSON-output result carrying the given JSON payload. +TestResult JsonResult(absl::string_view json) { + ConformanceResponse response; + response.set_json_payload(json); + return CreateResult("foo", /*input_format=*/::conformance::JSON, + /*output_format=*/::conformance::JSON, + std::move(response)); +} + +constexpr absl::string_view kJsonInt32 = R"json({"optionalInt32": 9})json"; + +TEST(JsonPayloadTest, Description) { + EXPECT_EQ(Describe(JsonPayload(HasJsonMember("optionalInt32"))), + R"(JSON payload has member "optionalInt32")"); + EXPECT_EQ(DescribeNegation(JsonPayload(HasJsonMember("optionalInt32"))), + R"(JSON payload doesn't have member "optionalInt32")"); + EXPECT_EQ(Describe(JsonPayload(_)), "JSON payload is anything"); +} + +TEST(JsonPayloadTest, Matches) { + TestResult result = JsonResult(kJsonInt32); + + EXPECT_THAT(Explain(JsonPayload(HasJsonMember("optionalInt32")), result), + Accepts(IsEmpty())); + YieldedResult yielded(result); + EXPECT_TRUE(Value(yielded, JsonPayload(_))); + EXPECT_TRUE(Value(yielded, JsonPayload(Truly([](const Json::Value& value) { + return value["optionalInt32"].asInt() == 9; + })))); + EXPECT_TRUE(Value(yielded, JsonPayload(SizeIs(1)))); +} + +TEST(JsonPayloadTest, InnerMatcherWithoutExplanation) { + // The description and the actual JSON text, quoted and escaped like + // RawPayload() does, are reported instead. + TestResult result = JsonResult(kJsonInt32); + + EXPECT_THAT(Explain(JsonPayload(HasJsonMember("optionalInt64")), result), + Rejects("Expect: JSON payload has member \"optionalInt64\", " + "but got: \"{\\\"optionalInt32\\\": 9}\"")); +} + +TEST(JsonPayloadTest, InnerMatcherExplanationIsUsed) { + TestResult result = JsonResult(kJsonInt32); + + EXPECT_THAT(Explain(JsonPayload(SizeIs(2)), result), + Rejects("whose size 1 doesn't match")); + // Truly() explains itself, tersely; this is what a lambda validator ported + // from the legacy suite records in the failure list. Only the gist is + // pinned, since the exact wording belongs to gMock. + EXPECT_THAT(Explain(JsonPayload(Truly([](const Json::Value& value) { + return value.isNull(); + })), + result), + Rejects(HasSubstr("didn't satisfy"))); +} + +TEST(JsonPayloadTest, InvalidJson) { + // The legacy validator's message, followed by jsoncpp's own diagnostic. + // The payload has to be valid JSON before any inner matcher gets a say. + TestResult result = JsonResult("{"); + + EXPECT_THAT( + Explain(JsonPayload(_), result), + Rejects(StartsWith("JSON payload cannot be parsed as valid JSON: "))); + EXPECT_THAT( + Explain(JsonPayload(_), JsonResult("")), + Rejects(StartsWith("JSON payload cannot be parsed as valid JSON: "))); + EXPECT_THAT( + Explain(JsonPayload(_), JsonResult("nonsense")), + Rejects(StartsWith("JSON payload cannot be parsed as valid JSON: "))); +} + +TEST(JsonPayloadTest, ParsesAnyJsonValueNotJustObjects) { + // Scalars and arrays are valid JSON documents to jsoncpp's reader (like the + // legacy validator); whether they are acceptable is up to the inner + // matcher. + TestResult null_result = JsonResult("null"); + EXPECT_TRUE(Value(YieldedResult(null_result), + JsonPayload(Truly([](const Json::Value& value) { + return value.isNull(); + })))); + TestResult array_result = JsonResult("[1, 2]"); + EXPECT_TRUE(Value(YieldedResult(array_result), JsonPayload(SizeIs(2)))); +} + +TEST(JsonPayloadTest, EmptyResponse) { + TestResult result = + CreateResult("foo", ::conformance::JSON, ConformanceResponse()); + + EXPECT_THAT(Explain(JsonPayload(_), result), + Rejects("Response didn't have any field in the Response.")); +} + +TEST_P(PayloadMatcherErrorResponseTest, IsAFailureForJsonPayload) { + TestResult result = CreateResult("foo", ::conformance::JSON, + ResponseFromText(GetParam().response)); + + EXPECT_THAT(Explain(JsonPayload(_), result), + Rejects("Failed to parse input or produce output.")); +} + +TEST(JsonPayloadTest, Skipped) { + TestResult result = + CreateResult("foo", ::conformance::JSON, + ResponseFromText(R"pb(skipped: "skipped message")pb")); + + EXPECT_THAT(Explain(JsonPayload(_), result), + Rejects("the testee skipped the test: skipped message")); +} + +TEST(JsonPayloadTest, WrongOutputFormat) { + // Same handling as the other payload matchers. + TestResult result = CreateResult("foo", ::conformance::JSON, + ProtobufPayload(VarintField(1, 9))); + + EXPECT_THAT( + Explain(JsonPayload(_), result), + Rejects("Test was asked for JSON output but provided PROTOBUF instead.")); + EXPECT_THAT( + Explain(JsonPayload(_), + CreateResult("bar", ::conformance::JSON, + ResponseFromText(R"pb(text_payload: "")pb"))), + Rejects( + "Test was asked for JSON output but provided TEXT_FORMAT instead.")); +} + +TEST(JsonPayloadTest, RequiresATestThatAskedForJsonOutput) { + // A test that asked for (and got) some other format can't be matched by + // JsonPayload() at all; that is a bug in the test. + TestResult result = CreateResult("foo", ::conformance::PROTOBUF, + ProtobufPayload(VarintField(1, 9))); + + EXPECT_THAT(Explain(JsonPayload(_), result), + Rejects("JsonPayload() needs JSON output, but the test asked " + "for PROTOBUF output.")); +} + +TEST(JsonPayloadTest, ComposesWithGMock) { + TestResult result = JsonResult(kJsonInt32); + + YieldedResult yielded(result); + EXPECT_TRUE(Value(yielded, Not(JsonPayload(HasJsonMember("optionalInt64"))))); + EXPECT_TRUE(Value( + yielded, + AnyOf(IsParseError(), JsonPayload(HasJsonMember("optionalInt32"))))); + EXPECT_TRUE(Value( + yielded, AllOf(JsonPayload(HasJsonMember("optionalInt32")), + WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb"))))); +} + +TEST_F(YieldsTest, JsonOutput) { + // JSON-input, JSON-output tests through the whole stack: the legacy + // messages are what gets recorded for the failure list. + auto run_json = [&](absl::string_view response, absl::string_view name) { + RespondWith(response); + return testee_.CreateTest(name, TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "{}") + .SerializeJson(); + }; + constexpr absl::string_view kJsonResponse = + R"pb(json_payload: "{\"optionalInt32\": 9}")pb"; + // Kept out of the EXPECT_YIELDS_FAILURE() arguments below: MSVC's + // traditional preprocessor mis-scans raw string literals containing \" or + // an odd number of quotes when they appear inside macro arguments. + constexpr absl::string_view kUnparseableJsonResponse = + R"pb(json_payload: "{\"nonsense\": 1}")pb"; + constexpr absl::string_view kInvalidJsonResponse = R"pb(json_payload: "{")pb"; + + EXPECT_THAT(run_json(kJsonResponse, "parsed"), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 9)pb")))); + EXPECT_THAT(run_json(kJsonResponse, "validated"), + Yields(JsonPayload(HasJsonMember("optionalInt32")))); + EXPECT_EQ(ReplayedCounts(), (Counts{/*expected_successes=*/2})); + + EXPECT_YIELDS_FAILURE( + EXPECT_THAT(run_json(kUnparseableJsonResponse, "unparseable"), + Yields(WhenParsed(_))), + "JSON output we received from test was unparseable."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(run_json(kInvalidJsonResponse, "invalid"), + Yields(JsonPayload(_))), + "JSON payload cannot be parsed as valid JSON: "); + EXPECT_YIELDS_FAILURE( + EXPECT_THAT(run_json(kJsonResponse, "mismatch"), + Yields(JsonPayload(HasJsonMember("optionalInt64")))), + "Expect: JSON payload has member \"optionalInt64\", but got: " + "\"{\\\"optionalInt32\\\": 9}\""); + EXPECT_EQ(ReplayedCounts(), + (Counts{/*expected_successes=*/2, /*expected_failures=*/0, + /*unexpected_successes=*/0, /*unexpected_failures=*/3})); + EXPECT_THAT( + ledger_.UnexpectedFailures(), + ElementsAre( + FieldsAre("Required.Proto2.JsonInput.invalid.JsonOutput", + StartsWith("JSON payload cannot be parsed as valid JSON: "), + absl::nullopt), + FieldsAre("Required.Proto2.JsonInput.mismatch.JsonOutput", + "Expect: JSON payload has member \"optionalInt64\", but " + "got: \"{\\\"optionalInt32\\\": 9}\"", + absl::nullopt), + FieldsAre("Required.Proto2.JsonInput.unparseable.JsonOutput", + "JSON output we received from test was unparseable.", + absl::nullopt))); +} + } // namespace } // namespace conformance } // namespace protobuf
diff --git a/conformance/matchers_testing.cc b/conformance/matchers_testing.cc new file mode 100644 index 0000000..19f22f4 --- /dev/null +++ b/conformance/matchers_testing.cc
@@ -0,0 +1,89 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/matchers_testing.h" + +#include <string> +#include <utility> +#include <vector> + +#include <gtest/gtest.h> +#include "absl/container/flat_hash_set.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/result_record.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +// The record in `part`, if it is a gtest success that carries one. +absl::optional<RecordedResult> RecordIn(const testing::TestPartResult& part) { + if (part.type() != testing::TestPartResult::kSuccess) return absl::nullopt; + absl::StatusOr<RecordedResult> recorded = + ParseResultRecordMessage(part.message()); + if (!recorded.ok()) return absl::nullopt; + return *std::move(recorded); +} + +} // namespace + +std::vector<std::pair<std::string, std::string>> RecordedResults( + const testing::TestResult& result) { + std::vector<std::pair<std::string, std::string>> results; + for (int i = 0; i < result.total_part_count(); ++i) { + if (absl::optional<RecordedResult> recorded = + RecordIn(result.GetTestPartResult(i)); + recorded.has_value()) { + results.emplace_back(recorded->test_name, recorded->record.ToString()); + } + } + return results; +} + +void ExpectYieldsFailure(const testing::TestPartResultArray& results, + absl::string_view substr) { + int failures = 0; + // gtest evaluates a failing matcher twice, and the second evaluation can't + // see what the first one recorded while both are intercepted; record each + // outcome once. + absl::flat_hash_set<std::string> recorded; + for (int i = 0; i < results.size(); ++i) { + const testing::TestPartResult& part = results.GetTestPartResult(i); + if (part.type() == testing::TestPartResult::kSuccess) { + if (absl::optional<RecordedResult> record = RecordIn(part); + record.has_value() && + recorded + .insert(ResultRecordMessage(record->test_name, record->record)) + .second) { + SUCCEED() << ResultRecordMessage(record->test_name, record->record); + } + continue; + } + ++failures; + if (part.type() == testing::TestPartResult::kNonFatalFailure && + absl::StrContains(part.message(), substr)) { + continue; + } + ADD_FAILURE() << "Expected: 1 non-fatal failure containing \"" << substr + << "\"\n Actual:\n" + << part; + } + if (failures != 1) { + ADD_FAILURE() << "Expected: 1 non-fatal failure containing \"" << substr + << "\"\n Actual: " << failures << " failures"; + } +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/matchers_testing.h b/conformance/matchers_testing.h new file mode 100644 index 0000000..6a0da0b --- /dev/null +++ b/conformance/matchers_testing.h
@@ -0,0 +1,64 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Helpers for unit tests of code that checks conformance results with +// Yields() (see matchers.h). + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_MATCHERS_TESTING_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_MATCHERS_TESTING_H__ + +#include <string> +#include <utility> +#include <vector> + +#include <gtest/gtest-spi.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// The outcomes Yields() recorded in `result` (see ResultRecordMessage() in +// result_record.h), in order, as (conformance test name, +// ResultRecord::ToString()) pairs. +std::vector<std::pair<std::string, std::string>> RecordedResults( + const testing::TestResult& result); + +// Implements EXPECT_YIELDS_FAILURE(): `results` is what the statement +// reported. +void ExpectYieldsFailure(const testing::TestPartResultArray& results, + absl::string_view substr); + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +// Like EXPECT_NONFATAL_FAILURE(statement, substr), for a statement that checks +// a conformance result with Yields(): `statement` must report exactly one +// non-fatal failure, whose message contains `substr`. EXPECT_NONFATAL_FAILURE +// would also intercept the gtest success Yields() records the outcome with +// (see ResultRecordMessage() in result_record.h) and complain about it. This +// records each outcome `statement` recorded on the running gtest test once it +// is done, once, so that the test environment still hears about the result. +#define EXPECT_YIELDS_FAILURE(statement, substr) \ + do { \ + ::testing::TestPartResultArray gtest_failures; \ + { \ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter( \ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, \ + >est_failures); \ + statement; \ + } \ + ::google::protobuf::conformance::internal::ExpectYieldsFailure(gtest_failures, \ + substr); \ + } while (false) + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_MATCHERS_TESTING_H__
diff --git a/conformance/message_type_fixtures.cc b/conformance/message_type_fixtures.cc new file mode 100644 index 0000000..93498c9 --- /dev/null +++ b/conformance/message_type_fixtures.cc
@@ -0,0 +1,27 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/message_type_fixtures.h" + +#include <string> + +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { + +std::string MessageTypeParamName( + const testing::TestParamInfo<const Descriptor*>& info) { + return ParamName(info.param); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/message_type_fixtures.h b/conformance/message_type_fixtures.h new file mode 100644 index 0000000..1f634be --- /dev/null +++ b/conformance/message_type_fixtures.h
@@ -0,0 +1,103 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_MESSAGE_TYPE_FIXTURES_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_MESSAGE_TYPE_FIXTURES_H__ + +#include <string> +#include <tuple> + +#include <gtest/gtest.h> +#include "absl/strings/str_join.h" +#include "conformance/binary_test_util.h" +#include "conformance/test_fixture.h" +#include "conformance/test_protos/test_messages_edition2023.pb.h" +#include "google/protobuf/descriptor.h" + +// Fixtures for conformance tests that know which message type they exercise, +// and the INSTANTIATE_TEST_SUITE_P name generators that go with them. The +// fixtures report the message type through ConformanceTest::MessageUnderTest(). +// The base SetUp() then skips the test when --maximum_edition doesn't cover +// the message's edition, so test bodies never have to gate themselves. +// +// Suites parameterized over more than the message type, with a std::tuple +// whose first element is the `const Descriptor*`, derive from ConformanceTest +// and override MessageUnderTest() themselves (see +// binary_premature_eof_test.cc). + +namespace google { +namespace protobuf { +namespace conformance { + +// The fixture for conformance tests parameterized over the test message type +// only. Give each group its own suite name with a type alias. Instantiate it +// over AllTestMessageTypes() or Proto3TestMessageTypes() with +// MessageTypeParamName(): +// +// using FooTest = MessageTypeConformanceTest; +// +// TEST_P(FooTest, Bar) { +// EXPECT_THAT(Testee("Bar").ParseBinary(message(), ...).ParseOnly(), +// Yields(IsParseError())); +// } +// +// INSTANTIATE_TEST_SUITE_P(All, FooTest, ValuesIn(AllTestMessageTypes()), +// MessageTypeParamName); +class MessageTypeConformanceTest + : public ConformanceTest, + public testing::WithParamInterface<const Descriptor*> { + protected: + const Descriptor* message() const { return GetParam(); } + + private: + const Descriptor* MessageUnderTest() const override { return GetParam(); } +}; + +// The fixture for conformance tests that only exist for +// TestAllTypesEdition2023, such as extensions and delimited fields. Like +// MessageTypeConformanceTest, alias it per group: +// +// using DelimitedExtensionTest = Edition2023ConformanceTest; +class Edition2023ConformanceTest : public ConformanceTest { + protected: + static const Descriptor* message() { + return protobuf_test_messages::editions::TestAllTypesEdition2023:: + descriptor(); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +// The INSTANTIATE_TEST_SUITE_P name generator for suites parameterized over +// the message type only: ParamName() of the descriptor, e.g. "EditionsProto2". +std::string MessageTypeParamName( + const testing::TestParamInfo<const Descriptor*>& info); + +// The INSTANTIATE_TEST_SUITE_P name generator for suites parameterized over a +// std::tuple: ParamName() of every element, joined with "_", e.g. +// "Proto3_INT32" for a (const Descriptor*, FieldDescriptor::Type) tuple. Every +// element type needs a ParamName() overload (see binary_test_util.h). +// +// INSTANTIATE_TEST_SUITE_P(All, FooTest, +// Combine(ValuesIn(AllTestMessageTypes()), +// ValuesIn(kTypes)), +// TupleParamName<FooTest::ParamType>); +template <typename Tuple> +std::string TupleParamName(const testing::TestParamInfo<Tuple>& info) { + return std::apply( + [](const auto&... params) { + return absl::StrJoin({ParamName(params)...}, "_"); + }, + info.param); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_MESSAGE_TYPE_FIXTURES_H__
diff --git a/conformance/message_type_fixtures_test.cc b/conformance/message_type_fixtures_test.cc new file mode 100644 index 0000000..b612888 --- /dev/null +++ b/conformance/message_type_fixtures_test.cc
@@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/message_type_fixtures.h" + +#include <cstddef> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/binary_test_util.h" +#include "conformance/mock_test_runner.h" +#include "conformance/test_environment.h" +#include "conformance/test_environment_testing.h" +#include "google/protobuf/descriptor.h" +#include "editions/golden/test_messages_proto2_editions.pb.h" +#include "google/protobuf/test_messages_proto2.pb.h" +#include "google/protobuf/test_messages_proto3.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; +using TestAllTypesProto2Editions = + ::protobuf_test_messages::editions::proto2::TestAllTypesProto2; +using ::testing::HasSubstr; +using ::testing::Not; +using ::testing::StrictMock; +using ::testing::TestParamInfo; +using ::testing::ValuesIn; + +// The fixtures under test consult the process-global ConformanceEnvironment in +// SetUp(), so this binary installs one for the duration of the run. It keeps +// the default maximum_edition (EDITION_PROTO3), under which every editions +// message is unsupported and must be skipped. No conformance test is ever run +// against the testee, so any call on the strict mock fails. +class GlobalEnvironment : public testing::Environment { + public: + void SetUp() override { + environment_.emplace( + internal::ConformanceEnvironmentOptions{/*runner=*/&runner_}); + } + void TearDown() override { environment_.reset(); } + + private: + StrictMock<MockTestRunner> runner_; + absl::optional<internal::ScopedGlobalConformanceEnvironment> environment_; +}; + +testing::Environment* const kGlobalEnvironment = + testing::AddGlobalTestEnvironment(new GlobalEnvironment); + +TEST(MessageTypeParamNameTest, IsParamNameOfTheDescriptor) { + size_t index = 0; + for (const Descriptor* message : AllTestMessageTypes()) { + EXPECT_EQ(MessageTypeParamName( + TestParamInfo<const Descriptor*>(message, index++)), + ParamName(message)); + } + EXPECT_EQ(MessageTypeParamName(TestParamInfo<const Descriptor*>( + TestAllTypesProto2Editions::descriptor(), 0)), + "EditionsProto2"); +} + +TEST(TupleParamNameTest, JoinsElementNamesWithUnderscore) { + using Param = std::tuple<const Descriptor*, FieldDescriptor::Type>; + EXPECT_EQ( + TupleParamName(TestParamInfo<Param>( + {TestAllTypesProto3::descriptor(), FieldDescriptor::TYPE_INT32}, 0)), + "Proto3_INT32"); + EXPECT_EQ(TupleParamName( + TestParamInfo<Param>({TestAllTypesProto2Editions::descriptor(), + FieldDescriptor::TYPE_MESSAGE}, + 1)), + "EditionsProto2_MESSAGE"); +} + +TEST(TupleParamNameTest, HandlesIntsAndSingleElementTuples) { + using Param = std::tuple<const Descriptor*, int, int, int>; + EXPECT_EQ(TupleParamName(TestParamInfo<Param>( + {TestAllTypesProto2::descriptor(), 6, 0, 3}, 0)), + "Proto2_6_0_3"); + EXPECT_EQ(TupleParamName(TestParamInfo<std::tuple<const Descriptor*>>( + {TestAllTypesProto3::descriptor()}, 0)), + "Proto3"); +} + +// Expects that every test of the current suite that ran was skipped iff +// `skipped_if(test name)` holds. For use from TearDownTestSuite(). +template <typename Predicate> +void ExpectSkippedIff(Predicate skipped_if) { + const testing::TestSuite& suite = + *testing::UnitTest::GetInstance()->current_test_suite(); + for (int i = 0; i < suite.total_test_count(); ++i) { + const testing::TestInfo& test = *suite.GetTestInfo(i); + if (!test.should_run()) continue; // Filtered out or sharded away. + EXPECT_EQ(test.result()->Skipped(), skipped_if(test.name())) << test.name(); + } +} + +// Instantiated over all four message types: SetUp() must skip the two editions +// instances (via MessageUnderTest()) before their bodies run, and let the +// proto2/proto3 ones through. +class MessageTypeSkipTest : public MessageTypeConformanceTest { + protected: + static void TearDownTestSuite() { + ExpectSkippedIff([](absl::string_view name) { + return absl::StrContains(name, "Editions"); + }); + } +}; + +TEST_P(MessageTypeSkipTest, BodyOnlyRunsForSupportedMessages) { + EXPECT_THAT(ParamName(message()), Not(HasSubstr("Editions"))); +} + +INSTANTIATE_TEST_SUITE_P(All, MessageTypeSkipTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +// TestAllTypesEdition2023 is never supported under EDITION_PROTO3, so SetUp() +// must skip every test of an Edition2023ConformanceTest suite. +class Edition2023SkipTest : public Edition2023ConformanceTest { + protected: + static void TearDownTestSuite() { + ExpectSkippedIff([](absl::string_view) { return true; }); + } +}; + +TEST_F(Edition2023SkipTest, BodyNeverRuns) { + FAIL() << "SetUp() must skip this test: " << message()->full_name() + << " isn't supported under --maximum_edition=PROTO3."; +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/recursion_limit_payloads.cc b/conformance/recursion_limit_payloads.cc new file mode 100644 index 0000000..91bf0b6 --- /dev/null +++ b/conformance/recursion_limit_payloads.cc
@@ -0,0 +1,120 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/recursion_limit_payloads.h" + +#include <cstddef> +#include <string> +#include <utility> +#include <vector> + +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "conformance/binary_wireformat.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +// One nesting level of such a payload, seen from outside: the bytes before the +// level below it (the level's own fields, then the tag of the field that holds +// the next level; its length prefix is added by Nested()) and the bytes after +// it (e.g. the end tag of a group). The innermost level holds no next level. +struct Level { + Wire before; + Wire after; +}; + +// The serialization of `levels[0]`, which holds the serialization of +// `levels[1]` as a length-prefixed submessage, and so on down to +// `levels.back()`. Two passes and no recursion: the sizes inside out, the +// bytes outside in. +Wire Nested(const std::vector<Level>& levels) { + std::vector<size_t> sizes(levels.size()); + for (size_t i = levels.size(); i-- > 0;) { + sizes[i] = levels[i].before.size() + levels[i].after.size(); + if (i + 1 < levels.size()) { + sizes[i] += Varint(sizes[i + 1]).size() + sizes[i + 1]; + } + } + std::string bytes; + bytes.reserve(sizes.empty() ? 0 : sizes[0]); + for (size_t i = 0; i < levels.size(); ++i) { + absl::StrAppend(&bytes, levels[i].before.data()); + if (i + 1 < levels.size()) { + absl::StrAppend(&bytes, Varint(sizes[i + 1]).data()); + } + } + for (size_t i = levels.size(); i-- > 0;) { + absl::StrAppend(&bytes, levels[i].after.data()); + } + return Wire(std::move(bytes)); +} + +} // namespace + +Wire DeepMapPayload(int depth) { + ABSL_DCHECK_GE(depth, 1); + std::vector<Level> levels; + levels.push_back({Tag(301, WireType::kLengthPrefixed)}); + for (int i = 0; i < depth; ++i) { + // The map entry: C++ writes the key even when it is the default. + levels.push_back( + {Wire(VarintField(1, 0), Tag(2, WireType::kLengthPrefixed))}); + // The nested message. + Wire fields = VarintField(1, 123); + if (i + 1 < depth) { + fields = Wire(fields, Tag(301, WireType::kLengthPrefixed)); + } + levels.push_back({fields}); + } + return Nested(levels); +} + +Wire DeepMapStringKeyPayload(int depth) { + ABSL_DCHECK_GE(depth, 1); + std::vector<Level> levels; + levels.push_back({Tag(71, WireType::kLengthPrefixed)}); + for (int i = 0; i < depth; ++i) { + // The map entry: C++ writes the key even when it is empty. + levels.push_back( + {Wire(LengthPrefixedField(1, ""), Tag(2, WireType::kLengthPrefixed))}); + // The NestedMessage. + levels.push_back({Tag(2, WireType::kLengthPrefixed)}); + // The nested TestAllTypesEdition2023. + Wire fields = VarintField(1, 123); + if (i + 1 < depth) { + fields = Wire(fields, Tag(71, WireType::kLengthPrefixed)); + } + levels.push_back({fields}); + } + return Nested(levels); +} + +// A message set item is a group: start tag (1), type_id (2), message (3), +// end tag. +Wire DeepMessageSetPayload(int depth) { + ABSL_DCHECK_GE(depth, 0); + std::vector<Level> levels; + levels.push_back({Tag(500, WireType::kLengthPrefixed)}); + for (int i = 0; i <= depth; ++i) { + // The MessageSetCorrect and its item. + levels.push_back( + {Wire(Tag(1, WireType::kStartGroup), VarintField(2, 4135312), + Tag(3, WireType::kLengthPrefixed)), + Tag(1, WireType::kEndGroup)}); + // The MessageSetCorrectExtension2. + levels.push_back( + {i < depth ? Tag(10, WireType::kLengthPrefixed) : VarintField(9, 123)}); + } + return Nested(levels); +} + +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/recursion_limit_payloads.h b/conformance/recursion_limit_payloads.h new file mode 100644 index 0000000..fe78242 --- /dev/null +++ b/conformance/recursion_limit_payloads.h
@@ -0,0 +1,47 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The deeply nested payloads of the recursion-limit performance tests +// (binary_recursion_limit_test.cc). They are built directly on the wire +// rather than serialized from a message object of that depth. ByteSizeLong(), +// the serializer and the destructor recurse once per nesting level, which +// overflows the stack under the sanitizers' larger frames (b/563731788). The +// bytes are identical to what the message objects serialize, which +// recursion_limit_payloads_test.cc checks at a depth the stack can take. + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_RECURSION_LIMIT_PAYLOADS_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_RECURSION_LIMIT_PAYLOADS_H__ + +#include "conformance/binary_wireformat.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// TestAllTypesEdition2023 whose map_recursive (301) entry {key 0, value} +// holds a TestAllTypesEdition2023 with optional_int32 (1) = 123 and, `depth` +// times in all, its own such entry. `depth` must be at least 1. +Wire DeepMapPayload(int depth); + +// TestAllTypesEdition2023 whose map_string_nested_message (71) entry {key "", +// value} holds a NestedMessage whose corecursive (2) is a +// TestAllTypesEdition2023 with optional_int32 (1) = 123 and, `depth` times in +// all, its own such entry. `depth` must be at least 1. +Wire DeepMapStringKeyPayload(int depth); + +// TestAllTypesProto2 whose message_set_correct (500) holds a MessageSetCorrect +// with one MessageSetCorrectExtension2 item (type id 4135312). The item's +// sub_msg (10) is again such a MessageSetCorrect, `depth` times. The +// innermost extension holds i (9) = 123 instead. With `depth` 0 it is the +// only one. +Wire DeepMessageSetPayload(int depth); + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_RECURSION_LIMIT_PAYLOADS_H__
diff --git a/conformance/recursion_limit_payloads_test.cc b/conformance/recursion_limit_payloads_test.cc new file mode 100644 index 0000000..9f22282 --- /dev/null +++ b/conformance/recursion_limit_payloads_test.cc
@@ -0,0 +1,84 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/recursion_limit_payloads.h" + +#include <gtest/gtest.h> +#include "conformance/binary_wireformat.h" +#include "conformance/test_protos/test_messages_edition2023.pb.h" +#include "google/protobuf/test_messages_proto2.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::editions::TestAllTypesEdition2023; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; + +// The same payloads built from message objects the way the legacy suite built +// them, at depths the stack can take: the shallowest, and one deep enough for +// the length prefixes to need two bytes. +class RecursionLimitPayloadsTest : public ::testing::TestWithParam<int> {}; + +TEST_P(RecursionLimitPayloadsTest, DeepMapMatchesTheMessageObject) { + const int depth = GetParam(); + TestAllTypesEdition2023 message; + TestAllTypesEdition2023* sub = &message; + for (int i = 0; i < depth; i++) { + sub = &(*sub->mutable_map_recursive())[0]; + sub->set_optional_int32(123); + } + EXPECT_EQ(DeepMapPayload(depth), Wire(message.SerializeAsString())); +} + +TEST_P(RecursionLimitPayloadsTest, DeepMapStringKeyMatchesTheMessageObject) { + const int depth = GetParam(); + TestAllTypesEdition2023 message; + TestAllTypesEdition2023* sub = &message; + for (int i = 0; i < depth; i++) { + sub = (*sub->mutable_map_string_nested_message())[""].mutable_corecursive(); + sub->set_optional_int32(123); + } + EXPECT_EQ(DeepMapStringKeyPayload(depth), Wire(message.SerializeAsString())); +} + +TEST_P(RecursionLimitPayloadsTest, DeepMessageSetMatchesTheMessageObject) { + const int depth = GetParam(); + TestAllTypesProto2 message; + TestAllTypesProto2::MessageSetCorrect* sub = + message.mutable_message_set_correct(); + for (int i = 0; i < depth; i++) { + sub = + sub->MutableExtension(TestAllTypesProto2::MessageSetCorrectExtension2:: + message_set_extension) + ->mutable_sub_msg(); + } + sub->MutableExtension( + TestAllTypesProto2::MessageSetCorrectExtension2::message_set_extension) + ->set_i(123); + EXPECT_EQ(DeepMessageSetPayload(depth), Wire(message.SerializeAsString())); +} + +INSTANTIATE_TEST_SUITE_P(Depths, RecursionLimitPayloadsTest, + ::testing::Values(1, 3, 50)); + +// A message set of depth 0 is a single extension holding i. (A plain TEST, so +// it can't share the parameterized fixture's suite name.) +TEST(RecursionLimitPayloadsDepthZeroTest, MessageSet) { + TestAllTypesProto2 message; + message.mutable_message_set_correct() + ->MutableExtension(TestAllTypesProto2::MessageSetCorrectExtension2:: + message_set_extension) + ->set_i(123); + EXPECT_EQ(DeepMessageSetPayload(0), Wire(message.SerializeAsString())); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_ledger.cc b/conformance/result_ledger.cc new file mode 100644 index 0000000..c25e341 --- /dev/null +++ b/conformance/result_ledger.cc
@@ -0,0 +1,515 @@ +#include "conformance/result_ledger.h" + +#include <algorithm> +#include <cstddef> +#include <fstream> +#include <ios> +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/failure_list.h" +#include "conformance/result_record.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +bool InsertUniqueTest(absl::flat_hash_set<std::string>& set, + absl::string_view value) { + if (!set.emplace(value).second) { + return false; + } + return true; +} + +void IncrementIfUnique(bool unique, int& counter) { + if (unique) { + ++counter; + } +} + +// Formats an entry line for SaveFailureList(): the test name padded to +// `alignment`, then " # " and the message, or the name alone if the line has +// no message. A comment or blank line is only stripped. +std::string ReformatLine(size_t alignment, absl::string_view line) { + size_t comment_pos = line.find('#'); + absl::string_view test_name = + absl::StripAsciiWhitespace(line.substr(0, comment_pos)); + if (test_name.empty()) { + return std::string(absl::StripAsciiWhitespace(line)); + } + absl::string_view message; + if (comment_pos != absl::string_view::npos) { + message = absl::StripAsciiWhitespace(line.substr(comment_pos + 1)); + } + if (message.empty()) { + return std::string(test_name); + } + ABSL_CHECK_GE(alignment, test_name.length()); + std::string whitespace(alignment - test_name.length(), ' '); + return absl::StrCat(test_name, whitespace, " # ", message); +} + +// The failure list line for `test_name` failing with `message`, which may be +// empty; ReformatLine() aligns it. +std::string EntryLine(absl::string_view test_name, absl::string_view message) { + if (message.empty()) return std::string(test_name); + return absl::StrCat(test_name, " # ", message); +} + +// The test name (or wildcard) of a failure list line, i.e. what precedes its +// comment, with all whitespace removed: the key entries are loaded and looked +// up by, so that LoadFailureList() and AppendSavedLines() agree on it. +std::string EntryKeyOf(absl::string_view line) { + std::string key(line.substr(0, line.find('#'))); + key.erase(std::remove_if(key.begin(), key.end(), absl::ascii_isspace), + key.end()); + return key; +} + +// The test name of a failure list line as written (an entry as listed, see +// EntryKeyOf() for the key it is looked up by), or empty for a comment or +// blank line. +absl::string_view EntryOf(absl::string_view line) { + return absl::StripAsciiWhitespace(line.substr(0, line.find('#'))); +} + +// The copybara markers that delimit a region the open source export drops. +// They are spelled in two pieces so that the export's scrubber, which +// processes every file, doesn't take these literals for markers. +constexpr absl::string_view kStripBeginMarker = + "copybara:" + "strip_begin"; +constexpr absl::string_view kStripEndMarker = + "copybara:" + "strip_end"; + +// Whether `line` is a comment whose text starts with `marker`, e.g. a +// `# <kStripBeginMarker>` line (copybara allows text after the marker). +bool IsCommentStartingWith(absl::string_view line, absl::string_view marker) { + line = absl::StripAsciiWhitespace(line); + if (!absl::ConsumePrefix(&line, "#")) return false; + return absl::StartsWith(absl::StripLeadingAsciiWhitespace(line), marker); +} + +bool IsStripBegin(absl::string_view line) { + return IsCommentStartingWith(line, kStripBeginMarker); +} + +// Also matches the `strip_end_and_replace_begin` marker, which closes the +// block as well. +bool IsStripEnd(absl::string_view line) { + return IsCommentStartingWith(line, kStripEndMarker); +} + +// A line SaveFailureList() writes, with the alignment scope it belongs to: 0 +// for the lines outside copybara strip blocks, k > 0 for the lines of the k-th +// block (its marker lines included). +struct SavedLine { + std::string text; + size_t scope; +}; + +} // namespace + +ResultLedger::~ResultLedger() { + if (!finalized_) { + ABSL_LOG(FATAL) + << "ResultLedger::Finalize() was not called before destruction."; + } +} + +absl::Status ResultLedger::LoadFailureList(absl::string_view filename) { + std::ifstream infile(std::string{filename}); + + if (!infile.is_open()) { + return absl::InternalError( + absl::StrCat("Couldn't open failure list file: ", filename)); + } + + for (std::string line; std::getline(infile, line);) { + failure_list_lines_.push_back(line); + + std::string test_name = EntryKeyOf(line); + + if (test_name.empty()) { // Skip empty lines and comments. + continue; + } + + // Find our failure message if it exists. Will be set to an empty string + // if no message is found. Empty failure messages also pass our tests. + size_t comment_pos = line.find('#'); + std::string message; + if (comment_pos != std::string::npos) { + message = line.substr(comment_pos + 1); // +1 to skip the delimiter + // If we had only whitespace after the delimiter, we will have an empty + // failure message and the test will still pass. + message = std::string(absl::StripAsciiWhitespace(message)); + } + if (absl::Status status = AddExpectedFailure(test_name, message); + !status.ok()) { + return status; + } + } + + return absl::OkStatus(); +} + +absl::Status ResultLedger::AddExpectedFailure( + absl::string_view test_name, absl::string_view failure_message) { + if (absl::Status status = failure_list_.Add(test_name, failure_message); + !status.ok()) { + return status; + } + ABSL_CHECK(unseen_expected_failures_.emplace(test_name).second); + ABSL_CHECK(unmatched_expected_failures_.emplace(test_name).second); + return absl::OkStatus(); +} + +absl::Status ResultLedger::SaveFailureList(absl::string_view filename) const { + std::ofstream outfile(std::string{filename}, std::ios::binary); + if (!outfile.is_open()) { + return absl::InternalError(absl::StrCat( + "Couldn't open failure list file for writing: ", filename)); + } + + // The loaded lines with unseen entries dropped and changed entries rewritten + // (see AppendSavedLines()), each tagged with its alignment scope; the new + // failures are merged in below. A copybara strip block runs from a + // strip_begin line to the next strip_end line; a strip_begin with no + // strip_end after it, a strip_end outside a block and a strip_begin inside + // one are ordinary comments, as they are to LoadFailureList(). + std::vector<SavedLine> lines; + size_t blocks = 0; + size_t scope = 0; + for (size_t i = 0; i < failure_list_lines_.size(); ++i) { + const std::string& line = failure_list_lines_[i]; + if (scope == 0 && IsStripBegin(line) && + std::any_of( + failure_list_lines_.begin() + static_cast<std::ptrdiff_t>(i), + failure_list_lines_.end(), IsStripEnd)) { + scope = ++blocks; + } + std::vector<std::string> saved; + AppendSavedLines(line, saved); + for (std::string& text : saved) { + lines.push_back({std::move(text), scope}); + } + if (scope != 0 && IsStripEnd(line)) { + scope = 0; + } + } + + // Calculate the alignment of each scope: the longest name among its lines + // and, outside the blocks, the new failures. + std::vector<size_t> alignment(blocks + 1, 0); + for (const SavedLine& line : lines) { + alignment[line.scope] = + std::max(alignment[line.scope], EntryOf(line.text).length()); + } + for (const auto& failure : new_failures_) { + alignment[0] = std::max(alignment[0], failure.first.length()); + } + + // Output the lines, inserting the new failures by name among the lines + // outside the blocks. + auto to_add = new_failures_.begin(); + for (const SavedLine& line : lines) { + if (line.scope == 0) { + absl::string_view test_name = EntryOf(line.text); + while (to_add != new_failures_.end() && test_name > to_add->first) { + outfile << ReformatLine(alignment[0], + EntryLine(to_add->first, to_add->second)) + << "\n"; + ++to_add; + } + } + outfile << ReformatLine(alignment[line.scope], line.text) << "\n"; + } + + // Add any remaining new failures. + while (to_add != new_failures_.end()) { + outfile << ReformatLine(alignment[0], + EntryLine(to_add->first, to_add->second)) + << "\n"; + ++to_add; + } + + outfile.close(); + if (!outfile.good()) { + return absl::InternalError( + absl::StrCat("Failed to write failure list file: ", filename)); + } + return absl::OkStatus(); +} + +void ResultLedger::AppendSavedLines(absl::string_view line, + std::vector<std::string>& lines) const { + const std::string entry = EntryKeyOf(line); + if (entry.empty()) { + // A comment or blank line. + lines.emplace_back(line); + return; + } + + auto it = entry_matches_.find(entry); + if (it == entry_matches_.end()) { + // No test matched the entry with a verdict: it is unseen and goes. + return; + } + const absl::btree_map<std::string, MatchOutcome>& matches = it->second; + + // Kept verbatim unless a matched test succeeded, was skipped or failed with + // a different message. + const bool changed = absl::c_any_of(matches, [](const auto& match) { + return match.second.kind != MatchOutcome::kExpectedFailure; + }); + if (!changed) { + lines.emplace_back(line); + return; + } + + // Kept as the entry (a wildcard included) with the new message if every + // matched test failed with that same message. + const MatchOutcome& first = matches.begin()->second; + const bool same_new_message = + first.kind == MatchOutcome::kOtherFailure && + absl::c_all_of(matches, [&](const auto& match) { + return match.second.kind == MatchOutcome::kOtherFailure && + match.second.message == first.message; + }); + if (same_new_message) { + lines.push_back(EntryLine(entry, first.message)); + return; + } + + // Otherwise the matched set diverged: one line per matched test that still + // fails (which for an exact entry that succeeded or was skipped is none). + const std::string expected_message = *failure_list_.ExpectedMessage(entry); + for (const auto& [test_name, outcome] : matches) { + switch (outcome.kind) { + case MatchOutcome::kSuccess: + case MatchOutcome::kSkip: + break; + case MatchOutcome::kOtherFailure: + lines.push_back(EntryLine(test_name, outcome.message)); + break; + case MatchOutcome::kExpectedFailure: + lines.push_back(EntryLine(test_name, expected_message)); + break; + } + } +} + +absl::Status ResultLedger::ReportSuccess(absl::string_view test_name) { + bool unique = InsertUniqueTest(seen_tests_, test_name); + absl::optional<std::string> failure_match = MarkMatched(test_name); + + if (failure_match.has_value()) { + // This was expected to fail, but it succeeded. + IncrementIfUnique(unique, number_of_matches_[*failure_match]); + IncrementIfUnique(unique, unexpected_successes_); + if (unique) { + unexpected_success_matches_[test_name] = *failure_match; + } + unseen_expected_failures_.erase(*failure_match); + entry_matches_[*failure_match][test_name] = { + /*kind=*/MatchOutcome::kSuccess}; + } else { + // This wasn't expected to fail. + IncrementIfUnique(unique, expected_successes_); + } + return failure_list_.VerdictOnSuccess(test_name); +} + +absl::Status ResultLedger::ReportFailure(absl::string_view test_name, + TestPriority priority, + absl::string_view failure_message) { + bool unique = InsertUniqueTest(seen_tests_, test_name); + absl::optional<std::string> failure_match = MarkMatched(test_name); + + std::string formatted_failure_message = FormatFailureMessage(failure_message); + // Counts the failure as unexpected and records it for UnexpectedFailures(). + auto record_unexpected_failure = [&] { + IncrementIfUnique(unique, unexpected_failures_); + if (unique) { + unexpected_failure_messages_[test_name] = formatted_failure_message; + } + }; + + if (!failure_match.has_value()) { + if (priority > failure_list_.enforcement_level()) { + // Tolerated: neither a failure nor a skip, and not written to the + // failure list; only counted. + IncrementIfUnique(unique, tolerated_failures_); + } else { + // This was not expected to fail. + record_unexpected_failure(); + new_failures_[test_name] = formatted_failure_message; + } + return failure_list_.VerdictOnFailure(test_name, priority, failure_message); + } + + // The entry is seen either way: Finalize() is about entries no test + // reached, and a message mismatch is already reported by the verdict, by + // UnexpectedFailures() and by SaveFailureList(). + unseen_expected_failures_.erase(*failure_match); + if (!absl::StartsWith(formatted_failure_message, + *failure_list_.ExpectedMessage(*failure_match))) { + record_unexpected_failure(); + entry_matches_[*failure_match][test_name] = { + /*kind=*/MatchOutcome::kOtherFailure, + /*message=*/formatted_failure_message}; + } else { + entry_matches_[*failure_match][test_name] = { + /*kind=*/MatchOutcome::kExpectedFailure}; + IncrementIfUnique(unique, number_of_matches_[*failure_match]); + IncrementIfUnique(unique, expected_failures_); + } + return failure_list_.VerdictOnFailure(test_name, priority, failure_message); +} + +absl::Status ResultLedger::ReportSkip(absl::string_view test_name, + absl::string_view skip_reason) { + bool unique = InsertUniqueTest(seen_tests_, test_name); + IncrementIfUnique(unique, skipped_); + absl::optional<std::string> failure_match = MarkMatched(test_name); + if (!failure_match.has_value()) { + return absl::OkStatus(); + } + + // A skipped test is a listed test that didn't fail, so its match is not an + // unseen entry for Finalize() but is dropped by SaveFailureList(), like a + // success (the verdict asks for the same by hand). It doesn't count toward + // the entry's wildcard expansion cap (number_of_matches_) either, unlike in + // the legacy runner, which counted every test a wildcard matched: the cap + // is about real outcomes hidden by a wildcard, and a skip isn't one. + unseen_expected_failures_.erase(*failure_match); + entry_matches_[*failure_match][test_name] = {/*kind=*/MatchOutcome::kSkip}; + IncrementIfUnique(unique, listed_skips_); + if (unique) { + listed_skip_matches_[test_name] = *failure_match; + } + return failure_list_.VerdictOnSkip(test_name, skip_reason); +} + +void ResultLedger::ReportNotSelected(absl::string_view test_name) { + seen_tests_.emplace(test_name); + MarkMatched(test_name); +} + +void ResultLedger::Report(absl::string_view test_name, + const ResultRecord& record) { + switch (record.status) { + case ResultRecord::Status::kPass: + ReportSuccess(test_name).IgnoreError(); + break; + case ResultRecord::Status::kFail: + case ResultRecord::Status::kCrash: + ReportFailure(test_name, record.priority, record.message).IgnoreError(); + break; + case ResultRecord::Status::kSkip: + ReportSkip(test_name, record.message).IgnoreError(); + break; + case ResultRecord::Status::kNotSelected: + ReportNotSelected(test_name); + break; + } +} + +absl::optional<std::string> ResultLedger::MarkMatched( + absl::string_view test_name) { + absl::optional<std::string> failure_match = + failure_list_.MatchingEntry(test_name); + if (failure_match.has_value()) { + unmatched_expected_failures_.erase(*failure_match); + } + return failure_match; +} + +std::vector<std::string> ResultLedger::OverexpandedWildcards() const { + std::vector<std::string> entries; + for (const auto& [entry, matches] : number_of_matches_) { + if (matches > kMaximumWildcardExpansions) entries.push_back(entry); + } + absl::c_sort(entries); + return entries; +} + +std::vector<std::string> ResultLedger::UnseenExpectedFailures() const { + std::vector<std::string> unseen(unseen_expected_failures_.begin(), + unseen_expected_failures_.end()); + absl::c_sort(unseen); + return unseen; +} + +std::vector<std::string> ResultLedger::UnmatchedExpectedFailures() const { + std::vector<std::string> unmatched(unmatched_expected_failures_.begin(), + unmatched_expected_failures_.end()); + absl::c_sort(unmatched); + return unmatched; +} + +std::vector<UnexpectedResult> ResultLedger::UnexpectedFailures() const { + std::vector<UnexpectedResult> failures; + failures.reserve(unexpected_failure_messages_.size()); + // The map is ordered by test name. + for (const auto& [test_name, failure_message] : + unexpected_failure_messages_) { + failures.push_back( + {/*test_name=*/test_name, /*failure_message=*/failure_message}); + } + return failures; +} + +std::vector<UnexpectedResult> ResultLedger::UnexpectedSuccesses() const { + std::vector<UnexpectedResult> successes; + successes.reserve(unexpected_success_matches_.size()); + // The map is ordered by test name. + for (const auto& [test_name, matched_entry] : unexpected_success_matches_) { + successes.push_back( + {/*test_name=*/test_name, + /*failure_message=*/*failure_list_.ExpectedMessage(matched_entry), + /*matched_entry=*/matched_entry}); + } + return successes; +} + +std::vector<std::pair<std::string, std::string>> ResultLedger::ListedSkips() + const { + // The map is ordered by test name. + return std::vector<std::pair<std::string, std::string>>( + listed_skip_matches_.begin(), listed_skip_matches_.end()); +} + +absl::Status ResultLedger::Finalize() { + finalized_ = true; + std::vector<std::string> unseen = UnseenExpectedFailures(); + if (!unseen.empty()) { + return absl::FailedPreconditionError( + absl::StrCat("The following expected failures were not seen: ", + absl::StrJoin(unseen, ", "))); + } + return absl::OkStatus(); +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_ledger.h b/conformance/result_ledger.h new file mode 100644 index 0000000..45cecc5 --- /dev/null +++ b/conformance/result_ledger.h
@@ -0,0 +1,306 @@ +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LEDGER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LEDGER_H__ + +#include <string> +#include <utility> +#include <vector> + +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/failure_list.h" +#include "conformance/result_record.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// A test whose result contradicted the failure list. See +// ResultLedger::UnexpectedFailures() and ResultLedger::UnexpectedSuccesses(). +// TODO: Remove with conformance_test_runner. +struct UnexpectedResult { + // The full name of the test, as reported to the ResultLedger. + std::string test_name; + // For an unexpected failure, the test's failure message as + // ResultLedger::SaveFailureList() would write it. For an unexpected success, + // the message of the failure list entry that matched the test. + std::string failure_message; + // For an unexpected success, the failure list entry that matched + // `test_name`, possibly a wildcard. Unset for an unexpected failure, which + // matched no entry or matched one with a different message. + absl::optional<std::string> matched_entry; +}; + +// How many tests one wildcard entry may cover with a verdict, that is an +// expected failure or an unexpected success. See OverexpandedWildcards(). +inline constexpr int kMaximumWildcardExpansions = 20; + +// The ledger of a conformance run: tallies the results of its tests against +// its failure list. It knows nothing of gtest; the test environment's +// ResultListener (result_listener.h) feeds it. The failure list is loaded +// once (see LoadFailureList()) into a FailureList, which decides each result: +// Yields() (matchers.h) asks it for the verdict and records the outcome, and +// the listener reports the outcome here (see Report()) and reads the counters +// back as test properties. The environment owns the ledger and, under --fix, +// writes the new failure list from it (see SaveFailureList()). +class ResultLedger { + public: + ResultLedger() = default; + ~ResultLedger(); + + // The lowest priority whose failures fail the suite (see TestPriority in + // testee.h). ReportFailure() tolerates a failing test of a lower priority + // unless it is in the failure list. Defaults to kLowestPriority, which + // enforces every priority. + void set_enforcement_level(TestPriority level) { + failure_list_.set_enforcement_level(level); + } + + // Loads a failure list from disk and adds its entries to the ones loaded so + // far. Each line that isn't blank or a comment names one expected failure, + // optionally followed by `#` and the expected failure message. Returns an + // error if the file can't be opened. Also returns an error if an entry is + // already present, including one from an earlier file, or contains an + // invalid wildcard. + absl::Status LoadFailureList(absl::string_view filename); + + // The failure list as loaded so far, which decides each result. Yields() + // reads the global environment's through GetGlobalFailureList(). + const FailureList& failure_list() const { return failure_list_; } + + // Saves an updated failure list to disk based on the reported results. + // Comment and blank lines stay where they are. Entries are updated as + // follows: + // + // - An entry that no test matched with a verdict is dropped. These are + // the entries Finalize() reports as unseen. + // - An entry whose matched tests all still fail with its message is kept + // verbatim, wildcards included. + // - An entry whose matched tests all failed with the same different + // message is kept with that message instead, wildcards included. + // - Any other entry is rewritten in place, one line per matched test in + // name order. A test that succeeded or was skipped (see ReportSkip()) + // is left out. A test that failed with a different message gets that + // message. A test that still fails with the entry's message keeps it. + // An exact entry therefore goes when its test succeeded or was skipped. + // A wildcard entry is expanded into the tests it matched, so that the + // tests that still fail stay listed. + // - Tests that failed but matched no entry are inserted in name order + // with their messages. + // + // Entry lines are aligned to the longest name. + // + // A copybara strip block (`# copybara` `:strip_begin` ... `:strip_end`) is + // the region the open source export drops. It is kept as one contiguous + // region and aligned on its own. Its entries are handled as above and stay + // inside it. No new failure is ever inserted into it. A new failure goes + // in front of the first entry outside any block that sorts after it, or at + // the end of the file. One that sorts between the last entry before a + // block and the first entry after it lands right after the block's + // strip_end. Unbalanced markers are ordinary comments, as they are to + // LoadFailureList(). + // + // Returns an error if `filename` can't be opened for writing or the write + // fails. + absl::Status SaveFailureList(absl::string_view filename) const; + + // Records a successful test run and returns + // FailureList::VerdictOnSuccess(). + absl::Status ReportSuccess(absl::string_view test_name); + + // Records a failed test run along with the failure message and returns + // FailureList::VerdictOnFailure(). A tolerated failure is only counted + // (see tolerated_failures()). An unexpected one is also kept for + // UnexpectedFailures() and SaveFailureList(). + absl::Status ReportFailure(absl::string_view test_name, TestPriority priority, + absl::string_view failure_message); + + // Records a test that the testee skipped for `skip_reason` and returns + // FailureList::VerdictOnSkip(). If the test matched a failure list entry, + // the entry counts as seen and matched, so Finalize() doesn't report it as + // unseen, and SaveFailureList() drops the match like a success. A listed + // test that is skipped is also recorded for ListedSkips(). + absl::Status ReportSkip(absl::string_view test_name, + absl::string_view skip_reason); + + // Records a test the runner didn't run because it wasn't selected (see + // ResultRecord::Status::kNotSelected). + // The legacy runner matched a test name against the failure list before + // checking whether the test was selected. Like it, this only marks the + // entry the name matches, if any, as matched for UnmatchedExpectedFailures(). + // The test is not counted by any statistic and its entry stays unseen. + // TODO: Remove with conformance_test_runner. + void ReportNotSelected(absl::string_view test_name); + + // Records `record`, the outcome Yields() recorded for `test_name`, through + // the Report*() method for its status. The verdict was applied when the + // result was checked, so it is dropped here. + void Report(absl::string_view test_name, const ResultRecord& record); + + // Whether any of the Report*() methods has been called for `test_name`. + // The ResultListener checks this before it reports a recorded outcome (see + // Report()), so that a test is counted once. + bool WasReported(absl::string_view test_name) const { + return seen_tests_.contains(test_name); + } + + // The sorted failure list entries that more than kMaximumWildcardExpansions + // reported tests matched with a verdict, that is as an expected failure or + // an unexpected success. Skips don't count. Such a wildcard hides too + // much; the test environment fails the run over it. + std::vector<std::string> OverexpandedWildcards() const; + + // Runs sanity checks over the failure list to make sure everything we + // expected to run was reported. Returns an error naming the sorted expected + // failure entries that have not been reported as a failure, an unexpected + // success or a skip. Must be called before destruction. + absl::Status Finalize(); + + // Returns the sorted expected failure entries that no reported test name has + // matched so far, regardless of the outcome. Names are reported through + // ReportSuccess(), ReportFailure(), ReportSkip() or ReportNotSelected(). A + // wildcard entry counts as matched once it has matched any test name. + // Unlike the unseen entries Finalize() reports, an entry whose test was not + // selected is not returned. It is a real test, only not run here. + // TODO: Remove with conformance_test_runner. + std::vector<std::string> UnmatchedExpectedFailures() const; + + // Returns the tests reported as unexpected failures so far, sorted by test + // name. These are the tests unexpected_failures() counts. Each result's + // `failure_message` is the message as SaveFailureList() would write it, and + // its `matched_entry` is unset. This includes tests that were expected to + // fail, but with a different message. SaveFailureList() rewrites their + // entry, so adding these lines to the failure list would duplicate it. + // TODO: Remove with conformance_test_runner. + std::vector<UnexpectedResult> UnexpectedFailures() const; + + // Returns the tests reported as unexpected successes so far, sorted by test + // name. These are the tests unexpected_successes() counts. Each result's + // `matched_entry` is the failure list entry the test matched, and + // `failure_message` is that entry's message. + // TODO: Remove with conformance_test_runner. + std::vector<UnexpectedResult> UnexpectedSuccesses() const; + + // Returns the tests the testee skipped although they are in the failure + // list, sorted by test name. These are the tests listed_skips() counts. + // Each pair is the test name and the failure list entry it matched. Such a + // test fails the gtest run: the matchers in matchers.h fail it with the + // error ReportSkip() returns. It is not an unexpected failure or success. + // SaveFailureList() under --fix drops its match like a passing test's, that + // is the whole entry, or the test's own line when a wildcard entry is + // expanded. + // TODO: Remove with conformance_test_runner. + std::vector<std::pair<std::string, std::string>> ListedSkips() const; + + // The number of tests that were reported skipped. This includes the listed + // skips. + int skipped() const { return skipped_; } + + // The number of skipped tests that are in the failure list. See + // ListedSkips(). + int listed_skips() const { return listed_skips_; } + + // The number of tests that failed but were tolerated because their priority + // is above the enforcement level. + int tolerated_failures() const { return tolerated_failures_; } + + // The number of tests that were in the failure list and failed with the + // expected message. + int expected_failures() const { return expected_failures_; } + + // The number of tests that failed but were not in the failure list, or + // failed with a different message than the one listed. + int unexpected_failures() const { return unexpected_failures_; } + + // The number of tests that were not in the failure list and succeeded. + int expected_successes() const { return expected_successes_; } + + // The number of tests that were in the failure list but succeeded. + int unexpected_successes() const { return unexpected_successes_; } + + private: + // Adds one failure list entry, as LoadFailureList() does for each line. + // Returns an error if the test name is already present or contains an + // invalid wildcard. + absl::Status AddExpectedFailure(absl::string_view test_name, + absl::string_view failure_message); + + // The sorted expected failure entries Finalize() complains about: those not + // reported as a failure, an unexpected success or a skip so far. + std::vector<std::string> UnseenExpectedFailures() const; + + // Marks the failure list entry `test_name` matches, if any, as matched (see + // UnmatchedExpectedFailures()) and returns it. + absl::optional<std::string> MarkMatched(absl::string_view test_name); + + // The verdict on a test that matched a failure list entry, as far as + // SaveFailureList() is concerned. + struct MatchOutcome { + enum Kind { + kExpectedFailure, // Failed with the entry's message. + kOtherFailure, // Failed with a different message (see `message`). + kSuccess, // Succeeded (see ReportSuccess()). + kSkip, // Was skipped by the testee (see ReportSkip()). + }; + Kind kind; + // The formatted failure message, for kOtherFailure. + std::string message; + }; + + // Appends to `lines` what SaveFailureList() writes for `line`, a line of the + // failure list as loaded. That is the line itself, nothing, or the entry's + // rewritten lines, possibly expanded. See SaveFailureList(). + void AppendSavedLines(absl::string_view line, + std::vector<std::string>& lines) const; + + FailureList failure_list_; + + absl::flat_hash_set<std::string> unseen_expected_failures_; + // Entries never matched by name by any Report*() call. + absl::flat_hash_set<std::string> unmatched_expected_failures_; + // The tests that matched each entry with a verdict, by test name, and how + // they came out. Entries are as listed, wildcards included. + // ReportNotSelected() is not a verdict and doesn't record anything here. + absl::flat_hash_map<std::string, absl::btree_map<std::string, MatchOutcome>> + entry_matches_; + // How many tests matched each entry as an expected failure or an unexpected + // success. See OverexpandedWildcards(). + absl::flat_hash_map<std::string, int> number_of_matches_; + + // Every test name reported so far (see WasReported()), so that a test + // reported more than once is only counted once. + absl::flat_hash_set<std::string> seen_tests_; + + // The tests counted by unexpected_failures_, mapped to their formatted + // failure message, and the tests counted by unexpected_successes_, mapped to + // the failure list entry they matched. + absl::btree_map<std::string, std::string> unexpected_failure_messages_; + absl::btree_map<std::string, std::string> unexpected_success_matches_; + // The tests counted by listed_skips_, mapped to the entry they matched. + absl::btree_map<std::string, std::string> listed_skip_matches_; + + std::vector<std::string> failure_list_lines_; + // The tests that failed but matched no entry, with their formatted messages. + absl::btree_map<std::string, std::string> new_failures_; + + int skipped_ = 0; + int listed_skips_ = 0; + int tolerated_failures_ = 0; + int expected_failures_ = 0; + int unexpected_failures_ = 0; + int expected_successes_ = 0; + int unexpected_successes_ = 0; + bool finalized_ = false; +}; + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LEDGER_H__
diff --git a/conformance/result_ledger_test.cc b/conformance/result_ledger_test.cc new file mode 100644 index 0000000..09b5bbe --- /dev/null +++ b/conformance/result_ledger_test.cc
@@ -0,0 +1,1916 @@ +#include "conformance/result_ledger.h" + +#include <initializer_list> +#include <string> + +#include "google/protobuf/testing/file.h" +#include "google/protobuf/testing/file.h" +#include "google/protobuf/testing/file.h" +#include <gmock/gmock.h> +#include "google/protobuf/testing/googletest.h" +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/types/optional.h" +#include "conformance/failure_list.h" +#include "conformance/result_record.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +using ::absl_testing::IsOk; +using ::absl_testing::StatusIs; +using ::testing::AllOf; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::FieldsAre; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Not; +using ::testing::Optional; +using ::testing::Pair; +using ::testing::SizeIs; + +class ResultLedgerTest : public ::testing::Test { + protected: + ResultLedgerTest() + : tmp_dir_(absl::StrCat(TestTempDir(), "/result_ledger_test/")), + failure_list_path_(absl::StrCat( + tmp_dir_, + testing::UnitTest::GetInstance()->current_test_info()->name())) { + if (!File::Exists(tmp_dir_)) { + ABSL_CHECK_OK( + File::RecursivelyCreateDir(tmp_dir_, 0777)); + } + } + + struct Failure { + absl::string_view name; + absl::string_view message; + + Failure( // NOLINT(google-explicit-constructor) + absl::string_view name) + : name(name), message("") {} + Failure(absl::string_view name, absl::string_view message) + : name(name), message(message) {} + }; + + void CreateFailureList( + std::initializer_list<const Failure> expected_failures) { + std::string content; + for (auto failure : expected_failures) { + absl::StrAppend(&content, failure.name, " # ", failure.message, "\n"); + } + ABSL_CHECK_OK( + File::SetContents(failure_list_path_, content, true)); + } + + void CreateFailureList(absl::string_view content) { + ABSL_CHECK_OK( + File::SetContents(failure_list_path_, content, true)); + } + + absl::string_view failure_list() const { return failure_list_path_; } + + ~ResultLedgerTest() override { + File::DeleteRecursively(failure_list_path_, NULL, NULL); + } + + private: + std::string tmp_dir_; + std::string failure_list_path_; + std::string output_path_; +}; + +TEST_F(ResultLedgerTest, RequiresFinalize) { + EXPECT_DEATH({ ResultLedger ledger; }, "Finalize"); +} + +TEST_F(ResultLedgerTest, ReportSuccess) { + CreateFailureList({}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportSuccess("foo"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 1); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportSkipped) { + CreateFailureList({}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportSkip("foo", "reason"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 1); + EXPECT_EQ(ledger.listed_skips(), 0); + EXPECT_THAT(ledger.ListedSkips(), IsEmpty()); +} + +TEST_F(ResultLedgerTest, ReportSkipMarksListedEntrySeenAndMatched) { + CreateFailureList({{"foo", "abc"}, {"bar.*", "abc"}, {"baz", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // A listed skip is reported like an unexpected success, naming the entry. + EXPECT_THAT(ledger.ReportSkip("foo", "not supported"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test foo (matched to foo) is in the failure list but " + "was skipped by the testee: not supported. Remove its " + "match from the failure list.")); + EXPECT_THAT(ledger.ReportSkip("bar.x", "no"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test bar.x (matched to bar.*) is in the failure list " + "but was skipped by the testee: no. Remove its match " + "from the failure list.")); + // Reporting a test twice returns the error again but counts it once. + EXPECT_THAT(ledger.ReportSkip("bar.x", "no"), Not(IsOk())); + + // A skip is only counted as a skip, but the entries it matched are neither + // unmatched nor unseen: dropping the match is left to SaveFailureList(). + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), ElementsAre("baz")); + EXPECT_THAT(ledger.Finalize(), StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("were not seen: baz"))); + EXPECT_EQ(ledger.skipped(), 2); + EXPECT_EQ(ledger.listed_skips(), 2); + EXPECT_THAT(ledger.ListedSkips(), + ElementsAre(Pair("bar.x", "bar.*"), Pair("foo", "foo"))); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger.UnexpectedSuccesses(), IsEmpty()); +} + +TEST_F(ResultLedgerTest, ReportNotSelected) { + CreateFailureList({{"foo"}, {"bar.*"}, {"baz"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ledger.ReportNotSelected("foo"); + ledger.ReportNotSelected("bar.x"); + ledger.ReportNotSelected("unlisted"); + + // Only the matched entries stop being unmatched; nothing else changes. + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), ElementsAre("baz")); + EXPECT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + "The following expected failures were not seen: bar.*, " + "baz, foo")); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); + EXPECT_EQ(ledger.tolerated_failures(), 0); +} + +TEST_F(ResultLedgerTest, WasReported) { + CreateFailureList({}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + EXPECT_FALSE(ledger.WasReported("success")); + + EXPECT_THAT(ledger.ReportSuccess("success"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("failure", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportSkip("skip", "reason"), IsOk()); + ledger.ReportNotSelected("not_selected"); + + EXPECT_TRUE(ledger.WasReported("success")); + EXPECT_TRUE(ledger.WasReported("failure")); + EXPECT_TRUE(ledger.WasReported("skip")); + EXPECT_TRUE(ledger.WasReported("not_selected")); + EXPECT_FALSE(ledger.WasReported("other")); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, FailureListVerdictsRecordNothing) { + CreateFailureList({{"foo", "abc"}, {"bar.*", "abc"}, {"baz", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ledger.set_enforcement_level(kP0); + + const FailureList& failure_list = ledger.failure_list(); + EXPECT_THAT(failure_list.VerdictOnSuccess("unlisted"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnSuccess("foo"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test foo (matched to foo) is in the failure list, but " + "test succeeded. Remove its match from the failure " + "list.")); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "abc"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("foo", kP0, "xyz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure message for test: foo expected: " + "abc actual: xyz")); + EXPECT_THAT(failure_list.VerdictOnFailure("bar.x", kP1, "abc"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnFailure("unlisted", kP0, "xyz"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "Unexpected failure for test: unlisted")); + EXPECT_THAT(failure_list.VerdictOnFailure("unlisted", kP1, "xyz"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnSkip("unlisted", "reason"), IsOk()); + EXPECT_THAT(failure_list.VerdictOnSkip("bar.x", "reason"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test bar.x (matched to bar.*) is in the failure list " + "but was skipped by the testee: reason. Remove its " + "match from the failure list.")); + + // Nothing was reported. + EXPECT_FALSE(ledger.WasReported("foo")); + EXPECT_FALSE(ledger.WasReported("unlisted")); + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), + ElementsAre("bar.*", "baz", "foo")); + EXPECT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("were not seen: bar.*, baz, foo"))); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); + EXPECT_EQ(ledger.tolerated_failures(), 0); +} + +TEST_F(ResultLedgerTest, FailureListLookupsRecordNothing) { + CreateFailureList({{"foo", "abc"}, {"bar.*", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + const FailureList& failure_list = ledger.failure_list(); + EXPECT_THAT(failure_list.MatchingEntry("foo"), Optional(std::string("foo"))); + EXPECT_THAT(failure_list.MatchingEntry("bar.x"), + Optional(std::string("bar.*"))); + EXPECT_EQ(failure_list.MatchingEntry("foo.x"), absl::nullopt); + EXPECT_EQ(failure_list.MatchingEntry("unlisted"), absl::nullopt); + EXPECT_THAT(failure_list.ExpectedMessage("foo"), + Optional(std::string("abc"))); + EXPECT_THAT(failure_list.ExpectedMessage("bar.*"), + Optional(std::string("abc"))); + // Entries are looked up as listed, not matched. + EXPECT_EQ(failure_list.ExpectedMessage("bar.x"), absl::nullopt); + + // Looking up a name doesn't match its entry. + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), ElementsAre("bar.*", "foo")); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, ReportRecordDispatchesOnTheStatus) { + CreateFailureList({{"fail", "abc"}, + {"crash", "abc"}, + {"skip", "abc"}, + {"not_selected", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ledger.Report("pass", {kP0, ResultRecord::Status::kPass, ""}); + ledger.Report("fail", {kP0, ResultRecord::Status::kFail, "abc"}); + ledger.Report("crash", {kP0, ResultRecord::Status::kCrash, "xyz"}); + ledger.Report("skip", {kP0, ResultRecord::Status::kSkip, "reason"}); + ledger.Report("not_selected", {kP0, ResultRecord::Status::kNotSelected, ""}); + // The verdict is dropped: a tolerated failure is recorded like any other. + ledger.set_enforcement_level(kP0); + ledger.Report("tolerated", {kP1, ResultRecord::Status::kFail, "xyz"}); + + EXPECT_TRUE(ledger.WasReported("pass")); + EXPECT_TRUE(ledger.WasReported("not_selected")); + EXPECT_EQ(ledger.expected_successes(), 1); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("crash", "xyz", absl::nullopt))); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_THAT(ledger.ListedSkips(), ElementsAre(Pair("skip", "skip"))); + EXPECT_EQ(ledger.skipped(), 1); + EXPECT_EQ(ledger.tolerated_failures(), 1); + EXPECT_EQ(ledger.unexpected_successes(), 0); + // A test the runner didn't select leaves its entry unseen. + EXPECT_THAT(ledger.Finalize(), StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("not_selected"))); +} + +TEST_F(ResultLedgerTest, ReportExpectedFailure) { + CreateFailureList({{"foo", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportDuplicates) { + CreateFailureList({{"foo", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportSuccess("bar"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("bar"), IsOk()); + ASSERT_THAT(ledger.ReportSkip("baz", "reason"), IsOk()); + ASSERT_THAT(ledger.ReportSkip("baz", "reason"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 1); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 1); +} + +TEST_F(ResultLedgerTest, ReportExpectedFailureWildcard) { + CreateFailureList({{"foo.*.bar", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo.baz.bar", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportUnseenFailure) { + CreateFailureList({{"foo.bar"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.Finalize(), StatusIs(absl::StatusCode::kFailedPrecondition, + AllOf(HasSubstr("were not seen"), + HasSubstr("foo.bar")))); + + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportUnseenFailureUnrelatedSkip) { + CreateFailureList({{"foo.bar"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // "foo" doesn't match the entry "foo.bar", so the entry stays unseen. + ASSERT_THAT(ledger.ReportSkip("foo", "reason"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), StatusIs(absl::StatusCode::kFailedPrecondition, + AllOf(HasSubstr("were not seen"), + HasSubstr("foo.bar")))); + + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 1); +} + +TEST_F(ResultLedgerTest, ReportUnexpectedFailure) { + CreateFailureList({}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportFailure("foo_failing", kP0, ""), + StatusIs(absl::StatusCode::kFailedPrecondition, + AllOf(HasSubstr("Unexpected failure"), + HasSubstr("foo_failing")))); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportUnexpectedFailureMismatchedName) { + CreateFailureList({{"foo"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportFailure("foo_failing", kP0, ""), + StatusIs(absl::StatusCode::kFailedPrecondition, + AllOf(HasSubstr("Unexpected failure"), + HasSubstr("foo_failing")))); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, ""), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportUnexpectedFailureMismatchedMessage) { + CreateFailureList({{"foo.*.bar", "message"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + AllOf(HasSubstr("Unexpected failure"), HasSubstr("foo"), + HasSubstr("message"), HasSubstr("abc")))); + EXPECT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "message"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_EQ(ledger.skipped(), 0); +} + +TEST_F(ResultLedgerTest, ReportFailureMatchesExpectedMessagePrefix) { + CreateFailureList({{"foo", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // Like the legacy runner, the actual message only has to start with the + // expected one. + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc: more details"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); +} + +TEST_F(ResultLedgerTest, ReportFailureRejectsExpectedMessageSuffix) { + CreateFailureList({{"foo", "abc: more details"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // The expected message is longer than the actual one, so it's not a prefix. + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc"), + StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("Unexpected failure message"))); + + // The mismatch is an unexpected failure (which --fix fixes); the entry + // itself was seen, so Finalize() has nothing to add. + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("foo", "abc", absl::nullopt))); + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 0); + EXPECT_EQ(ledger.unexpected_failures(), 1); +} + +TEST_F(ResultLedgerTest, ReportFailureEmptyExpectedMessageMatchesAnything) { + CreateFailureList({{"foo"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "any message at all"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); +} + +TEST_F(ResultLedgerTest, ReportFailurePrefixMatchIgnoresNewlinesAndWhitespace) { + CreateFailureList({{"foo", "abc def"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP0, " abc def\nghi\n"), IsOk()); + + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); +} + +TEST_F(ResultLedgerTest, PrefixMatchedFailureIsNotAnUnexpectedFailure) { + CreateFailureList({{"foo", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc: more details"), IsOk()); + + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, FinalizeReportsUnseenExpectedFailures) { + CreateFailureList({{"ccc"}, {"aaa"}, {"bbb.*.baz"}, {"ddd"}, {"eee"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // Failures, unexpected successes and skips all count as "seen". + ASSERT_THAT(ledger.ReportFailure("bbb.x.baz", kP0, ""), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("ccc"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSkip("ddd", "reason"), Not(IsOk())); + // Tests that weren't selected to run do not. + ledger.ReportNotSelected("eee"); + + EXPECT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + "The following expected failures were not seen: aaa, " + "eee")); +} + +TEST_F(ResultLedgerTest, UnmatchedExpectedFailures) { + CreateFailureList({{"ccc"}, {"aaa"}, {"bbb.*.baz"}, {"ddd"}, {"eee", "msg"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), + ElementsAre("aaa", "bbb.*.baz", "ccc", "ddd", "eee")); + + // Any report whose name matches an entry counts, whatever the outcome. + ASSERT_THAT(ledger.ReportFailure("bbb.x.baz", kP0, ""), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("ccc"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSkip("ddd", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("eee", kP0, "other message"), Not(IsOk())); + // Reports that match no entry change nothing. + ASSERT_THAT(ledger.ReportSuccess("aaa.child"), IsOk()); + ASSERT_THAT(ledger.ReportSkip("zzz", "reason"), IsOk()); + + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), ElementsAre("aaa")); + // A message mismatch is a verdict on the entry too: seen, not unseen. + EXPECT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + "The following expected failures were not seen: aaa")); +} + +TEST_F(ResultLedgerTest, UnmatchedExpectedFailuresEmpty) { + ResultLedger ledger; + EXPECT_THAT(ledger.UnmatchedExpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, UnexpectedFailures) { + CreateFailureList({{"foo.*.bar", "expected"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + + // Not in the failure list at all, in the list with another message, and the + // messages are formatted as SaveFailureList() would write them. + ASSERT_THAT(ledger.ReportFailure("zzz", kP0, " new\nfailure "), + Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "other message"), + Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, std::string(1000, 'x')), + Not(IsOk())); + // Expected failures, successes and skips are not reported. + ASSERT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "expected: more"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("bbb"), IsOk()); + ASSERT_THAT(ledger.ReportSkip("ccc", "reason"), IsOk()); + + EXPECT_THAT( + ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("aaa", std::string(128, 'x'), absl::nullopt), + FieldsAre("foo.a.bar", "other message", absl::nullopt), + FieldsAre("zzz", "newfailure", absl::nullopt))); + EXPECT_EQ(ledger.unexpected_failures(), 3); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, OverexpandedWildcardKeepsMatching) { + CreateFailureList({{"foo.*.bar"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + for (int i = 0; i < 100; ++i) { + ASSERT_THAT(ledger.ReportSuccess(absl::StrCat("foo.", i, ".bar")), + Not(IsOk())); + } + // The entry is far over the cap, but the verdicts don't change: the test + // environment fails the run over OverexpandedWildcards() instead. + ASSERT_THAT(ledger.ReportFailure("foo.baz.bar", kP0, "msg"), IsOk()); + + EXPECT_THAT(ledger.OverexpandedWildcards(), ElementsAre("foo.*.bar")); + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_THAT(ledger.UnexpectedSuccesses(), SizeIs(100)); + EXPECT_EQ(ledger.unexpected_successes(), 100); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, UnexpectedFailuresIgnoreDuplicateReports) { + ResultLedger ledger; + + // Only the first report of a test counts. + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "first"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "second"), Not(IsOk())); + + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("foo", "first", absl::nullopt))); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, UnexpectedSuccesses) { + CreateFailureList({{"foo.*.bar", "wildcard message"}, {"zzz", " zzz msg "}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.UnexpectedSuccesses(), IsEmpty()); + + ASSERT_THAT(ledger.ReportSuccess("zzz"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSuccess("foo.b.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSuccess("foo.a.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSuccess("foo.a.bar"), Not(IsOk())); + // Expected successes are not reported. + ASSERT_THAT(ledger.ReportSuccess("aaa"), IsOk()); + + // Sorted by test name; the message is the (normalized) one of the entry the + // test matched. + EXPECT_THAT( + ledger.UnexpectedSuccesses(), + ElementsAre(FieldsAre("foo.a.bar", "wildcard message", + Optional(std::string("foo.*.bar"))), + FieldsAre("foo.b.bar", "wildcard message", + Optional(std::string("foo.*.bar"))), + FieldsAre("zzz", "zzz msg", Optional(std::string("zzz"))))); + EXPECT_EQ(ledger.unexpected_successes(), 3); + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, OverexpandedWildcards) { + CreateFailureList({{"foo.*.bar", "abc"}, {"baz.*", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // Expected failures and unexpected successes count toward the cap, each + // test once however often it is reported. + for (int i = 0; i < kMaximumWildcardExpansions; ++i) { + ASSERT_THAT( + ledger.ReportFailure(absl::StrCat("foo.", i, ".bar"), kP0, "abc"), + IsOk()); + ASSERT_THAT(ledger.ReportSuccess(absl::StrCat("baz.", i)), Not(IsOk())); + } + ASSERT_THAT(ledger.ReportFailure("foo.0.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("baz.0"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), IsEmpty()); + + // Skips and failures with another message don't count. + ASSERT_THAT(ledger.ReportSkip("foo.skip.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.other.bar", kP0, "xyz"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), IsEmpty()); + + // One more verdict puts an entry over the cap. The list is sorted. + ASSERT_THAT(ledger.ReportFailure("foo.more.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("baz.more"), Not(IsOk())); + EXPECT_THAT(ledger.OverexpandedWildcards(), + ElementsAre("baz.*", "foo.*.bar")); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, LoadFailureListInvalidFile) { + ResultLedger ledger; + EXPECT_THAT(ledger.LoadFailureList(absl::StrCat(failure_list(), "/invalid")), + StatusIs(absl::StatusCode::kInternal, HasSubstr("/invalid"))); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, LoadFailureListDuplicateFailure) { + CreateFailureList(R"( + foo # abc + foo # zyx +)"); + ResultLedger ledger; + + EXPECT_THAT(ledger.LoadFailureList(failure_list()), + StatusIs(absl::StatusCode::kAlreadyExists, + AllOf(HasSubstr("already exists"), HasSubstr("foo")))); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, LoadFailureListOverlappingFailure) { + CreateFailureList(R"( + foo.*.bar # abc + foo.bar.* # zyx +)"); + ResultLedger ledger; + + EXPECT_THAT(ledger.LoadFailureList(failure_list()), + StatusIs(absl::StatusCode::kAlreadyExists, + AllOf(HasSubstr("foo.bar.*"), HasSubstr("foo.*.bar")))); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, UnexpectedSuccessMessageNamesTheEntry) { + CreateFailureList({{"foo", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // The message is the legacy runner's, including the matched entry. + EXPECT_THAT(ledger.ReportSuccess("foo"), + StatusIs(absl::StatusCode::kFailedPrecondition, + "test foo (matched to foo) is in the failure list, but " + "test succeeded. Remove its match from the failure " + "list.")); + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.unexpected_successes(), 1); +} + +TEST_F(ResultLedgerTest, UnexpectedSuccessMessageNamesTheWildcardEntry) { + CreateFailureList({{"foo.*.bar", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportSuccess("foo.a.bar"), + StatusIs(absl::StatusCode::kFailedPrecondition, + HasSubstr("test foo.a.bar (matched to foo.*.bar) is in " + "the failure list"))); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, EveryPriorityIsEnforcedByDefault) { + ResultLedger ledger; + EXPECT_THAT(ledger.ReportFailure("p0", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("p1", kP1, "abc"), Not(IsOk())); + EXPECT_EQ(ledger.unexpected_failures(), 2); + EXPECT_EQ(ledger.tolerated_failures(), 0); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, EnforcementLevelZeroToleratesP1) { + // Only kP0 is enforced: an unlisted kP1 failure is tolerated. + ResultLedger ledger; + ledger.set_enforcement_level(kP0); + EXPECT_THAT(ledger.ReportFailure("p0", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("p1", kP1, "abc"), IsOk()); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_EQ(ledger.tolerated_failures(), 1); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, ToleratedFailure) { + CreateFailureList({{"listed", "abc"}}); + ResultLedger ledger; + ledger.set_enforcement_level(kP0); + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + ASSERT_THAT(ledger.ReportFailure("foo", kP1, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bar", kP1, "xyz"), IsOk()); + // Duplicates only count once. + ASSERT_THAT(ledger.ReportFailure("foo", kP1, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("listed", kP0, "abc"), IsOk()); + + EXPECT_EQ(ledger.tolerated_failures(), 2); + // A tolerated failure is neither a skip nor a failure of any kind, and it + // isn't written to the failure list. + EXPECT_EQ(ledger.skipped(), 0); + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 0); + EXPECT_EQ(ledger.expected_successes(), 0); + EXPECT_EQ(ledger.unexpected_successes(), 0); + EXPECT_THAT(ledger.UnexpectedFailures(), IsEmpty()); + EXPECT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, "listed # abc\n"); +} + +TEST_F(ResultLedgerTest, ToleratedFailureIsCountedOnceAcrossKinds) { + ResultLedger ledger; + ledger.set_enforcement_level(kP0); + // The same test name can only be counted under one outcome. + ASSERT_THAT(ledger.ReportSuccess("foo"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo", kP1, "abc"), IsOk()); + + EXPECT_EQ(ledger.expected_successes(), 1); + EXPECT_EQ(ledger.tolerated_failures(), 0); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, ListedFailureIsCheckedWhateverItsPriority) { + // A listed test is never tolerated, so that the failure list can't go stale + // unnoticed. + CreateFailureList({{"foo", "abc"}, {"bar", "abc"}}); + ResultLedger ledger; + ledger.set_enforcement_level(kP0); + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP1, "abc"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("bar", kP1, "xyz"), Not(IsOk())); + + EXPECT_EQ(ledger.expected_failures(), 1); + EXPECT_EQ(ledger.unexpected_failures(), 1); + EXPECT_EQ(ledger.tolerated_failures(), 0); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, LoadFailureListInvalidWildcard) { + CreateFailureList({{"foo.b*r", "abc"}}); + ResultLedger ledger; + EXPECT_THAT(ledger.LoadFailureList(failure_list()), + StatusIs(absl::StatusCode::kInvalidArgument)); + ledger.Finalize().IgnoreError(); +} + +TEST_F(ResultLedgerTest, LoadFailureListTruncatesMessage) { + // Like a reported message, so that the two still compare equal. + std::string message(1000, 'b'); + CreateFailureList({{"foo", message}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP0, message), IsOk()); + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 1); +} + +TEST_F(ResultLedgerTest, LoadSecondFailureList) { + CreateFailureList({{"foo", "abc"}}); + const std::string second = absl::StrCat(failure_list(), ".2"); + ABSL_CHECK_OK(File::SetContents(second, "bar # zyx\n", true)); + const std::string third = absl::StrCat(failure_list(), ".3"); + ABSL_CHECK_OK(File::SetContents(third, "foo # zyx\n", true)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.LoadFailureList(second), IsOk()); + + // Entries must not repeat across files either. + EXPECT_THAT(ledger.LoadFailureList(third), + StatusIs(absl::StatusCode::kAlreadyExists)); + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("bar", kP0, "zyx"), IsOk()); + EXPECT_THAT(ledger.Finalize(), IsOk()); + EXPECT_EQ(ledger.expected_failures(), 2); +} + +TEST_F(ResultLedgerTest, ExpectedMessagesAreNormalized) { + CreateFailureList({{"foo", " abc "}, {"bar.*.baz", "xyz"}, {"qux"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + // Like reported messages (see ReportFailure()), so that SaveFailureList() + // compares like with like. + const FailureList& failure_list = ledger.failure_list(); + EXPECT_THAT(failure_list.ExpectedMessage("foo"), Optional(Eq("abc"))); + EXPECT_THAT(failure_list.ExpectedMessage("bar.*.baz"), Optional(Eq("xyz"))); + EXPECT_THAT(failure_list.ExpectedMessage("qux"), Optional(Eq(""))); + EXPECT_THAT(ledger.Finalize(), Not(IsOk())); +} + +TEST_F(ResultLedgerTest, FailureListMatchesWholeNameComponents) { + CreateFailureList({{"foo", "abc"}, {"bar.*.baz", "abc"}, {"qux", "abc"}}); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + + EXPECT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("qux", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("bar.a.baz", kP0, "abc"), IsOk()); + EXPECT_THAT(ledger.ReportFailure("bar.b.baz", kP0, "abc"), IsOk()); + // Neither a prefix nor an extension of an entry matches it, and a wildcard + // stands for exactly one component. + EXPECT_THAT(ledger.ReportFailure("foo.bar", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("bar", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("bar.a", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("bar.a.baz.qux", kP0, "abc"), Not(IsOk())); + EXPECT_THAT(ledger.ReportFailure("other", kP0, "abc"), Not(IsOk())); + + EXPECT_EQ(ledger.expected_failures(), 4); + EXPECT_EQ(ledger.unexpected_failures(), 5); + EXPECT_THAT(ledger.Finalize(), IsOk()); +} + +TEST_F(ResultLedgerTest, SaveFailureListNoop) { + CreateFailureList(R"( +foo # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListNoopWildcard) { + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.*.bar # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListKeepsEntryWithInternalWhitespace) { + // LoadFailureList() ignores whitespace anywhere in the name, so the entry + // matches "foo.bar"; SaveFailureList() must find it under the same key. + CreateFailureList(R"( +foo .bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo .bar # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListKeepsPrefixMatchedEntryVerbatim) { + CreateFailureList(R"( +foo # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + // The actual message is longer than the entry's, which is only a prefix. + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc: more details"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListRemoveNewPassing) { + CreateFailureList(R"( +foo # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("foo"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListDropsUnseenEntry) { + CreateFailureList(R"( +foo # abc +bar # def +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bar", kP0, "def"), IsOk()); + ASSERT_THAT(ledger.Finalize(), + StatusIs(absl::StatusCode::kFailedPrecondition, + "The following expected failures were not seen: foo")); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +bar # def +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpandsWildcardWhenOneMatchPasses) { + // The wildcard no longer describes the failures: the test that passed must + // go, but the ones that still fail must stay listed, so the wildcard is + // expanded into them, in its place. + CreateFailureList(R"( +# Header kept above the expansion. +foo.*.bar # abc +zzz # zzz +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.c.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("foo.b.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc: details"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("zzz", kP0, "zzz"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# Header kept above the expansion. +foo.a.bar # abc +foo.c.bar # abc +zzz # zzz +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListDropsWildcardWhenEveryMatchPasses) { + CreateFailureList(R"( +# Header left behind. +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("foo.a.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSuccess("foo.b.bar"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# Header left behind. +)"); +} + +TEST_F(ResultLedgerTest, + SaveFailureListExpandsWildcardWhenOneMatchChangesMessage) { + // One match fails with a different message while another still fails with + // the listed one: the wildcard is expanded so that each test gets its own + // message, and the result loads again (it used to contain the wildcard + // twice). + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "xyz"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.a.bar # xyz +foo.b.bar # abc +)"); + ResultLedger reloaded; + EXPECT_THAT(reloaded.LoadFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + EXPECT_THAT(reloaded.failure_list().ExpectedMessage("foo.a.bar"), + Optional(std::string("xyz"))); + EXPECT_THAT(reloaded.failure_list().ExpectedMessage("foo.b.bar"), + Optional(std::string("abc"))); + ASSERT_THAT(reloaded.Finalize(), Not(IsOk())); +} + +TEST_F(ResultLedgerTest, + SaveFailureListExpandsWildcardWhenMatchesChangeToDifferentMessages) { + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "xyz"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "uvw"), Not(IsOk())); + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("foo.a.bar", "xyz", absl::nullopt), + FieldsAre("foo.b.bar", "uvw", absl::nullopt))); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.a.bar # xyz +foo.b.bar # uvw +)"); +} + +TEST_F(ResultLedgerTest, + SaveFailureListKeepsWildcardWhenEveryMatchChangesToTheSameMessage) { + // The wildcard still describes the failures, only the message moved on, so + // the author's wildcard is kept with the new message. + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "xyz"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "xyz"), Not(IsOk())); + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("foo.a.bar", "xyz", absl::nullopt), + FieldsAre("foo.b.bar", "xyz", absl::nullopt))); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.*.bar # xyz +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpandedWildcardDropsSkippedMatch) { + // A skipped test can't be an expected failure, so when the wildcard is + // expanded (here because another match passed) the skipped test gets no + // line, like the one that passed. + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportSkip("foo.a.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSuccess("foo.b.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.c.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.c.bar # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpandsWildcardWhenOneMatchIsSkipped) { + // A skip alone is reason to expand the wildcard, just like a pass: the tests + // that still fail stay listed, the skipped one goes. + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSkip("foo.b.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.c.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.a.bar # abc +foo.c.bar # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListDropsWildcardWhenEveryMatchIsSkipped) { + CreateFailureList(R"( +# Header left behind. +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportSkip("foo.a.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.ReportSkip("foo.b.bar", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# Header left behind. +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListKeepsWildcardWithTooManyMatches) { + // Exceeding the wildcard expansion cap is reported separately (see + // OverexpandedWildcards()); the tests do still fail with the listed + // message, so the entry is kept. + CreateFailureList(R"( +foo.*.bar # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + for (int i = 0; i <= kMaximumWildcardExpansions; ++i) { + ASSERT_THAT( + ledger.ReportFailure(absl::StrCat("foo.", i, ".bar"), kP0, "abc"), + IsOk()); + } + ASSERT_THAT(ledger.OverexpandedWildcards(), ElementsAre("foo.*.bar")); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.*.bar # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpansionInterleavesNewFailures) { + // New failures are inserted by name among the expanded lines too, and the + // alignment accounts for the expanded names. + CreateFailureList(R"( +# Block one. +f.*.x # abc + +# Block two. +zz # zzz +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("f.aaaa.x", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("f.b.x"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("f.cc.x", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("f.bb.y", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("zz", kP0, "zzz"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# Block one. +f.aaaa.x # abc +f.bb.y # new +f.cc.x # abc + +# Block two. +zz # zzz +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListChangedMessageStaysInPlace) { + // An exact entry whose message changed is rewritten where it is, so that it + // stays under its comment block. + CreateFailureList(R"( +# About bbb. +bbb # abc + +# About aaa. +aaa # aaa +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "xyz"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("bbb", "xyz", absl::nullopt))); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# About bbb. +bbb # xyz + +# About aaa. +aaa # aaa +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListKeepsMessagelessWildcardVerbatim) { + CreateFailureList(R"( +foo.*.bar +zzz # zzz +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + // An empty expected message matches any failure. + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.b.bar", kP0, "xyz"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("zzz", kP0, "zzz"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.*.bar +zzz # zzz +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpandsMessagelessWildcard) { + // The expanded lines have no message either: no `#` and no trailing space. + CreateFailureList(R"( +foo.*.bar +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo.a.bar", kP0, "abc"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("foo.b.bar"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("foo.c.bar", kP0, ""), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo.a.bar +foo.c.bar +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListDropsSkippedEntry) { + // A skipped test can't be an expected failure, so its entry goes like that + // of a test that passed (whether the skip itself is acceptable is decided, + // and reported, by the caller). + CreateFailureList(R"( +foo # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportSkip("foo", "reason"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListUnwritablePath) { + ResultLedger ledger; + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + // A directory can't be opened for writing. + const std::string directory = absl::StrCat(failure_list(), ".dir"); + ASSERT_THAT(File::RecursivelyCreateDir(directory, 0777), + IsOk()); + EXPECT_THAT(ledger.SaveFailureList(directory), + StatusIs(absl::StatusCode::kInternal, HasSubstr(directory))); + // Neither can a file in a directory that doesn't exist. + const std::string missing_directory = + absl::StrCat(failure_list(), ".missing/list.txt"); + EXPECT_THAT( + ledger.SaveFailureList(missing_directory), + StatusIs(absl::StatusCode::kInternal, HasSubstr(missing_directory))); +} + +TEST_F(ResultLedgerTest, SaveFailureListWriteFailure) { + ResultLedger ledger; + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + // /dev/full opens fine but every write to it fails with ENOSPC, which is + // the only way to reach the write-failure path without a filesystem mock. + // It is a Linux device. +#ifdef __linux__ + EXPECT_THAT(ledger.SaveFailureList("/dev/full"), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("Failed to write failure list file"))); +#else + GTEST_SKIP() << "/dev/full is Linux-only"; +#endif +} + +TEST_F(ResultLedgerTest, SaveFailureListChangeFailureMessage) { + CreateFailureList(R"( +foo # abc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "zyx"), Not(IsOk())); + EXPECT_THAT(ledger.UnexpectedFailures(), + ElementsAre(FieldsAre("foo", "zyx", absl::nullopt))); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +foo # zyx +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListAddNewFailure) { + ResultLedger ledger; + ASSERT_THAT(ledger.ReportFailure("foo", kP0, "abc"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"(foo # abc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListAddNormalizedMessage) { + ResultLedger ledger; + std::string message = absl::StrCat("aa\n", std::string(1000, 'b')); + std::string normalized_message = absl::StrCat("aa", std::string(126, 'b')); + ASSERT_THAT(ledger.ReportFailure("foo", kP0, message), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::StrCat("foo # ", normalized_message, "\n")); +} + +TEST_F(ResultLedgerTest, SaveFailureListInsertAlphabetically) { + CreateFailureList(R"( +# This is a comment. +aaa # aaa +# This is another comment. +bbb # bbb + +ccc # ccc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "bbb"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("abc", kP0, "abc"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# This is a comment. +aaa # aaa +# This is another comment. +abc # abc +bbb # bbb + +ccc # ccc +)"); +} + +TEST_F(ResultLedgerTest, SaveFailureListInsertAligned) { + CreateFailureList(R"( +# This is a comment. +aa # aaa +# This is another comment. +bbbb #bbb + +cc # ccc +)"); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbbb", kP0, " bbb "), IsOk()); + ASSERT_THAT(ledger.ReportFailure("cc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("abcdef", kP0, "abc"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, R"( +# This is a comment. +aa # aaa +# This is another comment. +abcdef # abc +bbbb # bbb + +cc # ccc +)"); +} + +// The copybara marker lines of a strip block (the region the open source +// export drops), spelled in two pieces so that the export's scrubber, which +// processes every file, leaves this test's data alone. +constexpr absl::string_view kStripBegin = + "# copybara:" + "strip_begin"; +constexpr absl::string_view kStripEnd = + "# copybara:" + "strip_end"; + +TEST_F(ResultLedgerTest, SaveFailureListKeepsUnchangedStripBlockVerbatim) { + // The block is aligned on its own, so a list whose block is narrower than + // the rest is a no-op byte for byte. + CreateFailureList(absl::Substitute(R"( +aaa.long.name # aaa +zzz # zzz + +# Google-only failures. +$0 +bbb # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa.long.name", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("zzz", kP0, "zzz"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "bbb"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa.long.name # aaa +zzz # zzz + +# Google-only failures. +$0 +bbb # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, SaveFailureListRewritesEntryInsideStripBlockInPlace) { + CreateFailureList(absl::Substitute(R"( +aaa.long.name # aaa +$0 +bbb # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa.long.name", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "new message"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa.long.name # aaa +$0 +bbb # new message +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, SaveFailureListExpandsWildcardInsideStripBlock) { + // The expansion takes the wildcard's place and widens the block's column + // only. + CreateFailureList(absl::Substitute(R"( +aaa # aaa +$0 +b.*.b # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("b.long.b", kP0, "bbb"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("b.x.b"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa # aaa +$0 +b.long.b # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, SaveFailureListDropsPassingEntryInsideStripBlock) { + CreateFailureList(absl::Substitute(R"( +aaa # aaa +$0 +bbb # bbb +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportSuccess("bbb"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa # aaa +$0 +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, SaveFailureListInsertsNewFailuresOutsideStripBlock) { + // A new failure goes in front of the first entry outside the block that + // sorts after it: `aab` (which sorts before the block), `bbb` (which would + // sort into it) and `yyy` (which sorts after it) all land after the block, + // before `yyz`; `zzz` sorts last and goes at the end. The new names only + // widen the column outside the block. + CreateFailureList(absl::Substitute(R"( +aaa # aaa +$0 +bba # bba +bbc # bbc +$1 +yyz # yyz +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bba", kP0, "bba"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbc", kP0, "bbc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("yyz", kP0, "yyz"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aab", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("yyy.long", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("zzz", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa # aaa +$0 +bba # bba +bbc # bbc +$1 +aab # new +bbb # new +yyy.long # new +yyz # yyz +zzz # new +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, + SaveFailureListAppendsNewFailuresAfterTrailingStripBlock) { + CreateFailureList(absl::Substitute(R"( +aaa # aaa +# Keep this block last. +$0 +bbb # bbb +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "bbb"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("abc", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa # aaa +# Keep this block last. +$0 +bbb # bbb +$1 +abc # new +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, + SaveFailureListTreatsUnbalancedStripMarkersAsComments) { + // A strip_end with no open block and a strip_begin with no strip_end after + // it don't delimit anything: everything is one region, aligned together, + // and new failures are inserted by name as usual. + CreateFailureList(absl::Substitute(R"( +aaa # aaa +$1 +ccc.long # ccc +$0 +eee # eee +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("ccc.long", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("eee", kP0, "eee"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.ReportFailure("ddd", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +aaa # aaa +$1 +bbb # new +ccc.long # ccc +$0 +ddd # new +eee # eee +)", + kStripBegin, kStripEnd)); +} + +TEST_F(ResultLedgerTest, SaveFailureListIgnoresNestedStripBegin) { + // A second strip_begin inside a block is a plain comment; the block ends at + // the first strip_end, and the second strip_end is a plain comment too. + CreateFailureList(absl::Substitute(R"( +$0 +aaa.long # aaa +$0 +bbb # bbb +$1 +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); + ResultLedger ledger; + ASSERT_THAT(ledger.LoadFailureList(failure_list()), IsOk()); + ASSERT_THAT(ledger.ReportFailure("aaa.long", kP0, "aaa"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bbb", kP0, "bbb"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("ccc", kP0, "ccc"), IsOk()); + ASSERT_THAT(ledger.ReportFailure("bba", kP0, "new"), Not(IsOk())); + ASSERT_THAT(ledger.Finalize(), IsOk()); + + EXPECT_THAT(ledger.SaveFailureList(absl::StrCat(failure_list(), ".new")), + IsOk()); + + std::string content; + ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, + true), + IsOk()); + EXPECT_EQ(content, absl::Substitute(R"( +$0 +aaa.long # aaa +$0 +bbb # bbb +$1 +bba # new +ccc # ccc +$1 +)", + kStripBegin, kStripEnd)); +} + +} // namespace +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_listener.cc b/conformance/result_listener.cc new file mode 100644 index 0000000..628c8ee --- /dev/null +++ b/conformance/result_listener.cc
@@ -0,0 +1,130 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/result_listener.h" + +#include <iterator> +#include <string> +#include <utility> + +#include <gtest/gtest.h> +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "conformance/result_ledger.h" +#include "conformance/result_record.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +Statistics Statistics::From(const ResultLedger& ledger) { + Statistics statistics; + statistics.skipped_tests = ledger.skipped(); + statistics.listed_skips = ledger.listed_skips(); + statistics.tolerated_failures = ledger.tolerated_failures(); + statistics.expected_failures = ledger.expected_failures(); + statistics.unexpected_failures = ledger.unexpected_failures(); + statistics.expected_successes = ledger.expected_successes(); + statistics.unexpected_successes = ledger.unexpected_successes(); + return statistics; +} + +Statistics Statistics::operator-(const Statistics& other) const { + Statistics delta; + delta.skipped_tests = skipped_tests - other.skipped_tests; + delta.listed_skips = listed_skips - other.listed_skips; + delta.tolerated_failures = tolerated_failures - other.tolerated_failures; + delta.expected_failures = expected_failures - other.expected_failures; + delta.unexpected_failures = unexpected_failures - other.unexpected_failures; + delta.expected_successes = expected_successes - other.expected_successes; + delta.unexpected_successes = + unexpected_successes - other.unexpected_successes; + return delta; +} + +Statistics& Statistics::operator+=(const Statistics& other) { + skipped_tests += other.skipped_tests; + listed_skips += other.listed_skips; + tolerated_failures += other.tolerated_failures; + expected_failures += other.expected_failures; + unexpected_failures += other.unexpected_failures; + expected_successes += other.expected_successes; + unexpected_successes += other.unexpected_successes; + return *this; +} + +void Statistics::RecordProperties() const { + testing::Test::RecordProperty("skipped_tests", skipped_tests); + testing::Test::RecordProperty("listed_skips", listed_skips); + testing::Test::RecordProperty("tolerated_failures", tolerated_failures); + testing::Test::RecordProperty("expected_failures", expected_failures); + testing::Test::RecordProperty("unexpected_failures", unexpected_failures); + testing::Test::RecordProperty("expected_successes", expected_successes); + testing::Test::RecordProperty("unexpected_successes", unexpected_successes); +} + +void ResultListener::OnTestSuiteStart(const testing::TestSuite& /*suite*/) { + suite_start_ = Statistics::From(*ledger_); +} + +void ResultListener::OnTestStart(const testing::TestInfo& /*test_info*/) { + // Results checked in SetUpTestSuite() are the suite's, not this test's: + // hold them back until the suite ends. + suite_results_.insert(suite_results_.end(), + std::make_move_iterator(recorded_results_.begin()), + std::make_move_iterator(recorded_results_.end())); + recorded_results_.clear(); + test_start_ = Statistics::From(*ledger_); +} + +void ResultListener::OnTestPartResult(const testing::TestPartResult& part) { + if (part.type() != testing::TestPartResult::kSuccess) return; + absl::StatusOr<RecordedResult> recorded = + ParseResultRecordMessage(part.message()); + if (absl::IsNotFound(recorded.status())) return; // An ordinary success. + if (!recorded.ok()) { + undecodable_results_.emplace_back(recorded.status().message()); + return; + } + // The first outcome recorded for a test is the one that counts. + if (ledger_->WasReported(recorded->test_name)) return; + ledger_->Report(recorded->test_name, recorded->record); + recorded_results_.push_back(*std::move(recorded)); +} + +void ResultListener::OnTestEnd(const testing::TestInfo& /*test_info*/) { + ReportRecordedResults(); + (Statistics::From(*ledger_) - test_start_).RecordProperties(); +} + +void ResultListener::OnTestSuiteEnd(const testing::TestSuite& /*suite*/) { + // What SetUpTestSuite() checked, then what TearDownTestSuite() did. + recorded_results_.insert(recorded_results_.begin(), + std::make_move_iterator(suite_results_.begin()), + std::make_move_iterator(suite_results_.end())); + suite_results_.clear(); + ReportRecordedResults(); + (Statistics::From(*ledger_) - suite_start_).RecordProperties(); +} + +void ResultListener::ReportRecordedResults() { + for (const RecordedResult& recorded : recorded_results_) { + testing::Test::RecordProperty(recorded.test_name, + recorded.record.ToString()); + } + recorded_results_.clear(); + for (const std::string& error : undecodable_results_) { + ADD_FAILURE() << error << "\nThe outcome it recorded is lost."; + } + undecodable_results_.clear(); +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_listener.h b/conformance/result_listener.h new file mode 100644 index 0000000..6048b70 --- /dev/null +++ b/conformance/result_listener.h
@@ -0,0 +1,104 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LISTENER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LISTENER_H__ + +#include <string> +#include <vector> + +#include <gtest/gtest.h> +#include "absl/base/nullability.h" +#include "conformance/result_ledger.h" +#include "conformance/result_record.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// A snapshot of the ResultLedger's counters. The ResultListener reports the +// per-test and per-suite deltas, and the environment the run's total, as test +// properties. +struct Statistics { + int skipped_tests = 0; + int listed_skips = 0; + int tolerated_failures = 0; + int expected_failures = 0; + int unexpected_failures = 0; + int expected_successes = 0; + int unexpected_successes = 0; + + static Statistics From(const ResultLedger& ledger); + Statistics operator-(const Statistics& other) const; + Statistics& operator+=(const Statistics& other); + + // Records each statistic as a gtest property of the current test (or suite, + // or run) under the names skipped_tests, listed_skips, + // tolerated_failures, expected_failures, unexpected_failures, + // expected_successes and unexpected_successes. + void RecordProperties() const; +}; + +// How the conformance test environment hears what every conformance test did. +// Yields() (matchers.h) records each outcome as a gtest success in the running +// test (see ResultRecordMessage() in result_record.h). This listener picks +// those out of the test part results as gtest reports them and tallies them in +// the ResultLedger at once. At the end of every gtest test and test suite it +// records the outcomes checked there as properties named after the conformance +// tests (see ReportRecordedResults()) and the statistics the test or suite +// added as its properties. gtest still attributes properties to the test at +// that point (it forgets the current test only after the listeners ran), and +// likewise for the suite. +// +// ConformanceEnvironment (test_environment.h) registers one with gtest. The +// listener doesn't depend on the environment, so unit tests drive it directly. +class ResultListener : public testing::EmptyTestEventListener { + public: + explicit ResultListener(ResultLedger* absl_nonnull ledger) + : ledger_(ledger) {} + + void OnTestSuiteStart(const testing::TestSuite& suite) override; + void OnTestStart(const testing::TestInfo& test_info) override; + // gtest holds its lock while the listeners see a test part result, so this + // must not call back into it; it only tallies. + void OnTestPartResult(const testing::TestPartResult& part) override; + void OnTestEnd(const testing::TestInfo& test_info) override; + void OnTestSuiteEnd(const testing::TestSuite& suite) override; + + // Records the outcome of every conformance test checked since the last call + // as a property of the current gtest test (or suite, or run) named after the + // conformance test, with ResultRecord::ToString() as the value, so that the + // reports can tell what each test did. The outcomes themselves were tallied + // in the ledger as they were checked. Also fails the current test for every + // gtest success that claimed to carry an outcome but couldn't be decoded + // (see ParseResultRecordMessage() in result_record.h), so that no outcome + // goes missing quietly. Runs when a test or a suite ends, and from + // ConformanceEnvironment::TearDown() for anything checked outside a test + // suite. + void ReportRecordedResults(); + + private: + ResultLedger* const absl_nonnull ledger_; + // The counters when the current suite and test started. + Statistics suite_start_; + Statistics test_start_; + // The outcomes checked since ReportRecordedResults() last ran, in order, and + // the gtest successes that claimed to carry an outcome but couldn't be + // decoded. + std::vector<RecordedResult> recorded_results_; + std::vector<std::string> undecodable_results_; + // The results checked in SetUpTestSuite(), held back while the tests run. + std::vector<RecordedResult> suite_results_; +}; + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_RESULT_LISTENER_H__
diff --git a/conformance/result_listener_test.cc b/conformance/result_listener_test.cc new file mode 100644 index 0000000..6b2e5c2 --- /dev/null +++ b/conformance/result_listener_test.cc
@@ -0,0 +1,189 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/result_listener.h" + +#include <string> +#include <utility> +#include <vector> + +#include <gmock/gmock.h> +#include <gtest/gtest-spi.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/result_ledger.h" +#include "conformance/result_record.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::Key; +using ::testing::Not; +using ::testing::Pair; + +constexpr absl::string_view kFooTestName = + "Required.Proto3.ProtobufInput.Foo.ProtobufOutput"; +constexpr absl::string_view kSetUpTestName = + "Required.Proto3.ProtobufInput.SetUpTestSuite.ProtobufOutput"; +constexpr absl::string_view kTearDownTestName = + "Required.Proto3.ProtobufInput.TearDownTestSuite.ProtobufOutput"; + +ResultRecord Pass() { return {kP0, ResultRecord::Status::kPass, ""}; } +ResultRecord Fail() { return {kP0, ResultRecord::Status::kFail, "boom"}; } + +// A gtest success with `message`, as SUCCEED() << message reports it to the +// test event listeners. +testing::TestPartResult Success(absl::string_view message) { + return testing::TestPartResult(testing::TestPartResult::kSuccess, __FILE__, + __LINE__, + absl::StrCat("Succeeded\n", message).c_str()); +} + +// The success Yields() reports when it records `record` for `test_name` (see +// ResultRecordMessage() in result_record.h). +testing::TestPartResult RecordedSuccess(absl::string_view test_name, + const ResultRecord& record) { + return Success(ResultRecordMessage(test_name, record)); +} + +// Drives a listener by hand, with the gtest test and suite it hears about +// being this one. Whatever it records lands on the running test, including +// what it records when told that the suite ended. +class ResultListenerTest : public testing::Test { + protected: + ~ResultListenerTest() override { + // ResultLedger insists on being finalized before destruction. + ledger_.Finalize().IgnoreError(); + } + + static const testing::TestSuite& CurrentSuite() { + return *testing::UnitTest::GetInstance()->current_test_suite(); + } + static const testing::TestInfo& CurrentTest() { + return *testing::UnitTest::GetInstance()->current_test_info(); + } + + // The properties of the running test so far, as (key, value) pairs. + static std::vector<std::pair<std::string, std::string>> Properties() { + const testing::TestResult& result = *CurrentTest().result(); + std::vector<std::pair<std::string, std::string>> properties; + for (int i = 0; i < result.test_property_count(); ++i) { + const testing::TestProperty& property = result.GetTestProperty(i); + properties.emplace_back(property.key(), property.value()); + } + return properties; + } + + // The outcomes among Properties(): those whose value is a ResultRecord. + static std::vector<std::pair<std::string, std::string>> Outcomes() { + std::vector<std::pair<std::string, std::string>> outcomes; + for (const auto& property : Properties()) { + if (ResultRecord::Parse(property.second).has_value()) { + outcomes.push_back(property); + } + } + return outcomes; + } + + ResultLedger ledger_; + ResultListener listener_{&ledger_}; +}; + +TEST_F(ResultListenerTest, TalliesAtOnceAndReportsWhenTheTestEnds) { + listener_.OnTestSuiteStart(CurrentSuite()); + listener_.OnTestStart(CurrentTest()); + + listener_.OnTestPartResult(RecordedSuccess(kFooTestName, Pass())); + EXPECT_EQ(ledger_.expected_successes(), 1); + EXPECT_THAT(Properties(), IsEmpty()); + + listener_.OnTestEnd(CurrentTest()); + EXPECT_THAT(Properties(), AllOf(Contains(Pair(kFooTestName, "P0 PASS")), + Contains(Pair("expected_successes", "1")), + Contains(Pair("unexpected_failures", "0")))); +} + +TEST_F(ResultListenerTest, TheFirstOutcomeOfATestCounts) { + listener_.OnTestStart(CurrentTest()); + listener_.OnTestPartResult(RecordedSuccess(kFooTestName, Fail())); + listener_.OnTestPartResult(RecordedSuccess(kFooTestName, Pass())); + EXPECT_EQ(ledger_.unexpected_failures(), 1); + EXPECT_EQ(ledger_.expected_successes(), 0); + + listener_.OnTestEnd(CurrentTest()); + EXPECT_THAT(Outcomes(), ElementsAre(Pair(kFooTestName, "P0 FAIL: boom"))); +} + +TEST_F(ResultListenerTest, IgnoresOtherTestPartResults) { + listener_.OnTestStart(CurrentTest()); + // A success that merely looks like a record, and a failure that carries one. + listener_.OnTestPartResult(Success("P0 FAIL: not a record")); + listener_.OnTestPartResult(testing::TestPartResult( + testing::TestPartResult::kNonFatalFailure, __FILE__, __LINE__, + ResultRecordMessage(kFooTestName, Fail()).c_str())); + listener_.OnTestEnd(CurrentTest()); + EXPECT_EQ(ledger_.unexpected_failures(), 0); + EXPECT_THAT(Outcomes(), IsEmpty()); +} + +TEST_F(ResultListenerTest, ReportsRecordedResultsOnRequest) { + // What ConformanceEnvironment::TearDown() does for the results checked + // outside any test suite. + listener_.OnTestPartResult(RecordedSuccess(kFooTestName, Fail())); + EXPECT_THAT(Properties(), Not(Contains(Key(kFooTestName)))); + listener_.ReportRecordedResults(); + EXPECT_THAT(Properties(), Contains(Pair(kFooTestName, "P0 FAIL: boom"))); +} + +TEST_F(ResultListenerTest, UndecodableRecordFailsTheTest) { + // A success marked as a record that can't be read is a bug somewhere, and + // the outcome it carried is lost: the test fails rather than staying quiet. + listener_.OnTestPartResult( + Success(absl::StrCat(kResultRecordMarker, "\tnot a record"))); + EXPECT_NONFATAL_FAILURE(listener_.ReportRecordedResults(), + "Can't decode the conformance result record"); + // Once. + listener_.ReportRecordedResults(); +} + +TEST_F(ResultListenerTest, HoldsSetUpTestSuiteResultsBackUntilTheSuiteEnds) { + listener_.OnTestSuiteStart(CurrentSuite()); + listener_.OnTestPartResult(RecordedSuccess(kSetUpTestName, Pass())); + + listener_.OnTestStart(CurrentTest()); + listener_.OnTestPartResult(RecordedSuccess(kFooTestName, Fail())); + listener_.OnTestEnd(CurrentTest()); + // The test reports its own outcome and statistics only. + EXPECT_THAT(Outcomes(), ElementsAre(Pair(kFooTestName, "P0 FAIL: boom"))); + EXPECT_THAT(Properties(), AllOf(Contains(Pair("expected_successes", "0")), + Contains(Pair("unexpected_failures", "1")))); + + listener_.OnTestPartResult(RecordedSuccess(kTearDownTestName, Pass())); + listener_.OnTestSuiteEnd(CurrentSuite()); + // The suite reports what SetUpTestSuite() checked, then what + // TearDownTestSuite() did, and the statistics of the whole suite. + EXPECT_THAT(Outcomes(), ElementsAre(Pair(kFooTestName, "P0 FAIL: boom"), + Pair(kSetUpTestName, "P0 PASS"), + Pair(kTearDownTestName, "P0 PASS"))); + EXPECT_THAT(Properties(), AllOf(Contains(Pair("expected_successes", "2")), + Contains(Pair("unexpected_failures", "1")))); +} + +} // namespace +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_record.cc b/conformance/result_record.cc new file mode 100644 index 0000000..216b846 --- /dev/null +++ b/conformance/result_record.cc
@@ -0,0 +1,109 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/result_record.h" + +#include <iterator> +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +// The status words of ResultRecord::ToString(), indexed by Status. +constexpr absl::string_view kStatusNames[] = {"PASS", "FAIL", "CRASH", "SKIP", + "NOT_SELECTED"}; + +} // namespace + +std::string ResultRecord::ToString() const { + std::string value = absl::StrCat(PriorityName(priority), " ", + kStatusNames[static_cast<int>(status)]); + absl::string_view stripped = absl::StripTrailingAsciiWhitespace(message); + if (!stripped.empty()) absl::StrAppend(&value, ": ", stripped); + return value; +} + +absl::optional<ResultRecord> ResultRecord::Parse(absl::string_view value) { + ResultRecord record; + bool found = false; + for (TestPriority priority : {kP0, kP1}) { + if (absl::ConsumePrefix(&value, + absl::StrCat(PriorityName(priority), " "))) { + record.priority = priority; + found = true; + break; + } + } + if (!found) return absl::nullopt; + absl::string_view status = value.substr(0, value.find(':')); + const absl::string_view* name = absl::c_find(kStatusNames, status); + if (name == std::end(kStatusNames)) return absl::nullopt; + record.status = static_cast<Status>(name - std::begin(kStatusNames)); + value.remove_prefix(status.size()); + if (!value.empty() && !absl::ConsumePrefix(&value, ": ")) { + return absl::nullopt; + } + record.message = std::string(value); + return record; +} + +std::string ResultRecordMessage(absl::string_view test_name, + const ResultRecord& record) { + return absl::StrCat(kResultRecordMarker, "\t", absl::CEscape(test_name), "\t", + absl::CEscape(record.ToString())); +} + +absl::StatusOr<RecordedResult> ParseResultRecordMessage( + absl::string_view message) { + for (absl::string_view line : absl::StrSplit(message, '\n')) { + if (!absl::StartsWith(line, kResultRecordMarker)) continue; + auto invalid = [line] { + return absl::InvalidArgumentError( + absl::StrCat("Can't decode the conformance result record in a gtest " + "success: ", + line)); + }; + std::vector<absl::string_view> fields = absl::StrSplit(line, '\t'); + if (fields.size() != 3 || fields[0] != kResultRecordMarker) { + return invalid(); + } + RecordedResult result; + std::string value; + if (!absl::CUnescape(fields[1], &result.test_name) || + !absl::CUnescape(fields[2], &value)) { + return invalid(); + } + absl::optional<ResultRecord> record = ResultRecord::Parse(value); + if (!record.has_value()) return invalid(); + result.record = *std::move(record); + return result; + } + return absl::NotFoundError("Not a conformance result record."); +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/result_record.h b/conformance/result_record.h new file mode 100644 index 0000000..50ae98c --- /dev/null +++ b/conformance/result_record.h
@@ -0,0 +1,86 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_RESULT_RECORD_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_RESULT_RECORD_H__ + +#include <string> + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// The outcome of one conformance test, as Yields() (matchers.h) records it +// when it checks the test's result and as the test environment tallies it +// (see ResultLedger::Report() in result_ledger.h) and reports it, as a gtest +// property named after the test whose value is ToString(), e.g. "P0 PASS" or +// "P1 FAIL: Failed to parse input.". +struct ResultRecord { + enum class Status { + kPass, // The result matched the expectation. + kFail, // It didn't. + kCrash, // The testee hit a runtime error or timed out. + kSkip, // The testee declined to run the test. + kNotSelected, // The runner filtered the test out, so it never ran. + }; + + TestPriority priority; + Status status; + // The failure message or skip reason, if any. + std::string message; + + // "<priority> <status>[: <message>]": the priority as PriorityName() spells + // it, the status as PASS, FAIL, CRASH, SKIP or NOT_SELECTED, and the message + // stripped of trailing whitespace and left out when empty. + std::string ToString() const; + + // The inverse of ToString(). Returns nullopt for a value in any other + // form. + static absl::optional<ResultRecord> Parse(absl::string_view value); +}; + +// How Yields() hands a record to the test environment: as the message of a +// gtest success (SUCCEED()) in the running test. gtest prints nothing for a +// success and reports it to the test event listeners at once, so the +// environment's listener sees the record as soon as the result is checked, +// while the record stays with the gtest result it belongs to. +// +// The first line that starts with kResultRecordMarker carries the record: +// the marker, the test name and ToString(), tab-separated and C-escaped onto +// that one line, so that what gtest appends to the message (SCOPED_TRACE +// output) doesn't get in the way. +inline constexpr absl::string_view kResultRecordMarker = "[conformance result]"; + +std::string ResultRecordMessage(absl::string_view test_name, + const ResultRecord& record); + +// A record as ParseResultRecordMessage() finds it. +struct RecordedResult { + std::string test_name; + ResultRecord record; +}; + +// The inverse of ResultRecordMessage(): the record in `message`, the message +// of a test part result. Returns kNotFound if there is none, that is if the +// part is an ordinary success, and kInvalidArgument if the message claims to +// carry a record that can't be decoded. The test environment fails the test +// over the latter, so that no outcome goes missing quietly. +absl::StatusOr<RecordedResult> ParseResultRecordMessage( + absl::string_view message); + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_RESULT_RECORD_H__
diff --git a/conformance/result_record_test.cc b/conformance/result_record_test.cc new file mode 100644 index 0000000..987078d --- /dev/null +++ b/conformance/result_record_test.cc
@@ -0,0 +1,153 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/result_record.h" + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/status/status.h" +#include "absl/status/status_matchers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { +namespace { + +using ::absl_testing::IsOkAndHolds; +using ::absl_testing::StatusIs; +using ::testing::FieldsAre; +using ::testing::HasSubstr; +using ::testing::Not; +using ::testing::Optional; +using ::testing::Property; +using ::testing::StartsWith; + +TEST(ResultRecordTest, ToString) { + EXPECT_EQ((ResultRecord{kP0, ResultRecord::Status::kPass, ""}).ToString(), + "P0 PASS"); + EXPECT_EQ((ResultRecord{kP1, ResultRecord::Status::kFail, "boom"}).ToString(), + "P1 FAIL: boom"); + EXPECT_EQ((ResultRecord{kP0, ResultRecord::Status::kCrash, "x"}).ToString(), + "P0 CRASH: x"); + EXPECT_EQ((ResultRecord{kP0, ResultRecord::Status::kSkip, "why"}).ToString(), + "P0 SKIP: why"); + EXPECT_EQ( + (ResultRecord{kP0, ResultRecord::Status::kNotSelected, ""}).ToString(), + "P0 NOT_SELECTED"); +} + +TEST(ResultRecordTest, ToStringStripsTrailingWhitespace) { + EXPECT_EQ( + (ResultRecord{kP0, ResultRecord::Status::kFail, "a: b \n\n"}).ToString(), + "P0 FAIL: a: b"); + EXPECT_EQ((ResultRecord{kP0, ResultRecord::Status::kFail, " \n"}).ToString(), + "P0 FAIL"); +} + +TEST(ResultRecordTest, ParseInvertsToString) { + for (TestPriority priority : {kP0, kP1}) { + for (ResultRecord::Status status : + {ResultRecord::Status::kPass, ResultRecord::Status::kFail, + ResultRecord::Status::kCrash, ResultRecord::Status::kSkip, + ResultRecord::Status::kNotSelected}) { + for (absl::string_view message : {"", "msg", "a: b: c", "multi\nline"}) { + ResultRecord record{priority, status, std::string(message)}; + EXPECT_THAT(ResultRecord::Parse(record.ToString()), + Optional(FieldsAre(priority, status, message))) + << record.ToString(); + } + } + } +} + +TEST(ResultRecordTest, ParseRejectsOtherValues) { + EXPECT_EQ(ResultRecord::Parse(""), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("garbage"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 "), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P2 PASS"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 BOGUS"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 pass"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 PASS extra"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 FAIL:"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse("P0 FAIL:msg"), absl::nullopt); + EXPECT_EQ(ResultRecord::Parse(" P0 PASS"), absl::nullopt); +} + +TEST(ResultRecordMessageTest, IsOneMarkedLine) { + EXPECT_EQ(ResultRecordMessage("Required.Foo", + {kP0, ResultRecord::Status::kPass, ""}), + absl::StrCat(kResultRecordMarker, "\tRequired.Foo\tP0 PASS")); + // Names and messages are escaped onto the one line. + std::string message = ResultRecordMessage( + "odd\tname", {kP1, ResultRecord::Status::kFail, "two\nlines\twith tab"}); + EXPECT_THAT(message, StartsWith(kResultRecordMarker)); + EXPECT_THAT(message, Not(HasSubstr("\n"))); +} + +TEST(ResultRecordMessageTest, ParseInvertsResultRecordMessage) { + for (absl::string_view name : {"Required.Foo", "a b\tc", "multi\nline"}) { + for (absl::string_view message : + {"", "msg", "a: b: c", "tab\tbed", "multi\nline\n", "quote\"d\\"}) { + ResultRecord record{kP1, ResultRecord::Status::kFail, + std::string(message)}; + EXPECT_THAT(ParseResultRecordMessage(ResultRecordMessage(name, record)), + IsOkAndHolds(FieldsAre(name, Property(&ResultRecord::ToString, + record.ToString())))) + << ResultRecordMessage(name, record); + } + } +} + +TEST(ResultRecordMessageTest, ParseFindsTheRecordAmongGtestsOwnLines) { + // SUCCEED() prefixes its message and appends SCOPED_TRACE() context. + ResultRecord record{kP0, ResultRecord::Status::kSkip, "not supported"}; + EXPECT_THAT( + ParseResultRecordMessage(absl::StrCat( + "Succeeded\n", ResultRecordMessage("Required.Foo", record), + "\nGoogle Test trace:\nfoo_test.cc:12: P0 PASS looks like one")), + IsOkAndHolds(FieldsAre( + "Required.Foo", + Property(&ResultRecord::ToString, "P0 SKIP: not supported")))); +} + +TEST(ResultRecordMessageTest, ParseTellsOrdinarySuccessesApart) { + EXPECT_THAT(ParseResultRecordMessage(""), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(ParseResultRecordMessage("Succeeded"), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(ParseResultRecordMessage("Succeeded\nP0 PASS"), + StatusIs(absl::StatusCode::kNotFound)); + EXPECT_THAT(ParseResultRecordMessage( + absl::StrCat("Succeeded\n ", kResultRecordMarker, "\ta\tb")), + StatusIs(absl::StatusCode::kNotFound)); +} + +TEST(ResultRecordMessageTest, ParseRejectsAMarkedLineItCantDecode) { + for (absl::string_view rest : + {"", "\tnot a record", "\tname\tP0 BOGUS", "\tname\tP0 PASS\textra", + "x\tname\tP0 PASS", "\tbad\\escape\tP0 PASS"}) { + EXPECT_THAT( + ParseResultRecordMessage(absl::StrCat(kResultRecordMarker, rest)), + StatusIs(absl::StatusCode::kInvalidArgument, + HasSubstr("Can't decode the conformance result record"))) + << rest; + } +} + +} // namespace +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/sample_conformance_test.cc b/conformance/sample_conformance_test.cc new file mode 100644 index 0000000..e38c32a --- /dev/null +++ b/conformance/sample_conformance_test.cc
@@ -0,0 +1,80 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A tiny conformance suite used to exercise the gtest conformance framework end +// to end against the real C++ testee. Some of these tests fail on purpose; see +// the sample_failure_list_*.txt files and test_environment_integration_test.cc. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/test_messages_proto3.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::proto3::TestAllTypesProto3; + +using SampleConformanceTest = ConformanceTest; + +TEST_F(SampleConformanceTest, SamplePass) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +// Valid input is expected to parse, so this always fails against a conforming +// testee. +TEST_F(SampleConformanceTest, SampleDeliberateFailure) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(IsParseError())); +} + +// Same as above, but only P1. +TEST_F(SampleConformanceTest, SampleP1Failure) { + EXPECT_THAT( + Testee(kP1) + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(IsParseError())); +} + +// Only runs with gtest's also_run_disabled_tests flag. Its presence must not +// make a normal run count as partial (see IsPartialRun() in +// test_environment.cc). +TEST_F(SampleConformanceTest, DISABLED_SampleDisabled) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +// Runs a conformance test but never checks its result with Yields(), which +// fails the test. Only runs when asked for by name (see +// test_environment_integration_test.cc), so that the other runs stay clean. +TEST_F(SampleConformanceTest, DISABLED_SampleUncheckedResult) { + auto unchecked = + Testee("SampleUncheckedResult") + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/sample_failure_list_empty.txt b/conformance/sample_failure_list_empty.txt new file mode 100644 index 0000000..a4f32fb --- /dev/null +++ b/conformance/sample_failure_list_empty.txt
@@ -0,0 +1,2 @@ +# Intentionally empty: every deliberate failure in sample_conformance_test is +# unexpected with this list.
diff --git a/conformance/sample_failure_list_ok.txt b/conformance/sample_failure_list_ok.txt new file mode 100644 index 0000000..d2429a2 --- /dev/null +++ b/conformance/sample_failure_list_ok.txt
@@ -0,0 +1,3 @@ +# Expected failures of sample_conformance_test. These tests fail on purpose. +Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput # Should have failed to parse, but didn't.
diff --git a/conformance/sample_failure_list_unseen.txt b/conformance/sample_failure_list_unseen.txt new file mode 100644 index 0000000..1ff44a8 --- /dev/null +++ b/conformance/sample_failure_list_unseen.txt
@@ -0,0 +1,4 @@ +# Like sample_failure_list_ok.txt, plus an entry for a test that never runs. +Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleNeverRuns.ProtobufOutput # This test doesn't exist.
diff --git a/conformance/test_environment.cc b/conformance/test_environment.cc index 98c69ef..45a5915 100644 --- a/conformance/test_environment.cc +++ b/conformance/test_environment.cc
@@ -7,6 +7,12 @@ #include "conformance/test_environment.h" +#ifdef _WIN32 +#include <direct.h> // for _getcwd +#else +#include <unistd.h> // for getcwd +#endif + #include <algorithm> #include <cstdlib> #include <memory> @@ -24,9 +30,12 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/span.h" +#include "conformance/failure_list.h" #include "conformance/fork_pipe_runner.h" #include "conformance/global_test_environment.h" -#include "conformance/test_manager.h" +#include "conformance/result_ledger.h" +#include "conformance/result_listener.h" #include "conformance/test_runner.h" #include "conformance/testee.h" @@ -36,6 +45,21 @@ namespace internal { namespace { +// The current working directory, or "" if it can't be determined. +std::string CurrentWorkingDirectory() { + // Both glibc/macOS (as an extension) and the MSVC CRT malloc() a buffer of + // the right size when given a null one. +#ifdef _WIN32 + char* cwd = _getcwd(nullptr, 0); +#else + char* cwd = getcwd(nullptr, 0); +#endif + if (cwd == nullptr) return ""; + std::string result = cwd; + std::free(cwd); + return result; +} + // Whether `path` is absolute on this platform. Only paths that start with a // slash are absolute on POSIX; Windows also has drive letters ("C:\\...", // "C:/...") and root-relative backslash paths. @@ -112,54 +136,26 @@ } // namespace -Statistics Statistics::From(const TestManager& manager) { - Statistics statistics; - statistics.skipped_tests = manager.skipped(); - statistics.listed_skips = manager.listed_skips(); - statistics.tolerated_failures = manager.tolerated_failures(); - statistics.expected_failures = manager.expected_failures(); - statistics.unexpected_failures = manager.unexpected_failures(); - statistics.expected_successes = manager.expected_successes(); - statistics.unexpected_successes = manager.unexpected_successes(); - return statistics; -} +// Hooks ReportUncheckedResults() into the end of every gtest test. gtest +// still attributes failures to the test at that point (it forgets the current +// test only after the listeners ran). +class ConformanceEnvironment::UncheckedResultReporter + : public testing::EmptyTestEventListener { + public: + explicit UncheckedResultReporter( + ConformanceEnvironment* absl_nonnull environment) + : environment_(environment) {} -Statistics Statistics::operator-(const Statistics& other) const { - Statistics delta; - delta.skipped_tests = skipped_tests - other.skipped_tests; - delta.listed_skips = listed_skips - other.listed_skips; - delta.tolerated_failures = tolerated_failures - other.tolerated_failures; - delta.expected_failures = expected_failures - other.expected_failures; - delta.unexpected_failures = unexpected_failures - other.unexpected_failures; - delta.expected_successes = expected_successes - other.expected_successes; - delta.unexpected_successes = - unexpected_successes - other.unexpected_successes; - return delta; -} + void OnTestEnd(const testing::TestInfo& /*test_info*/) override { + environment_->ReportUncheckedResults(); + } -Statistics& Statistics::operator+=(const Statistics& other) { - skipped_tests += other.skipped_tests; - listed_skips += other.listed_skips; - tolerated_failures += other.tolerated_failures; - expected_failures += other.expected_failures; - unexpected_failures += other.unexpected_failures; - expected_successes += other.expected_successes; - unexpected_successes += other.unexpected_successes; - return *this; -} + private: + ConformanceEnvironment* const absl_nonnull environment_; +}; -void Statistics::RecordProperties() const { - testing::Test::RecordProperty("skipped_tests", skipped_tests); - testing::Test::RecordProperty("listed_skips", listed_skips); - testing::Test::RecordProperty("tolerated_failures", tolerated_failures); - testing::Test::RecordProperty("expected_failures", expected_failures); - testing::Test::RecordProperty("unexpected_failures", unexpected_failures); - testing::Test::RecordProperty("expected_successes", expected_successes); - testing::Test::RecordProperty("unexpected_successes", unexpected_successes); -} - -TestManager& GetGlobalTestManager() { - return ConformanceEnvironment::Get().test_manager(); +const FailureList& GetGlobalFailureList() { + return ConformanceEnvironment::Get().ledger().failure_list(); } ConformanceEnvironment::ConformanceEnvironment( @@ -167,24 +163,34 @@ : options_(std::move(options)), owned_runner_(TakeOwnedRunner(options_)), testee_(std::make_unique<Testee>( - options_.runner != nullptr ? options_.runner : owned_runner_.get())) { + options_.runner != nullptr ? options_.runner : owned_runner_.get())), + result_listener_(new ResultListener(&ledger_)), + unchecked_result_reporter_(new UncheckedResultReporter(this)) { ABSL_CHECK(global_environment == nullptr) << "A global ConformanceEnvironment is already set."; global_environment = this; - // Anything below proto3 (including the legacy runner's EDITION_UNKNOWN + // Anything below proto3 (including conformance_test_runner's EDITION_UNKNOWN // default) is meaningless, since proto2 and proto3 tests always run. This is // the one place the maximum edition is clamped; see the option's comment. options_.maximum_edition = std::max(options_.maximum_edition, EDITION_PROTO3); - test_manager_.set_enforcement_level(options_.enforcement_level); + ledger_.set_enforcement_level(options_.enforcement_level); + testing::TestEventListeners& listeners = + testing::UnitTest::GetInstance()->listeners(); + listeners.Append(result_listener_); + listeners.Append(unchecked_result_reporter_); } ConformanceEnvironment::~ConformanceEnvironment() { - // TestManager insists on being finalized before destruction. An installed + testing::TestEventListeners& listeners = + testing::UnitTest::GetInstance()->listeners(); + delete listeners.Release(unchecked_result_reporter_); + delete listeners.Release(result_listener_); + // ResultLedger insists on being finalized before destruction. An installed // environment is never destroyed (see Install()), so this only runs for // environments constructed by ScopedGlobalConformanceEnvironment in unit // tests, and those don't necessarily call TearDown(); make sure we don't // crash on the way out. - test_manager_.Finalize().IgnoreError(); + ledger_.Finalize().IgnoreError(); ABSL_CHECK_EQ(global_environment, this); global_environment = nullptr; } @@ -194,12 +200,13 @@ ABSL_CHECK(global_environment == nullptr) << "ConformanceEnvironment::Install() must only be called once."; // gtest deletes the environments it owns at the end of RUN_ALL_TESTS(), but - // callers (e.g. the merged conformance_test_runner) still need the test - // manager afterwards. So although this class is a testing::Environment, it - // isn't registered with gtest itself: gtest only gets a forwarding proxy, - // and the real environment intentionally lives until the process exits. - // Only the environment object leaks, though; TearDown() releases the testee. - // TODO: b/410122039 - register it directly once the legacy suites are gone. + // the merged conformance_test_runner still needs the ledger + // afterwards. So although this class is a testing::Environment, it isn't + // registered with gtest itself: gtest only gets a forwarding proxy, and the + // real environment intentionally lives until the process exits. Only the + // environment object leaks, though; TearDown() releases the testee. + // TODO: register it directly once conformance_test_main.cc no + // longer needs it after RUN_ALL_TESTS() (see the class comment). auto* environment = new ConformanceEnvironment(std::move(options)); testing::AddGlobalTestEnvironment(new ForwardingEnvironment(environment)); return *environment; @@ -209,10 +216,32 @@ ABSL_CHECK(global_environment != nullptr) << "No ConformanceEnvironment has been installed. Conformance test " "binaries must call ConformanceEnvironment::Install() before " - "RUN_ALL_TESTS()."; + "RUN_ALL_TESTS(); the usual way to do that is to depend on the " + "test_environment_main library instead of a generic gtest main."; return *global_environment; } +void ConformanceEnvironment::ReportUncheckedResults() { + if (testee_ == nullptr) { + return; // TearDown() has already run. + } + absl::Span<const std::string> tests_run = testee_->tests_run(); + absl::Span<const std::string> new_tests = + tests_run.subspan(results_reconciled_); + results_reconciled_ = tests_run.size(); + // Don't pile a second failure onto a test that bailed out before it could + // check its results. + if (testing::Test::HasFatalFailure()) { + return; + } + for (const std::string& test_name : new_tests) { + if (!ledger_.WasReported(test_name)) { + ADD_FAILURE() << "TestResult for " << test_name + << " was never checked; wrap the matcher in Yields()"; + } + } +} + Testee& ConformanceEnvironment::testee() { ABSL_CHECK(testee_ != nullptr) << "The testee has already been shut down by " @@ -223,7 +252,7 @@ void ConformanceEnvironment::SetUp() { for (const std::string& failure_list : options_.failure_list_files) { - absl::Status status = test_manager_.LoadFailureList(failure_list); + absl::Status status = ledger_.LoadFailureList(failure_list); ASSERT_TRUE(status.ok()) << "Failed to load failure list " << failure_list << ": " << status; } @@ -231,7 +260,18 @@ } void ConformanceEnvironment::TearDown() { - Statistics::From(test_manager_).RecordProperties(); + // What's left after the last test: results checked outside any test suite + // are only reported here, and results run after the last test (e.g. in a + // TearDownTestSuite()) are only checked here. + result_listener_->ReportRecordedResults(); + ReportUncheckedResults(); + Statistics::From(ledger_).RecordProperties(); + for (const std::string& entry : ledger_.OverexpandedWildcards()) { + ADD_FAILURE() << "The failure list entry " << entry << " matched more than " + << kMaximumWildcardExpansions + << " tests, which hides too much. List the failing tests " + "individually, or narrow the wildcard."; + } const bool partial_run = partial_run_override_.value_or(IsPartialRun()); // The path the failure list was rewritten to, if --fix succeeded. @@ -245,6 +285,7 @@ "run. Rerun with --fix and without the filter."; } else { std::string output_file = options_.fix_output_file; + const char* workspace = std::getenv("BUILD_WORKSPACE_DIRECTORY"); if (output_file.empty()) { if (options_.failure_list_files.empty()) { ADD_FAILURE() << "--fix needs --fix_output_file when no " @@ -254,9 +295,7 @@ "--failure_list is given."; } else if (IsAbsolutePath(options_.failure_list_files[0])) { output_file = options_.failure_list_files[0]; - } else if (const char* workspace = - std::getenv("BUILD_WORKSPACE_DIRECTORY"); - workspace != nullptr) { + } else if (workspace != nullptr) { output_file = absl::StrCat(workspace, "/", options_.failure_list_files[0]); } else { @@ -264,26 +303,55 @@ "$BUILD_WORKSPACE_DIRECTORY (which `bazel run` " "sets) to locate the failure list to rewrite."; } + } else if (!IsAbsolutePath(output_file) && workspace != nullptr) { + // Like a relative --failure_list, a relative --fix_output_file names a + // source file: under `bazel run` the working directory is the + // runfiles tree, so resolve it against the workspace instead. This is + // how conformance_test() (conformance.bzl) lets --fix create a failure + // list that doesn't exist yet. + output_file = absl::StrCat(workspace, "/", output_file); } if (!output_file.empty()) { - absl::Status status = test_manager_.SaveFailureList(output_file); + const bool relative = !IsAbsolutePath(output_file); + absl::Status status = ledger_.SaveFailureList(output_file); if (status.ok()) { - ABSL_LOG(INFO) << "Wrote updated failure list to " << output_file; - fixed_list = output_file; + // Report where the file actually landed: a relative path outside + // `bazel run` is relative to whatever the working directory is. + std::string absolute_path = output_file; + if (relative) { + std::string cwd = CurrentWorkingDirectory(); + if (!cwd.empty()) { + absolute_path = absl::StrCat(cwd, "/", output_file); + } + } + ABSL_LOG(INFO) << "Wrote updated failure list to " << absolute_path; + if (relative && workspace == nullptr && + (std::getenv("TEST_SRCDIR") != nullptr || + std::getenv("TEST_TMPDIR") != nullptr)) { + ABSL_LOG(WARNING) + << "The updated failure list was written inside the test " + "sandbox, not into the source tree, because " + "$BUILD_WORKSPACE_DIRECTORY is unset. To update the " + "checked-in list use `bazel run <target> -- --fix` instead " + "of `bazel test`."; + } + fixed_list = absolute_path; } else { ADD_FAILURE() << "Failed to write updated failure list to " - << output_file << ": " << status; + << output_file << ": " << status + << " (does the directory exist?)"; } } } } - absl::Status status = test_manager_.Finalize(); + absl::Status status = ledger_.Finalize(); if (!status.ok() && set_up_succeeded_ && options_.check_unseen_expected_failures) { if (partial_run) { // Only a subset of the tests ran, so most expected failures were never - // exercised. The legacy runner's --test flag behaves the same way. + // exercised. The conformance_test_runner --test flag behaves the same + // way. ABSL_LOG(INFO) << "Not checking for unseen expected failures because " "only some of the tests ran (e.g. because of " "--gtest_filter or sharding):\n"
diff --git a/conformance/test_environment.h b/conformance/test_environment.h index f66d25a..1c624ec 100644 --- a/conformance/test_environment.h +++ b/conformance/test_environment.h
@@ -8,24 +8,29 @@ // The process-wide plumbing shared by every gtest-based conformance suite. // // ConformanceEnvironment is the process-global state of a conformance test -// binary. It owns the testee connection and the TestManager, records -// statistics, and checks or regenerates the failure list at the end of the -// run. Exactly one is installed per test binary, before RUN_ALL_TESTS(); -// Install() hooks it into gtest. Nothing in this header is meant for the -// conformance suites themselves. +// binary. It owns the testee connection, the ResultLedger and the +// ResultListener that feeds it (see result_ledger.h and result_listener.h), +// records the run's statistics, and checks or regenerates the failure list at +// the end of the run. Exactly one is installed per test binary, normally by +// test_environment_main.cc from command-line flags (see +// test_environment_flags.h). Install() hooks it into gtest. Nothing in this +// header is meant for the conformance suites themselves. // // Everything here is single-threaded. Use it only from gtest's main thread: // test bodies, fixtures and the environment hooks. The global environment is -// a plain pointer and TestManager isn't thread-safe, so tests must not create -// or run conformance Tests from other threads. +// a plain pointer and ResultLedger isn't thread-safe, so tests must not create +// or run conformance Tests from other threads. Yields() only reads the +// failure list; the environment's ResultListener is what tallies the +// outcomes, from gtest's listener hooks. // -// --gtest_repeat is not supported. Each conformance test result is recorded -// in the TestManager exactly once, so a repeated run would see every test as -// already recorded and the statistics and failure-list checks would be wrong. +// --gtest_repeat is not supported. The testee connection rejects a test name +// it has already run, and the ResultLedger counts each test once, so a +// repeated run would crash or report wrong statistics. #ifndef GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_H__ #define GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_H__ +#include <cstddef> #include <memory> #include <string> #include <vector> @@ -34,7 +39,8 @@ #include <gtest/gtest.h> #include "absl/base/nullability.h" #include "absl/types/optional.h" -#include "conformance/test_manager.h" +#include "conformance/result_ledger.h" +#include "conformance/result_listener.h" #include "conformance/test_runner.h" #include "conformance/testee.h" @@ -43,7 +49,9 @@ namespace conformance { namespace internal { -// Options for a ConformanceEnvironment. +// Options for a ConformanceEnvironment. Normally populated from command-line +// flags by test_environment_main.cc (see OptionsFromFlags() in +// test_environment_flags.h). struct ConformanceEnvironmentOptions { // Exactly one of `runner`, `owned_runner` or `testee_binary` must be set. // @@ -63,7 +71,7 @@ std::vector<std::string> testee_args; // Failure list files to load. All of them are loaded into a single - // TestManager, so entries must not overlap between files. SetUp() fails + // ResultLedger, so entries must not overlap between files. SetUp() fails // fatally if they do. std::vector<std::string> failure_list_files; @@ -86,21 +94,10 @@ // EDITION_PROTO3. That is the one place the clamp lives. Edition maximum_edition = EDITION_PROTO3; - // When true, only performance tests run; when false, only regular tests run. - // Because the two kinds of tests are mutually exclusive within one run, a - // performance run needs its own failure list, like conformance_test_runner's - // *_performance.txt lists: with a shared list, every regular test's entry - // would be "unseen" (and dropped by `fix`). - // - // TODO: b/410126673 - This in-binary mode is transitional. It only mirrors - // conformance_test_runner's --performance flag until the performance tests - // are a suite of their own, which BUILD files enable or disable like any - // other suite. Remove it together with the --performance flag at that - // point. - bool performance = false; - // Rewrites the failure list at the end of the run. Entries that no longer - // fail are dropped. New failures are added with their messages. + // fail are dropped. New failures are added with their messages. The rules, + // wildcard entries and copybara strip blocks included, are those of + // ResultLedger::SaveFailureList(). // // Refused when only a subset of the binary's tests ran, because of // --gtest_filter, sharding or --gtest_fail_fast. A test failure is reported @@ -110,8 +107,9 @@ // Where to write the fixed failure list. Required when `fix` is set and // `failure_list_files` doesn't have exactly one entry. Otherwise defaults - // to that single file under $BUILD_WORKSPACE_DIRECTORY, that is the source - // file under `bazel run`. + // to that single file. Relative paths, here and in `failure_list_files`, + // are resolved under $BUILD_WORKSPACE_DIRECTORY when it is set. Under + // `bazel run` that is the source file. std::string fix_output_file; // Whether TearDown() fails the run if some expected failures were never @@ -120,33 +118,13 @@ // sharding, since a partial run can't see every expected failure. The // conformance_test_runner --test flag behaves the same way. // - // The merged runner sets this to false. It reports the unmatched entries - // itself (see TestManager::UnmatchedExpectedFailures). + // The merged runner (conformance_test_main.cc) sets this to false. It + // reports the unmatched entries itself, per failure list flag and in the + // conformance_test_runner report format (see + // ResultLedger::UnmatchedExpectedFailures). bool check_unseen_expected_failures = true; }; -// A snapshot of the TestManager's counters. Used to report per-test and -// per-suite deltas as test properties. -struct Statistics { - int skipped_tests = 0; - int listed_skips = 0; - int tolerated_failures = 0; - int expected_failures = 0; - int unexpected_failures = 0; - int expected_successes = 0; - int unexpected_successes = 0; - - static Statistics From(const TestManager& manager); - Statistics operator-(const Statistics& other) const; - Statistics& operator+=(const Statistics& other); - - // Records each statistic as a gtest property of the current test (or suite, - // or run) under the names skipped_tests, listed_skips, - // tolerated_failures, expected_failures, unexpected_failures, - // expected_successes and unexpected_successes. - void RecordProperties() const; -}; - // Test helpers, defined in test_environment_testing.h. class ScopedGlobalConformanceEnvironment; class ScopedPartialRunOverride; @@ -160,15 +138,25 @@ // to make sure of that. gtest deletes the environments registered with it at // the end of RUN_ALL_TESTS() (see RunAllTests in gtest.cc). The installed // instance must outlive that, because the transitional merged runner -// (conformance_test_main.cc) reads test_manager() afterwards. Install() -// therefore registers a private proxy with gtest that forwards to SetUp() and -// TearDown(). The environment object itself is intentionally leaked. The -// testee is not: TearDown() releases it. +// (conformance_test_main.cc) reads ledger() afterwards to print the +// results in the conformance_test_runner report format. Install() therefore +// registers a private proxy with gtest that forwards to SetUp() and TearDown(). +// The environment object itself is intentionally leaked. The testee is not: +// TearDown() releases it. // TODO: b/563707827 - once the merged runner has retired (its users moved to // the conformance_test() macro), nothing needs the environment after // RUN_ALL_TESTS(); register it directly with AddGlobalTestEnvironment() and // delete the proxy. // +// Every conformance test a gtest test runs must be checked with Yields() (see +// matchers.h), which records its outcome as a gtest success in the running +// test. The ResultListener (result_listener.h) the environment registers +// with gtest tallies the outcome in the ResultLedger at once, and at the end +// of each gtest test (and test suite) records the outcomes checked in it and +// its statistics as its properties. The environment itself fails the test +// for every result the testee ran that was never checked (see +// ReportUncheckedResults()). +// // Single-threaded. See the file comment. class ConformanceEnvironment : public testing::Environment { public: @@ -181,7 +169,7 @@ // makes it the process-global instance. Must be called exactly once, before // RUN_ALL_TESTS(): // - // ConformanceEnvironment::Install(std::move(options)); + // ConformanceEnvironment::Install(OptionsFromFlags()); // return RUN_ALL_TESTS(); // // The environment itself is never destroyed (see the class comment), but @@ -189,7 +177,7 @@ static ConformanceEnvironment& Install(ConformanceEnvironmentOptions options); // Returns the process-global instance. Check-fails if Install() hasn't been - // called. + // called, which typically means test_environment_main wasn't linked in. static ConformanceEnvironment& Get(); // Loads the failure lists. Any failure here is fatal, so no tests run. @@ -198,10 +186,15 @@ // Finishes the run. gtest runs this after the last test (see Install()). // - // Records the run's statistics as test properties. Rewrites the failure - // list if `fix` was requested and all of the tests ran. Fails if expected - // failures were never seen, unless `check_unseen_expected_failures` is false - // or only a subset of the tests ran (see `fix`). Finally releases + // Reports the results checked outside any test suite (see + // ResultListener::ReportRecordedResults()), fails for every result run after + // the last test that was never checked (see ReportUncheckedResults()) and + // records the run's statistics as test properties. Fails for every failure + // list entry that matched too many tests (see + // ResultLedger::OverexpandedWildcards()). Rewrites the failure list if + // `fix` was requested and all of the tests ran. Fails if expected failures + // were never seen, unless `check_unseen_expected_failures` is false or only + // a subset of the tests ran (see `fix`). Finally releases // the testee. An owned runner is destroyed, which for a ForkPipeRunner // stops the process. For a caller-owned runner only our reference is // dropped, so the caller may destroy the runner right after @@ -212,9 +205,19 @@ // releases the testee. conformance_test_main relies on this. void TearDown() override; + // Fails the current gtest test for every conformance test the testee has + // run since the previous call whose result was never checked with Yields(). + // Otherwise the outcome of such a test would silently bypass the failure + // list. A checked result is one whose outcome the ResultListener has + // tallied in the ledger. A listener the environment registers with gtest + // calls it when a test ends, and TearDown() for anything run after the last + // test. A test that failed fatally is left alone: an ASSERT_* that didn't + // hold returned before the test could check its results. + void ReportUncheckedResults(); + // Where the tests' outcomes are recorded. The transitional merged runner // (conformance_test_main.cc) reads it after RUN_ALL_TESTS(). - TestManager& test_manager() { return test_manager_; } + const ResultLedger& ledger() const { return ledger_; } // The connection to the testee. Check-fails once TearDown() has released // it. @@ -225,6 +228,7 @@ private: friend class ScopedGlobalConformanceEnvironment; friend class ScopedPartialRunOverride; + class UncheckedResultReporter; // Creates an environment and makes it the process-global one. Check-fails if // there already is one. The testee connection is created eagerly, though a @@ -238,7 +242,7 @@ // Null when using the caller's `runner`. Released in TearDown(). std::unique_ptr<ConformanceTestRunner> owned_runner_; - TestManager test_manager_; + ResultLedger ledger_; // Null once TearDown() has released it. std::unique_ptr<Testee> testee_; @@ -248,6 +252,18 @@ absl::optional<bool> partial_run_override_; bool set_up_succeeded_ = false; + // Tallies the outcomes Yields() records in `ledger_` as gtest reports them, + // and reports them and the statistics at the end of every gtest test and + // suite. gtest owns it while it is registered; the destructor takes it + // back. + ResultListener* absl_nonnull result_listener_; + + // Calls ReportUncheckedResults() at the end of every gtest test. gtest owns + // it like result_listener_. + UncheckedResultReporter* absl_nonnull unchecked_result_reporter_; + + // How many of testee().tests_run() ReportUncheckedResults() has looked at. + size_t results_reconciled_ = 0; }; } // namespace internal
diff --git a/conformance/test_environment_flags.cc b/conformance/test_environment_flags.cc new file mode 100644 index 0000000..061a9f9 --- /dev/null +++ b/conformance/test_environment_flags.cc
@@ -0,0 +1,82 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Defines the global --testee_binary, --testee_args, --failure_list, +// --enforcement_level, --maximum_edition, --fix and --fix_output_file flags. +// Being process-global ABSL_FLAGs with short, generic names, they must only be +// linked into binaries with a test_environment_main-style main (see +// test_environment_flags.h), never into conformance_test_runner, whose command +// line parses flags of the same names itself. + +#include "conformance/test_environment_flags.h" + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include "absl/flags/flag.h" +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +ABSL_FLAG(std::string, testee_binary, "", + "The testee program to run conformance tests against."); +ABSL_FLAG(std::vector<std::string>, testee_args, {}, + "Comma-separated arguments to pass to --testee_binary. Positional " + "arguments are appended to these."); +ABSL_FLAG(std::vector<std::string>, failure_list, {}, + "Comma-separated list of files containing the tests that are " + "expected to fail, one test name per line. Use '#' for comments."); +ABSL_FLAG(google::protobuf::conformance::TestPriority, enforcement_level, + google::protobuf::conformance::kLowestPriority, + "The lowest priority whose failures fail the run, as its number: 0 " + "enforces only P0, 1 enforces P0 and P1. The default enforces every " + "priority. Failures of tests of a lower priority are tolerated " + "(counted, not failed) unless the test is in the failure list, which " + "is checked whatever the level."); +ABSL_FLAG(google::protobuf::Edition, maximum_edition, google::protobuf::EDITION_PROTO3, + "Only run conformance tests up to and including this edition, e.g. " + "EDITION_2023. Defaults to proto2 and proto3 only."); +ABSL_FLAG(bool, fix, false, + "Rewrite the failure list at the end of the run to match the " + "observed results. See --fix_output_file. Refused when only some " + "of the tests run (--gtest_filter, sharding). Use it via `bazel run " + "<target> -- --fix`: under `bazel test` the file is written inside " + "the test sandbox and discarded."); +ABSL_FLAG(std::string, fix_output_file, "", + "Where --fix writes the updated failure list. Required when more " + "than one --failure_list is given; otherwise defaults to the " + "failure list itself. A relative path (this flag's or the failure " + "list's) is resolved under $BUILD_WORKSPACE_DIRECTORY when that is " + "set, so that `bazel run ... -- --fix` creates or updates the file " + "in your workspace."); + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +ConformanceEnvironmentOptions OptionsFromFlags( + absl::Span<char* const> positional_args) { + ConformanceEnvironmentOptions options; + options.testee_binary = absl::GetFlag(FLAGS_testee_binary); + options.testee_args = absl::GetFlag(FLAGS_testee_args); + for (const char* positional_arg : positional_args) { + options.testee_args.push_back(positional_arg); + } + options.failure_list_files = absl::GetFlag(FLAGS_failure_list); + options.enforcement_level = absl::GetFlag(FLAGS_enforcement_level); + options.maximum_edition = absl::GetFlag(FLAGS_maximum_edition); + options.fix = absl::GetFlag(FLAGS_fix); + options.fix_output_file = absl::GetFlag(FLAGS_fix_output_file); + return options; +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_flags.h b/conformance/test_environment_flags.h new file mode 100644 index 0000000..6262008 --- /dev/null +++ b/conformance/test_environment_flags.h
@@ -0,0 +1,59 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The command-line flags of a conformance test binary and their translation +// into ConformanceEnvironmentOptions. test_environment_main.cc is the usual +// consumer. An in-process testee needs no main of its own: it defines +// MakeTesteeRunner() (see testee_runner.h). A binary with its own main can +// link this library instead to get the same flags. It then calls +// OptionsFromFlags() and sets `owned_runner` before installing the +// environment. + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__ + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include "absl/flags/declare.h" +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// Builds the environment options from the flags declared below. +// `positional_args` are the non-flag command-line arguments, excluding +// argv[0]. They are appended to --testee_args. +// +// ConformanceEnvironment::Install(OptionsFromFlags()); +// +// This doesn't insist on --testee_binary. Whether it is required (forked +// testee) or must be absent (in-process testee) is up to the linked +// MakeTesteeRunner() (see testee_runner.h), which test_environment_main.cc +// hands the options to. +ConformanceEnvironmentOptions OptionsFromFlags( + absl::Span<char* const> positional_args = {}); + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +ABSL_DECLARE_FLAG(std::string, testee_binary); +ABSL_DECLARE_FLAG(std::vector<std::string>, testee_args); +ABSL_DECLARE_FLAG(std::vector<std::string>, failure_list); +ABSL_DECLARE_FLAG(google::protobuf::conformance::TestPriority, enforcement_level); +ABSL_DECLARE_FLAG(google::protobuf::Edition, maximum_edition); +ABSL_DECLARE_FLAG(bool, fix); +ABSL_DECLARE_FLAG(std::string, fix_output_file); + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__
diff --git a/conformance/test_environment_flags_test.cc b/conformance/test_environment_flags_test.cc new file mode 100644 index 0000000..0d1aea8 --- /dev/null +++ b/conformance/test_environment_flags_test.cc
@@ -0,0 +1,241 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/test_environment_flags.h" + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/flags/reflection.h" +#include "absl/strings/string_view.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::google::protobuf::conformance::internal::ConformanceEnvironmentOptions; +using ::google::protobuf::conformance::internal::OptionsFromFlags; +using ::testing::ElementsAre; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::IsNull; + +// The flag-parsing step of a test_environment_main-style main, on the given +// command line (without argv[0]): parses the absl flags with the same parser +// the main relies on (InitGoogle() / absl::ParseCommandLine()) and returns the +// remaining positional arguments, which the main hands to OptionsFromFlags(). +// Exits the process on a malformed command line, like the main would, so only +// well-formed command lines are tested this way. +// +// The returned pointers point into `args` (argv is made of its strings' data, +// hence the non-const reference), so `args` must outlive them: pass a local. +std::vector<char*> ParseCommandLine(std::vector<std::string>& args) { + std::string argv0 = "conformance_test"; + std::vector<char*> argv = {argv0.data()}; + for (std::string& arg : args) argv.push_back(arg.data()); + std::vector<char*> positional_args; + std::vector<absl::UnrecognizedFlag> unrecognized_flags; + absl::ParseAbseilFlagsOnly(static_cast<int>(argv.size()), argv.data(), + positional_args, unrecognized_flags); + positional_args.erase(positional_args.begin()); // argv[0] + return positional_args; +} + +// --- OptionsFromFlags() ------------------------------------------------------ + +TEST(OptionsFromFlagsTest, Defaults) { + absl::FlagSaver saver; + + ConformanceEnvironmentOptions options = OptionsFromFlags(); + + EXPECT_THAT(options.runner, IsNull()); + EXPECT_THAT(options.owned_runner, IsNull()); + EXPECT_THAT(options.testee_binary, IsEmpty()); + EXPECT_THAT(options.testee_args, IsEmpty()); + EXPECT_THAT(options.failure_list_files, IsEmpty()); + EXPECT_EQ(options.enforcement_level, kLowestPriority); + EXPECT_EQ(options.maximum_edition, EDITION_PROTO3); + EXPECT_FALSE(options.fix); + EXPECT_THAT(options.fix_output_file, IsEmpty()); + EXPECT_TRUE(options.check_unseen_expected_failures); +} + +TEST(OptionsFromFlagsTest, ForwardsEveryFlag) { + absl::FlagSaver saver; + absl::SetFlag(&FLAGS_testee_binary, "testee"); + absl::SetFlag(&FLAGS_testee_args, {"--a", "--b"}); + absl::SetFlag(&FLAGS_failure_list, {"x.txt", "y.txt"}); + absl::SetFlag(&FLAGS_enforcement_level, kP0); + absl::SetFlag(&FLAGS_maximum_edition, EDITION_2023); + absl::SetFlag(&FLAGS_fix, true); + absl::SetFlag(&FLAGS_fix_output_file, "out.txt"); + + ConformanceEnvironmentOptions options = OptionsFromFlags(); + + EXPECT_EQ(options.testee_binary, "testee"); + EXPECT_THAT(options.testee_args, ElementsAre("--a", "--b")); + EXPECT_THAT(options.failure_list_files, ElementsAre("x.txt", "y.txt")); + EXPECT_EQ(options.enforcement_level, kP0); + EXPECT_EQ(options.maximum_edition, EDITION_2023); + EXPECT_TRUE(options.fix); + EXPECT_EQ(options.fix_output_file, "out.txt"); + // Not flags: the main sets the runner, the environment the rest. + EXPECT_THAT(options.runner, IsNull()); + EXPECT_THAT(options.owned_runner, IsNull()); + EXPECT_TRUE(options.check_unseen_expected_failures); +} + +TEST(OptionsFromFlagsTest, AppendsPositionalArgsToTesteeArgs) { + absl::FlagSaver saver; + absl::SetFlag(&FLAGS_testee_args, {"--from_flag"}); + std::vector<std::string> positional = {"--from_command_line", "value"}; + std::vector<char*> positional_args; + for (std::string& arg : positional) positional_args.push_back(arg.data()); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(options.testee_args, + ElementsAre("--from_flag", "--from_command_line", "value")); +} + +TEST(OptionsFromFlagsTest, PositionalArgsAloneBecomeTesteeArgs) { + absl::FlagSaver saver; + std::vector<std::string> positional = {"--only_positional"}; + std::vector<char*> positional_args; + for (std::string& arg : positional) positional_args.push_back(arg.data()); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(options.testee_args, ElementsAre("--only_positional")); +} + +TEST(OptionsFromFlagsTest, FixWithoutOutputFileLeavesTheOutputFileEmpty) { + // Defaulting --fix_output_file to the single --failure_list (and resolving + // it under $BUILD_WORKSPACE_DIRECTORY) is the environment's job, not the + // flags'. + absl::FlagSaver saver; + absl::SetFlag(&FLAGS_failure_list, {"failure_lists/cpp_binary.txt"}); + absl::SetFlag(&FLAGS_fix, true); + + ConformanceEnvironmentOptions options = OptionsFromFlags(); + + EXPECT_TRUE(options.fix); + EXPECT_THAT(options.fix_output_file, IsEmpty()); + EXPECT_THAT(options.failure_list_files, + ElementsAre("failure_lists/cpp_binary.txt")); +} + +TEST(OptionsFromFlagsTest, FixOutputFileWithoutFixIsForwardedAsIs) { + absl::FlagSaver saver; + absl::SetFlag(&FLAGS_fix_output_file, "out.txt"); + + ConformanceEnvironmentOptions options = OptionsFromFlags(); + + EXPECT_FALSE(options.fix); + EXPECT_EQ(options.fix_output_file, "out.txt"); +} + +// --- The flags as conformance_test() (conformance.bzl) passes them ----------- + +TEST(CommandLineTest, ParsesTheMacroCommandLine) { + absl::FlagSaver saver; + std::vector<std::string> args = { + "--testee_binary=google/protobuf/conformance/conformance_cpp", + "--failure_list=google/protobuf/conformance/failure_lists/" + "cpp_binary.txt", + "--maximum_edition=EDITION_2023"}; + std::vector<char*> positional_args = ParseCommandLine(args); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(positional_args, IsEmpty()); + EXPECT_EQ(options.testee_binary, + "google/protobuf/conformance/conformance_cpp"); + EXPECT_THAT(options.failure_list_files, + ElementsAre("google/protobuf/conformance/failure_lists/" + "cpp_binary.txt")); + EXPECT_EQ(options.maximum_edition, EDITION_2023); + EXPECT_EQ(options.enforcement_level, kLowestPriority); + EXPECT_FALSE(options.fix); +} + +TEST(CommandLineTest, ParsesTheFixFlags) { + absl::FlagSaver saver; + std::vector<std::string> args = {"--fix", "--fix_output_file=new.txt", + "--enforcement_level=0"}; + std::vector<char*> positional_args = ParseCommandLine(args); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_TRUE(options.fix); + EXPECT_EQ(options.fix_output_file, "new.txt"); + EXPECT_EQ(options.enforcement_level, kP0); +} + +TEST(CommandLineTest, FailureListIsCommaSeparated) { + absl::FlagSaver saver; + std::vector<std::string> args = {"--failure_list=a.txt,b.txt"}; + std::vector<char*> positional_args = ParseCommandLine(args); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(options.failure_list_files, ElementsAre("a.txt", "b.txt")); +} + +TEST(CommandLineTest, RepeatedFailureListFlagReplacesThePreviousValue) { + // absl vector flags don't accumulate: the last occurrence wins, so two + // lists must be given as --failure_list=a.txt,b.txt (see the flag's help). + absl::FlagSaver saver; + std::vector<std::string> args = {"--failure_list=a.txt", + "--failure_list=b.txt"}; + std::vector<char*> positional_args = ParseCommandLine(args); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(options.failure_list_files, ElementsAre("b.txt")); +} + +TEST(CommandLineTest, TesteeArgsAreCommaSeparatedAndPositionalArgsAppended) { + absl::FlagSaver saver; + std::vector<std::string> args = {"--testee_binary=testee", + "--testee_args=--x,--y", "--", "--z", "w"}; + std::vector<char*> positional_args = ParseCommandLine(args); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + + EXPECT_THAT(positional_args, + ElementsAre(absl::string_view("--z"), absl::string_view("w"))); + EXPECT_THAT(options.testee_args, ElementsAre("--x", "--y", "--z", "w")); +} + +TEST(EnforcementLevelFlagTest, ParsesThePriorityNumber) { + TestPriority priority = kLowestPriority; + std::string error; + EXPECT_TRUE(AbslParseFlag("0", &priority, &error)); + EXPECT_EQ(priority, kP0); + EXPECT_TRUE(AbslParseFlag("1", &priority, &error)); + EXPECT_EQ(priority, kP1); + EXPECT_EQ(AbslUnparseFlag(kP1), "1"); + + for (absl::string_view bad : {"-1", "2", "P1", ""}) { + EXPECT_FALSE(AbslParseFlag(bad, &priority, &error)) << bad; + EXPECT_THAT(error, HasSubstr("from 0 to 1")) << bad; + } +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_integration_test.cc b/conformance/test_environment_integration_test.cc new file mode 100644 index 0000000..354eb42 --- /dev/null +++ b/conformance/test_environment_integration_test.cc
@@ -0,0 +1,444 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// End-to-end checks of test_environment_main and ConformanceEnvironment, +// running the deliberately failing sample_conformance_test against the real +// C++ testee. Only what needs a real test process is checked here: the +// command line handling of the main, what gtest itself contributes to a run +// (partial-run detection, the XML report) and the exit code. Everything else +// is covered in-process by test_environment_test. + +#include <algorithm> +#include <cstddef> +#include <cstdio> +#include <cstdlib> +#include <initializer_list> +#include <string> +#include <vector> + +#ifndef _WIN32 +#include <sys/wait.h> // for WEXITSTATUS +#endif + +#include "tools/cpp/runfiles/runfiles.h" +#include "google/protobuf/testing/file.h" +#include "google/protobuf/testing/file.h" +#include <gmock/gmock.h> +#include "google/protobuf/testing/googletest.h" +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::Not; + +// The names of the sample tests, as they appear in failure lists. +constexpr absl::string_view kPass = + "Required.Proto3.ProtobufInput.SamplePass.ProtobufOutput"; +constexpr absl::string_view kDeliberateFailure = + "Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput"; +constexpr absl::string_view kP1Failure = + "Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput"; +constexpr absl::string_view kNeverRuns = + "Required.Proto3.ProtobufInput.SampleNeverRuns.ProtobufOutput"; +constexpr absl::string_view kUncheckedResult = + "Required.Proto3.ProtobufInput.SampleUncheckedResult.ProtobufOutput"; +constexpr absl::string_view kFailureMessage = + "Should have failed to parse, but didn't."; + +// gtest spells its flags --gtest_*, Google's internal build --gunit_*. +constexpr absl::string_view kGtestFlag = "--" GTEST_FLAG_PREFIX_; + +std::string Rlocation(const std::string& path) { + static const bazel::tools::cpp::runfiles::Runfiles* const runfiles = [] { + std::string error; + bazel::tools::cpp::runfiles::Runfiles* runfiles = + bazel::tools::cpp::runfiles::Runfiles::CreateForTest(&error); + ABSL_CHECK(runfiles != nullptr) << error; + return runfiles; + }(); + return runfiles->Rlocation(path); +} + +// The BUILD rule passes the sample test and the testee as $(rlocationpath) +// in these variables. +std::string RunfilesPathFromEnvironment(const char* name) { + const char* path = std::getenv(name); + ABSL_CHECK(path != nullptr) << name << " is not set"; + return path; +} + +const std::string& SampleTest() { + static const std::string* const path = + new std::string(Rlocation(RunfilesPathFromEnvironment("SAMPLE_TEST"))); + return *path; +} + +const std::string& TesteeBinary() { + static const std::string* const path = + new std::string(Rlocation(RunfilesPathFromEnvironment("TESTEE"))); + return *path; +} + +// The path of sample_failure_list_<name>.txt, a data dependency from the +// sample test's package. +std::string FailureListPath(absl::string_view name) { + std::string package = RunfilesPathFromEnvironment("SAMPLE_TEST"); + package.erase(package.rfind('/')); + return Rlocation( + absl::StrCat(package, "/sample_failure_list_", name, ".txt")); +} + +std::string FailureListFlag(absl::string_view name) { + return absl::StrCat("--failure_list=", FailureListPath(name)); +} + +std::string TmpPath(absl::string_view name) { + return absl::StrCat(TestTempDir(), "/", name); +} + +std::string WriteFile(absl::string_view name, absl::string_view content) { + std::string path = TmpPath(name); + ABSL_CHECK_OK(File::SetContents(path, content, true)); + return path; +} + +// `path` is a std::string (not a string_view) for the open source +// File::GetContents() shim. +std::string ReadFile(const std::string& path) { + std::string content; + ABSL_CHECK_OK(File::GetContents(path, &content, true)); + return content; +} + +// Quotes `arg` for the shell that popen() runs commands through. +std::string ShellQuote(absl::string_view arg) { +#ifdef _WIN32 + return absl::StrCat("\"", arg, "\""); +#else + return absl::StrCat("'", absl::StrReplaceAll(arg, {{"'", "'\\''"}}), "'"); +#endif +} + +struct RunResult { + int exit_code; + std::string output; // stdout and stderr +}; + +// Runs `binary` with `args` and returns its exit code and output. +RunResult RunBinary(const std::string& binary, + absl::Span<const std::string> args) { + std::string command = ShellQuote(binary); + for (const std::string& arg : args) { + absl::StrAppend(&command, " ", ShellQuote(arg)); + } + absl::StrAppend(&command, " 2>&1"); +#ifdef _WIN32 + // cmd.exe strips the first and last quote of a command line that starts + // with one, so give it a pair to strip. + FILE* pipe = _popen(absl::StrCat("\"", command, "\"").c_str(), "r"); +#else + FILE* pipe = popen(command.c_str(), "r"); +#endif + ABSL_CHECK(pipe != nullptr) << "Failed to run " << command; + + RunResult result; + char buffer[4096]; + size_t n; + while ((n = fread(buffer, 1, sizeof(buffer), pipe)) > 0) { + result.output.append(buffer, n); + } +#ifdef _WIN32 + result.exit_code = _pclose(pipe); +#else + int status = pclose(pipe); + result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; +#endif + return result; +} + +// Runs the sample test against the C++ testee with `args`. +RunResult RunSample(std::vector<std::string> args) { + args.insert(args.begin(), absl::StrCat("--testee_binary=", TesteeBinary())); + return RunBinary(SampleTest(), args); +} + +// Returns the part of the gtest XML `report` from the element opening with +// `open` up to the first of `ends`. gtest writes an element's properties +// before its children, so the right `ends` isolate one element's properties. +std::string XmlScope(const std::string& report, absl::string_view open, + std::initializer_list<absl::string_view> ends) { + size_t begin = report.find(open); + if (begin == std::string::npos) return ""; + size_t end = report.size(); + for (absl::string_view terminator : ends) { + end = std::min(end, report.find(terminator, begin + open.size())); + } + return report.substr(begin, end - begin); +} + +std::string XmlTestCase(const std::string& report, absl::string_view name) { + return XmlScope(report, absl::StrCat("<testcase name=\"", name, "\""), + {"<testcase", "</testsuite>"}); +} + +std::string XmlTestSuite(const std::string& report, absl::string_view name) { + return XmlScope(report, absl::StrCat("<testsuite name=\"", name, "\""), + {"<testcase", "</testsuite>"}); +} + +std::string XmlTestRun(const std::string& report) { + return XmlScope(report, "<testsuites", {"<testsuite "}); +} + +// Sets `name` to `value` in this process's environment, which the binaries +// the tests run inherit. The MSVC CRT has no setenv()/unsetenv(). +void SetEnvironmentVariable(const char* name, const char* value) { +#ifdef _WIN32 + ABSL_CHECK_EQ(_putenv_s(name, value), 0); +#else + ABSL_CHECK_EQ(setenv(name, value, /*overwrite=*/1), 0); +#endif +} + +void UnsetEnvironmentVariable(const char* name) { +#ifdef _WIN32 + ABSL_CHECK_EQ(_putenv_s(name, ""), 0); // An empty value removes it. +#else + ABSL_CHECK_EQ(unsetenv(name), 0); +#endif +} + +class TestEnvironmentIntegrationTest : public testing::Test { + protected: + static void SetUpTestSuite() { + // The test runner configures this binary's gtest through these variables + // (its filter, sharding, XML report, ...). gtest has read them by now, + // and they must not reach the binaries the tests run. + for (const char* name : + {"TESTBRIDGE_TEST_ONLY", "TESTBRIDGE_TEST_RUNNER_FAIL_FAST", + "XML_OUTPUT_FILE", "TEST_PREMATURE_EXIT_FILE", + "TEST_WARNINGS_OUTPUT_FILE", "TEST_TOTAL_SHARDS", "TEST_SHARD_INDEX", + "TEST_SHARD_STATUS_FILE", "GTEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", + "GTEST_SHARD_STATUS_FILE"}) { + UnsetEnvironmentVariable(name); + } + } +}; + +TEST_F(TestEnvironmentIntegrationTest, PassesWithAMatchingFailureList) { + RunResult run = RunSample({FailureListFlag("ok")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ OK ] SampleConformanceTest.SamplePass"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleDeliberateFailure"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleP1Failure"))); +} + +TEST_F(TestEnvironmentIntegrationTest, FailsWithAnEmptyFailureList) { + RunResult run = RunSample({FailureListFlag("empty")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ FAILED ] " + "SampleConformanceTest.SampleDeliberateFailure"), + HasSubstr(kDeliberateFailure), HasSubstr(kFailureMessage))); +} + +TEST_F(TestEnvironmentIntegrationTest, AListedTestThatPassesFailsTheRun) { + // The list also expects the test that passes to fail. + std::string list = WriteFile( + "pass_listed.txt", absl::StrCat(ReadFile(FailureListPath("ok")), kPass, + " # Should have failed to parse\n")); + RunResult run = RunSample({absl::StrCat("--failure_list=", list)}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ FAILED ] SampleConformanceTest.SamplePass"), + HasSubstr("is in the failure list, but test succeeded"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleDeliberateFailure"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleP1Failure"))); +} + +TEST_F(TestEnvironmentIntegrationTest, EnforcementLevelToleratesP1Failures) { + // Only the P0 failure is listed. + std::string list = + WriteFile("p0_only.txt", + absl::StrCat(kDeliberateFailure, " # ", kFailureMessage, "\n")); + std::string xml = TmpPath("tolerated.xml"); + RunResult run = + RunSample({absl::StrCat("--failure_list=", list), "--enforcement_level=0", + absl::StrCat(kGtestFlag, "output=xml:", xml)}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, + HasSubstr("[ OK ] SampleConformanceTest.SampleP1Failure")); + EXPECT_THAT(XmlTestCase(ReadFile(xml), "SampleP1Failure"), + HasSubstr("name=\"tolerated_failures\" value=\"1\"")); + + // Every priority is enforced by default. + run = RunSample({absl::StrCat("--failure_list=", list)}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT( + run.output, + AllOf(HasSubstr("[ FAILED ] SampleConformanceTest.SampleP1Failure"), + HasSubstr( + absl::StrCat("Unexpected failure for test: ", kP1Failure)))); +} + +TEST_F(TestEnvironmentIntegrationTest, AResultThatIsNeverCheckedFailsItsTest) { + // The sample test that forgets Yields() is disabled, so that the other runs + // stay clean; this run asks for it by name. + RunResult run = + RunSample({absl::StrCat(kGtestFlag, "also_run_disabled_tests"), + absl::StrCat(kGtestFlag, "filter=*SampleUncheckedResult")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT( + run.output, + AllOf(HasSubstr("[ FAILED ] " + "SampleConformanceTest.DISABLED_SampleUncheckedResult"), + HasSubstr(absl::StrCat( + "TestResult for ", kUncheckedResult, + " was never checked; wrap the matcher in Yields()")))); +} + +TEST_F(TestEnvironmentIntegrationTest, ListsTestsWithoutATestee) { + RunResult run = + RunBinary(SampleTest(), {absl::StrCat(kGtestFlag, "list_tests")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, HasSubstr("SampleDeliberateFailure")); +} + +TEST_F(TestEnvironmentIntegrationTest, PassesPositionalArgumentsToTheTestee) { + // The testee ignores them; this checks that flag parsing lets them through. + RunResult run = + RunSample({FailureListFlag("ok"), "--", "--ignored_by_testee"}); + EXPECT_EQ(run.exit_code, 0) << run.output; +} + +// Partial runs are detected from gtest's own test counts and state (see +// IsPartialRun() in test_environment.cc), which only a real run sets up. + +TEST_F(TestEnvironmentIntegrationTest, UnseenExpectedFailuresFailAFullRun) { + // The DISABLED_ sample test doesn't run either; that mustn't make the run + // count as partial. + RunResult run = RunSample({FailureListFlag("unseen")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT( + run.output, + HasSubstr(absl::StrCat("expected failures were not seen: ", kNeverRuns))); +} + +TEST_F(TestEnvironmentIntegrationTest, UnseenExpectedFailuresPassAFilteredRun) { + RunResult run = + RunSample({FailureListFlag("unseen"), + absl::StrCat(kGtestFlag, + "filter=SampleConformanceTest.Sample*Failure")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, Not(HasSubstr("were not seen"))); +} + +TEST_F(TestEnvironmentIntegrationTest, UnseenExpectedFailuresPassAShardedRun) { + // gtest takes the shard from the environment: TEST_* in Google's internal + // build, GTEST_* in open source. + for (const char* name : {"TEST_TOTAL_SHARDS", "GTEST_TOTAL_SHARDS"}) { + SetEnvironmentVariable(name, "2"); + } + for (const char* name : {"TEST_SHARD_INDEX", "GTEST_SHARD_INDEX"}) { + SetEnvironmentVariable(name, "0"); + } + RunResult run = RunSample({FailureListFlag("unseen")}); + for (const char* name : {"TEST_TOTAL_SHARDS", "GTEST_TOTAL_SHARDS", + "TEST_SHARD_INDEX", "GTEST_SHARD_INDEX"}) { + UnsetEnvironmentVariable(name); + } + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, AllOf(HasSubstr("Note: This is test shard 1 of 2"), + Not(HasSubstr("were not seen")))); +} + +TEST_F(TestEnvironmentIntegrationTest, AnEarlyStopUnderFailFastIsAPartialRun) { + // With a list that only names the never-run test, the deliberate failure + // stops the run before the other tests, and the unseen entry isn't reported + // on top of it. + std::string list = + WriteFile("never_runs_only.txt", absl::StrCat(kNeverRuns, "\n")); + RunResult run = RunSample({absl::StrCat("--failure_list=", list), + absl::StrCat(kGtestFlag, "fail_fast")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ FAILED ] " + "SampleConformanceTest.SampleDeliberateFailure"), + Not(HasSubstr("were not seen")))); +} + +TEST_F(TestEnvironmentIntegrationTest, FixRewritesTheFailureListInPlace) { + std::string list = WriteFile("failure_list.txt", ""); + RunResult run = RunSample({absl::StrCat("--failure_list=", list), "--fix"}); + // The failures were unexpected in this run; the rewritten list expects them. + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(ReadFile(list), + AllOf(HasSubstr(kDeliberateFailure), HasSubstr(kP1Failure), + HasSubstr(absl::StrCat("# ", kFailureMessage)))); + run = RunSample({absl::StrCat("--failure_list=", list)}); + EXPECT_EQ(run.exit_code, 0) << run.output; +} + +TEST_F(TestEnvironmentIntegrationTest, RecordsStatisticsInTheXmlReport) { + std::string xml = TmpPath("report.xml"); + RunResult run = RunSample( + {FailureListFlag("ok"), absl::StrCat(kGtestFlag, "output=xml:", xml)}); + ASSERT_EQ(run.exit_code, 0) << run.output; + std::string report = ReadFile(xml); + // Each level gets its own numbers: the P0 and P1 failures are one expected + // failure each, two on their suite and on the run. + EXPECT_THAT(XmlTestCase(report, "SampleDeliberateFailure"), + HasSubstr("name=\"expected_failures\" value=\"1\"")); + EXPECT_THAT(XmlTestCase(report, "SamplePass"), + HasSubstr("name=\"expected_successes\" value=\"1\"")); + EXPECT_THAT(XmlTestSuite(report, "SampleConformanceTest"), + HasSubstr("name=\"expected_failures\" value=\"2\"")); + EXPECT_THAT(XmlTestRun(report), + AllOf(HasSubstr("name=\"expected_failures\" value=\"2\""), + HasSubstr("name=\"unexpected_failures\" value=\"0\""))); +} + +TEST_F(TestEnvironmentIntegrationTest, RecordsEachOutcomeInTheXmlReport) { + std::string xml = TmpPath("outcomes.xml"); + RunResult run = RunSample( + {FailureListFlag("ok"), absl::StrCat(kGtestFlag, "output=xml:", xml)}); + ASSERT_EQ(run.exit_code, 0) << run.output; + std::string report = ReadFile(xml); + // Each conformance test's outcome is a property, named after it, of the + // gtest test that checked it. gtest writes ' as ' in attributes. + EXPECT_THAT( + XmlTestCase(report, "SamplePass"), + HasSubstr(absl::StrCat("name=\"", kPass, "\" value=\"P0 PASS\""))); + EXPECT_THAT( + XmlTestCase(report, "SampleDeliberateFailure"), + HasSubstr(absl::StrCat("name=\"", kDeliberateFailure, + "\" value=\"P0 FAIL: Should have failed to parse, " + "but didn't.\""))); + EXPECT_THAT( + XmlTestCase(report, "SampleP1Failure"), + HasSubstr(absl::StrCat("name=\"", kP1Failure, "\" value=\"P1 FAIL: "))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_main.cc b/conformance/test_environment_main.cc new file mode 100644 index 0000000..00c7b22 --- /dev/null +++ b/conformance/test_environment_main.cc
@@ -0,0 +1,63 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A gtest main for conformance test binaries. It configures a +// ConformanceEnvironment from the command-line flags declared in +// test_environment_flags.h and installs it before running all tests. +// Conformance suites should depend on the test_environment_main library +// instead of a generic gtest main. +// +// The testee is whatever the library defining MakeTesteeRunner() (see +// testee_runner.h) makes of --testee_binary and --testee_args; the binary must +// link exactly one such library, :forked_testee by default. Any positional +// arguments left after flag parsing are appended to --testee_args, so +// `conformance_test --testee_binary=foo -- --bar` passes `--bar` to a forked +// testee. + +#include <utility> +#include <vector> + +#include <gtest/gtest.h> +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/test_environment_flags.h" +#include "conformance/testee_runner.h" + +#include "absl/flags/parse.h" +#include "absl/log/initialize.h" + +// TODO: A custom main only because the stock gtest main parses +// absl flags only when gtest is built with Abseil (GTEST_HAS_ABSL), which +// open-source protobuf doesn't require yet. Once it can be required, install +// the environment from a static initializer and use the stock main instead. +int main(int argc, char* argv[]) { + // In Google's internal build, InitGoogleTest() also runs InitGoogle(), which + // parses absl flags, leaves only positional arguments in argv and + // initializes logging. In open source we do that ourselves, after gtest has + // removed its own flags from argv. + testing::InitGoogleTest(&argc, argv); + std::vector<char*> positional_args(argv, argv + argc); + positional_args = absl::ParseCommandLine(argc, argv); + absl::InitializeLog(); + + // Listing tests doesn't need a testee, and shouldn't require any flags. + if (!GTEST_FLAG_GET(list_tests)) { + google::protobuf::conformance::internal::ConformanceEnvironmentOptions options = + google::protobuf::conformance::internal::OptionsFromFlags( + absl::MakeConstSpan(positional_args).subspan(1)); + // The linked testee decides what the flags mean, and the environment + // takes its runner instead of the flags (it would otherwise spawn + // --testee_binary itself). + options.owned_runner = google::protobuf::conformance::MakeTesteeRunner( + options.testee_binary, options.testee_args); + options.testee_binary.clear(); + options.testee_args.clear(); + google::protobuf::conformance::internal::ConformanceEnvironment::Install( + std::move(options)); + } + return RUN_ALL_TESTS(); +}
diff --git a/conformance/test_environment_test.cc b/conformance/test_environment_test.cc index 91280e2..3325918 100644 --- a/conformance/test_environment_test.cc +++ b/conformance/test_environment_test.cc
@@ -36,9 +36,12 @@ #include "conformance/conformance.pb.h" #include "conformance/global_test_environment.h" #include "conformance/matchers.h" +#include "conformance/matchers_testing.h" #include "conformance/mock_test_runner.h" +#include "conformance/result_ledger.h" +#include "conformance/result_listener.h" +#include "conformance/result_record.h" #include "conformance/test_environment_testing.h" -#include "conformance/test_manager.h" #include "conformance/testee.h" #include "google/protobuf/test_messages_proto3.pb.h" #include "google/protobuf/text_format.h" @@ -51,11 +54,20 @@ using ::absl_testing::IsOk; using ::google::protobuf::conformance::internal::ConformanceEnvironment; using ::google::protobuf::conformance::internal::ConformanceEnvironmentOptions; +using ::google::protobuf::conformance::internal::ResultRecord; +using ::google::protobuf::conformance::internal::ResultRecordMessage; using ::protobuf_test_messages::proto3::TestAllTypesProto3; using ::testing::_; +using ::testing::Contains; +using ::testing::ElementsAre; +using ::testing::FieldsAre; using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Key; using ::testing::NiceMock; using ::testing::Not; +using ::testing::Pair; +using ::testing::Property; using ::testing::Return; // The name the testee sees for a kP0 test named "Foo" that parses @@ -64,6 +76,8 @@ "Required.Proto3.ProtobufInput.Foo.ProtobufOutput"; constexpr absl::string_view kBarTestName = "Required.Proto3.ProtobufInput.Bar.ProtobufOutput"; +constexpr absl::string_view kSetUpTestSuiteTestName = + "Required.Proto3.ProtobufInput.SetUpTestSuite.ProtobufOutput"; std::string SerializedResponse(absl::string_view textproto) { ::conformance::ConformanceResponse response; @@ -160,6 +174,47 @@ std::string old_directory_; }; +// The messages of the failures in `failures`. +std::vector<std::string> Messages( + const testing::TestPartResultArray& failures) { + std::vector<std::string> messages; + for (int i = 0; i < failures.size(); ++i) { + messages.push_back(failures.GetTestPartResult(i).message()); + } + return messages; +} + +// The properties of `result`, as (key, value) pairs. +std::vector<std::pair<std::string, std::string>> Properties( + const testing::TestResult& result) { + std::vector<std::pair<std::string, std::string>> properties; + for (int i = 0; i < result.test_property_count(); ++i) { + const testing::TestProperty& property = result.GetTestProperty(i); + properties.emplace_back(property.key(), property.value()); + } + return properties; +} + +// Runs ReportUncheckedResults() on `env` and returns the messages of the +// failures it reports. +std::vector<std::string> UncheckedResultFailures(ConformanceEnvironment& env) { + testing::TestPartResultArray failures; + { + testing::ScopedFakeTestPartResultReporter reporter( + testing::ScopedFakeTestPartResultReporter:: + INTERCEPT_ONLY_CURRENT_THREAD, + &failures); + env.ReportUncheckedResults(); + } + return Messages(failures); +} + +// Creates a kP0 test named `name` against the global testee, as the fixture's +// Testee() would. +internal::Test CreateP0Test(absl::string_view name) { + return ConformanceEnvironment::Get().testee().CreateTest(name, kP0); +} + // Tears down `env` in a full run, expecting it to report unseen expected // failures with a message containing `expected_message`. void TearDownExpectingUnseenFailures(ConformanceEnvironment& env, @@ -233,23 +288,26 @@ TEST_F(ConformanceEnvironmentTest, ScopedEnvironmentIsTheGlobalOne) { internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); - EXPECT_EQ(&internal::GetGlobalTestManager(), &env->test_manager()); + EXPECT_EQ(&internal::GetGlobalFailureList(), &env->ledger().failure_list()); } -TEST_F(ConformanceEnvironmentTest, EnforcementLevelIsPassedToTestManager) { +TEST_F(ConformanceEnvironmentTest, EnforcementLevelIsPassedToTheFailureList) { { // Every priority is enforced by default. internal::ScopedGlobalConformanceEnvironment env( {/*runner=*/&fake_runner_}); - EXPECT_THAT(env->test_manager().ReportFailure("P1.x", kP1, "boom"), - Not(IsOk())); + EXPECT_THAT( + env->ledger().failure_list().VerdictOnFailure("P1.x", kP1, "boom"), + Not(IsOk())); } { ConformanceEnvironmentOptions options; options.runner = &fake_runner_; options.enforcement_level = kP0; internal::ScopedGlobalConformanceEnvironment env(std::move(options)); - EXPECT_THAT(env->test_manager().ReportFailure("P1.x", kP1, "boom"), IsOk()); + EXPECT_THAT( + env->ledger().failure_list().VerdictOnFailure("P1.x", kP1, "boom"), + IsOk()); } } @@ -262,9 +320,8 @@ .WillOnce(Return(SerializedResponse(R"pb(protobuf_payload: "\010c")pb"))); EXPECT_THAT(RunP0("Foo"), Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); - - EXPECT_EQ(env->test_manager().expected_successes(), 1); - EXPECT_EQ(env->test_manager().unexpected_failures(), 0); + EXPECT_EQ(env->ledger().expected_successes(), 1); + EXPECT_EQ(env->ledger().unexpected_failures(), 0); env->TearDown(); } @@ -272,13 +329,167 @@ internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); env->SetUp(); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); - EXPECT_EQ(env->test_manager().unexpected_failures(), 1); + EXPECT_EQ(env->ledger().unexpected_failures(), 1); env->TearDown(); } +TEST_F(ConformanceEnvironmentTest, ResultsThatAreNeverCheckedFailTheTest) { + internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); + env->SetUp(); + + // A result that is stored but never checked and one matched without + // Yields() both leave the ResultLedger in the dark; only what Yields() + // records reaches it. (Dropping a result or applying a leaf matcher to it + // without Yields() does not compile; see YieldedResult in matchers.h.) + internal::TestResult unchecked = RunP0("Foo"); + EXPECT_THAT(RunP0("Bar"), + Property(&internal::TestResult::name, HasSubstr("Bar"))); + EXPECT_THAT(RunP0("Baz"), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); + + EXPECT_THAT( + UncheckedResultFailures(env.environment()), + ElementsAre(HasSubstr(absl::StrCat( + "TestResult for ", kFooTestName, + " was never checked; wrap the matcher in Yields()")), + HasSubstr(absl::StrCat( + "TestResult for ", kBarTestName, + " was never checked; wrap the matcher in Yields()")))); + // Each result is reported once. + EXPECT_THAT(UncheckedResultFailures(env.environment()), IsEmpty()); + env->TearDown(); +} + +TEST_F(ConformanceEnvironmentTest, TearDownReportsResultsThatWereNeverChecked) { + internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); + env->SetUp(); + internal::TestResult unchecked = RunP0("Foo"); + EXPECT_NONFATAL_FAILURE( + env->TearDown(), + absl::StrCat("TestResult for ", kFooTestName, + " was never checked; wrap the matcher in Yields()")); +} + +TEST_F(ConformanceEnvironmentTest, ReportUncheckedResultsIsANoOpAfterTearDown) { + internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); + env->SetUp(); + env->TearDown(); + EXPECT_THAT(UncheckedResultFailures(env.environment()), IsEmpty()); +} + +TEST_F(ConformanceEnvironmentTest, TalliesRecordedResultsAsTheyArrive) { + internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); + env->SetUp(); + + // What Yields() records (see ResultRecordMessage() in result_record.h), by + // hand, next to a success that isn't a record. + SUCCEED() << ResultRecordMessage(kFooTestName, + {kP0, ResultRecord::Status::kFail, "boom"}); + SUCCEED() << "P0 FAIL: not a record"; + EXPECT_THAT(env->ledger().UnexpectedFailures(), + ElementsAre(FieldsAre(kFooTestName, "boom", absl::nullopt))); + EXPECT_EQ(env->ledger().expected_successes(), 0); + + // The first outcome recorded for a test is the one that counts. + SUCCEED() << ResultRecordMessage(kFooTestName, + {kP0, ResultRecord::Status::kPass, ""}); + EXPECT_EQ(env->ledger().unexpected_failures(), 1); + EXPECT_EQ(env->ledger().expected_successes(), 0); + env->TearDown(); +} + +// The environment's ResultListener reports the results Yields() records and +// the statistics when a gtest test or test suite ends (result_listener_test.cc +// drives it by hand), so this only shows through an environment that outlives +// the test body, like the one a test binary installs. This suite keeps one +// alive across its tests, and until the suite itself has ended, and inspects +// what the listener did after each body has returned. The failure the +// environment adds for an unchecked result is a real one, so that is checked +// end to end in test_environment_integration_test.cc instead. +class LongLivedEnvironmentTest : public testing::Test { + protected: + struct SuiteState { + FakeTestRunner runner; + internal::ScopedGlobalConformanceEnvironment env{{/*runner=*/&runner}}; + }; + + // Inspects the suite's properties once the suite has ended, then destroys + // the environment. gtest tells its listeners about a suite ending in + // reverse order of registration, so this runs after the environment's + // listener as long as it is registered first. + class SuiteEndChecker : public testing::EmptyTestEventListener { + public: + explicit SuiteEndChecker(const testing::TestSuite* suite) + : suite_under_test_(suite) {} + + void OnTestSuiteEnd(const testing::TestSuite& suite) override { + if (&suite != suite_under_test_ || suite_ == nullptr) return; + // The suite's own properties hold what SetUpTestSuite() checked and the + // statistics of the whole suite. + std::vector<std::pair<std::string, std::string>> properties = + Properties(suite.ad_hoc_test_result()); + EXPECT_THAT(properties, + Contains(Pair(kSetUpTestSuiteTestName, "P0 PASS"))); + EXPECT_THAT(properties, Contains(Pair("expected_successes", "2"))); + EXPECT_THAT(properties, Contains(Pair("unexpected_failures", "0"))); + delete suite_; + suite_ = nullptr; + } + + private: + const testing::TestSuite* const suite_under_test_; + }; + + static void SetUpTestSuite() { + testing::UnitTest& unit_test = *testing::UnitTest::GetInstance(); + // gtest owns the checker and deletes it when the program ends. + unit_test.listeners().Append( + new SuiteEndChecker(unit_test.current_test_suite())); + suite_ = new SuiteState(); + // A result checked outside any test is the suite's. It is tallied at + // once, before the first test starts, so it is neither counted toward that + // test's statistics nor reported as unchecked when it ends, and it is + // reported as a property of the suite, not of the test. + EXPECT_THAT( + ConformanceEnvironment::Get() + .testee() + .CreateTest("SetUpTestSuite", kP0) + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); + } + + static void TearDownTestSuite() { + // The test's properties hold what Yields() recorded for Bar and the + // statistics of the test alone. + const testing::TestSuite& suite = + *testing::UnitTest::GetInstance()->current_test_suite(); + ASSERT_EQ(suite.total_test_count(), 1); + std::vector<std::pair<std::string, std::string>> properties = + Properties(*suite.GetTestInfo(0)->result()); + EXPECT_THAT(properties, Contains(Pair(kBarTestName, "P0 PASS"))); + EXPECT_THAT(properties, Contains(Pair("expected_successes", "1"))); + EXPECT_THAT(properties, Contains(Pair("unexpected_failures", "0"))); + EXPECT_THAT(properties, Not(Contains(Key(kSetUpTestSuiteTestName)))); + } + + static SuiteState* suite_; +}; + +LongLivedEnvironmentTest::SuiteState* LongLivedEnvironmentTest::suite_ = + nullptr; + +TEST_F(LongLivedEnvironmentTest, ReportsResultsWhenTheTestEnds) { + EXPECT_THAT( + CreateP0Test("Bar") + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + TEST_F(ConformanceEnvironmentTest, ExpectedFailureFromFailureListPasses) { std::string failure_list = WriteFile("failures.txt", @@ -291,8 +502,8 @@ EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())); - EXPECT_EQ(env->test_manager().expected_failures(), 1); - EXPECT_EQ(env->test_manager().unexpected_failures(), 0); + EXPECT_EQ(env->ledger().expected_failures(), 1); + EXPECT_EQ(env->ledger().unexpected_failures(), 0); env->TearDown(); } @@ -309,7 +520,7 @@ EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())); EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())); - EXPECT_EQ(env->test_manager().expected_failures(), 2); + EXPECT_EQ(env->ledger().expected_failures(), 2); env->TearDown(); } @@ -348,9 +559,9 @@ EXPECT_THAT(RunP1("Foo"), Yields(IsParseError())); - EXPECT_EQ(env->test_manager().tolerated_failures(), 1); - EXPECT_EQ(env->test_manager().skipped(), 0); - EXPECT_EQ(env->test_manager().unexpected_failures(), 0); + EXPECT_EQ(env->ledger().tolerated_failures(), 1); + EXPECT_EQ(env->ledger().skipped(), 0); + EXPECT_EQ(env->ledger().unexpected_failures(), 0); env->TearDown(); } @@ -369,8 +580,8 @@ EXPECT_THAT(RunP1("Foo"), Yields(IsParseError())); - EXPECT_EQ(env->test_manager().expected_failures(), 1); - EXPECT_EQ(env->test_manager().tolerated_failures(), 0); + EXPECT_EQ(env->ledger().expected_failures(), 1); + EXPECT_EQ(env->ledger().tolerated_failures(), 0); // ...and so it isn't unseen either. internal::ScopedPartialRunOverride full_run(false); env->TearDown(); @@ -415,6 +626,28 @@ "\nRemove them from the failure list, or rerun with --fix")); } +TEST_F(ConformanceEnvironmentTest, TearDownFailsForAnOverexpandedWildcard) { + std::string entry = "Required.Proto3.ProtobufInput.*.ProtobufOutput"; + std::string failure_list = WriteFile( + "failures.txt", absl::StrCat(entry, " # Should have failed to parse\n")); + ConformanceEnvironmentOptions options; + options.runner = &fake_runner_; + options.failure_list_files = {failure_list}; + internal::ScopedGlobalConformanceEnvironment env(std::move(options)); + env->SetUp(); + + // Each test is an expected failure; only their number is the problem. + for (int i = 0; i <= internal::kMaximumWildcardExpansions; ++i) { + EXPECT_THAT(RunP0(absl::StrCat("Test", i)), Yields(IsParseError())); + } + + internal::ScopedPartialRunOverride full_run(false); + EXPECT_NONFATAL_FAILURE( + env->TearDown(), + absl::StrCat("The failure list entry ", entry, " matched more than ", + internal::kMaximumWildcardExpansions, " tests")); +} + TEST_F(ConformanceEnvironmentTest, UnseenExpectedFailuresAreFineInPartialRun) { std::string failure_list = WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n")); @@ -426,8 +659,8 @@ internal::ScopedPartialRunOverride partial_run(true); env->TearDown(); - EXPECT_THAT(env->test_manager().Finalize().message(), - HasSubstr(absl::StrCat("were not seen: ", kFooTestName))); + EXPECT_THAT(env->ledger().UnmatchedExpectedFailures(), + ElementsAre(kFooTestName)); } TEST_F(ConformanceEnvironmentTest, UnseenCheckCanBeDisabled) { @@ -442,8 +675,8 @@ internal::ScopedPartialRunOverride full_run(false); env->TearDown(); - EXPECT_THAT(env->test_manager().Finalize().message(), - HasSubstr(absl::StrCat("were not seen: ", kFooTestName))); + EXPECT_THAT(env->ledger().UnmatchedExpectedFailures(), + ElementsAre(kFooTestName)); } TEST_F(ConformanceEnvironmentTest, FixWritesToTheOutputFile) { @@ -459,8 +692,8 @@ env->SetUp(); // Foo (listed) is never run; Bar (unlisted) fails. - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); // The unseen entry is still reported, but the message points at the // rewritten list instead of suggesting --fix. TearDownExpectingUnseenFailures( @@ -489,8 +722,8 @@ internal::ScopedGlobalConformanceEnvironment env(std::move(options)); env->SetUp(); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); TearDownExpectingUnseenFailures( env.environment(), absl::StrCat("updated failure list written to ", failure_list)); @@ -520,8 +753,8 @@ internal::ScopedGlobalConformanceEnvironment env(std::move(options)); env->SetUp(); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); TearDownExpectingUnseenFailures( env.environment(), absl::StrCat("updated failure list written to ", workspace, "/failures.txt")); @@ -553,6 +786,58 @@ EXPECT_EQ(ReadFile(TmpPath("runfiles/failures.txt")), ""); } +TEST_F(ConformanceEnvironmentTest, + FixResolvesRelativeOutputFileUnderBuildWorkspaceDirectory) { + // conformance_test() passes a relative --fix_output_file (and no + // --failure_list) when the failure list doesn't exist yet; --fix must then + // create it in the source tree, not in the runfiles. + std::string runfiles = CreateTmpDir("runfiles"); + std::string workspace = CreateTmpDir("workspace"); + ScopedWorkingDirectory cwd(runfiles); + ScopedEnvironmentVariable build_workspace_directory( + "BUILD_WORKSPACE_DIRECTORY", workspace); + + ConformanceEnvironmentOptions options; + options.runner = &fake_runner_; + options.fix = true; + options.fix_output_file = "failures.txt"; + internal::ScopedGlobalConformanceEnvironment env(std::move(options)); + env->SetUp(); + + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); + internal::ScopedPartialRunOverride full_run(false); + env->TearDown(); + + EXPECT_THAT(ReadFile(TmpPath("workspace/failures.txt")), + HasSubstr(kBarTestName)); + EXPECT_FALSE( + File::Exists(TmpPath("runfiles/failures.txt"))); +} + +TEST_F(ConformanceEnvironmentTest, + FixWithRelativeOutputFileWritesToTheWorkingDirectoryOutsideBlazeRun) { + std::string runfiles = CreateTmpDir("runfiles"); + ScopedWorkingDirectory cwd(runfiles); + ScopedEnvironmentVariable build_workspace_directory( + "BUILD_WORKSPACE_DIRECTORY", absl::nullopt); + + ConformanceEnvironmentOptions options; + options.runner = &fake_runner_; + options.fix = true; + options.fix_output_file = "failures.txt"; + internal::ScopedGlobalConformanceEnvironment env(std::move(options)); + env->SetUp(); + + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); + internal::ScopedPartialRunOverride full_run(false); + env->TearDown(); + + EXPECT_THAT(ReadFile(TmpPath("runfiles/failures.txt")), + HasSubstr(kBarTestName)); +} + TEST_F(ConformanceEnvironmentTest, FixWithMultipleListsNeedsOutputFile) { std::string first = WriteFile("first.txt", ""); std::string second = WriteFile("second.txt", ""); @@ -593,8 +878,8 @@ internal::ScopedGlobalConformanceEnvironment env(std::move(options)); env->SetUp(); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); // Rewriting would drop Foo, whose test simply wasn't selected. The unseen // check itself stays quiet in a partial run, so this is the only failure. @@ -615,8 +900,8 @@ internal::ScopedGlobalConformanceEnvironment env(std::move(options)); env->SetUp(); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse, but didn't."); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse, but didn't."); internal::ScopedPartialRunOverride full_run(false); EXPECT_NONFATAL_FAILURE( @@ -624,18 +909,43 @@ absl::StrCat("Failed to write updated failure list to ", not_a_file)); } +TEST_F(ConformanceEnvironmentTest, FixDropsListedSkips) { + std::string failure_list = + WriteFile("failures.txt", absl::StrCat(kFooTestName, " # listed\n")); + std::string fixed = TmpPath("fixed.txt"); + MockTestRunner mock; + ConformanceEnvironmentOptions options; + options.runner = &mock; + options.failure_list_files = {failure_list}; + options.fix = true; + options.fix_output_file = fixed; + internal::ScopedGlobalConformanceEnvironment env(std::move(options)); + env->SetUp(); + + // Foo (listed) is skipped by the testee, which fails it (see matchers.h); + // the rewritten list no longer has its entry, so the fixed run passes. + EXPECT_CALL(mock, RunTest(kFooTestName, _)) + .WillOnce(Return(SerializedResponse(R"pb(skipped: "not supported")pb"))); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())), + "is in the failure list but was skipped by the testee"); + internal::ScopedPartialRunOverride full_run(false); + env->TearDown(); + + EXPECT_THAT(ReadFile(fixed), Not(HasSubstr(kFooTestName))); +} + TEST_F(ConformanceEnvironmentTest, StatisticsSnapshotAndDelta) { internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_}); env->SetUp(); - internal::Statistics before = internal::Statistics::From(env->test_manager()); + internal::Statistics before = internal::Statistics::From(env->ledger()); EXPECT_THAT(RunP0("Foo"), Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); - EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), - "Should have failed to parse"); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())), + "Should have failed to parse"); internal::Statistics delta = - internal::Statistics::From(env->test_manager()) - before; + internal::Statistics::From(env->ledger()) - before; EXPECT_EQ(delta.expected_successes, 1); EXPECT_EQ(delta.unexpected_failures, 1); EXPECT_EQ(delta.expected_failures, 0); @@ -661,18 +971,17 @@ options.failure_list_files = {failure_list}; internal::ScopedGlobalConformanceEnvironment env(std::move(options)); env->SetUp(); - internal::Statistics before = internal::Statistics::From(env->test_manager()); + internal::Statistics before = internal::Statistics::From(env->ledger()); // A listed test the testee skips fails (see matchers.h) and is both a skip // and a listed skip, but neither an expected nor an unexpected anything. EXPECT_CALL(mock, RunTest(kFooTestName, _)) .WillOnce(Return(SerializedResponse(R"pb(skipped: "not supported")pb"))); - EXPECT_NONFATAL_FAILURE( - EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())), - "is in the failure list but was skipped by the testee"); + EXPECT_YIELDS_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())), + "is in the failure list but was skipped by the testee"); internal::Statistics delta = - internal::Statistics::From(env->test_manager()) - before; + internal::Statistics::From(env->ledger()) - before; EXPECT_EQ(delta.skipped_tests, 1); EXPECT_EQ(delta.listed_skips, 1); EXPECT_EQ(delta.expected_failures, 0); @@ -731,7 +1040,7 @@ env->TearDown(); EXPECT_TRUE(destroyed); // The statistics survive the testee. - EXPECT_EQ(env->test_manager().expected_successes(), 1); + EXPECT_EQ(env->ledger().expected_successes(), 1); } TEST_F(ConformanceEnvironmentTest, TearDownWithoutSetUpOnlyReleasesTheTestee) { @@ -755,7 +1064,7 @@ EXPECT_TRUE(destroyed); EXPECT_FALSE(File::Exists(fixed)); EXPECT_THAT(ReadFile(failure_list), HasSubstr("never seen")); - EXPECT_EQ(env->test_manager().expected_failures(), 0); + EXPECT_EQ(env->ledger().expected_failures(), 0); } TEST_F(ConformanceEnvironmentTest, OwnedRunnerDiesWithTheEnvironment) { @@ -773,6 +1082,32 @@ EXPECT_DEATH(ConformanceEnvironment::Get(), "Install"); } +TEST(ConformanceEnvironmentDeathTest, + UncheckedResultsAreNotReportedAfterAFatalFailure) { + // An ASSERT_* that doesn't hold returns from the test before it can check + // its results. A real (non-intercepted) fatal failure would fail this test, + // so the scenario runs in a death test child that exits with the number of + // stray failures. + FakeTestRunner runner; + EXPECT_EXIT( + { + internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner}); + internal::TestResult unchecked = + CreateP0Test("Foo") + .ParseBinary(TestAllTypesProto3::descriptor(), + VarintField(1, 99)) + .SerializeBinary(); + // What ASSERT_* records before returning. + [] { FAIL() << "fatal failure"; }(); + ABSL_CHECK(testing::Test::HasFatalFailure()); + // std::exit() runs no destructors, so the mock is leaked on purpose. + testing::Mock::AllowLeak(&runner); + std::exit(static_cast<int>( + UncheckedResultFailures(env.environment()).size())); + }, + testing::ExitedWithCode(0), ""); +} + TEST(ConformanceEnvironmentDeathTest, RequiresExactlyOneTestee) { FakeTestRunner runner; EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env({}),
diff --git a/conformance/test_fixture.cc b/conformance/test_fixture.cc index 5958da9..8749585 100644 --- a/conformance/test_fixture.cc +++ b/conformance/test_fixture.cc
@@ -11,7 +11,6 @@ #include "google/protobuf/descriptor.pb.h" #include <gtest/gtest.h> -#include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/strings/string_view.h" #include "conformance/test_environment.h" @@ -25,32 +24,27 @@ namespace { using ::google::protobuf::conformance::internal::ConformanceEnvironment; -using ::google::protobuf::conformance::internal::Statistics; // Whether tests for `descriptor`'s message type run under `maximum_edition`: // the file's edition must not be newer than it. bool IsSupported(const Descriptor& descriptor, Edition maximum_edition) { Edition edition = FileDescriptorLegacy(descriptor.file()).edition(); if (edition == EDITION_UNSTABLE) { - // `maximum_edition` is the maximum *stable* edition. Like the legacy - // runner, run the unstable tests alongside any editions tests. - // TODO: this runs the (newer-than-anything) unstable tests - // at --maximum_edition=2023 while a stable 2024 message would be skipped; - // gate EDITION_UNSTABLE on its own flag once legacy parity no longer - // matters. + // `maximum_edition` is the maximum *stable* edition; EDITION_UNSTABLE is + // newer than any of them, so it is gated on the editions tier instead: the + // unstable tests run whenever the 2023 tests do, as in the original suites + // and the open-source runner. Every test message the suites use is 2023 + // or older, so nothing is skipped that the unstable tests would leapfrog. + // TODO: once a test message of a newer stable edition + // exists, give EDITION_UNSTABLE its own opt-in (an `unstable_edition` + // option / flag / conformance_test() attribute) so that + // --maximum_edition=2023 keeps skipping the newer stable message while + // still running these. return maximum_edition >= EDITION_2023; } return edition <= maximum_edition; } -// Statistics accumulated by ConformanceTest::TearDown() for each test suite, -// keyed by suite name, until TearDownTestSuite() reports them. -absl::flat_hash_map<std::string, Statistics>& SuiteStatistics() { - static auto* suite_statistics = - new absl::flat_hash_map<std::string, Statistics>(); - return *suite_statistics; -} - std::string CurrentTestName() { const testing::TestInfo* test_info = testing::UnitTest::GetInstance()->current_test_info(); @@ -60,56 +54,18 @@ return test_info->name(); } -std::string CurrentTestSuiteName() { - const testing::TestSuite* test_suite = - testing::UnitTest::GetInstance()->current_test_suite(); - ABSL_CHECK(test_suite != nullptr) - << "ConformanceTest's TearDown hooks must run inside a test suite."; - return test_suite->name(); -} - } // namespace void ConformanceTest::SetUp() { - ConformanceEnvironment& environment = ConformanceEnvironment::Get(); - initial_statistics_ = Statistics::From(environment.test_manager()); - // TODO: Transitional. Goes away with the in-binary - // performance mode, see ConformanceEnvironmentOptions::performance. - if (IsPerformanceTest() && !environment.options().performance) { - GTEST_SKIP() << "Performance tests only run with --performance."; - } - if (!IsPerformanceTest() && environment.options().performance) { - GTEST_SKIP() << "Only performance tests run with --performance."; - } if (const Descriptor* message = MessageUnderTest(); message != nullptr && - !IsSupported(*message, environment.options().maximum_edition)) { + !IsSupported(*message, + ConformanceEnvironment::Get().options().maximum_edition)) { GTEST_SKIP() << "Skipping " << message->full_name() << " because its edition is newer than --maximum_edition."; } } -void ConformanceTest::TearDown() { - Statistics delta = - Statistics::From(ConformanceEnvironment::Get().test_manager()) - - initial_statistics_; - delta.RecordProperties(); - SuiteStatistics()[CurrentTestSuiteName()] += delta; -} - -void ConformanceTest::TearDownTestSuite() { - auto node = SuiteStatistics().extract(CurrentTestSuiteName()); - if (!node) { - // gtest only runs TearDownTestSuite() for suites with at least one test to - // run, and TearDown() runs even for tests skipped in SetUp(), so this only - // happens when a fatal failure in a SetUpTestSuite() skipped every test - // outright (which is already reported), or when a subclass's TearDown() - // doesn't call ConformanceTest::TearDown(). - return; - } - node.mapped().RecordProperties(); -} - internal::Test ConformanceTest::Testee() { return Testee(CurrentTestName()); } internal::Test ConformanceTest::Testee(absl::string_view name) {
diff --git a/conformance/test_fixture.h b/conformance/test_fixture.h index 4a0dcfd..38ab3f7 100644 --- a/conformance/test_fixture.h +++ b/conformance/test_fixture.h
@@ -8,9 +8,8 @@ // The gtest fixture for conformance test suites. // // ConformanceTest is the fixture every conformance test should use. It -// handles edition gating, per-test statistics and, for now, performance-test -// filtering. A suite overrides DefaultPriority() (see TestPriority in -// testee.h) to say how important its tests are. +// handles edition gating. A suite overrides DefaultPriority() (see +// TestPriority in testee.h) to say how important its tests are. // // Its Testee() builds a test against the global testee, at the suite's // priority or at the one it is given. @@ -52,7 +51,6 @@ #include <gtest/gtest.h> #include "absl/base/nullability.h" #include "absl/strings/string_view.h" -#include "conformance/test_environment.h" #include "conformance/testee.h" #include "google/protobuf/descriptor.h" @@ -60,9 +58,8 @@ namespace protobuf { namespace conformance { -// The fixture for all conformance tests. Tests that aren't performance tests -// derive from this directly. Performance tests derive from the transitional -// PerformanceConformanceTest. A suite overrides the defaults below as needed: +// The fixture for all conformance tests. A suite subclasses it and overrides +// the defaults below as needed: // // class FooTest : public ConformanceTest { // private: @@ -74,12 +71,19 @@ // Yields(WhenParsed(EqualsBinaryProto(input)))); // } // -// Subclasses that override SetUp(), TearDown() or TearDownTestSuite() must -// call the base implementation, first thing in SetUp() and last thing in -// TearDown() and TearDownTestSuite(), so that the statistics recorded there -// cover the whole test. The base SetUp() may skip the test. gtest only skips -// the test body on GTEST_SKIP(), so an overriding SetUp() must bail out itself -// afterwards: +// Naming rule: a fixture that belongs to the performance conformance suite +// (conformance_suite(name = "performance") in the BUILD file) must be named +// `*PerformanceTest`, and no fixture in any other suite may be. The +// performance suite's own test binaries need no such rule. +// conformance_test_runner, however, links every suite into one binary and +// selects the performance tests for its --performance flag by that suffix +// (see PerformanceGtestFilter() in conformance_test_runner.h). Nothing checks +// the rule mechanically. A misnamed fixture silently runs with the wrong flag. +// TODO: the rule goes away with conformance_test_runner. +// +// Subclasses that override SetUp() must call the base implementation first +// thing. It may skip the test. gtest only skips the test body on +// GTEST_SKIP(), so an overriding SetUp() must bail out itself afterwards: // // void SetUp() override { // ConformanceTest::SetUp(); @@ -96,25 +100,10 @@ // A single test can say otherwise with Testee(priority). virtual TestPriority DefaultPriority() const { return kP0; } - // Skips the test if it isn't the kind (performance or regular) selected by - // the environment or if MessageUnderTest() isn't supported under - // --maximum_edition, and snapshots statistics for TearDown(). + // Skips the test if MessageUnderTest() isn't supported under + // --maximum_edition. void SetUp() override; - // Records this test's statistics as test properties and accumulates them for - // the suite. - void TearDown() override; - - // Records the suite's accumulated statistics as suite properties. - static void TearDownTestSuite(); - - // Whether this is a performance test. Performance tests only run when the - // environment was configured with `performance = true`, and regular tests - // only run when it wasn't. - // TODO: Transitional. Goes away with the in-binary - // performance mode, see ConformanceEnvironmentOptions::performance. - virtual bool IsPerformanceTest() const { return false; } - // The message type this test exercises, if the fixture knows it. When // non-null, SetUp() skips the test unless the message is supported under // the current --maximum_edition, so that the test can use it @@ -144,17 +133,6 @@ internal::Test Testee(absl::string_view name); internal::Test Testee(TestPriority priority); internal::Test Testee(TestPriority priority, absl::string_view name); - - private: - internal::Statistics initial_statistics_; -}; - -// The fixture for performance conformance tests. -// TODO: Transitional. Goes away with the in-binary -// performance mode, see ConformanceEnvironmentOptions::performance. -class PerformanceConformanceTest : public ConformanceTest { - protected: - bool IsPerformanceTest() const override { return true; } }; } // namespace conformance
diff --git a/conformance/test_fixture_test.cc b/conformance/test_fixture_test.cc index 9442d39..93588b9 100644 --- a/conformance/test_fixture_test.cc +++ b/conformance/test_fixture_test.cc
@@ -105,12 +105,11 @@ Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); } -// A ConformanceTest whose SetUp() and TearDown() leave the global environment -// alone, so that Testee() can be called without one. +// A ConformanceTest whose SetUp() leaves the global environment alone, so +// that Testee() can be called without one. class TesteeDeathTest : public ConformanceTest { protected: void SetUp() override {} - void TearDown() override {} }; TEST_F(TesteeDeathTest, WithoutInstall) { EXPECT_DEATH(Testee(), "Install"); } @@ -209,17 +208,6 @@ FAIL() << "SetUp() should have skipped this test."; } -// TODO: Transitional. Goes away with the in-binary -// performance mode, see ConformanceEnvironmentOptions::performance. -class PerformanceFixtureTest : public FixtureTest { - protected: - bool IsPerformanceTest() const override { return true; } -}; - -TEST_F(PerformanceFixtureTest, IsSkippedWithoutPerformanceMode) { - FAIL() << "SetUp() should have skipped this test."; -} - // Checks, from an overriding SetUp(), that the documented `if (IsSkipped()) // return;` pattern sees the base class's skip. class SkipAwareSetUpTest : public Edition2023MessageTest {
diff --git a/conformance/test_manager.cc b/conformance/test_manager.cc deleted file mode 100644 index cb626c8..0000000 --- a/conformance/test_manager.cc +++ /dev/null
@@ -1,398 +0,0 @@ -#include "conformance/test_manager.h" - -#include <algorithm> -#include <cctype> -#include <cstddef> -#include <fstream> -#include <ios> -#include <string> -#include <utility> -#include <vector> - -#include "absl/algorithm/container.h" -#include "absl/container/flat_hash_set.h" -#include "absl/log/absl_check.h" -#include "absl/log/absl_log.h" -#include "absl/status/status.h" -#include "absl/strings/ascii.h" -#include "absl/strings/match.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "conformance/testee.h" - -namespace google { -namespace protobuf { -namespace conformance { -namespace internal { -namespace { - -// How many failures one wildcard entry may cover. As in the legacy runner the -// check precedes the count of the current match, so the limit is reached one -// failure later than the number says: the (kMaximumWildcardExpansions + 2)th -// is the first one reported. -constexpr int kMaximumWildcardExpansions = 20; -constexpr int kFailureMessageLengthLimit = 128; - -bool InsertUniqueTest(absl::flat_hash_set<std::string>& set, - absl::string_view value) { - if (!set.emplace(value).second) { - return false; - } - return true; -} - -void IncrementIfUnique(bool unique, int& counter) { - if (unique) { - ++counter; - } -} - -void Normalize(std::string& input) { - input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); -} - -// Sets up a failure message properly for our failure lists. -std::string FormatFailureMessage(absl::string_view input) { - std::string result(absl::StripAsciiWhitespace(input)); - // Remove newlines - Normalize(result); - // Truncate failure message if needed - if (result.length() > kFailureMessageLengthLimit) { - result = result.substr(0, kFailureMessageLengthLimit); - } - return result; -} - -std::string ReformatLine(size_t alignment, absl::string_view line) { - size_t comment_pos = line.find('#'); - absl::string_view test_name = - absl::StripAsciiWhitespace(line.substr(0, comment_pos)); - if (test_name.empty()) { - return std::string(absl::StripAsciiWhitespace(line)); - } - absl::string_view message = - absl::StripAsciiWhitespace(line.substr(comment_pos + 1)); - ABSL_CHECK_GE(alignment, test_name.length()); - std::string whitespace(alignment - test_name.length(), ' '); - return absl::StrCat(test_name, whitespace, " # ", message); -} - -} // namespace - -TestManager::~TestManager() { - if (!finalized_) { - ABSL_LOG(FATAL) - << "TestManager::Finalize() was not called before destruction."; - } -} - -absl::Status TestManager::LoadFailureList(absl::string_view filename) { - std::ifstream infile(std::string{filename}); - - if (!infile.is_open()) { - return absl::InternalError( - absl::StrCat("Couldn't open failure list file: ", filename)); - } - - for (std::string line; std::getline(infile, line);) { - failure_list_lines_.push_back(line); - - // Remove comments. - std::string test_name = line.substr(0, line.find('#')); - - test_name.erase( - std::remove_if(test_name.begin(), test_name.end(), ::isspace), - test_name.end()); - - if (test_name.empty()) { // Skip empty lines. - continue; - } - - // If we remove whitespace from the beginning of a line, and what we have - // left at first is a '#', then we have a comment. - if (test_name[0] != '#') { - // Find our failure message if it exists. Will be set to an empty string - // if no message is found. Empty failure messages also pass our tests. - size_t comment_pos = line.find('#'); - std::string message; - if (comment_pos != std::string::npos) { - message = line.substr(comment_pos + 1); // +1 to skip the delimiter - // If we had only whitespace after the delimiter, we will have an empty - // failure message and the test will still pass. - message = std::string(absl::StripAsciiWhitespace(message)); - } - if (absl::Status status = AddExpectedFailure(test_name, message); - !status.ok()) { - return status; - } - } - } - - return absl::OkStatus(); -} - -absl::Status TestManager::AddExpectedFailure( - absl::string_view test_name, absl::string_view failure_message) { - if (absl::Status status = expected_failure_list_.Insert(test_name); - !status.ok()) { - return status; - } - // Normalize the message the same way reported failures are normalized so - // that the two compare equal in ReportFailure. - ABSL_CHECK(expected_failure_messages_ - .emplace(test_name, FormatFailureMessage(failure_message)) - .second); - ABSL_CHECK(unseen_expected_failures_.emplace(test_name).second); - ABSL_CHECK(unmatched_expected_failures_.emplace(test_name).second); - return absl::OkStatus(); -} - -absl::Status TestManager::SaveFailureList(absl::string_view filename) const { - std::ofstream outfile(std::string{filename}, std::ios::binary); - if (!outfile.is_open()) { - return absl::InternalError(absl::StrCat( - "Couldn't open failure list file for writing: ", filename)); - } - - // Calculate alignment. - size_t alignment = 0; - for (const std::string& line : failure_list_lines_) { - absl::string_view test_name = absl::StripAsciiWhitespace( - absl::string_view(line).substr(0, line.find('#'))); - alignment = std::max(alignment, test_name.length()); - } - for (const auto& failure : new_failures_) { - alignment = std::max(alignment, failure.first.length()); - } - - // Output existing failure list, stripping out unseen tests and inserting - // new failures. - auto to_add = new_failures_.begin(); - for (const std::string& line : failure_list_lines_) { - absl::string_view test_name = absl::StripAsciiWhitespace( - absl::string_view(line).substr(0, line.find('#'))); - if (unseen_expected_failures_.contains(test_name) || - seen_unexpected_successes_.contains(test_name)) { - continue; - } - while (to_add != new_failures_.end() && test_name > to_add->first) { - outfile << ReformatLine(alignment, absl::StrCat(to_add->first, " # ", - to_add->second)) - << "\n"; - ++to_add; - } - outfile << ReformatLine(alignment, line) << "\n"; - } - - // Add any remaining new failures. - while (to_add != new_failures_.end()) { - outfile << ReformatLine(alignment, - absl::StrCat(to_add->first, " # ", to_add->second)) - << "\n"; - ++to_add; - } - - outfile.close(); - if (!outfile.good()) { - return absl::InternalError( - absl::StrCat("Failed to write failure list file: ", filename)); - } - return absl::OkStatus(); -} - -absl::Status TestManager::ReportSuccess(absl::string_view test_name) { - bool unique = InsertUniqueTest(seen_tests_, test_name); - absl::optional<std::string> failure_match = MarkMatched(test_name); - - if (failure_match.has_value()) { - // This was expected to fail, but it succeeded. - IncrementIfUnique(unique, number_of_matches_[*failure_match]); - IncrementIfUnique(unique, unexpected_successes_); - if (unique) { - unexpected_success_matches_[test_name] = *failure_match; - } - unseen_expected_failures_.erase(*failure_match); - seen_unexpected_successes_.insert(*failure_match); - return absl::FailedPreconditionError(absl::StrCat( - "test ", test_name, " (matched to ", *failure_match, - ") is in the failure list, but test succeeded. Remove its match from " - "the failure list.")); - } - - // This wasn't expected to fail. - IncrementIfUnique(unique, expected_successes_); - return absl::OkStatus(); -} - -absl::Status TestManager::ReportFailure(absl::string_view test_name, - TestPriority priority, - absl::string_view failure_message) { - bool unique = InsertUniqueTest(seen_tests_, test_name); - absl::optional<std::string> failure_match = MarkMatched(test_name); - - std::string formatted_failure_message = FormatFailureMessage(failure_message); - // Counts the failure as unexpected and records it for UnexpectedFailures(). - auto record_unexpected_failure = [&] { - IncrementIfUnique(unique, unexpected_failures_); - if (unique) { - unexpected_failure_messages_[test_name] = formatted_failure_message; - } - }; - - if (!failure_match.has_value()) { - if (priority > enforcement_level_) { - // Tolerated: neither a failure nor a skip, and not written to the - // failure list; only counted. - IncrementIfUnique(unique, tolerated_failures_); - ABSL_LOG(WARNING) << "WARNING, test=" << test_name << ": " - << absl::StripTrailingAsciiWhitespace(failure_message); - return absl::OkStatus(); - } - // This was not expected to fail. - record_unexpected_failure(); - new_failures_[test_name] = formatted_failure_message; - return absl::FailedPreconditionError( - absl::StrCat("Unexpected failure for test: ", test_name)); - } - - // Mirror the legacy runner, which only requires the actual failure message to - // start with the expected one. - const std::string& expected_failure_message = - expected_failure_messages_.at(*failure_match); - if (!absl::StartsWith(formatted_failure_message, expected_failure_message)) { - record_unexpected_failure(); - // TODO: b/563659620 - Keying the replacement by the (possibly wildcard) - // entry duplicates the line if another test later matches the entry with - // the expected message: with `foo.*.bar # abc` listed, - // ReportFailure("foo.a.bar", "xyz") + ReportFailure("foo.b.bar", "abc") - // makes SaveFailureList() keep the original line and add a second - // `foo.*.bar` line, which LoadFailureList() then rejects. - new_failures_[*failure_match] = formatted_failure_message; - return absl::FailedPreconditionError(absl::StrCat( - "Unexpected failure message for test: ", test_name, " expected: ", - expected_failure_message, " actual: ", formatted_failure_message)); - } - - unseen_expected_failures_.erase(*failure_match); - - if (number_of_matches_[*failure_match] > kMaximumWildcardExpansions) { - record_unexpected_failure(); - return absl::FailedPreconditionError( - absl::StrCat("The wildcard ", *failure_match, - " served as matches to too many test " - "names exceeding the max amount of ", - kMaximumWildcardExpansions, " for test: ", test_name)); - } - - IncrementIfUnique(unique, number_of_matches_[*failure_match]); - IncrementIfUnique(unique, expected_failures_); - ABSL_LOG(INFO) << "Ignoring expected failure for test " << test_name << ": " - << absl::StripTrailingAsciiWhitespace(failure_message); - - return absl::OkStatus(); -} - -absl::Status TestManager::ReportSkip(absl::string_view test_name, - absl::string_view skip_reason) { - bool unique = InsertUniqueTest(seen_tests_, test_name); - IncrementIfUnique(unique, skipped_); - absl::optional<std::string> failure_match = MarkMatched(test_name); - if (!failure_match.has_value()) { - return absl::OkStatus(); - } - - // A skip says nothing about whether the entry is still needed, so it is - // neither reported as unseen by Finalize() nor dropped by SaveFailureList(). - // Nor does it count toward the entry's wildcard expansion cap - // (number_of_matches_), unlike in the legacy runner, which counted every - // test a wildcard matched: the cap is about real outcomes hidden by a - // wildcard, and a skip isn't one. - unseen_expected_failures_.erase(*failure_match); - IncrementIfUnique(unique, listed_skips_); - if (unique) { - listed_skip_matches_[test_name] = *failure_match; - } - return absl::FailedPreconditionError(absl::StrCat( - "test ", test_name, " (matched to ", *failure_match, - ") is in the failure list but was skipped by the testee: ", skip_reason, - ". Remove its match from the failure list.")); -} - -void TestManager::ReportNotSelected(absl::string_view test_name) { - seen_tests_.emplace(test_name); - MarkMatched(test_name); -} - -absl::optional<std::string> TestManager::MarkMatched( - absl::string_view test_name) { - absl::optional<std::string> failure_match = - expected_failure_list_.WalkDownMatch(test_name); - if (failure_match.has_value()) { - unmatched_expected_failures_.erase(*failure_match); - } - return failure_match; -} - -std::vector<std::string> TestManager::UnseenExpectedFailures() const { - std::vector<std::string> unseen(unseen_expected_failures_.begin(), - unseen_expected_failures_.end()); - absl::c_sort(unseen); - return unseen; -} - -std::vector<std::string> TestManager::UnmatchedExpectedFailures() const { - std::vector<std::string> unmatched(unmatched_expected_failures_.begin(), - unmatched_expected_failures_.end()); - absl::c_sort(unmatched); - return unmatched; -} - -std::vector<UnexpectedResult> TestManager::UnexpectedFailures() const { - std::vector<UnexpectedResult> failures; - failures.reserve(unexpected_failure_messages_.size()); - // The map is ordered by test name. - for (const auto& [test_name, failure_message] : - unexpected_failure_messages_) { - failures.push_back( - {/*test_name=*/test_name, /*failure_message=*/failure_message}); - } - return failures; -} - -std::vector<UnexpectedResult> TestManager::UnexpectedSuccesses() const { - std::vector<UnexpectedResult> successes; - successes.reserve(unexpected_success_matches_.size()); - // The map is ordered by test name. - for (const auto& [test_name, matched_entry] : unexpected_success_matches_) { - successes.push_back( - {/*test_name=*/test_name, - /*failure_message=*/expected_failure_messages_.at(matched_entry), - /*matched_entry=*/matched_entry}); - } - return successes; -} - -std::vector<std::pair<std::string, std::string>> TestManager::ListedSkips() - const { - // The map is ordered by test name. - return std::vector<std::pair<std::string, std::string>>( - listed_skip_matches_.begin(), listed_skip_matches_.end()); -} - -absl::Status TestManager::Finalize() { - finalized_ = true; - std::vector<std::string> unseen = UnseenExpectedFailures(); - if (!unseen.empty()) { - return absl::FailedPreconditionError( - absl::StrCat("The following expected failures were not seen: ", - absl::StrJoin(unseen, ", "))); - } - return absl::OkStatus(); -} - -} // namespace internal -} // namespace conformance -} // namespace protobuf -} // namespace google
diff --git a/conformance/test_manager.h b/conformance/test_manager.h deleted file mode 100644 index fcf5830..0000000 --- a/conformance/test_manager.h +++ /dev/null
@@ -1,232 +0,0 @@ -#ifndef GOOGLE_PROTOBUF_CONFORMANCE_TEST_MANAGER_H__ -#define GOOGLE_PROTOBUF_CONFORMANCE_TEST_MANAGER_H__ - -#include <string> -#include <utility> -#include <vector> - -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" -#include "absl/status/status.h" -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "conformance/conformance.pb.h" -#include "conformance/failure_list_trie_node.h" -#include "conformance/testee.h" - -namespace google { -namespace protobuf { -namespace conformance { -namespace internal { - -// A test whose result contradicted the failure list. See -// TestManager::UnexpectedFailures() and TestManager::UnexpectedSuccesses(). -// TODO: b/563707827 - Remove with conformance_test_runner. -struct UnexpectedResult { - // The full name of the test, as reported to the TestManager. - std::string test_name; - // For an unexpected failure, the test's failure message as - // TestManager::SaveFailureList() would write it. For an unexpected success, - // the message of the failure list entry that matched the test. - std::string failure_message; - // For an unexpected success, the failure list entry that matched - // `test_name`, possibly a wildcard. Unset for an unexpected failure, which - // matched no entry or matched one with a different message. - absl::optional<std::string> matched_entry; -}; - -// Tracks the expected failures and the actual results of a test suite. The -// conformance matchers (see matchers.h) report the outcome of every test -// here. The test environment reads the results back and reports them as test -// properties. Under --fix the results also populate the new failure list. -class TestManager { - public: - TestManager() : expected_failure_list_("root") {} - ~TestManager(); - - // The lowest priority whose failures fail the suite (see TestPriority in - // testee.h). ReportFailure() tolerates a failing test of a lower priority - // unless it is in the failure list. Defaults to kLowestPriority, which - // enforces every priority. - void set_enforcement_level(TestPriority level) { enforcement_level_ = level; } - - // Loads a failure list from disk and adds its entries to the ones loaded so - // far. Each line that isn't blank or a comment names one expected failure, - // optionally followed by `#` and the expected failure message. Returns an - // error if the file can't be opened. Also returns an error if an entry is - // already present, including one from an earlier file, or contains an - // invalid wildcard. - absl::Status LoadFailureList(absl::string_view filename); - - // Saves an updated failure list to disk based on the reported results. - // Returns an error if `filename` can't be opened for writing or the write - // fails. - absl::Status SaveFailureList(absl::string_view filename) const; - - // Reports a successful test run. This will return an error if the test was - // expected to fail. - absl::Status ReportSuccess(absl::string_view test_name); - - // Reports a failed test run along with the failure message. Returns OK if - // the failure doesn't fail the suite. That is the case when the test is in - // the failure list with a matching message (an expected failure, logged at - // INFO), or when it isn't listed and `priority` is above the enforcement - // level (a tolerated failure, logged as a WARNING and only counted by - // tolerated_failures()). Otherwise the failure is unexpected and the error - // says why. A listed test is checked whatever its priority, so that the - // failure list can't go stale unnoticed. The normalized actual message - // only needs to start with the expected message, so an empty expected - // message matches any failure. - absl::Status ReportFailure(absl::string_view test_name, TestPriority priority, - absl::string_view failure_message); - - // Reports a test that the testee skipped for `skip_reason`. A skip is not - // a verdict on the test's failure list entry, if any. The entry counts as - // seen and matched, so that Finalize() doesn't report it and - // SaveFailureList() keeps it, but the skip is not an expected failure. A - // listed test that is skipped is recorded (see ListedSkips()) and returns an - // error that names the matched entry, like an unexpected success. Whether - // that error fails the test is the caller's policy. The matchers fail such - // a test. - absl::Status ReportSkip(absl::string_view test_name, - absl::string_view skip_reason); - - // Reports a test the runner didn't run because it wasn't selected. - // The original conformance_test_runner matches a test name against the - // failure list before checking whether the test was selected. Like it, - // this only marks the entry the name matches, if any, as matched for - // UnmatchedExpectedFailures(). The test is not counted by any statistic - // and its entry stays unseen. - // TODO: b/563707827 - Remove with conformance_test_runner. - void ReportNotSelected(absl::string_view test_name); - - // Whether any of the Report*() methods has been called for `test_name`. - // Yields() checks this to reject a second check of the same test. - bool WasReported(absl::string_view test_name) const { - return seen_tests_.contains(test_name); - } - - // Runs sanity checks over the failure list to make sure everything we - // expected to run was reported. Returns an error naming the sorted expected - // failure entries that have not been reported as a failure, an unexpected - // success or a skip. Must be called before destruction. - absl::Status Finalize(); - - // Returns the sorted expected failure entries that no reported test name has - // matched so far, regardless of the outcome. Names are reported through - // ReportSuccess(), ReportFailure(), ReportSkip() or ReportNotSelected(). A - // wildcard entry counts as matched once it has matched any test name. - // Unlike the unseen entries Finalize() reports, an entry whose test was not - // selected is not returned. It is a real test, only not run here. - // TODO: b/563707827 - Remove with conformance_test_runner. - std::vector<std::string> UnmatchedExpectedFailures() const; - - // Returns the tests reported as unexpected failures so far, sorted by test - // name. These are the tests unexpected_failures() counts. Each result's - // `failure_message` is the message as SaveFailureList() would write it, and - // its `matched_entry` is unset. This includes tests that were expected to - // fail, but with a different message. SaveFailureList() replaces their - // entry, so adding these lines to the failure list would duplicate it. - // TODO: b/563707827 - Remove with conformance_test_runner. - std::vector<UnexpectedResult> UnexpectedFailures() const; - - // Returns the tests reported as unexpected successes so far, sorted by test - // name. These are the tests unexpected_successes() counts. Each result's - // `matched_entry` is the failure list entry the test matched, and - // `failure_message` is that entry's message. - // TODO: b/563707827 - Remove with conformance_test_runner. - std::vector<UnexpectedResult> UnexpectedSuccesses() const; - - // Returns the tests the testee skipped although they are in the failure - // list, sorted by test name. These are the tests listed_skips() counts. - // Each pair is the test name and the failure list entry it matched. Such a - // test fails the gtest run: the matchers fail it with the error - // ReportSkip() returns. It is not an unexpected failure or success. Its - // entry is kept as is, also by SaveFailureList() under --fix, so removing - // the entry is up to the user. - // TODO: b/563707827 - Remove with conformance_test_runner. - std::vector<std::pair<std::string, std::string>> ListedSkips() const; - - // The number of tests that were reported skipped. This includes the listed - // skips. - int skipped() const { return skipped_; } - - // The number of skipped tests that are in the failure list. See - // ListedSkips(). - int listed_skips() const { return listed_skips_; } - - // The number of tests that failed but were tolerated because their priority - // is above the enforcement level. - int tolerated_failures() const { return tolerated_failures_; } - - // The number of tests that were in the failure list and failed with the - // expected message. - int expected_failures() const { return expected_failures_; } - - // The number of tests that failed but were not in the failure list, or - // failed with a different message than the one listed. - int unexpected_failures() const { return unexpected_failures_; } - - // The number of tests that were not in the failure list and succeeded. - int expected_successes() const { return expected_successes_; } - - // The number of tests that were in the failure list but succeeded. - int unexpected_successes() const { return unexpected_successes_; } - - private: - // Adds one failure list entry, as LoadFailureList() does for each line. - // Returns an error if the test name is already present or contains an - // invalid wildcard. - absl::Status AddExpectedFailure(absl::string_view test_name, - absl::string_view failure_message); - - // The sorted expected failure entries Finalize() complains about: those not - // reported as a failure, an unexpected success or a skip so far. - std::vector<std::string> UnseenExpectedFailures() const; - - // Marks the failure list entry `test_name` matches, if any, as matched (see - // UnmatchedExpectedFailures()) and returns it. - absl::optional<std::string> MarkMatched(absl::string_view test_name); - - FailureListTrieNode expected_failure_list_; - absl::flat_hash_map<std::string, std::string> expected_failure_messages_; - - absl::flat_hash_set<std::string> unseen_expected_failures_; - // Entries never matched by name by any Report*() call. - absl::flat_hash_set<std::string> unmatched_expected_failures_; - absl::flat_hash_set<std::string> seen_unexpected_successes_; - absl::flat_hash_map<std::string, int> number_of_matches_; - - // Every test name reported so far (see WasReported()), so that a test - // reported more than once is only counted once. - absl::flat_hash_set<std::string> seen_tests_; - - // The tests counted by unexpected_failures_, mapped to their formatted - // failure message, and the tests counted by unexpected_successes_, mapped to - // the failure list entry they matched. - absl::btree_map<std::string, std::string> unexpected_failure_messages_; - absl::btree_map<std::string, std::string> unexpected_success_matches_; - // The tests counted by listed_skips_, mapped to the entry they matched. - absl::btree_map<std::string, std::string> listed_skip_matches_; - - std::vector<std::string> failure_list_lines_; - absl::btree_map<std::string, std::string> new_failures_; - - int skipped_ = 0; - int listed_skips_ = 0; - int tolerated_failures_ = 0; - int expected_failures_ = 0; - int unexpected_failures_ = 0; - int expected_successes_ = 0; - int unexpected_successes_ = 0; - bool finalized_ = false; - TestPriority enforcement_level_ = kLowestPriority; -}; - -} // namespace internal -} // namespace conformance -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_CONFORMANCE_TEST_MANAGER_H__
diff --git a/conformance/test_manager_test.cc b/conformance/test_manager_test.cc deleted file mode 100644 index 1b33d18..0000000 --- a/conformance/test_manager_test.cc +++ /dev/null
@@ -1,1073 +0,0 @@ -#include "conformance/test_manager.h" - -#include <initializer_list> -#include <string> - -#include "google/protobuf/testing/file.h" -#include "google/protobuf/testing/file.h" -#include "google/protobuf/testing/file.h" -#include <gmock/gmock.h> -#include "google/protobuf/testing/googletest.h" -#include <gtest/gtest.h> -#include "absl/log/absl_check.h" -#include "absl/status/status.h" -#include "absl/status/status_matchers.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "conformance/testee.h" - -namespace google { -namespace protobuf { -namespace conformance { -namespace internal { -namespace { - -using ::absl_testing::IsOk; -using ::absl_testing::StatusIs; -using ::testing::AllOf; -using ::testing::ElementsAre; -using ::testing::FieldsAre; -using ::testing::HasSubstr; -using ::testing::IsEmpty; -using ::testing::Not; -using ::testing::Optional; -using ::testing::Pair; -using ::testing::SizeIs; - -class TestManagerTest : public ::testing::Test { - protected: - TestManagerTest() - : tmp_dir_(absl::StrCat(TestTempDir(), "/test_manager_test/")), - failure_list_path_(absl::StrCat( - tmp_dir_, - testing::UnitTest::GetInstance()->current_test_info()->name())) { - if (!File::Exists(tmp_dir_)) { - ABSL_CHECK_OK( - File::RecursivelyCreateDir(tmp_dir_, 0777)); - } - } - - struct Failure { - absl::string_view name; - absl::string_view message; - - Failure( // NOLINT(google-explicit-constructor) - absl::string_view name) - : name(name), message("") {} - Failure(absl::string_view name, absl::string_view message) - : name(name), message(message) {} - }; - - void CreateFailureList( - std::initializer_list<const Failure> expected_failures) { - std::string content; - for (auto failure : expected_failures) { - absl::StrAppend(&content, failure.name, " # ", failure.message, "\n"); - } - ABSL_CHECK_OK( - File::SetContents(failure_list_path_, content, true)); - } - - void CreateFailureList(absl::string_view content) { - ABSL_CHECK_OK( - File::SetContents(failure_list_path_, content, true)); - } - - absl::string_view failure_list() const { return failure_list_path_; } - - ~TestManagerTest() override { - File::DeleteRecursively(failure_list_path_, NULL, NULL); - } - - private: - std::string tmp_dir_; - std::string failure_list_path_; - std::string output_path_; -}; - -TEST_F(TestManagerTest, RequiresFinalize) { - EXPECT_DEATH({ TestManager manager; }, "Finalize"); -} - -TEST_F(TestManagerTest, ReportSuccess) { - CreateFailureList({}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportSuccess("foo"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 1); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportSkipped) { - CreateFailureList({}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportSkip("foo", "reason"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 1); - EXPECT_EQ(manager.listed_skips(), 0); - EXPECT_THAT(manager.ListedSkips(), IsEmpty()); -} - -TEST_F(TestManagerTest, ReportSkipMarksListedEntrySeenAndMatched) { - CreateFailureList({{"foo", "abc"}, {"bar.*", "abc"}, {"baz", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // A listed skip is reported like an unexpected success, naming the entry. - EXPECT_THAT(manager.ReportSkip("foo", "not supported"), - StatusIs(absl::StatusCode::kFailedPrecondition, - "test foo (matched to foo) is in the failure list but " - "was skipped by the testee: not supported. Remove its " - "match from the failure list.")); - EXPECT_THAT(manager.ReportSkip("bar.x", "no"), - StatusIs(absl::StatusCode::kFailedPrecondition, - "test bar.x (matched to bar.*) is in the failure list " - "but was skipped by the testee: no. Remove its match " - "from the failure list.")); - // Reporting a test twice returns the error again but counts it once. - EXPECT_THAT(manager.ReportSkip("bar.x", "no"), Not(IsOk())); - - // A skip is only counted as a skip, but the entries it matched are neither - // unmatched nor unseen: whether the entry is still needed is unknown. - EXPECT_THAT(manager.UnmatchedExpectedFailures(), ElementsAre("baz")); - EXPECT_THAT(manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - HasSubstr("were not seen: baz"))); - EXPECT_EQ(manager.skipped(), 2); - EXPECT_EQ(manager.listed_skips(), 2); - EXPECT_THAT(manager.ListedSkips(), - ElementsAre(Pair("bar.x", "bar.*"), Pair("foo", "foo"))); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_THAT(manager.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(manager.UnexpectedSuccesses(), IsEmpty()); -} - -TEST_F(TestManagerTest, ReportNotSelected) { - CreateFailureList({{"foo"}, {"bar.*"}, {"baz"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - manager.ReportNotSelected("foo"); - manager.ReportNotSelected("bar.x"); - manager.ReportNotSelected("unlisted"); - - // Only the matched entries stop being unmatched; nothing else changes. - EXPECT_THAT(manager.UnmatchedExpectedFailures(), ElementsAre("baz")); - EXPECT_THAT(manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - "The following expected failures were not seen: bar.*, " - "baz, foo")); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); - EXPECT_EQ(manager.tolerated_failures(), 0); -} - -TEST_F(TestManagerTest, WasReported) { - CreateFailureList({}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - EXPECT_FALSE(manager.WasReported("success")); - - EXPECT_THAT(manager.ReportSuccess("success"), IsOk()); - EXPECT_THAT(manager.ReportFailure("failure", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportSkip("skip", "reason"), IsOk()); - manager.ReportNotSelected("not_selected"); - - EXPECT_TRUE(manager.WasReported("success")); - EXPECT_TRUE(manager.WasReported("failure")); - EXPECT_TRUE(manager.WasReported("skip")); - EXPECT_TRUE(manager.WasReported("not_selected")); - EXPECT_FALSE(manager.WasReported("other")); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, ReportExpectedFailure) { - CreateFailureList({{"foo", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.Finalize(), IsOk()); - - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportDuplicates) { - CreateFailureList({{"foo", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportSuccess("bar"), IsOk()); - ASSERT_THAT(manager.ReportSuccess("bar"), IsOk()); - ASSERT_THAT(manager.ReportSkip("baz", "reason"), IsOk()); - ASSERT_THAT(manager.ReportSkip("baz", "reason"), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 1); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 1); -} - -TEST_F(TestManagerTest, ReportExpectedFailureWildcard) { - CreateFailureList({{"foo.*.bar", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo.baz.bar", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.Finalize(), IsOk()); - - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportUnseenFailure) { - CreateFailureList({{"foo.bar"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT( - manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("were not seen"), HasSubstr("foo.bar")))); - - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportUnseenFailureUnrelatedSkip) { - CreateFailureList({{"foo.bar"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // "foo" doesn't match the entry "foo.bar", so the entry stays unseen. - ASSERT_THAT(manager.ReportSkip("foo", "reason"), IsOk()); - - EXPECT_THAT( - manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("were not seen"), HasSubstr("foo.bar")))); - - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 1); -} - -TEST_F(TestManagerTest, ReportUnexpectedFailure) { - CreateFailureList({}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportFailure("foo_failing", kP0, ""), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("Unexpected failure"), - HasSubstr("foo_failing")))); - - EXPECT_THAT(manager.Finalize(), IsOk()); - - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportUnexpectedFailureMismatchedName) { - CreateFailureList({{"foo"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportFailure("foo_failing", kP0, ""), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("Unexpected failure"), - HasSubstr("foo_failing")))); - ASSERT_THAT(manager.ReportFailure("foo", kP0, ""), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportUnexpectedFailureMismatchedMessage) { - CreateFailureList({{"foo.*.bar", "message"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportFailure("foo.a.bar", kP0, "abc"), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("Unexpected failure"), HasSubstr("foo"), - HasSubstr("message"), HasSubstr("abc")))); - EXPECT_THAT(manager.ReportFailure("foo.b.bar", kP0, "message"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, ReportFailureMatchesExpectedMessagePrefix) { - CreateFailureList({{"foo", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // Like the legacy runner, the actual message only has to start with the - // expected one. - EXPECT_THAT(manager.ReportFailure("foo", kP0, "abc: more details"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); -} - -TEST_F(TestManagerTest, ReportFailureRejectsExpectedMessageSuffix) { - CreateFailureList({{"foo", "abc: more details"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // The expected message is longer than the actual one, so it's not a prefix. - EXPECT_THAT(manager.ReportFailure("foo", kP0, "abc"), - StatusIs(absl::StatusCode::kFailedPrecondition, - HasSubstr("Unexpected failure message"))); - - // Like any other message mismatch, the stale entry is still reported as - // unseen so that --fix replaces it. - EXPECT_THAT(manager.Finalize(), Not(IsOk())); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 1); -} - -TEST_F(TestManagerTest, ReportFailureEmptyExpectedMessageMatchesAnything) { - CreateFailureList({{"foo"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP0, "any message at all"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); -} - -TEST_F(TestManagerTest, ReportFailurePrefixMatchIgnoresNewlinesAndWhitespace) { - CreateFailureList({{"foo", "abc def"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP0, " abc def\nghi\n"), IsOk()); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); -} - -TEST_F(TestManagerTest, PrefixMatchedFailureIsNotAnUnexpectedFailure) { - CreateFailureList({{"foo", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc: more details"), IsOk()); - - EXPECT_THAT(manager.UnexpectedFailures(), IsEmpty()); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, FinalizeReportsUnseenExpectedFailures) { - CreateFailureList({{"ccc"}, {"aaa"}, {"bbb.*.baz"}, {"ddd"}, {"eee"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // Failures, unexpected successes and skips all count as "seen". - ASSERT_THAT(manager.ReportFailure("bbb.x.baz", kP0, ""), IsOk()); - ASSERT_THAT(manager.ReportSuccess("ccc"), Not(IsOk())); - ASSERT_THAT(manager.ReportSkip("ddd", "reason"), Not(IsOk())); - // Tests that weren't selected to run do not. - manager.ReportNotSelected("eee"); - - EXPECT_THAT(manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - "The following expected failures were not seen: aaa, " - "eee")); -} - -TEST_F(TestManagerTest, UnmatchedExpectedFailures) { - CreateFailureList({{"ccc"}, {"aaa"}, {"bbb.*.baz"}, {"ddd"}, {"eee", "msg"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.UnmatchedExpectedFailures(), - ElementsAre("aaa", "bbb.*.baz", "ccc", "ddd", "eee")); - - // Any report whose name matches an entry counts, whatever the outcome. - ASSERT_THAT(manager.ReportFailure("bbb.x.baz", kP0, ""), IsOk()); - ASSERT_THAT(manager.ReportSuccess("ccc"), Not(IsOk())); - ASSERT_THAT(manager.ReportSkip("ddd", "reason"), Not(IsOk())); - ASSERT_THAT(manager.ReportFailure("eee", kP0, "other message"), Not(IsOk())); - // Reports that match no entry change nothing. - ASSERT_THAT(manager.ReportSuccess("aaa.child"), IsOk()); - ASSERT_THAT(manager.ReportSkip("zzz", "reason"), IsOk()); - - EXPECT_THAT(manager.UnmatchedExpectedFailures(), ElementsAre("aaa")); - // Message mismatches are still unseen, though. - EXPECT_THAT(manager.Finalize(), - StatusIs(absl::StatusCode::kFailedPrecondition, - "The following expected failures were not seen: aaa, " - "eee")); -} - -TEST_F(TestManagerTest, UnmatchedExpectedFailuresEmpty) { - TestManager manager; - EXPECT_THAT(manager.UnmatchedExpectedFailures(), IsEmpty()); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, UnexpectedFailures) { - CreateFailureList({{"foo.*.bar", "expected"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.UnexpectedFailures(), IsEmpty()); - - // Not in the failure list at all, in the list with another message, and the - // messages are formatted as SaveFailureList() would write them. - ASSERT_THAT(manager.ReportFailure("zzz", kP0, " new\nfailure "), - Not(IsOk())); - ASSERT_THAT(manager.ReportFailure("foo.a.bar", kP0, "other message"), - Not(IsOk())); - ASSERT_THAT(manager.ReportFailure("aaa", kP0, std::string(1000, 'x')), - Not(IsOk())); - // Expected failures, successes and skips are not reported. - ASSERT_THAT(manager.ReportFailure("foo.b.bar", kP0, "expected: more"), - IsOk()); - ASSERT_THAT(manager.ReportSuccess("bbb"), IsOk()); - ASSERT_THAT(manager.ReportSkip("ccc", "reason"), IsOk()); - - EXPECT_THAT( - manager.UnexpectedFailures(), - ElementsAre(FieldsAre("aaa", std::string(128, 'x'), absl::nullopt), - FieldsAre("foo.a.bar", "other message", absl::nullopt), - FieldsAre("zzz", "newfailure", absl::nullopt))); - EXPECT_EQ(manager.unexpected_failures(), 3); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, UnexpectedFailuresIncludeExceededWildcardMatches) { - CreateFailureList({{"foo.*.bar"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - for (int i = 0; i < 100; ++i) { - ASSERT_THAT(manager.ReportSuccess(absl::StrCat("foo.", i, ".bar")), - Not(IsOk())); - } - ASSERT_THAT(manager.ReportFailure("foo.baz.bar", kP0, "msg"), Not(IsOk())); - - EXPECT_THAT(manager.UnexpectedFailures(), - ElementsAre(FieldsAre("foo.baz.bar", "msg", absl::nullopt))); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_THAT(manager.UnexpectedSuccesses(), SizeIs(100)); - EXPECT_EQ(manager.unexpected_successes(), 100); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, UnexpectedFailuresIgnoreDuplicateReports) { - TestManager manager; - - // Only the first report of a test counts. - ASSERT_THAT(manager.ReportFailure("foo", kP0, "first"), Not(IsOk())); - ASSERT_THAT(manager.ReportFailure("foo", kP0, "second"), Not(IsOk())); - - EXPECT_THAT(manager.UnexpectedFailures(), - ElementsAre(FieldsAre("foo", "first", absl::nullopt))); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, UnexpectedSuccesses) { - CreateFailureList({{"foo.*.bar", "wildcard message"}, {"zzz", " zzz msg "}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.UnexpectedSuccesses(), IsEmpty()); - - ASSERT_THAT(manager.ReportSuccess("zzz"), Not(IsOk())); - ASSERT_THAT(manager.ReportSuccess("foo.b.bar"), Not(IsOk())); - ASSERT_THAT(manager.ReportSuccess("foo.a.bar"), Not(IsOk())); - ASSERT_THAT(manager.ReportSuccess("foo.a.bar"), Not(IsOk())); - // Expected successes are not reported. - ASSERT_THAT(manager.ReportSuccess("aaa"), IsOk()); - - // Sorted by test name; the message is the (normalized) one of the entry the - // test matched. - EXPECT_THAT( - manager.UnexpectedSuccesses(), - ElementsAre(FieldsAre("foo.a.bar", "wildcard message", - Optional(std::string("foo.*.bar"))), - FieldsAre("foo.b.bar", "wildcard message", - Optional(std::string("foo.*.bar"))), - FieldsAre("zzz", "zzz msg", Optional(std::string("zzz"))))); - EXPECT_EQ(manager.unexpected_successes(), 3); - EXPECT_THAT(manager.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, ReportUnexpectedFailureTooManyWildcardMatches) { - CreateFailureList({{"foo.*.bar"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - for (int i = 0; i < 100; ++i) { - EXPECT_THAT(manager.ReportSuccess(absl::StrCat("foo.", i, ".bar")), - StatusIs(absl::StatusCode::kFailedPrecondition)); - } - ASSERT_THAT(manager.ReportFailure("foo.baz.bar", kP0, ""), - StatusIs(absl::StatusCode::kFailedPrecondition, - AllOf(HasSubstr("too many test names"), - HasSubstr("foo.baz.bar")))); - - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 0); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 100); - EXPECT_EQ(manager.skipped(), 0); -} - -TEST_F(TestManagerTest, LoadFailureListInvalidFile) { - TestManager manager; - EXPECT_THAT(manager.LoadFailureList(absl::StrCat(failure_list(), "/invalid")), - StatusIs(absl::StatusCode::kInternal, HasSubstr("/invalid"))); - manager.Finalize().IgnoreError(); -} - -TEST_F(TestManagerTest, LoadFailureListDuplicateFailure) { - CreateFailureList(R"( - foo # abc - foo # zyx -)"); - TestManager manager; - - EXPECT_THAT(manager.LoadFailureList(failure_list()), - StatusIs(absl::StatusCode::kAlreadyExists, - AllOf(HasSubstr("already exists"), HasSubstr("foo")))); - manager.Finalize().IgnoreError(); -} - -TEST_F(TestManagerTest, LoadFailureListOverlappingFailure) { - CreateFailureList(R"( - foo.*.bar # abc - foo.bar.* # zyx -)"); - TestManager manager; - - EXPECT_THAT(manager.LoadFailureList(failure_list()), - StatusIs(absl::StatusCode::kAlreadyExists, - AllOf(HasSubstr("foo.bar.*"), HasSubstr("foo.*.bar")))); - manager.Finalize().IgnoreError(); -} - -TEST_F(TestManagerTest, UnexpectedSuccessMessageNamesTheEntry) { - CreateFailureList({{"foo", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - // The message is the legacy runner's, including the matched entry. - EXPECT_THAT(manager.ReportSuccess("foo"), - StatusIs(absl::StatusCode::kFailedPrecondition, - "test foo (matched to foo) is in the failure list, but " - "test succeeded. Remove its match from the failure " - "list.")); - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.unexpected_successes(), 1); -} - -TEST_F(TestManagerTest, UnexpectedSuccessMessageNamesTheWildcardEntry) { - CreateFailureList({{"foo.*.bar", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportSuccess("foo.a.bar"), - StatusIs(absl::StatusCode::kFailedPrecondition, - HasSubstr("test foo.a.bar (matched to foo.*.bar) is in " - "the failure list"))); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, EveryPriorityIsEnforcedByDefault) { - TestManager manager; - EXPECT_THAT(manager.ReportFailure("p0", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("p1", kP1, "abc"), Not(IsOk())); - EXPECT_EQ(manager.unexpected_failures(), 2); - EXPECT_EQ(manager.tolerated_failures(), 0); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, EnforcementLevelZeroToleratesP1) { - // Only kP0 is enforced: an unlisted kP1 failure is tolerated. - TestManager manager; - manager.set_enforcement_level(kP0); - EXPECT_THAT(manager.ReportFailure("p0", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("p1", kP1, "abc"), IsOk()); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.tolerated_failures(), 1); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, ToleratedFailure) { - CreateFailureList({{"listed", "abc"}}); - TestManager manager; - manager.set_enforcement_level(kP0); - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - ASSERT_THAT(manager.ReportFailure("foo", kP1, "abc"), IsOk()); - ASSERT_THAT(manager.ReportFailure("bar", kP1, "xyz"), IsOk()); - // Duplicates only count once. - ASSERT_THAT(manager.ReportFailure("foo", kP1, "abc"), IsOk()); - ASSERT_THAT(manager.ReportFailure("listed", kP0, "abc"), IsOk()); - - EXPECT_EQ(manager.tolerated_failures(), 2); - // A tolerated failure is neither a skip nor a failure of any kind, and it - // isn't written to the failure list. - EXPECT_EQ(manager.skipped(), 0); - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 0); - EXPECT_EQ(manager.expected_successes(), 0); - EXPECT_EQ(manager.unexpected_successes(), 0); - EXPECT_THAT(manager.UnexpectedFailures(), IsEmpty()); - EXPECT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, "listed # abc\n"); -} - -TEST_F(TestManagerTest, ToleratedFailureIsCountedOnceAcrossKinds) { - TestManager manager; - manager.set_enforcement_level(kP0); - // The same test name can only be counted under one outcome. - ASSERT_THAT(manager.ReportSuccess("foo"), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo", kP1, "abc"), IsOk()); - - EXPECT_EQ(manager.expected_successes(), 1); - EXPECT_EQ(manager.tolerated_failures(), 0); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, ListedFailureIsCheckedWhateverItsPriority) { - // A listed test is never tolerated, so that the failure list can't go stale - // unnoticed. - CreateFailureList({{"foo", "abc"}, {"bar", "abc"}}); - TestManager manager; - manager.set_enforcement_level(kP0); - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP1, "abc"), IsOk()); - EXPECT_THAT(manager.ReportFailure("bar", kP1, "xyz"), Not(IsOk())); - - EXPECT_EQ(manager.expected_failures(), 1); - EXPECT_EQ(manager.unexpected_failures(), 1); - EXPECT_EQ(manager.tolerated_failures(), 0); - manager.Finalize().IgnoreError(); -} - -TEST_F(TestManagerTest, LoadFailureListInvalidWildcard) { - CreateFailureList({{"foo.b*r", "abc"}}); - TestManager manager; - EXPECT_THAT(manager.LoadFailureList(failure_list()), - StatusIs(absl::StatusCode::kInvalidArgument)); - manager.Finalize().IgnoreError(); -} - -TEST_F(TestManagerTest, LoadFailureListTruncatesMessage) { - // Like a reported message, so that the two still compare equal. - std::string message(1000, 'b'); - CreateFailureList({{"foo", message}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP0, message), IsOk()); - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 1); -} - -TEST_F(TestManagerTest, LoadSecondFailureList) { - CreateFailureList({{"foo", "abc"}}); - const std::string second = absl::StrCat(failure_list(), ".2"); - ABSL_CHECK_OK(File::SetContents(second, "bar # zyx\n", true)); - const std::string third = absl::StrCat(failure_list(), ".3"); - ABSL_CHECK_OK(File::SetContents(third, "foo # zyx\n", true)); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.LoadFailureList(second), IsOk()); - - // Entries must not repeat across files either. - EXPECT_THAT(manager.LoadFailureList(third), - StatusIs(absl::StatusCode::kAlreadyExists)); - EXPECT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.ReportFailure("bar", kP0, "zyx"), IsOk()); - EXPECT_THAT(manager.Finalize(), IsOk()); - EXPECT_EQ(manager.expected_failures(), 2); -} - -TEST_F(TestManagerTest, FailureListMatchesWholeNameComponents) { - CreateFailureList({{"foo", "abc"}, {"bar.*.baz", "abc"}, {"qux", "abc"}}); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - - EXPECT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.ReportFailure("qux", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.ReportFailure("bar.a.baz", kP0, "abc"), IsOk()); - EXPECT_THAT(manager.ReportFailure("bar.b.baz", kP0, "abc"), IsOk()); - // Neither a prefix nor an extension of an entry matches it, and a wildcard - // stands for exactly one component. - EXPECT_THAT(manager.ReportFailure("foo.bar", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("bar", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("bar.a", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("bar.a.baz.qux", kP0, "abc"), Not(IsOk())); - EXPECT_THAT(manager.ReportFailure("other", kP0, "abc"), Not(IsOk())); - - EXPECT_EQ(manager.expected_failures(), 4); - EXPECT_EQ(manager.unexpected_failures(), 5); - EXPECT_THAT(manager.Finalize(), IsOk()); -} - -TEST_F(TestManagerTest, SaveFailureListNoop) { - CreateFailureList(R"( -foo # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), IsOk()); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -foo # abc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListNoopWildcard) { - CreateFailureList(R"( -foo.*.bar # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo.a.bar", kP0, "abc"), IsOk()); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -foo.*.bar # abc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListKeepsPrefixMatchedEntryVerbatim) { - CreateFailureList(R"( -foo # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - // The actual message is longer than the entry's, which is only a prefix. - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc: more details"), IsOk()); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -foo # abc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListRemoveNewPassing) { - CreateFailureList(R"( -foo # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportSuccess("foo"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -)"); -} - -TEST_F(TestManagerTest, SaveFailureListRemovePartialPassingWildcard) { - // This is a weird case where the wildcard no longer fully matches the - // failure. Users would have to updated the failure list twice to get the - // failure list to be correct. - CreateFailureList(R"( -foo.*.bar # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportSuccess("foo.a.bar"), Not(IsOk())); - ASSERT_THAT(manager.ReportFailure("foo.b.bar", kP0, "abc"), IsOk()); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -)"); -} - -TEST_F(TestManagerTest, SaveFailureListKeepsSkippedEntry) { - // A skipped test says nothing about whether its entry is still needed, so - // the entry is kept verbatim (whether the skip itself is acceptable is - // decided, and reported, by the caller). - CreateFailureList(R"( -foo # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportSkip("foo", "reason"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -foo # abc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListUnwritablePath) { - TestManager manager; - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - // A directory can't be opened for writing. - const std::string directory = absl::StrCat(failure_list(), ".dir"); - ASSERT_THAT(File::RecursivelyCreateDir(directory, 0777), - IsOk()); - EXPECT_THAT(manager.SaveFailureList(directory), - StatusIs(absl::StatusCode::kInternal, HasSubstr(directory))); - // Neither can a file in a directory that doesn't exist. - const std::string missing_directory = - absl::StrCat(failure_list(), ".missing/list.txt"); - EXPECT_THAT( - manager.SaveFailureList(missing_directory), - StatusIs(absl::StatusCode::kInternal, HasSubstr(missing_directory))); -} - -TEST_F(TestManagerTest, SaveFailureListWriteFailure) { - TestManager manager; - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - // /dev/full opens fine but every write to it fails with ENOSPC, which is - // the only way to reach the write-failure path without a filesystem mock. - // It is a Linux device. -#ifdef __linux__ - EXPECT_THAT(manager.SaveFailureList("/dev/full"), - StatusIs(absl::StatusCode::kInternal, - HasSubstr("Failed to write failure list file"))); -#else - GTEST_SKIP() << "/dev/full is Linux-only"; -#endif -} - -TEST_F(TestManagerTest, SaveFailureListChangeFailureMessage) { - CreateFailureList(R"( -foo # abc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportFailure("foo", kP0, "zyx"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), Not(IsOk())); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -foo # zyx -)"); -} - -TEST_F(TestManagerTest, SaveFailureListAddNewFailure) { - TestManager manager; - ASSERT_THAT(manager.ReportFailure("foo", kP0, "abc"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"(foo # abc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListAddNormalizedMessage) { - TestManager manager; - std::string message = absl::StrCat("aa\n", std::string(1000, 'b')); - std::string normalized_message = absl::StrCat("aa", std::string(126, 'b')); - ASSERT_THAT(manager.ReportFailure("foo", kP0, message), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, absl::StrCat("foo # ", normalized_message, "\n")); -} - -TEST_F(TestManagerTest, SaveFailureListInsertAlphabetically) { - CreateFailureList(R"( -# This is a comment. -aaa # aaa -# This is another comment. -bbb # bbb - -ccc # ccc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportFailure("aaa", kP0, "aaa"), IsOk()); - ASSERT_THAT(manager.ReportFailure("bbb", kP0, "bbb"), IsOk()); - ASSERT_THAT(manager.ReportFailure("ccc", kP0, "ccc"), IsOk()); - ASSERT_THAT(manager.ReportFailure("abc", kP0, "abc"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -# This is a comment. -aaa # aaa -# This is another comment. -abc # abc -bbb # bbb - -ccc # ccc -)"); -} - -TEST_F(TestManagerTest, SaveFailureListInsertAligned) { - CreateFailureList(R"( -# This is a comment. -aa # aaa -# This is another comment. -bbbb #bbb - -cc # ccc -)"); - TestManager manager; - ASSERT_THAT(manager.LoadFailureList(failure_list()), IsOk()); - ASSERT_THAT(manager.ReportFailure("aa", kP0, "aaa"), IsOk()); - ASSERT_THAT(manager.ReportFailure("bbbb", kP0, " bbb "), IsOk()); - ASSERT_THAT(manager.ReportFailure("cc", kP0, "ccc"), IsOk()); - ASSERT_THAT(manager.ReportFailure("abcdef", kP0, "abc"), Not(IsOk())); - ASSERT_THAT(manager.Finalize(), IsOk()); - - EXPECT_THAT(manager.SaveFailureList(absl::StrCat(failure_list(), ".new")), - IsOk()); - - std::string content; - ASSERT_THAT(File::GetContents(absl::StrCat(failure_list(), ".new"), &content, - true), - IsOk()); - EXPECT_EQ(content, R"( -# This is a comment. -aa # aaa -# This is another comment. -abcdef # abc -bbbb # bbb - -cc # ccc -)"); -} - -} // namespace -} // namespace internal -} // namespace conformance -} // namespace protobuf -} // namespace google
diff --git a/conformance/testee.cc b/conformance/testee.cc index 9ec8524..4a6bc6c 100644 --- a/conformance/testee.cc +++ b/conformance/testee.cc
@@ -1,3 +1,10 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2025 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + #include "conformance/testee.h" #include <string> @@ -5,6 +12,7 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "conformance/binary_wireformat.h" @@ -36,16 +44,21 @@ return ::conformance::UNSPECIFIED; } +// Builds the full test name according to `name_style` (see NameStyle): +// "<Level>.<Edition>.<InputFormat>Input.<name>", followed by +// ".<OutputFormat>Output" unless the style omits it. std::string GetTestName(absl::string_view test_name, TestPriority priority, const ::conformance::ConformanceRequest& request, - const Descriptor& message) { - absl::string_view level = PriorityLevelName(priority); - std::string syntax_identifier = GetEditionIdentifier(message); - - return absl::StrCat( - level, ".", syntax_identifier, ".", - GetFormatIdentifier(GetInputFormat(request)), "Input.", test_name, ".", - GetFormatIdentifier(request.requested_output_format()), "Output"); + const Descriptor& message, NameStyle name_style) { + std::string full_name = absl::StrCat( + PriorityLevelName(priority), ".", GetEditionIdentifier(message), ".", + GetFormatIdentifier(GetInputFormat(request)), "Input.", test_name); + if (name_style == NameStyle::kWithOutputFormat) { + absl::StrAppend(&full_name, ".", + GetFormatIdentifier(request.requested_output_format()), + "Output"); + } + return full_name; } } // namespace @@ -54,6 +67,7 @@ absl::string_view test_name, const ConformanceRequest& request) { ABSL_CHECK(test_names_ran_.emplace(test_name).second) << "Duplicated test name: " << test_name; + tests_run_.emplace_back(test_name); std::string serialized_request; // TODO: Remove this suppression. @@ -101,7 +115,7 @@ } TestResult InMemoryMessage::SerializeBinary() && { - return SerializeImpl(::conformance::PROTOBUF); + return Finish(::conformance::PROTOBUF, NameStyle::kWithOutputFormat); } TestResult InMemoryMessage::SerializeText(TextSerializationOptions options) && { @@ -109,23 +123,45 @@ request_.set_print_unknown_fields(true); } - return SerializeImpl(::conformance::TEXT_FORMAT); + return Finish(::conformance::TEXT_FORMAT, NameStyle::kWithOutputFormat); } TestResult InMemoryMessage::SerializeJson() && { - return SerializeImpl(::conformance::JSON); + return Finish(::conformance::JSON, NameStyle::kWithOutputFormat); } -TestResult InMemoryMessage::SerializeImpl( - const ::conformance::WireFormat format) { - request_.set_requested_output_format(format); +TestResult InMemoryMessage::ParseOnly(ParseOnlyOptions options) && { + // We don't expect output, but if the testee erroneously accepts the input we + // let it send its response in the input's format (unless the test says + // otherwise). We must not leave it unspecified. This is exactly what the + // legacy runner did. + return Finish(options.output_format.value_or(GetInputFormat(request_)), + NameStyle::kWithoutOutputFormat); +} - std::string full_name = GetTestName(name_, priority_, request_, *type_); +InMemoryMessage&& InMemoryMessage::OverrideTestCategory( + ::conformance::TestCategory category) && { + ABSL_DCHECK_EQ(category, ::conformance::TEXT_FORMAT_TEST) + << "OverrideTestCategory() only relabels the text-format suite's " + "binary-input tests as TEXT_FORMAT_TEST"; + ABSL_DCHECK(request_.has_protobuf_payload()) + << "OverrideTestCategory() is for PROTOBUF-input tests"; + request_.set_test_category(category); + return std::move(*this); +} + +TestResult InMemoryMessage::Finish( + const ::conformance::WireFormat output_format, const NameStyle name_style) { + request_.set_requested_output_format(output_format); + + std::string full_name = + GetTestName(name_, priority_, request_, *type_, name_style); ::conformance::ConformanceResponse response = testee_->Run(full_name, request_); - return TestResult(full_name, priority_, type_, format, std::move(response)); + return TestResult(full_name, priority_, type_, output_format, + std::move(response)); } } // namespace internal @@ -140,6 +176,23 @@ return "Unknown"; } +bool AbslParseFlag(absl::string_view text, TestPriority* priority, + std::string* error) { + int number; + if (!absl::SimpleAtoi(text, &number) || number < 0 || + number > static_cast<int>(kLowestPriority)) { + *error = absl::StrCat("expected a priority number from 0 to ", + static_cast<int>(kLowestPriority)); + return false; + } + *priority = static_cast<TestPriority>(number); + return true; +} + +std::string AbslUnparseFlag(TestPriority priority) { + return absl::StrCat(static_cast<int>(priority)); +} + absl::string_view PriorityLevelName(TestPriority priority) { // TODO: b/564550230 - return PriorityName() once the tests are renamed. return priority == TestPriority::kP1 ? "Recommended" : "Required";
diff --git a/conformance/testee.h b/conformance/testee.h index 2757d36..3a3e924 100644 --- a/conformance/testee.h +++ b/conformance/testee.h
@@ -1,11 +1,21 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2025 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + #ifndef GOOGLE_PROTOBUF_CONFORMANCE_TESTEE_H__ #define GOOGLE_PROTOBUF_CONFORMANCE_TESTEE_H__ #include <string> #include <utility> +#include <vector> #include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" #include "conformance/binary_wireformat.h" #include "conformance/conformance.pb.h" #include "conformance/test_runner.h" @@ -25,7 +35,7 @@ // .SerializeBinary(), // Yields(WhenParsed(EqualsBinaryProto(input)))); -// TODO Possible future APIs to expand conformance coverage: +// TODO: Possible future APIs to expand conformance coverage: // - Add ClearUnknownFields() to InMemoryMessage // - Add MergeFrom() method to InMemoryMessage to merge raw binary // - Remove && qualifiers on Parse* and add InMemoryMessage::Merge that merges @@ -56,22 +66,34 @@ inline constexpr TestPriority kP0 = TestPriority::kP0; inline constexpr TestPriority kP1 = TestPriority::kP1; -// The lowest priority there is. An enforcement level (see TestManager in -// test_manager.h) is the lowest priority whose unlisted failures fail the run; +// The lowest priority there is. An enforcement level (see ResultLedger in +// result_ledger.h) is the lowest priority whose unlisted failures fail the run; // kLowestPriority, the default, enforces every priority. inline constexpr TestPriority kLowestPriority = kP1; // The name of a priority: "P0" or "P1". absl::string_view PriorityName(TestPriority priority); +// Flag support for TestPriority, as the priority's number: "0" is kP0, "1" is +// kP1. +bool AbslParseFlag(absl::string_view text, TestPriority* priority, + std::string* error); +std::string AbslUnparseFlag(TestPriority priority); + // The level a priority is named with in test names, until the rename (see // TestPriority): "Required" for kP0, "Recommended" for kP1. absl::string_view PriorityLevelName(TestPriority priority); namespace internal { -// The final result of a conformance test, to be processed by a matcher. -class TestResult { +// The final result of a conformance test: the testee's response and what the +// test asked of it. Hand it to exactly one EXPECT_THAT(..., Yields(...)), +// which records the outcome against the failure list (see matchers.h). +// Dropping a result does not compile, and neither does a leaf matcher applied +// to one without Yields() (see YieldedResult in matchers.h). The test +// environment fails a gtest test that runs a conformance test but never +// checks its result in any other way (see Testee::tests_run()). +class [[nodiscard]] TestResult { public: // The name of the test that was run, useful for failure matching and // reporting. @@ -117,6 +139,21 @@ bool print_unknown_fields = false; }; +// Options for InMemoryMessage::ParseOnly(). +struct ParseOnlyOptions { + // The output format to request from the testee. Defaults to the input's + // format. + absl::optional<::conformance::WireFormat> output_format; +}; + +// How a test's full name is derived from the name it was created with: +// kWithOutputFormat <S>.<E>.<Input>Input.<name>.<Output>Output +// kWithoutOutputFormat <S>.<E>.<Input>Input.<name> (ParseOnly()) +enum class NameStyle { + kWithOutputFormat, + kWithoutOutputFormat, +}; + // This class represents a message held in memory by the testee that can be // manipulated in various ways. class InMemoryMessage { @@ -129,6 +166,51 @@ TestResult SerializeText(TextSerializationOptions options = {}) &&; TestResult SerializeJson() &&; + // Finishes a test whose outcome depends only on parsing, for example one + // that expects a parse error: + // + // EXPECT_THAT(Testee().ParseBinary(type, input).ParseOnly(), + // Yields(IsParseError())); + // + // There is no parse-only request, so the testee is still asked to + // serialize, by default in the input's format. The test name carries no + // output-format suffix, for example "Required.Proto3.ProtobufInput.<name>". + // + // `options.output_format` requests a different output format without + // changing the name. It exists for the JSON tests that need PROTOBUF + // output from JSON input while still being named + // "<S>.<E>.JsonInput.<name>", for example in + // RunValidJsonTestOrParseFailure. See ParseOnlyOptions. + // + // Like the Serialize*() methods, this consumes the message and must remain + // the terminal call of a test. Once further operations on an + // InMemoryMessage exist, such as Merge or reflection accessors (see the + // TODO at the top of this file), they have to come before it, not after. + TestResult ParseOnly(ParseOnlyOptions options = {}) &&; + + // Relabels a PROTOBUF-input request as TEXT_FORMAT_TEST. That is the only + // category accepted. Anything else is a DCHECK failure. + // + // Testee().ParseBinary(type, input) + // .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + // .SerializeText() + // + // ParseBinary() categorizes its request as BINARY_TEST, but the text-format + // suite's binary-input tests (text_unknown_field_test.cc) have to stay + // TEXT_FORMAT_TEST. No testee tells BINARY_TEST and JSON_TEST apart on a + // binary payload. The OSS Ruby testee, however, skips every + // TEXT_FORMAT_TEST request and cannot produce text-format output. + // Relabeling those tests would turn its skips into failures that no failure + // list covers. The JSON suite's binary-input tests are plain BINARY_TEST + // requests. + // + // Returns *this as an rvalue. Must be chained in the same expression. + // A reference bound to the result would outlive the temporary it refers to. + // TODO: drop once the Ruby testee skips text-format tests by + // payload and requested output format rather than by category. + InMemoryMessage&& OverrideTestCategory( + ::conformance::TestCategory category) &&; + private: InMemoryMessage(class Testee* testee, absl::string_view name, TestPriority priority, const Descriptor* type, @@ -140,7 +222,11 @@ request_(std::move(request)) {} friend class Test; - TestResult SerializeImpl(::conformance::WireFormat format); + // Finishes the test: requests `output_format` from the testee and runs it + // under the name built according to `name_style`. All public terminal + // methods funnel into this. + TestResult Finish(::conformance::WireFormat output_format, + NameStyle name_style); class Testee* testee_; std::string name_; @@ -187,6 +273,13 @@ return Test(this, name, priority); } + // The full names of the tests run so far, in the order they ran. Each of + // them must be checked with Yields() (see matchers.h). + // ConformanceEnvironment (test_environment.h) compares this with the tests + // the ResultLedger has heard about and fails a gtest test that ran a + // conformance test but never checked its result. + absl::Span<const std::string> tests_run() const { return tests_run_; } + private: ::conformance::ConformanceResponse Run( absl::string_view test_name, @@ -195,6 +288,8 @@ ConformanceTestRunner* runner_; + // The tests run so far, in order, and as a set for Run()'s duplicate check. + std::vector<std::string> tests_run_; absl::flat_hash_set<std::string> test_names_ran_; };
diff --git a/conformance/testee_runner.h b/conformance/testee_runner.h new file mode 100644 index 0000000..1717025 --- /dev/null +++ b/conformance/testee_runner.h
@@ -0,0 +1,52 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The link-time seam that decides which testee a conformance test binary +// (one with test_environment_main.cc's main) runs against. +// +// Exactly one library linked into the binary defines MakeTesteeRunner(): +// +// * :forked_testee, which conformance_test() (conformance.bzl) links by +// default, spawns --testee_binary with --testee_args and talks to it over +// pipes (ForkPipeRunner). +// * An in-process testee library answers the requests in this process. That +// makes the testee debuggable from the test binary and needs no fork or +// pipe. conformance_test(in_process = True) links it in place of +// :forked_testee. + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_TESTEE_RUNNER_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_TESTEE_RUNNER_H__ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "conformance/test_runner.h" + +namespace google { +namespace protobuf { +namespace conformance { + +// Returns the runner the conformance suites of this binary talk to, given the +// values of --testee_binary and --testee_args. The latter includes any +// positional arguments. Never returns null. Check-fails, with a message +// naming the flag, on a command line the linked testee can't make sense of. +// A forked testee needs `testee_binary`. An in-process testee rejects a +// non-empty `testee_binary` or `testee_args` rather than silently running the +// linked testee instead of the requested one. +// +// Called once, before the tests run (see test_environment_main.cc), and the +// result is destroyed by ConformanceEnvironment::TearDown(). +std::unique_ptr<ConformanceTestRunner> MakeTesteeRunner( + absl::string_view testee_binary, absl::Span<const std::string> testee_args); + +} // namespace conformance +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_TESTEE_RUNNER_H__
diff --git a/conformance/testee_test.cc b/conformance/testee_test.cc index 124bf35..a1dd4b9 100644 --- a/conformance/testee_test.cc +++ b/conformance/testee_test.cc
@@ -22,7 +22,13 @@ namespace { using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::testing::_; +using ::testing::AnyNumber; +using ::testing::DoAll; +using ::testing::ElementsAre; +using ::testing::IsEmpty; using ::testing::Return; +using ::testing::SaveArg; MATCHER_P(RequestEquals, expected_textproto, "") { ::conformance::ConformanceRequest request, expected; @@ -65,7 +71,7 @@ EXPECT_EQ(result.name(), "Required.Proto2.ProtobufInput.foo.ProtobufOutput"); EXPECT_EQ(result.priority(), TestPriority::kP0); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::PROTOBUF); + EXPECT_EQ(result.format(), ::conformance::PROTOBUF); EXPECT_THAT(result.response(), EqualsProto(R"pb(runtime_error: "error")pb")); } @@ -91,7 +97,7 @@ "Recommended.Proto2.TextFormatInput.foo.TextFormatOutput"); EXPECT_EQ(result.priority(), TestPriority::kP1); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::TEXT_FORMAT); + EXPECT_EQ(result.format(), ::conformance::TEXT_FORMAT); EXPECT_THAT(result.response(), EqualsProto(R"pb(runtime_error: "error")pb")); } @@ -119,7 +125,7 @@ "Required.Proto2.ProtobufInput.foo.TextFormatOutput"); EXPECT_EQ(result.priority(), TestPriority::kP0); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::TEXT_FORMAT); + EXPECT_EQ(result.format(), ::conformance::TEXT_FORMAT); EXPECT_THAT(result.response(), EqualsProto(R"pb(runtime_error: "error")pb")); } @@ -143,7 +149,7 @@ EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo.JsonOutput"); EXPECT_EQ(result.priority(), TestPriority::kP0); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::JSON); + EXPECT_EQ(result.format(), ::conformance::JSON); EXPECT_THAT(result.response(), EqualsProto(R"pb(runtime_error: "error")pb")); } @@ -169,7 +175,7 @@ EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo.ProtobufOutput"); EXPECT_EQ(result.priority(), TestPriority::kP0); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::PROTOBUF); + EXPECT_EQ(result.format(), ::conformance::PROTOBUF); EXPECT_THAT(result.response(), EqualsProto(R"pb(runtime_error: "error")pb")); } @@ -195,7 +201,7 @@ EXPECT_EQ(result.name(), "Required.Proto2.ProtobufInput.foo.ProtobufOutput"); EXPECT_EQ(result.priority(), TestPriority::kP0); EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); - EXPECT_THAT(result.format(), ::conformance::PROTOBUF); + EXPECT_EQ(result.format(), ::conformance::PROTOBUF); EXPECT_THAT( result.response(), EqualsProto( @@ -222,12 +228,339 @@ .SerializeBinary(); EXPECT_DEATH( - testee.CreateTest("foo", TestPriority::kP0) + (void)testee.CreateTest("foo", TestPriority::kP0) .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) .SerializeBinary(), "Duplicated test name: Required.Proto2.ProtobufInput.foo.ProtobufOutput"); } +TEST(TesteeTest, RecordsTheTestsRun) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL(mock, RunTest).WillRepeatedly(Return(std::string("\004"))); + EXPECT_THAT(testee.tests_run(), IsEmpty()); + + (void)testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .SerializeBinary(); + (void)testee.CreateTest("bar", TestPriority::kP1) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .SerializeText(); + + EXPECT_THAT( + testee.tests_run(), + ElementsAre("Required.Proto2.ProtobufInput.foo.ProtobufOutput", + "Recommended.Proto2.ProtobufInput.bar.TextFormatOutput")); +} + +TEST(TesteeTest, ParseOnlyBinary) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Required.Proto2.ProtobufInput.foo", RequestEquals(R"pb( + protobuf_payload: "wire" + requested_output_format: PROTOBUF + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: BINARY_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .ParseOnly(); + + EXPECT_EQ(result.name(), "Required.Proto2.ProtobufInput.foo"); + EXPECT_EQ(result.priority(), TestPriority::kP0); + EXPECT_EQ(result.type(), TestAllTypesProto2::descriptor()); + EXPECT_EQ(result.format(), ::conformance::PROTOBUF); + EXPECT_THAT(result.response(), EqualsProto(R"pb(parse_error: "error")pb")); +} + +// The request of a ParseOnly() test must be byte-identical to the one +// SerializeBinary() sends for the same input; only the test name differs. +TEST(TesteeTest, ParseOnlyBinarySendsSameRequestAsSerializeBinary) { + MockTestRunner mock; + Testee testee(&mock); + std::string serialized_request, parse_only_request; + EXPECT_CALL(mock, + RunTest("Required.Proto2.ProtobufInput.foo.ProtobufOutput", _)) + .WillOnce(DoAll(SaveArg<1>(&serialized_request), + RespondWith(R"pb(parse_error: "error")pb"))); + EXPECT_CALL(mock, RunTest("Required.Proto2.ProtobufInput.foo", _)) + .WillOnce(DoAll(SaveArg<1>(&parse_only_request), + RespondWith(R"pb(parse_error: "error")pb"))); + + TestResult serialized = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .SerializeBinary(); + TestResult parse_only = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .ParseOnly(); + + EXPECT_EQ(serialized.name(), + "Required.Proto2.ProtobufInput.foo.ProtobufOutput"); + EXPECT_EQ(parse_only.name(), "Required.Proto2.ProtobufInput.foo"); + EXPECT_EQ(parse_only_request, serialized_request); +} + +TEST(TesteeTest, ParseOnlyText) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Recommended.Proto2.TextFormatInput.foo", RequestEquals(R"pb( + text_payload: "text" + requested_output_format: TEXT_FORMAT + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: TEXT_FORMAT_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = testee.CreateTest("foo", TestPriority::kP1) + .ParseText(TestAllTypesProto2::descriptor(), "text") + .ParseOnly(); + + EXPECT_EQ(result.name(), "Recommended.Proto2.TextFormatInput.foo"); + EXPECT_EQ(result.priority(), TestPriority::kP1); + EXPECT_EQ(result.format(), ::conformance::TEXT_FORMAT); + EXPECT_THAT(result.response(), EqualsProto(R"pb(parse_error: "error")pb")); +} + +TEST(TesteeTest, ParseOnlyJson) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Required.Proto2.JsonInput.foo", RequestEquals(R"pb( + json_payload: "json" + requested_output_format: JSON + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: JSON_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .ParseOnly(); + + EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo"); + EXPECT_EQ(result.format(), ::conformance::JSON); + EXPECT_THAT(result.response(), EqualsProto(R"pb(parse_error: "error")pb")); +} + +TEST(TesteeDeathTest, ParseOnlyDuplicateTestName) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL(mock, RunTest("Required.Proto2.ProtobufInput.foo", _)) + .WillRepeatedly(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .ParseOnly(); + + EXPECT_DEATH((void)testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .ParseOnly(), + "Duplicated test name: Required.Proto2.ProtobufInput.foo"); +} + +TEST(TesteeTest, OverrideTestCategory) { + MockTestRunner mock; + Testee testee(&mock); + // The category changes, the name (derived from the input format) doesn't. + EXPECT_CALL( + mock, + RunTest("Required.Proto2.ProtobufInput.foo.TextFormatOutput", + RequestEquals(R"pb( + protobuf_payload: "wire" + requested_output_format: TEXT_FORMAT + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: TEXT_FORMAT_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(text_payload: "")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText(); + + EXPECT_EQ(result.name(), + "Required.Proto2.ProtobufInput.foo.TextFormatOutput"); +} + +TEST(TesteeTest, OverrideTestCategoryParseOnly) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Required.Proto2.ProtobufInput.foo", RequestEquals(R"pb( + protobuf_payload: "wire" + requested_output_format: PROTOBUF + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: TEXT_FORMAT_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .ParseOnly(); + + EXPECT_EQ(result.name(), "Required.Proto2.ProtobufInput.foo"); +} + +TEST(TesteeDeathTest, OverrideTestCategoryAcceptsOnlyTextFormatTest) { + MockTestRunner mock; + Testee testee(&mock); + // Without the DCHECK (opt builds) the request goes through as relabeled. + EXPECT_CALL(mock, RunTest) + .Times(AnyNumber()) + .WillRepeatedly(RespondWith(R"pb(parse_error: "error")pb")); + EXPECT_DEBUG_DEATH( + (void)testee.CreateTest("foo", TestPriority::kP0) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .OverrideTestCategory(::conformance::JSON_TEST) + .ParseOnly(), + "only relabels the text-format suite's binary-input tests"); +} + +TEST(TesteeDeathTest, OverrideTestCategoryNeedsBinaryInput) { + MockTestRunner mock; + Testee testee(&mock); + // Without the DCHECK (opt builds) the request goes through as relabeled. + EXPECT_CALL(mock, RunTest) + .Times(AnyNumber()) + .WillRepeatedly(RespondWith(R"pb(parse_error: "error")pb")); + EXPECT_DEBUG_DEATH((void)testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "{}") + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .ParseOnly(), + "is for PROTOBUF-input tests"); +} + +// --------------------------------------------------------------------------- +// JSON test names and ParseOnly() output override +// --------------------------------------------------------------------------- + +TEST(TesteeTest, SerializeJsonNamesTheTestWithInputAndOutputFormat) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL(mock, RunTest("Required.Proto2.JsonInput.foo.JsonOutput", _)) + .WillOnce(RespondWith(R"pb(json_payload: "{}")pb")); + + TestResult result = testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .SerializeJson(); + + EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo.JsonOutput"); + EXPECT_EQ(result.format(), ::conformance::JSON); +} + +TEST(TesteeTest, SerializeJsonFromBinaryInputNamesTheInputFormat) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Recommended.Proto2.ProtobufInput.foo.JsonOutput", + RequestEquals(R"pb( + protobuf_payload: "wire" + requested_output_format: JSON + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: BINARY_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(serialize_error: "error")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP1) + .ParseBinary(TestAllTypesProto2::descriptor(), Wire("wire")) + .SerializeJson(); + + EXPECT_EQ(result.name(), "Recommended.Proto2.ProtobufInput.foo.JsonOutput"); + EXPECT_EQ(result.format(), ::conformance::JSON); + EXPECT_THAT(result.response(), + EqualsProto(R"pb(serialize_error: "error")pb")); +} + +TEST(TesteeTest, ParseOnlyJsonWithProtobufOutput) { + // The legacy RunValidJsonTestOrParseFailure asked for PROTOBUF output but + // named the test like a parse-failure test. + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Required.Proto2.JsonInput.foo", RequestEquals(R"pb( + json_payload: "json" + requested_output_format: PROTOBUF + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: JSON_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = + testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}); + + EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo"); + EXPECT_EQ(result.format(), ::conformance::PROTOBUF); + EXPECT_THAT(result.response(), EqualsProto(R"pb(parse_error: "error")pb")); +} + +// With the output format overridden, the request is byte-identical to the one +// SerializeBinary() sends for the same input; only the name differs. +TEST(TesteeTest, ParseOnlyWithProtobufOutputSendsSameRequestAsSerializeBinary) { + MockTestRunner mock; + Testee testee(&mock); + std::string serialized_request, parse_only_request; + EXPECT_CALL(mock, RunTest("Required.Proto2.JsonInput.foo.ProtobufOutput", _)) + .WillOnce(DoAll(SaveArg<1>(&serialized_request), + RespondWith(R"pb(parse_error: "error")pb"))); + EXPECT_CALL(mock, RunTest("Required.Proto2.JsonInput.foo", _)) + .WillOnce(DoAll(SaveArg<1>(&parse_only_request), + RespondWith(R"pb(parse_error: "error")pb"))); + + TestResult serialized = + testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .SerializeBinary(); + TestResult parse_only = + testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .ParseOnly({/*output_format=*/::conformance::PROTOBUF}); + + EXPECT_EQ(serialized.name(), "Required.Proto2.JsonInput.foo.ProtobufOutput"); + EXPECT_EQ(parse_only.name(), "Required.Proto2.JsonInput.foo"); + EXPECT_EQ(parse_only_request, serialized_request); +} + +TEST(TesteeTest, ParseOnlyWithExplicitInputFormatIsTheDefault) { + MockTestRunner mock; + Testee testee(&mock); + EXPECT_CALL( + mock, + RunTest("Required.Proto2.JsonInput.foo", RequestEquals(R"pb( + json_payload: "json" + requested_output_format: JSON + message_type: "protobuf_test_messages.proto2.TestAllTypesProto2" + test_category: JSON_TEST + )pb"))) + .WillOnce(RespondWith(R"pb(parse_error: "error")pb")); + + TestResult result = testee.CreateTest("foo", TestPriority::kP0) + .ParseJson(TestAllTypesProto2::descriptor(), "json") + .ParseOnly({/*output_format=*/::conformance::JSON}); + + EXPECT_EQ(result.name(), "Required.Proto2.JsonInput.foo"); + EXPECT_EQ(result.format(), ::conformance::JSON); +} + TEST(TestPriorityTest, PriorityName) { EXPECT_EQ(PriorityName(kP0), "P0"); EXPECT_EQ(PriorityName(kP1), "P1");
diff --git a/conformance/text_any_test.cc b/conformance/text_any_test.cc new file mode 100644 index 0000000..fd96dd5 --- /dev/null +++ b/conformance/text_any_test.cc
@@ -0,0 +1,199 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for google.protobuf.Any fields: the +// `[type_url] { ... }` expansion, raw type_url/value pairs, and the type URL +// prefixes and separators a parser must accept or reject. This replaces the +// legacy TextFormatConformanceTestSuiteImpl<M>::RunAnyTests(), which ran for +// the proto3-style message types only; the test names and the requests sent to +// the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// The inputs below are the legacy raw string literals verbatim: their +// surrounding whitespace and line breaks are part of the request bytes. The +// packed message is always protobuf_test_messages.proto3.TestAllTypesProto3, +// whichever message type the test runs for, as in the legacy suite. + +using TextAnyTest = MessageTypeConformanceTest; + +TEST_P(TextAnyTest, AnyField) { + constexpr absl::string_view kInput = R"( + optional_any: { + [type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] + { optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyField").ParseText(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyField").ParseText(message(), kInput).SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithRawBytes) { + constexpr absl::string_view kInput = R"( + optional_any: { + type_url: + "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3" value: + "\b\271`" + } + )"; + EXPECT_THAT(Testee("AnyFieldWithRawBytes") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithRawBytes") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithInvalidType) { + constexpr absl::string_view kInput = R"( + optional_any: { + [type.googleapis.com/unknown] { + optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyFieldWithInvalidType") + .ParseText(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextAnyTest, WithCustomTypeUrlPrefix) { + constexpr absl::string_view kInput = R"( + optional_any: { + [non.default.domain/protobuf_test_messages.proto3.TestAllTypesProto3] + { + optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyFieldWithCustomTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithCustomTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithSpecialCharactersInTypeUrlPrefix) { + constexpr absl::string_view kInput = R"( + optional_any: { + [non.default.domain/sub-path_0/~!$&()*+,;=/protobuf_test_messages.proto3.TestAllTypesProto3] + { + optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyFieldWithSpecialCharactersInTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithSpecialCharactersInTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithValidUrlPercentEscapeInTypeUrlPrefix) { + constexpr absl::string_view kInput = R"( + optional_any: { + [non.default.domain/%2F/protobuf_test_messages.proto3.TestAllTypesProto3] + { + optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyFieldWithValidUrlPercentEscapeInTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithValidUrlPercentEscapeInTypeUrlPrefix") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithInvalidUrlPercentEscapeInTypeUrlPrefix) { + constexpr absl::string_view kInput = R"( + optional_any: { + [non.default.domain/%ZZ/protobuf_test_messages.proto3.TestAllTypesProto3] + { + optional_int32: 12345 + } + } + )"; + EXPECT_THAT(Testee("AnyFieldWithInvalidUrlPercentEscapeInTypeUrlPrefix") + .ParseText(message(), kInput) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextAnyTest, WithWhitespaceInTypeUrl) { + constexpr absl::string_view kInput = + "optional_any: {\n" + " [ ty pe.go\nogleap\tis.com/\n" + " proto buf_te\nst_messages.proto3.Test\tAllTypesProto3 ]\n" + " {\n" + " optional_int32: 12345\n" + " }\n" + "}"; + EXPECT_THAT(Testee("AnyFieldWithWhitespaceInTypeUrl") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithWhitespaceInTypeUrl") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextAnyTest, WithCommentsInTypeUrl) { + constexpr absl::string_view kInput = + "optional_any: {\n" + " [type.google # comment \napis.com/" + "protobuf_test_messages.proto3.Test # comment \nAllTypesProto3]" + " {\n" + " optional_int32: 12345\n" + " }\n" + "}"; + EXPECT_THAT(Testee("AnyFieldWithCommentsInTypeUrl") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AnyFieldWithCommentsInTypeUrl") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextAnyTest, ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_enum_test.cc b/conformance/text_enum_test.cc new file mode 100644 index 0000000..c0cf932 --- /dev/null +++ b/conformance/text_enum_test.cc
@@ -0,0 +1,94 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for enum fields given by number. An open +// enum (proto3-style message types) keeps any number, a closed enum +// (proto2-style message types) rejects numbers that aren't one of its values. +// This replaces the legacy TextFormatConformanceTestSuiteImpl<M>:: +// RunOpenEnumTests() and RunClosedEnumTests(); the test names and the +// requests sent to the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. +constexpr absl::string_view kKnownNumberInput = + "\n optional_nested_enum: 1\n "; +constexpr absl::string_view kUnknownNumberInput = + "\n optional_nested_enum: 42\n "; + +using TextOpenEnumTest = MessageTypeConformanceTest; + +TEST_P(TextOpenEnumTest, ByNumber) { + EXPECT_THAT(Testee("ClosedEnumFieldByNumber") + .ParseText(message(), kKnownNumberInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kKnownNumberInput)))); + EXPECT_THAT(Testee("ClosedEnumFieldByNumber") + .ParseText(message(), kKnownNumberInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kKnownNumberInput)))); +} + +TEST_P(TextOpenEnumTest, WithUnknownNumber) { + EXPECT_THAT(Testee("ClosedEnumFieldWithUnknownNumber") + .ParseText(message(), kUnknownNumberInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kUnknownNumberInput)))); + EXPECT_THAT(Testee("ClosedEnumFieldWithUnknownNumber") + .ParseText(message(), kUnknownNumberInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kUnknownNumberInput)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextOpenEnumTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +using TextClosedEnumTest = MessageTypeConformanceTest; + +TEST_P(TextClosedEnumTest, ByNumber) { + EXPECT_THAT(Testee("ClosedEnumFieldByNumber") + .ParseText(message(), kKnownNumberInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kKnownNumberInput)))); + EXPECT_THAT(Testee("ClosedEnumFieldByNumber") + .ParseText(message(), kKnownNumberInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kKnownNumberInput)))); +} + +TEST_P(TextClosedEnumTest, WithUnknownNumber) { + EXPECT_THAT(Testee("ClosedEnumFieldWithUnknownNumber") + .ParseText(message(), kUnknownNumberInput) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextClosedEnumTest, + ValuesIn(Proto2TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_float_test.cc b/conformance/text_float_test.cc new file mode 100644 index 0000000..6058415 --- /dev/null +++ b/conformance/text_float_test.cc
@@ -0,0 +1,537 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for float and double literals: the decimal +// forms with and without the "f"/"F" type suffix, the case-insensitive special +// values, the rejected hex and octal forms, and the literals that overflow or +// underflow the field's range. This replaces the floating point section of +// the legacy TextFormatConformanceTestSuiteImpl<M>::RunAllTests(), which ran +// for the proto3-style message types only; the test names and the requests +// sent to the testee are identical to the legacy ones. + +#include <ostream> +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::Values; +using ::testing::ValuesIn; + +// The type suffix a float literal is tested with. The legacy suite ran every +// literal test three times: bare, and with the "f" and "F" suffixes, the +// latter two with "_f" / "_F" appended to the test name. +struct FloatSuffix { + absl::string_view suffix; + // The component of the gtest parameter name (ParamName() requires a + // non-empty one, so the bare case can't just use `suffix`). + absl::string_view param_name; +}; + +constexpr FloatSuffix kFloatSuffixes[] = { + {"", "NoSuffix"}, + {"f", "f"}, + {"F", "F"}, +}; + +std::string ParamName(const FloatSuffix& suffix) { + return std::string(suffix.param_name); +} + +// So a failing test prints the suffix by name instead of as raw bytes. +void PrintTo(const FloatSuffix& suffix, std::ostream* os) { + *os << suffix.param_name; +} + +// Parameterized over (test message type, float suffix). Reporting the +// message type through MessageUnderTest() makes the base SetUp() skip the +// editions instance when --maximum_edition doesn't cover it. TestName() and +// Input() only build values; every expectation is in the test bodies. +class TextFloatLiteralTest : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, FloatSuffix>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + absl::string_view suffix() const { return std::get<1>(GetParam()).suffix; } + + // The legacy test name: `base` as is for the bare literal, otherwise + // "<base>_<suffix>", e.g. "FloatFieldMaxValue_f". + std::string TestName(absl::string_view base) const { + return suffix().empty() ? std::string(base) + : absl::StrCat(base, "_", suffix()); + } + + // The text-format input setting optional_float to `literal` with the + // suffix appended, e.g. "optional_float: 3.192837f". + std::string Input(absl::string_view literal) const { + return absl::StrCat("optional_float: ", literal, suffix()); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(TextFloatLiteralTest, Positive) { + const std::string input = Input("3.192837"); + EXPECT_THAT(Testee(TestName("FloatField")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatField")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// The field has implicit presence, so zero means unset. +TEST_P(TextFloatLiteralTest, Zero) { + EXPECT_THAT(Testee(TestName("FloatFieldZero")) + .ParseText(message(), Input("0")) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); + EXPECT_THAT(Testee(TestName("FloatFieldZero")) + .ParseText(message(), Input("0")) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(TextFloatLiteralTest, Negative) { + const std::string input = Input("-3.192837"); + EXPECT_THAT(Testee(TestName("FloatFieldNegative")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldNegative")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, VeryPreciseNumber) { + const std::string input = Input("3.123456789123456789"); + EXPECT_THAT(Testee(TestName("FloatFieldWithVeryPreciseNumber")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldWithVeryPreciseNumber")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, MaxValue) { + const std::string input = Input("3.4028235e+38"); + EXPECT_THAT(Testee(TestName("FloatFieldMaxValue")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldMaxValue")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, MinValue) { + const std::string input = Input("1.17549e-38"); + EXPECT_THAT(Testee(TestName("FloatFieldMinValue")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldMinValue")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, WithInt32Max) { + const std::string input = Input("4294967296"); + EXPECT_THAT(Testee(TestName("FloatFieldWithInt32Max")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldWithInt32Max")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, LargerThanInt64) { + const std::string input = Input("9223372036854775808"); + EXPECT_THAT(Testee(TestName("FloatFieldLargerThanInt64")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldLargerThanInt64")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, TooLarge) { + const std::string input = Input("3.4028235e+39"); + EXPECT_THAT(Testee(TestName("FloatFieldTooLarge")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldTooLarge")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, TooSmall) { + const std::string input = Input("1.17549e-39"); + EXPECT_THAT(Testee(TestName("FloatFieldTooSmall")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldTooSmall")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, LargerThanUint64) { + const std::string input = Input("18446744073709551616"); + EXPECT_THAT(Testee(TestName("FloatFieldLargerThanUint64")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldLargerThanUint64")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// https://protobuf.dev/reference/protobuf/textformat-spec/#literals says +// "-0" is a valid float literal. -0 should be considered not the same as 0 +// when considering implicit presence, and so should round trip. +TEST_P(TextFloatLiteralTest, NegativeZero) { + const std::string input = Input("-0"); + EXPECT_THAT(Testee(TestName("FloatFieldNegativeZero")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldNegativeZero")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// https://protobuf.dev/reference/protobuf/textformat-spec/#literals says +// ".123", "-.123", ".123e2" are a valid float literal. +TEST_P(TextFloatLiteralTest, NoLeadingZero) { + const std::string input = Input(".123"); + EXPECT_THAT(Testee(TestName("FloatFieldNoLeadingZero")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldNoLeadingZero")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, NegativeNoLeadingZero) { + const std::string input = Input("-.123"); + EXPECT_THAT(Testee(TestName("FloatFieldNegativeNoLeadingZero")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldNegativeNoLeadingZero")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatLiteralTest, NoLeadingZeroWithExponent) { + const std::string input = Input(".123e2"); + EXPECT_THAT(Testee(TestName("FloatFieldNoLeadingZeroWithExponent")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FloatFieldNoLeadingZeroWithExponent")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextFloatLiteralTest, + Combine(ValuesIn(Proto3TestMessageTypes()), + ValuesIn(kFloatSuffixes)), + TupleParamName<TextFloatLiteralTest::ParamType>); + +// https://protobuf.dev/reference/protobuf/textformat-spec/#value say case +// doesn't matter for special values, test a few. +// +// Parameterized over (test message type, spelling of the special value); the +// two suites below only differ in the spellings they are instantiated with. +class TextFloatSpecialValueTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, absl::string_view>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + absl::string_view spelling() const { return std::get<1>(GetParam()); } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +using TextFloatNanTest = TextFloatSpecialValueTest; + +TEST_P(TextFloatNanTest, Spelling) { + const std::string input = absl::StrCat("optional_float: ", spelling()); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_", spelling())) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_", spelling())) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextFloatNanTest, + Combine(ValuesIn(Proto3TestMessageTypes()), + Values("nan", "NaN", "nAn")), + TupleParamName<TextFloatNanTest::ParamType>); + +using TextFloatInfinityTest = TextFloatSpecialValueTest; + +TEST_P(TextFloatInfinityTest, Positive) { + const std::string input = absl::StrCat("optional_float: ", spelling()); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_Pos", spelling())) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_Pos", spelling())) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFloatInfinityTest, Negative) { + const std::string input = absl::StrCat("optional_float: -", spelling()); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_Neg", spelling())) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(absl::StrCat("FloatFieldValue_Neg", spelling())) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextFloatInfinityTest, + Combine(ValuesIn(Proto3TestMessageTypes()), + Values("inf", "infinity", "INF", "INFINITY", + "iNF", "inFINITY")), + TupleParamName<TextFloatInfinityTest::ParamType>); + +// The remaining float tests, and their double counterparts, run once per +// message type. +using TextFloatTest = MessageTypeConformanceTest; + +// https://protobuf.dev/reference/protobuf/textformat-spec/#numeric and +// https://protobuf.dev/reference/protobuf/textformat-spec/#value says +// hex or octal float literals are invalid. +TEST_P(TextFloatTest, NoHex) { + EXPECT_THAT(Testee("FloatFieldNoHex") + .ParseText(message(), R"pb(optional_float: 0x1)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextFloatTest, NoNegativeHex) { + EXPECT_THAT(Testee("FloatFieldNoNegativeHex") + .ParseText(message(), R"pb(optional_float: -0x1)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextFloatTest, NoOctal) { + EXPECT_THAT(Testee("FloatFieldNoOctal") + .ParseText(message(), R"pb(optional_float: 012)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextFloatTest, NoNegativeOctal) { + EXPECT_THAT(Testee("FloatFieldNoNegativeOctal") + .ParseText(message(), R"pb(optional_float: -012)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +// https://protobuf.dev/reference/protobuf/textformat-spec/#value says +// overflows are mapped to infinity/-infinity. +TEST_P(TextFloatTest, OverflowInfinity) { + EXPECT_THAT( + Testee("FloatFieldOverflowInfinity") + .ParseText(message(), R"pb(optional_float: 1e50)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: inf)pb")))); + EXPECT_THAT( + Testee("FloatFieldOverflowInfinity") + .ParseText(message(), R"pb(optional_float: 1e50)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: inf)pb")))); +} + +TEST_P(TextFloatTest, OverflowNegativeInfinity) { + EXPECT_THAT( + Testee("FloatFieldOverflowNegativeInfinity") + .ParseText(message(), R"pb(optional_float: -1e50)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: -inf)pb")))); + EXPECT_THAT( + Testee("FloatFieldOverflowNegativeInfinity") + .ParseText(message(), R"pb(optional_float: -1e50)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: -inf)pb")))); +} + +// Exponent is one more than uint64 max. +TEST_P(TextFloatTest, OverflowInfinityHugeExponent) { + EXPECT_THAT( + Testee("FloatFieldOverflowInfinityHugeExponent") + .ParseText(message(), R"pb(optional_float: 1e18446744073709551616)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: inf)pb")))); + EXPECT_THAT( + Testee("FloatFieldOverflowInfinityHugeExponent") + .ParseText(message(), R"pb(optional_float: 1e18446744073709551616)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: inf)pb")))); +} + +TEST_P(TextFloatTest, LargeNegativeExponentParsesAsZero) { + EXPECT_THAT(Testee("FloatFieldLargeNegativeExponentParsesAsZero") + .ParseText(message(), R"pb(optional_float: 1e-50)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); + EXPECT_THAT(Testee("FloatFieldLargeNegativeExponentParsesAsZero") + .ParseText(message(), R"pb(optional_float: 1e-50)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(TextFloatTest, NegativeLargeNegativeExponentParsesAsNegativeZero) { + EXPECT_THAT(Testee("NegFloatFieldLargeNegativeExponentParsesAsNegZero") + .ParseText(message(), R"pb(optional_float: -1e-50)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: -0)pb")))); + EXPECT_THAT(Testee("NegFloatFieldLargeNegativeExponentParsesAsNegZero") + .ParseText(message(), R"pb(optional_float: -1e-50)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_float: -0)pb")))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextFloatTest, ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +using TextDoubleTest = MessageTypeConformanceTest; + +// https://protobuf.dev/reference/protobuf/textformat-spec/#value says +// overflows are mapped to infinity/-infinity. +TEST_P(TextDoubleTest, OverflowInfinity) { + EXPECT_THAT( + Testee("DoubleFieldOverflowInfinity") + .ParseText(message(), R"pb(optional_double: 1e9999)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: inf)pb")))); + EXPECT_THAT( + Testee("DoubleFieldOverflowInfinity") + .ParseText(message(), R"pb(optional_double: 1e9999)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: inf)pb")))); +} + +TEST_P(TextDoubleTest, OverflowNegativeInfinity) { + EXPECT_THAT( + Testee("DoubleFieldOverflowNegativeInfinity") + .ParseText(message(), R"pb(optional_double: -1e9999)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: -inf)pb")))); + EXPECT_THAT( + Testee("DoubleFieldOverflowNegativeInfinity") + .ParseText(message(), R"pb(optional_double: -1e9999)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: -inf)pb")))); +} + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. + +// Exponent is one more than uint64 max. +TEST_P(TextDoubleTest, OverflowInfinityHugeExponent) { + EXPECT_THAT( + Testee("DoubleFieldOverflowInfinityHugeExponent") + .ParseText(message(), "optional_double: 1e18446744073709551616") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: inf)pb")))); + EXPECT_THAT( + Testee("DoubleFieldOverflowInfinityHugeExponent") + .ParseText(message(), "optional_double: 1e18446744073709551616") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: inf)pb")))); +} + +TEST_P(TextDoubleTest, LargeNegativeExponentParsesAsZero) { + EXPECT_THAT( + Testee("DoubleFieldLargeNegativeExponentParsesAsZero") + .ParseText(message(), "optional_double: 1e-18446744073709551616") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("")))); + EXPECT_THAT( + Testee("DoubleFieldLargeNegativeExponentParsesAsZero") + .ParseText(message(), "optional_double: 1e-18446744073709551616") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("")))); +} + +TEST_P(TextDoubleTest, NegativeLargeNegativeExponentParsesAsNegativeZero) { + EXPECT_THAT( + Testee("NegDoubleFieldLargeNegativeExponentParsesAsNegZero") + .ParseText(message(), + R"pb(optional_double: -1e-18446744073709551616)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: -0)pb")))); + EXPECT_THAT( + Testee("NegDoubleFieldLargeNegativeExponentParsesAsNegZero") + .ParseText(message(), + R"pb(optional_double: -1e-18446744073709551616)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_double: -0)pb")))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextDoubleTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc deleted file mode 100644 index 44e1728..0000000 --- a/conformance/text_format_conformance_suite.cc +++ /dev/null
@@ -1,991 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#include "conformance/text_format_conformance_suite.h" - -#include <cstddef> -#include <string> -#include <utility> -#include <vector> - -#include "absl/log/absl_log.h" -#include "absl/log/die_if_null.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "conformance/conformance_test.h" -#include "conformance/test_protos/test_messages_edition2023.pb.h" -#include "conformance/test_protos/test_messages_edition_unstable.pb.h" -#include "editions/golden/test_messages_proto2_editions.pb.h" -#include "editions/golden/test_messages_proto3_editions.pb.h" -#include "google/protobuf/test_messages_proto2.pb.h" -#include "google/protobuf/test_messages_proto3.pb.h" -#include "google/protobuf/text_format.h" - -using conformance::ConformanceRequest; -using conformance::ConformanceResponse; -using conformance::TestStatus; -using conformance::WireFormat; -using protobuf_test_messages::editions::TestAllTypesEdition2023; -using protobuf_test_messages::proto2::TestAllTypesProto2; -using protobuf_test_messages::proto2::UnknownToTestAllTypes; -using protobuf_test_messages::proto3::TestAllTypesProto3; -using TestAllTypesProto2Editions = - protobuf_test_messages::editions::proto2::TestAllTypesProto2; -using TestAllTypesProto3Editions = - protobuf_test_messages::editions::proto3::TestAllTypesProto3; - -namespace google { -namespace protobuf { -namespace conformance { - -// The number of repetitions to use for performance tests. -// Corresponds approx to 500KB wireformat bytes. -static const size_t kPerformanceRepeatCount = 50000; - -TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() { - SetFailureListFlagName("--text_format_failure_list"); -} - -bool TextFormatConformanceTestSuite::ParseTextFormatResponse( - const ConformanceResponse& response, - const ConformanceRequestSetting& setting, Message* test_message) { - TextFormat::Parser parser; - const ConformanceRequest& request = setting.GetRequest(); - if (request.print_unknown_fields()) { - parser.AllowFieldNumber(true); - } - if (!parser.ParseFromString(response.text_payload(), test_message)) { - ABSL_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode " - << "yielded unparseable proto. Text payload: " - << response.text_payload(); - return false; - } - - return true; -} - -bool TextFormatConformanceTestSuite::ParseResponse( - const ConformanceResponse& response, - const ConformanceRequestSetting& setting, Message* test_message) { - const ConformanceRequest& request = setting.GetRequest(); - WireFormat requested_output = request.requested_output_format(); - const std::string& test_name = setting.GetTestName(); - ConformanceLevel level = setting.GetLevel(); - - TestStatus test; - test.set_name(test_name); - switch (response.result_case()) { - case ConformanceResponse::kProtobufPayload: { - if (requested_output != ::conformance::PROTOBUF) { - test.set_failure_message(absl::StrCat( - "Test was asked for ", WireFormatToString(requested_output), - " output but provided PROTOBUF instead.")); - ReportFailure(test, level, request, response); - return false; - } - - if (!test_message->ParseFromString(response.protobuf_payload())) { - test.set_failure_message( - "Protobuf output we received from test was unparseable."); - ReportFailure(test, level, request, response); - return false; - } - - break; - } - - case ConformanceResponse::kTextPayload: { - if (requested_output != ::conformance::TEXT_FORMAT) { - test.set_failure_message(absl::StrCat( - "Test was asked for ", WireFormatToString(requested_output), - " output but provided TEXT_FORMAT instead.")); - ReportFailure(test, level, request, response); - return false; - } - - if (!ParseTextFormatResponse(response, setting, test_message)) { - test.set_failure_message( - "TEXT_FORMAT output we received from test was unparseable."); - ReportFailure(test, level, request, response); - return false; - } - - break; - } - - default: - ABSL_LOG(FATAL) << test_name - << ": unknown payload type: " << response.result_case(); - } - - return true; -} - -void TextFormatConformanceTestSuite::RunSuiteImpl() { - TextFormatConformanceTestSuiteImpl<TestAllTypesProto2>(this); - TextFormatConformanceTestSuiteImpl<TestAllTypesProto3>(this); - if (maximum_edition_ >= Edition::EDITION_2023) { - TextFormatConformanceTestSuiteImpl<TestAllTypesProto2Editions>(this); - TextFormatConformanceTestSuiteImpl<TestAllTypesProto3Editions>(this); - TextFormatConformanceTestSuiteImpl<TestAllTypesEdition2023>(this); - } -} - -template <typename MessageType> -TextFormatConformanceTestSuiteImpl<MessageType>:: - TextFormatConformanceTestSuiteImpl(TextFormatConformanceTestSuite* suite) - : suite_(*ABSL_DIE_IF_NULL(suite)) { - // Flag control performance tests to keep them internal and opt-in only - if (suite_.performance_) { - if (MessageType::GetDescriptor()->name() == "TestAllTypesEdition2023") { - // There are no editions-sensitive performance tests. - return; - } - if (MessageType::GetDescriptor()->name() == "TestAllTypesEditionUnstable") { - // There are no editions-sensitive performance tests. - return; - } - RunTextFormatPerformanceTests(); - } else { - if (MessageType::GetDescriptor()->name() == "TestAllTypesProto2") { - RunGroupTests(); - RunClosedEnumTests(); - } - if (MessageType::GetDescriptor()->name() == "TestAllTypesEdition2023") { - RunDelimitedTests(); - } - if (MessageType::GetDescriptor()->name() == "TestAllTypesProto3") { - RunAnyTests(); - RunOpenEnumTests(); - // TODO Run these over proto2 also. - RunAllTests(); - } - } -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::ExpectParseFailure( - const std::string& test_name, ConformanceLevel level, - const std::string& input) { - MessageType prototype; - // We don't expect output, but if the program erroneously accepts the protobuf - // we let it send its response as this. We must not leave it unspecified. - ConformanceRequestSetting setting( - level, ::conformance::TEXT_FORMAT, ::conformance::TEXT_FORMAT, - ::conformance::TEXT_FORMAT_TEST, prototype, test_name, input); - const ConformanceRequest& request = setting.GetRequest(); - ConformanceResponse response; - std::string effective_test_name = absl::StrCat( - setting.ConformanceLevelToString(level), ".", - setting.GetSyntaxIdentifier(), ".TextFormatInput.", test_name); - - if (!suite_.RunTest(effective_test_name, request, &response)) { - return; - } - - TestStatus test; - test.set_name(effective_test_name); - if (response.result_case() == ConformanceResponse::kParseError) { - suite_.ReportSuccess(test); - } else if (response.result_case() == ConformanceResponse::kSkipped) { - suite_.ReportSkip(test, request, response); - } else if (response.result_case() == ConformanceResponse::kRuntimeError) { - test.set_failure_message( - "Should have failed to parse, but raised an error instead."); - suite_.ReportFailure(test, level, request, response); - } else { - test.set_failure_message("Should have failed to parse, but didn't."); - suite_.ReportFailure(test, level, request, response); - } -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunValidTextFormatTest( - const std::string& test_name, ConformanceLevel level, - const std::string& input_text) { - MessageType prototype; - RunValidTextFormatTestWithMessage(test_name, level, input_text, prototype); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>:: - RunValidTextFormatTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const std::string& input_text, - const Message& message) { - ConformanceRequestSetting setting1( - level, ::conformance::TEXT_FORMAT, ::conformance::PROTOBUF, - ::conformance::TEXT_FORMAT_TEST, message, test_name, input_text); - suite_.RunValidInputTest(setting1, input_text); - ConformanceRequestSetting setting2( - level, ::conformance::TEXT_FORMAT, ::conformance::TEXT_FORMAT, - ::conformance::TEXT_FORMAT_TEST, message, test_name, input_text); - suite_.RunValidInputTest(setting2, input_text); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>:: - RunValidTextFormatTestWithExpected(const std::string& test_name, - ConformanceLevel level, - const std::string& input_text, - const std::string& expected_text) { - MessageType prototype; - ConformanceRequestSetting setting1( - level, ::conformance::TEXT_FORMAT, ::conformance::PROTOBUF, - ::conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text); - suite_.RunValidInputTest(setting1, expected_text); - ConformanceRequestSetting setting2( - level, ::conformance::TEXT_FORMAT, ::conformance::TEXT_FORMAT, - ::conformance::TEXT_FORMAT_TEST, prototype, test_name, input_text); - suite_.RunValidInputTest(setting2, expected_text); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl< - MessageType>::RunValidUnknownTextFormatTest(const std::string& test_name, - const Message& message) { - std::string serialized_input; - // TODO: Remove this suppression. - (void)message.SerializeToString(&serialized_input); - MessageType prototype; - ConformanceRequestSetting setting1( - RECOMMENDED, ::conformance::PROTOBUF, ::conformance::TEXT_FORMAT, - ::conformance::TEXT_FORMAT_TEST, prototype, - absl::StrCat(test_name, "_Drop"), serialized_input); - setting1.SetPrototypeMessageForCompare(message); - suite_.RunValidBinaryInputTest(setting1, ""); - - ConformanceRequestSetting setting2( - RECOMMENDED, ::conformance::PROTOBUF, ::conformance::TEXT_FORMAT, - ::conformance::TEXT_FORMAT_TEST, prototype, - absl::StrCat(test_name, "_Print"), serialized_input); - setting2.SetPrototypeMessageForCompare(message); - setting2.SetPrintUnknownFields(true); - suite_.RunValidBinaryInputTest(setting2, serialized_input); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunDelimitedTests() { - RunValidTextFormatTest("GroupFieldNoColon", REQUIRED, - "GroupLikeType { group_int32: 1 }"); - RunValidTextFormatTest("GroupFieldWithColon", REQUIRED, - "GroupLikeType: { group_int32: 1 }"); - RunValidTextFormatTest("GroupFieldEmpty", REQUIRED, "GroupLikeType {}"); - RunValidTextFormatTest( - "GroupFieldExtension", REQUIRED, - "[protobuf_test_messages.editions.groupliketype] { c: 1 }"); - RunValidTextFormatTest( - "DelimitedFieldExtension", REQUIRED, - "[protobuf_test_messages.editions.delimited_ext] { c: 1 }"); - - - // Test that lower-cased group name (i.e. implicit field name) are accepted. - RunValidTextFormatTest("DelimitedFieldLowercased", REQUIRED, - "groupliketype { group_int32: 1 }"); - RunValidTextFormatTest("DelimitedFieldLowercasedDifferent", REQUIRED, - "delimited_field { group_int32: 1 }"); - - // Extensions always used the field name, and should never accept the message - // name. - ExpectParseFailure( - "DelimitedFieldExtensionMessageName", REQUIRED, - "[protobuf_test_messages.editions.GroupLikeType] { group_int32: 1 }"); - - // Extension names can contain whitespace and comments. - RunValidTextFormatTest( - "ExtensionNameWithWhitespace", REQUIRED, - "[protobuf _test_messages.edit\tions.exten\nsion_int32]: 1"); - RunValidTextFormatTest( - "ExtensionNameWithComment", REQUIRED, - "[protobuf_test_messages.edit # comment \nions.extension_int32]: 1"); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunGroupTests() { - RunValidTextFormatTest("GroupFieldNoColon", REQUIRED, - "Data { group_int32: 1 }"); - RunValidTextFormatTest("GroupFieldWithColon", REQUIRED, - "Data: { group_int32: 1 }"); - RunValidTextFormatTest("GroupFieldEmpty", REQUIRED, "Data {}"); - RunValidTextFormatTest("GroupFieldMultiWord", REQUIRED, - "MultiWordGroupField { group_int32: 1 }"); - - // Test that lower-cased group name (i.e. implicit field name) is accepted - RunValidTextFormatTest("GroupFieldLowercased", REQUIRED, - "data { group_int32: 1 }"); - RunValidTextFormatTest("GroupFieldLowercasedMultiWord", REQUIRED, - "multiwordgroupfield { group_int32: 1 }"); - - // Test extensions of group type - RunValidTextFormatTest("GroupFieldExtension", REQUIRED, - absl::StrFormat("[%s] { group_int32: 1 }", - MessageType::GetDescriptor() - ->file() - ->FindExtensionByName("groupfield") - ->PrintableNameForExtension())); - ExpectParseFailure("GroupFieldExtensionGroupName", REQUIRED, - absl::StrFormat("[%s] { group_int32: 1 }", - MessageType::GetDescriptor() - ->file() - ->FindMessageTypeByName("GroupField") - ->full_name())); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunAllTests() { - RunValidTextFormatTest("HelloWorld", REQUIRED, - "optional_string: 'Hello, World!'"); - // Integer fields. - RunValidTextFormatTest("Int32FieldMaxValue", REQUIRED, - "optional_int32: 2147483647"); - RunValidTextFormatTest("Int32FieldMinValue", REQUIRED, - "optional_int32: -2147483648"); - RunValidTextFormatTest("Uint32FieldMaxValue", REQUIRED, - "optional_uint32: 4294967295"); - RunValidTextFormatTest("Int64FieldMaxValue", REQUIRED, - "optional_int64: 9223372036854775807"); - RunValidTextFormatTest("Int64FieldMinValue", REQUIRED, - "optional_int64: -9223372036854775808"); - RunValidTextFormatTest("Uint64FieldMaxValue", REQUIRED, - "optional_uint64: 18446744073709551615"); - // Integer fields - Hex - RunValidTextFormatTestWithExpected("Int32FieldMaxValueHex", REQUIRED, - "optional_int32: 0x7FFFFFFF", - "optional_int32: 2147483647"); - RunValidTextFormatTestWithExpected("Int32FieldMinValueHex", REQUIRED, - "optional_int32: -0x80000000", - "optional_int32: -2147483648"); - RunValidTextFormatTestWithExpected("Uint32FieldMaxValueHex", REQUIRED, - "optional_uint32: 0xFFFFFFFF", - "optional_uint32: 4294967295"); - RunValidTextFormatTestWithExpected("Int64FieldMaxValueHex", REQUIRED, - "optional_int64: 0x7FFFFFFFFFFFFFFF", - "optional_int64: 9223372036854775807"); - RunValidTextFormatTestWithExpected("Int64FieldMinValueHex", REQUIRED, - "optional_int64: -0x8000000000000000", - "optional_int64: -9223372036854775808"); - RunValidTextFormatTestWithExpected("Uint64FieldMaxValueHex", REQUIRED, - "optional_uint64: 0xFFFFFFFFFFFFFFFF", - "optional_uint64: 18446744073709551615"); - // Integer fields - Octal - RunValidTextFormatTestWithExpected("Int32FieldMaxValueOctal", REQUIRED, - "optional_int32: 017777777777", - "optional_int32: 2147483647"); - RunValidTextFormatTestWithExpected("Int32FieldMinValueOctal", REQUIRED, - "optional_int32: -020000000000", - "optional_int32: -2147483648"); - RunValidTextFormatTestWithExpected("Uint32FieldMaxValueOctal", REQUIRED, - "optional_uint32: 037777777777", - "optional_uint32: 4294967295"); - RunValidTextFormatTestWithExpected("Int64FieldMaxValueOctal", REQUIRED, - "optional_int64: 0777777777777777777777", - "optional_int64: 9223372036854775807"); - RunValidTextFormatTestWithExpected("Int64FieldMinValueOctal", REQUIRED, - "optional_int64: -01000000000000000000000", - "optional_int64: -9223372036854775808"); - RunValidTextFormatTestWithExpected("Uint64FieldMaxValueOctal", REQUIRED, - "optional_uint64: 01777777777777777777777", - "optional_uint64: 18446744073709551615"); - - // Parsers reject out-of-bound integer values. - ExpectParseFailure("Int32FieldTooLarge", REQUIRED, - "optional_int32: 2147483648"); - ExpectParseFailure("Int32FieldTooSmall", REQUIRED, - "optional_int32: -2147483649"); - ExpectParseFailure("Uint32FieldTooLarge", REQUIRED, - "optional_uint32: 4294967296"); - ExpectParseFailure("Int64FieldTooLarge", REQUIRED, - "optional_int64: 9223372036854775808"); - ExpectParseFailure("Int64FieldTooSmall", REQUIRED, - "optional_int64: -9223372036854775809"); - ExpectParseFailure("Uint64FieldTooLarge", REQUIRED, - "optional_uint64: 18446744073709551616"); - // Parsers reject out-of-bound integer values - Hex - ExpectParseFailure("Int32FieldTooLargeHex", REQUIRED, - "optional_int32: 0x80000000"); - ExpectParseFailure("Int32FieldTooSmallHex", REQUIRED, - "optional_int32: -0x80000001"); - ExpectParseFailure("Uint32FieldTooLargeHex", REQUIRED, - "optional_uint32: 0x100000000"); - ExpectParseFailure("Int64FieldTooLargeHex", REQUIRED, - "optional_int64: 0x8000000000000000"); - ExpectParseFailure("Int64FieldTooSmallHex", REQUIRED, - "optional_int64: -0x8000000000000001"); - ExpectParseFailure("Uint64FieldTooLargeHex", REQUIRED, - "optional_uint64: 0x10000000000000000"); - // Parsers reject out-of-bound integer values - Octal - ExpectParseFailure("Int32FieldTooLargeOctal", REQUIRED, - "optional_int32: 020000000000"); - ExpectParseFailure("Int32FieldTooSmallOctal", REQUIRED, - "optional_int32: -020000000001"); - ExpectParseFailure("Uint32FieldTooLargeOctal", REQUIRED, - "optional_uint32: 040000000000"); - ExpectParseFailure("Int64FieldTooLargeOctal", REQUIRED, - "optional_int64: 01000000000000000000000"); - ExpectParseFailure("Int64FieldTooSmallOctal", REQUIRED, - "optional_int64: -01000000000000000000001"); - ExpectParseFailure("Uint64FieldTooLargeOctal", REQUIRED, - "optional_uint64: 02000000000000000000000"); - - // Floating point fields - for (const auto& suffix : std::vector<std::string>{"", "f", "F"}) { - const std::string name_suffix = - suffix.empty() ? "" : absl::StrCat("_", suffix); - - RunValidTextFormatTest(absl::StrCat("FloatField", name_suffix), REQUIRED, - absl::StrCat("optional_float: 3.192837", suffix)); - RunValidTextFormatTestWithExpected( - absl::StrCat("FloatFieldZero", name_suffix), REQUIRED, - absl::StrCat("optional_float: 0", suffix), - "" /* implicit presence, so zero means unset*/); - RunValidTextFormatTest(absl::StrCat("FloatFieldNegative", name_suffix), - REQUIRED, - absl::StrCat("optional_float: -3.192837", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldWithVeryPreciseNumber", name_suffix), REQUIRED, - absl::StrCat("optional_float: 3.123456789123456789", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldMaxValue", name_suffix), REQUIRED, - absl::StrCat("optional_float: 3.4028235e+38", suffix)); - RunValidTextFormatTest(absl::StrCat("FloatFieldMinValue", name_suffix), - REQUIRED, - absl::StrCat("optional_float: 1.17549e-38", suffix)); - RunValidTextFormatTest(absl::StrCat("FloatFieldWithInt32Max", name_suffix), - REQUIRED, - absl::StrCat("optional_float: 4294967296", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldLargerThanInt64", name_suffix), REQUIRED, - absl::StrCat("optional_float: 9223372036854775808", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldTooLarge", name_suffix), REQUIRED, - absl::StrCat("optional_float: 3.4028235e+39", suffix)); - RunValidTextFormatTest(absl::StrCat("FloatFieldTooSmall", name_suffix), - REQUIRED, - absl::StrCat("optional_float: 1.17549e-39", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldLargerThanUint64", name_suffix), REQUIRED, - absl::StrCat("optional_float: 18446744073709551616", suffix)); - // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says - // "-0" is a valid float literal. -0 should be considered not the same as 0 - // when considering implicit presence, and so should round trip. - RunValidTextFormatTest(absl::StrCat("FloatFieldNegativeZero", name_suffix), - REQUIRED, - absl::StrCat("optional_float: -0", suffix)); - // https://protobuf.dev/reference/protobuf/textformat-spec/#literals says - // ".123", "-.123", ".123e2" are a valid float literal. - RunValidTextFormatTest(absl::StrCat("FloatFieldNoLeadingZero", name_suffix), - REQUIRED, - absl::StrCat("optional_float: .123", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldNegativeNoLeadingZero", name_suffix), REQUIRED, - absl::StrCat("optional_float: -.123", suffix)); - RunValidTextFormatTest( - absl::StrCat("FloatFieldNoLeadingZeroWithExponent", name_suffix), - REQUIRED, absl::StrCat("optional_float: .123e2", suffix)); - } - // https://protobuf.dev/reference/protobuf/textformat-spec/#value say case - // doesn't matter for special values, test a few - for (const auto& value : std::vector<std::string>{"nan", "NaN", "nAn"}) { - RunValidTextFormatTest(absl::StrCat("FloatFieldValue_", value), REQUIRED, - absl::StrCat("optional_float: ", value)); - } - for (const auto& value : std::vector<std::string>{ - "inf", "infinity", "INF", "INFINITY", "iNF", "inFINITY"}) { - RunValidTextFormatTest(absl::StrCat("FloatFieldValue_Pos", value), REQUIRED, - absl::StrCat("optional_float: ", value)); - RunValidTextFormatTest(absl::StrCat("FloatFieldValue_Neg", value), REQUIRED, - absl::StrCat("optional_float: -", value)); - } - // https://protobuf.dev/reference/protobuf/textformat-spec/#numeric and - // https://protobuf.dev/reference/protobuf/textformat-spec/#value says - // hex or octal float literals are invalid. - ExpectParseFailure("FloatFieldNoHex", REQUIRED, "optional_float: 0x1"); - ExpectParseFailure("FloatFieldNoNegativeHex", REQUIRED, - "optional_float: -0x1"); - ExpectParseFailure("FloatFieldNoOctal", REQUIRED, "optional_float: 012"); - ExpectParseFailure("FloatFieldNoNegativeOctal", REQUIRED, - "optional_float: -012"); - // https://protobuf.dev/reference/protobuf/textformat-spec/#value says - // overflows are mapped to infinity/-infinity. - RunValidTextFormatTestWithExpected("FloatFieldOverflowInfinity", REQUIRED, - "optional_float: 1e50", - "optional_float: inf"); - RunValidTextFormatTestWithExpected("FloatFieldOverflowNegativeInfinity", - REQUIRED, "optional_float: -1e50", - "optional_float: -inf"); - RunValidTextFormatTestWithExpected("DoubleFieldOverflowInfinity", REQUIRED, - "optional_double: 1e9999", - "optional_double: inf"); - RunValidTextFormatTestWithExpected("DoubleFieldOverflowNegativeInfinity", - REQUIRED, "optional_double: -1e9999", - "optional_double: -inf"); - // Exponent is one more than uint64 max. - RunValidTextFormatTestWithExpected( - "FloatFieldOverflowInfinityHugeExponent", REQUIRED, - "optional_float: 1e18446744073709551616", "optional_float: inf"); - RunValidTextFormatTestWithExpected( - "DoubleFieldOverflowInfinityHugeExponent", REQUIRED, - "optional_double: 1e18446744073709551616", "optional_double: inf"); - RunValidTextFormatTestWithExpected( - "DoubleFieldLargeNegativeExponentParsesAsZero", REQUIRED, - "optional_double: 1e-18446744073709551616", ""); - RunValidTextFormatTestWithExpected( - "NegDoubleFieldLargeNegativeExponentParsesAsNegZero", REQUIRED, - "optional_double: -1e-18446744073709551616", "optional_double: -0"); - - RunValidTextFormatTestWithExpected( - "FloatFieldLargeNegativeExponentParsesAsZero", REQUIRED, - "optional_float: 1e-50", ""); - RunValidTextFormatTestWithExpected( - "NegFloatFieldLargeNegativeExponentParsesAsNegZero", REQUIRED, - "optional_float: -1e-50", "optional_float: -0"); - - // String literals x {Strings, Bytes} - for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) { - const std::string field_name = - field_type == "String" ? "optional_string" : "optional_bytes"; - RunValidTextFormatTest( - absl::StrCat("StringLiteralConcat", field_type), REQUIRED, - absl::StrCat(field_name, ": 'first' \"second\"\n'third'")); - RunValidTextFormatTest( - absl::StrCat("StringLiteralBasicEscapes", field_type), REQUIRED, - absl::StrCat(field_name, ": '\\a\\b\\f\\n\\r\\t\\v\\?\\\\\\'\\\"'")); - RunValidTextFormatTest( - absl::StrCat("StringLiteralOctalEscapes", field_type), REQUIRED, - absl::StrCat(field_name, ": '\\341\\210\\264'")); - RunValidTextFormatTest(absl::StrCat("StringLiteralHexEscapes", field_type), - REQUIRED, - absl::StrCat(field_name, ": '\\xe1\\x88\\xb4'")); - RunValidTextFormatTest( - absl::StrCat("StringLiteralShortUnicodeEscape", field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\u1234'")); - RunValidTextFormatTest( - absl::StrCat("StringLiteralLongUnicodeEscapes", field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\U00001234\\U00010437'")); - // String literals don't include line feeds. - ExpectParseFailure(absl::StrCat("StringLiteralIncludesLF", field_type), - REQUIRED, - absl::StrCat(field_name, ": 'first line\nsecond line'")); - // Unicode escapes don't include code points that lie beyond the planes - // (> 0x10ffff). - ExpectParseFailure( - absl::StrCat("StringLiteralLongUnicodeEscapeTooLarge", field_type), - REQUIRED, absl::StrCat(field_name, ": '\\U00110000'")); - // Unicode escapes don't include surrogates. - ExpectParseFailure( - absl::StrCat("StringLiteralShortUnicodeEscapeSurrogatePair", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\ud801\\udc37'")); - ExpectParseFailure( - absl::StrCat("StringLiteralShortUnicodeEscapeSurrogateFirstOnly", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\ud800'")); - ExpectParseFailure( - absl::StrCat("StringLiteralShortUnicodeEscapeSurrogateSecondOnly", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\udc00'")); - ExpectParseFailure( - absl::StrCat("StringLiteralLongUnicodeEscapeSurrogateFirstOnly", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d800'")); - ExpectParseFailure( - absl::StrCat("StringLiteralLongUnicodeEscapeSurrogateSecondOnly", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\U0000dc00'")); - ExpectParseFailure( - absl::StrCat("StringLiteralLongUnicodeEscapeSurrogatePair", field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d801\\U00000dc37'")); - ExpectParseFailure( - absl::StrCat("StringLiteralUnicodeEscapeSurrogatePairLongShort", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\U0000d801\\udc37'")); - ExpectParseFailure( - absl::StrCat("StringLiteralUnicodeEscapeSurrogatePairShortLong", - field_type), - RECOMMENDED, absl::StrCat(field_name, ": '\\ud801\\U0000dc37'")); - - // The following method depend on the type of field, as strings have extra - // validation. - const auto test_method = - field_type == "String" - ? &TextFormatConformanceTestSuiteImpl::ExpectParseFailure - : &TextFormatConformanceTestSuiteImpl::RunValidTextFormatTest; - - // String fields reject invalid UTF-8 byte sequences; bytes fields don't. - (this->*test_method)(absl::StrCat(field_type, "FieldBadUTF8Octal"), - REQUIRED, absl::StrCat(field_name, ": '\\300'")); - (this->*test_method)(absl::StrCat(field_type, "FieldBadUTF8Hex"), REQUIRED, - absl::StrCat(field_name, ": '\\xc0'")); - } - - // Separators - for (const auto& test_case : std::vector<std::pair<std::string, std::string>>{ - {"string", "\"abc\""}, - {"bytes", "\"abc\""}, - {"int32", "123"}, - {"bool", "true"}, - {"double", "1.23"}, - {"fixed32", "0x123"}, - }) { - // Optional Field Separators - for (const auto& field_type : - std::vector<std::string>{"Single", "Repeated"}) { - std::string field_name, field_value; - if (field_type == "Single") { - field_name = absl::StrCat("optional_", test_case.first); - field_value = test_case.second; - } else { - field_name = absl::StrCat("repeated_", test_case.first); - field_value = absl::StrCat("[", test_case.second, "]"); - } - - RunValidTextFormatTest(absl::StrCat("FieldSeparatorCommaTopLevel", - field_type, "_", test_case.first), - REQUIRED, - absl::StrCat(field_name, ": ", field_value, ",")); - RunValidTextFormatTest(absl::StrCat("FieldSeparatorSemiTopLevelSingle", - field_type, "_", test_case.first), - REQUIRED, - absl::StrCat(field_name, ": ", field_value, ";")); - - ExpectParseFailure( - absl::StrCat("FieldSeparatorCommaTopLevelDuplicatesFails", field_type, - "_", test_case.first), - REQUIRED, absl::StrCat(field_name, ": ", field_value, ",,")); - ExpectParseFailure( - absl::StrCat("FieldSeparatorSemiTopLevelDuplicateFails", field_type, - "_", test_case.first), - REQUIRED, absl::StrCat(field_name, ": ", field_value, ";;")); - } - - // Required List Separators - RunValidTextFormatTest( - absl::StrCat("ListSeparator_", test_case.first), REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ",", - test_case.second, "]")); - ExpectParseFailure( - absl::StrCat("ListSeparatorSemiFails_", test_case.first), REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ";", - test_case.second, "]")); - // For string and bytes, if we skip the separator, the parser will treat - // the two values as a single value. - if (test_case.first == "string" || test_case.first == "bytes") { - RunValidTextFormatTest( - absl::StrCat("ListSeparatorMissingIsOneValue_", test_case.first), - REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, - " ", test_case.second, "]")); - } else { - ExpectParseFailure( - absl::StrCat("ListSeparatorMissingFails_", test_case.first), REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, - " ", test_case.second, "]")); - } - ExpectParseFailure( - absl::StrCat("ListSeparatorDuplicateFails_", test_case.first), REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, - ",,", test_case.second, "]")); - ExpectParseFailure( - absl::StrCat("ListSeparatorSingleTrailingFails_", test_case.first), - REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, - ",]")); - ExpectParseFailure( - absl::StrCat("ListSeparatorTwoValuesTrailingFails_", test_case.first), - REQUIRED, - absl::StrCat("repeated_", test_case.first, ": [", test_case.second, ",", - test_case.second, ",]")); - } - // The test message don't really have all types nested, so just check one - // data type for the nested field separator support - RunValidTextFormatTest("FieldSeparatorCommaNested", REQUIRED, - "optional_nested_message: { a: 123, }"); - RunValidTextFormatTest("FieldSeparatorSemiNested", REQUIRED, - "optional_nested_message: { a: 123; }"); - ExpectParseFailure("FieldSeparatorCommaNestedDuplicates", REQUIRED, - "optional_nested_message: { a: 123,, }"); - ExpectParseFailure("FieldSeparatorSemiNestedDuplicates", REQUIRED, - "optional_nested_message: { a: 123;; }"); - - // Unknown Fields - UnknownToTestAllTypes message; - // Unable to print unknown Fixed32/Fixed64 fields as if they are known. - // Fixed32/Fixed64 fields are not added in the tests. - message.set_optional_int32(123); - message.set_optional_string("hello"); - message.set_optional_bool(true); - RunValidUnknownTextFormatTest("ScalarUnknownFields", message); - - message.Clear(); - message.mutable_nested_message()->set_c(111); - RunValidUnknownTextFormatTest("MessageUnknownFields", message); - - message.Clear(); - message.mutable_optionalgroup()->set_a(321); - RunValidUnknownTextFormatTest("GroupUnknownFields", message); - - message.add_repeated_int32(1); - message.add_repeated_int32(2); - message.add_repeated_int32(3); - RunValidUnknownTextFormatTest("RepeatedUnknownFields", message); - - // Reserved field names - for (const auto& test_case : std::vector<std::pair<std::string, std::string>>{ - {"Boolean", "true"}, - {"Integer", "-123"}, - {"Float", "0.123"}, - {"Enum", "ENUM_VALUE"}, - {"String", "\"hello\""}, - {"Message", "{ a: 123 }"}, - {"MessageAngleBrackets", "< a: 123 >"}, - {"RepeatedInteger", "[-123, 456]"}, - {"RepeatedFloat", "[0.123, 1e-10] "}, - {"RepeatedString", R"pb(["hello", "world"])pb"}, - }) { - RunValidTextFormatTest(absl::StrCat("ReservedFieldName.", test_case.first), - REQUIRED, - absl::StrCat("reserved_field: ", test_case.second)); - // TODO: Add a test for reserved field numbers on 999999. - } - - // Map fields - MessageType prototype; - (*prototype.mutable_map_string_string())["c"] = "value"; - (*prototype.mutable_map_string_string())["b"] = "value"; - (*prototype.mutable_map_string_string())["a"] = "value"; - RunValidTextFormatTestWithMessage("AlphabeticallySortedMapStringKeys", - REQUIRED, - R"( - map_string_string { - key: "a" - value: "value" - } - map_string_string { - key: "b" - value: "value" - } - map_string_string { - key: "c" - value: "value" - } - )", - prototype); - - prototype.Clear(); - (*prototype.mutable_map_int32_int32())[3] = 0; - (*prototype.mutable_map_int32_int32())[2] = 0; - (*prototype.mutable_map_int32_int32())[1] = 0; - RunValidTextFormatTestWithMessage("AlphabeticallySortedMapIntKeys", REQUIRED, - R"( - map_int32_int32 { - key: 1 - value: 0 - } - map_int32_int32 { - key: 2 - value: 0 - } - map_int32_int32 { - key: 3 - value: 0 - } - )", - prototype); - - prototype.Clear(); - (*prototype.mutable_map_bool_bool())[true] = false; - (*prototype.mutable_map_bool_bool())[false] = false; - RunValidTextFormatTestWithMessage("AlphabeticallySortedMapBoolKeys", REQUIRED, - R"( - map_bool_bool { - key: false - value: false - } - map_bool_bool { - key: true - value: false - } - )", - prototype); - - prototype.Clear(); - ConformanceRequestSetting setting_map( - REQUIRED, ::conformance::TEXT_FORMAT, ::conformance::PROTOBUF, - ::conformance::TEXT_FORMAT_TEST, prototype, "DuplicateMapKey", R"( - map_string_nested_message { - key: "duplicate" - value: { a: 123 } - } - map_string_nested_message { - key: "duplicate" - value: { corecursive: {} } - } - )"); - // The last-specified value will be retained in a parsed map - suite_.RunValidInputTest(setting_map, R"( - map_string_nested_message { - key: "duplicate" - value: { corecursive: {} } - } - )"); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunAnyTests() { - // Any fields - RunValidTextFormatTest("AnyField", REQUIRED, - R"( - optional_any: { - [type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] - { optional_int32: 12345 - } - } - )"); - RunValidTextFormatTest("AnyFieldWithRawBytes", REQUIRED, - R"( - optional_any: { - type_url: - "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3" value: - "\b\271`" - } - )"); - ExpectParseFailure("AnyFieldWithInvalidType", REQUIRED, - R"( - optional_any: { - [type.googleapis.com/unknown] { - optional_int32: 12345 - } - } - )"); - RunValidTextFormatTest("AnyFieldWithCustomTypeUrlPrefix", REQUIRED, - R"( - optional_any: { - [non.default.domain/protobuf_test_messages.proto3.TestAllTypesProto3] - { - optional_int32: 12345 - } - } - )"); - RunValidTextFormatTest("AnyFieldWithSpecialCharactersInTypeUrlPrefix", - REQUIRED, - R"( - optional_any: { - [non.default.domain/sub-path_0/~!$&()*+,;=/protobuf_test_messages.proto3.TestAllTypesProto3] - { - optional_int32: 12345 - } - } - )"); - RunValidTextFormatTest("AnyFieldWithValidUrlPercentEscapeInTypeUrlPrefix", - REQUIRED, - R"( - optional_any: { - [non.default.domain/%2F/protobuf_test_messages.proto3.TestAllTypesProto3] - { - optional_int32: 12345 - } - } - )"); - ExpectParseFailure("AnyFieldWithInvalidUrlPercentEscapeInTypeUrlPrefix", - REQUIRED, - R"( - optional_any: { - [non.default.domain/%ZZ/protobuf_test_messages.proto3.TestAllTypesProto3] - { - optional_int32: 12345 - } - } - )"); - RunValidTextFormatTest( - "AnyFieldWithWhitespaceInTypeUrl", REQUIRED, - "optional_any: {\n" - " [ ty pe.go\nogleap\tis.com/\n" - " proto buf_te\nst_messages.proto3.Test\tAllTypesProto3 ]\n" - " {\n" - " optional_int32: 12345\n" - " }\n" - "}"); - RunValidTextFormatTest( - "AnyFieldWithCommentsInTypeUrl", REQUIRED, - "optional_any: {\n" - " [type.google # comment \napis.com/" - "protobuf_test_messages.proto3.Test # comment \nAllTypesProto3]" - " {\n" - " optional_int32: 12345\n" - " }\n" - "}"); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl< - MessageType>::RunTextFormatPerformanceTests() { - TestTextFormatPerformanceMergeMessageWithRepeatedField("Bool", - "repeated_bool: true"); - TestTextFormatPerformanceMergeMessageWithRepeatedField( - "Double", "repeated_double: 123"); - TestTextFormatPerformanceMergeMessageWithRepeatedField( - "Int32", "repeated_uint32: 123"); - TestTextFormatPerformanceMergeMessageWithRepeatedField( - "Int64", "repeated_uint64: 123"); - TestTextFormatPerformanceMergeMessageWithRepeatedField( - "String", R"(repeated_string: "foo")"); - TestTextFormatPerformanceMergeMessageWithRepeatedField( - "Bytes", R"(repeated_bytes: "foo")"); -} - -// This is currently considered valid input by some languages but not others -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>:: - TestTextFormatPerformanceMergeMessageWithRepeatedField( - const std::string& test_type_name, const std::string& message_field) { - std::string recursive_message = - absl::StrCat("recursive_message { ", message_field, " }"); - - std::string input; - for (size_t i = 0; i < kPerformanceRepeatCount; i++) { - absl::StrAppend(&input, recursive_message); - } - - std::string expected = "recursive_message { "; - for (size_t i = 0; i < kPerformanceRepeatCount; i++) { - absl::StrAppend(&expected, message_field, " "); - } - absl::StrAppend(&expected, "}"); - - RunValidTextFormatTestWithExpected( - absl::StrCat("TestTextFormatPerformanceMergeMessageWithRepeatedField", - test_type_name), - RECOMMENDED, input, expected); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunOpenEnumTests() { - RunValidTextFormatTest("ClosedEnumFieldByNumber", REQUIRED, - R"( - optional_nested_enum: 1 - )"); - RunValidTextFormatTest("ClosedEnumFieldWithUnknownNumber", REQUIRED, - R"( - optional_nested_enum: 42 - )"); -} - -template <typename MessageType> -void TextFormatConformanceTestSuiteImpl<MessageType>::RunClosedEnumTests() { - RunValidTextFormatTest("ClosedEnumFieldByNumber", REQUIRED, - R"( - optional_nested_enum: 1 - )"); - ExpectParseFailure("ClosedEnumFieldWithUnknownNumber", REQUIRED, - R"( - optional_nested_enum: 42 - )"); -} - -} // namespace conformance -} // namespace protobuf -} // namespace google
diff --git a/conformance/text_format_conformance_suite.h b/conformance/text_format_conformance_suite.h deleted file mode 100644 index dd5cffc..0000000 --- a/conformance/text_format_conformance_suite.h +++ /dev/null
@@ -1,84 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file or at -// https://developers.google.com/open-source/licenses/bsd - -#ifndef GOOGLE_PROTOBUF_CONFORMANCE_TEXT_FORMAT_CONFORMANCE_SUITE_H__ -#define GOOGLE_PROTOBUF_CONFORMANCE_TEXT_FORMAT_CONFORMANCE_SUITE_H__ - -#include <string> - -#include "conformance/conformance_test.h" -#include "google/protobuf/message.h" - -namespace google { -namespace protobuf { -namespace conformance { - -class TextFormatConformanceTestSuite : public ConformanceTestSuite { - public: - TextFormatConformanceTestSuite(); - - private: - void RunSuiteImpl() override; - - bool ParseTextFormatResponse( - const ::conformance::ConformanceResponse& response, - const ConformanceRequestSetting& setting, Message* test_message); - bool ParseResponse(const ::conformance::ConformanceResponse& response, - const ConformanceRequestSetting& setting, - Message* test_message) override; - - template <typename MessageType> - friend class TextFormatConformanceTestSuiteImpl; -}; - -template <typename MessageType> -class TextFormatConformanceTestSuiteImpl { - public: - explicit TextFormatConformanceTestSuiteImpl( - TextFormatConformanceTestSuite* suite); - - private: - using ConformanceRequestSetting = - TextFormatConformanceTestSuite::ConformanceRequestSetting; - using ConformanceLevel = TextFormatConformanceTestSuite::ConformanceLevel; - constexpr static ConformanceLevel RECOMMENDED = ConformanceLevel::RECOMMENDED; - constexpr static ConformanceLevel REQUIRED = ConformanceLevel::REQUIRED; - - void RunAllTests(); - - void RunDelimitedTests(); - void RunGroupTests(); - void RunAnyTests(); - void RunOpenEnumTests(); - void RunClosedEnumTests(); - - void RunTextFormatPerformanceTests(); - void RunValidTextFormatTest(const std::string& test_name, - ConformanceLevel level, const std::string& input); - void RunValidTextFormatTestWithExpected(const std::string& test_name, - ConformanceLevel level, - const std::string& input_text, - const std::string& expected_text); - void RunValidUnknownTextFormatTest(const std::string& test_name, - const Message& message); - void RunValidTextFormatTestWithMessage(const std::string& test_name, - ConformanceLevel level, - const std::string& input_text, - const Message& message); - void ExpectParseFailure(const std::string& test_name, ConformanceLevel level, - const std::string& input); - void TestTextFormatPerformanceMergeMessageWithRepeatedField( - const std::string& test_type_name, const std::string& message_field); - - TextFormatConformanceTestSuite& suite_; -}; - -} // namespace conformance -} // namespace protobuf -} // namespace google - -#endif // GOOGLE_PROTOBUF_CONFORMANCE_TEXT_FORMAT_CONFORMANCE_SUITE_H__
diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt index 80dcee2..485591f 100644 --- a/conformance/text_format_failure_list_java.txt +++ b/conformance/text_format_failure_list_java.txt
@@ -1,7 +1,7 @@ -Recommended.*.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. -Recommended.*.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. -Recommended.*.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. -Recommended.*.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # TEXT_FORMAT output we received from test was unparseable. +Recommended.*.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput # Output was not equivalent to reference message: added: optionalgroup: { a: 321 } +Recommended.*.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput # Output was not equivalent to reference message: added: nested_message: { c: 111 } +Recommended.*.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput # Output was not equivalent to reference message: added: optionalgroup: { a: 321 }added: repeated_int32[0]: 1added: repeated_int32 +Recommended.*.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput # Output was not equivalent to reference message: added: optional_int32: 123added: optional_string: "hello"added: optional_bool: t Required.*.TextFormatInput.AnyField.ProtobufOutput # Failed to parse input or produce output. Required.*.TextFormatInput.AnyField.TextFormatOutput # Failed to parse input or produce output. Required.*.TextFormatInput.FloatFieldNoNegativeOctal # Should have failed to parse, but didn't.
diff --git a/conformance/text_group_test.cc b/conformance/text_group_test.cc new file mode 100644 index 0000000..dea3cae --- /dev/null +++ b/conformance/text_group_test.cc
@@ -0,0 +1,330 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for group fields (proto2-style message types) +// and for their editions successor, delimited-encoded message fields +// (TestAllTypesEdition2023). This replaces the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunGroupTests() and +// RunDelimitedTests(); the test names and the requests sent to the testee are +// identical to the legacy ones. + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// Group fields. In the test message types, "Data" is a group field with an +// int32 field group_int32, "MultiWordGroupField" is a group with the same +// field, and "groupfield" is an extension of group type GroupField. +using TextGroupTest = MessageTypeConformanceTest; + +TEST_P(TextGroupTest, NoColon) { + EXPECT_THAT( + Testee("GroupFieldNoColon") + .ParseText(message(), R"pb(Data { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldNoColon") + .ParseText(message(), R"pb(Data { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data { group_int32: 1 })pb")))); +} + +TEST_P(TextGroupTest, WithColon) { + EXPECT_THAT( + Testee("GroupFieldWithColon") + .ParseText(message(), R"pb(Data: { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data: { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldWithColon") + .ParseText(message(), R"pb(Data: { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data: { group_int32: 1 })pb")))); +} + +TEST_P(TextGroupTest, Empty) { + EXPECT_THAT(Testee("GroupFieldEmpty") + .ParseText(message(), R"pb(Data {})pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data {})pb")))); + EXPECT_THAT(Testee("GroupFieldEmpty") + .ParseText(message(), R"pb(Data {})pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(Data {})pb")))); +} + +TEST_P(TextGroupTest, MultiWord) { + EXPECT_THAT( + Testee("GroupFieldMultiWord") + .ParseText(message(), R"pb(MultiWordGroupField { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb( + MultiWordGroupField { group_int32: 1 } + )pb")))); + EXPECT_THAT( + Testee("GroupFieldMultiWord") + .ParseText(message(), R"pb(MultiWordGroupField { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb( + MultiWordGroupField { group_int32: 1 } + )pb")))); +} + +// The lower-cased group name (i.e. the implicit field name) is accepted. +TEST_P(TextGroupTest, Lowercased) { + EXPECT_THAT( + Testee("GroupFieldLowercased") + .ParseText(message(), R"pb(data { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(data { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldLowercased") + .ParseText(message(), R"pb(data { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(data { group_int32: 1 })pb")))); +} + +TEST_P(TextGroupTest, LowercasedMultiWord) { + EXPECT_THAT( + Testee("GroupFieldLowercasedMultiWord") + .ParseText(message(), R"pb(multiwordgroupfield { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb( + multiwordgroupfield { group_int32: 1 } + )pb")))); + EXPECT_THAT( + Testee("GroupFieldLowercasedMultiWord") + .ParseText(message(), R"pb(multiwordgroupfield { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb( + multiwordgroupfield { group_int32: 1 } + )pb")))); +} + +// An extension of group type is named by the extension field. +TEST_P(TextGroupTest, Extension) { + const FieldDescriptor* extension = + message()->file()->FindExtensionByName("groupfield"); + ASSERT_NE(extension, nullptr); + const std::string input = absl::StrCat( + "[", extension->PrintableNameForExtension(), "] { group_int32: 1 }"); + EXPECT_THAT(Testee("GroupFieldExtension") + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT( + Testee("GroupFieldExtension").ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// ... and never by the name of its group message type. +TEST_P(TextGroupTest, ExtensionGroupName) { + const Descriptor* group = + message()->file()->FindMessageTypeByName("GroupField"); + ASSERT_NE(group, nullptr); + EXPECT_THAT(Testee("GroupFieldExtensionGroupName") + .ParseText(message(), absl::StrCat("[", group->full_name(), + "] { group_int32: 1 }")) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextGroupTest, ValuesIn(Proto2TestMessageTypes()), + MessageTypeParamName); + +// Delimited-encoded message fields of TestAllTypesEdition2023 (message()): +// groupliketype and delimited_field are fields of the nested message type +// GroupLikeType (whose group-like name may stand in for the field name), and +// the groupliketype and delimited_ext extensions are of the top-level +// GroupLikeType. The fixture skips itself when --maximum_edition doesn't +// cover editions. +using TextDelimitedTest = Edition2023ConformanceTest; + +TEST_F(TextDelimitedTest, GroupFieldNoColon) { + EXPECT_THAT( + Testee("GroupFieldNoColon") + .ParseText(message(), R"pb(GroupLikeType { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(GroupLikeType { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldNoColon") + .ParseText(message(), R"pb(GroupLikeType { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(GroupLikeType { group_int32: 1 })pb")))); +} + +TEST_F(TextDelimitedTest, GroupFieldWithColon) { + EXPECT_THAT( + Testee("GroupFieldWithColon") + .ParseText(message(), R"pb(GroupLikeType: { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(GroupLikeType: { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldWithColon") + .ParseText(message(), R"pb(GroupLikeType: { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(GroupLikeType: { group_int32: 1 })pb")))); +} + +TEST_F(TextDelimitedTest, GroupFieldEmpty) { + EXPECT_THAT(Testee("GroupFieldEmpty") + .ParseText(message(), R"pb(GroupLikeType {})pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(GroupLikeType {})pb")))); + EXPECT_THAT(Testee("GroupFieldEmpty") + .ParseText(message(), R"pb(GroupLikeType {})pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(GroupLikeType {})pb")))); +} + +TEST_F(TextDelimitedTest, GroupFieldExtension) { + EXPECT_THAT( + Testee("GroupFieldExtension") + .ParseText( + message(), + R"pb([protobuf_test_messages.editions.groupliketype] { c: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + R"pb([protobuf_test_messages.editions.groupliketype] { c: 1 })pb")))); + EXPECT_THAT( + Testee("GroupFieldExtension") + .ParseText( + message(), + R"pb([protobuf_test_messages.editions.groupliketype] { c: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto( + R"pb([protobuf_test_messages.editions.groupliketype] { c: 1 })pb")))); +} + +TEST_F(TextDelimitedTest, DelimitedFieldExtension) { + EXPECT_THAT( + Testee("DelimitedFieldExtension") + .ParseText( + message(), + R"pb([protobuf_test_messages.editions.delimited_ext] { c: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + R"pb([protobuf_test_messages.editions.delimited_ext] { c: 1 })pb")))); + EXPECT_THAT( + Testee("DelimitedFieldExtension") + .ParseText( + message(), + R"pb([protobuf_test_messages.editions.delimited_ext] { c: 1 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto( + R"pb([protobuf_test_messages.editions.delimited_ext] { c: 1 })pb")))); +} + + +// The lower-cased group name (i.e. the implicit field name) is accepted. +TEST_F(TextDelimitedTest, DelimitedFieldLowercased) { + EXPECT_THAT( + Testee("DelimitedFieldLowercased") + .ParseText(message(), R"pb(groupliketype { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(groupliketype { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("DelimitedFieldLowercased") + .ParseText(message(), R"pb(groupliketype { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(groupliketype { group_int32: 1 })pb")))); +} + +TEST_F(TextDelimitedTest, DelimitedFieldLowercasedDifferent) { + EXPECT_THAT( + Testee("DelimitedFieldLowercasedDifferent") + .ParseText(message(), R"pb(delimited_field { group_int32: 1 })pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(delimited_field { group_int32: 1 })pb")))); + EXPECT_THAT( + Testee("DelimitedFieldLowercasedDifferent") + .ParseText(message(), R"pb(delimited_field { group_int32: 1 })pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(delimited_field { group_int32: 1 })pb")))); +} + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. + +// Extensions always use the field name, and never accept the message name. +TEST_F(TextDelimitedTest, DelimitedFieldExtensionMessageName) { + EXPECT_THAT(Testee("DelimitedFieldExtensionMessageName") + .ParseText(message(), + "[protobuf_test_messages.editions.GroupLikeType] " + "{ group_int32: 1 }") + .ParseOnly(), + Yields(IsParseError())); +} + +// Extension names can contain whitespace and comments. +TEST_F(TextDelimitedTest, ExtensionNameWithWhitespace) { + EXPECT_THAT( + Testee("ExtensionNameWithWhitespace") + .ParseText( + message(), + "[protobuf _test_messages.edit\tions.exten\nsion_int32]: 1") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto( + "[protobuf _test_messages.edit\tions.exten\nsion_int32]: 1")))); + EXPECT_THAT( + Testee("ExtensionNameWithWhitespace") + .ParseText( + message(), + "[protobuf _test_messages.edit\tions.exten\nsion_int32]: 1") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto( + "[protobuf _test_messages.edit\tions.exten\nsion_int32]: 1")))); +} + +TEST_F(TextDelimitedTest, ExtensionNameWithComment) { + EXPECT_THAT(Testee("ExtensionNameWithComment") + .ParseText(message(), + "[protobuf_test_messages.edit # comment " + "\nions.extension_int32]: 1") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("[protobuf_test_messages.edit # comment " + "\nions.extension_int32]: 1")))); + EXPECT_THAT(Testee("ExtensionNameWithComment") + .ParseText(message(), + "[protobuf_test_messages.edit # comment " + "\nions.extension_int32]: 1") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("[protobuf_test_messages.edit # comment " + "\nions.extension_int32]: 1")))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_hello_world_test.cc b/conformance/text_hello_world_test.cc new file mode 100644 index 0000000..d812355 --- /dev/null +++ b/conformance/text_hello_world_test.cc
@@ -0,0 +1,58 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The smoke test of the text-format conformance suite: a single string field +// round-trips through the testee. This replaces the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunAllTests()'s "HelloWorld" test, +// which ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// Like the legacy RunValidTextFormatTest(), every valid-input test here and in +// the other text_*_test.cc files has two legs, in the legacy order: the input +// is parsed and serialized as binary ("<name>.ProtobufOutput"), then parsed +// and serialized as text format ("<name>.TextFormatOutput"), and each result +// must be equivalent to the expected message. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using TextHelloWorldTest = MessageTypeConformanceTest; + +TEST_P(TextHelloWorldTest, HelloWorld) { + EXPECT_THAT( + Testee("HelloWorld") + .ParseText(message(), R"pb(optional_string: 'Hello, World!')pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_string: 'Hello, World!')pb")))); + EXPECT_THAT( + Testee("HelloWorld") + .ParseText(message(), R"pb(optional_string: 'Hello, World!')pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_string: 'Hello, World!')pb")))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextHelloWorldTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_integer_test.cc b/conformance/text_integer_test.cc new file mode 100644 index 0000000..140f583 --- /dev/null +++ b/conformance/text_integer_test.cc
@@ -0,0 +1,431 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for integer fields: the extreme values of +// each integer type in decimal, hex and octal, and rejection of values just +// past them. This replaces the integer blocks of the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunAllTests(); the test names and +// the requests sent to the testee are identical to the legacy ones. +// +// The legacy suite ran RunAllTests() only for TestAllTypesProto3, which its +// RunSuiteImpl() instantiated in its proto3 and editions flavours, so these +// run over Proto3TestMessageTypes() (like HelloWorld). + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using TextIntegerTest = MessageTypeConformanceTest; + +// Extreme values in decimal. +TEST_P(TextIntegerTest, Int32FieldMaxValue) { + EXPECT_THAT( + Testee("Int32FieldMaxValue") + .ParseText(message(), R"pb(optional_int32: 2147483647)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); + EXPECT_THAT( + Testee("Int32FieldMaxValue") + .ParseText(message(), R"pb(optional_int32: 2147483647)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); +} + +TEST_P(TextIntegerTest, Int32FieldMinValue) { + EXPECT_THAT(Testee("Int32FieldMinValue") + .ParseText(message(), R"pb(optional_int32: -2147483648)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); + EXPECT_THAT(Testee("Int32FieldMinValue") + .ParseText(message(), R"pb(optional_int32: -2147483648)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); +} + +TEST_P(TextIntegerTest, Uint32FieldMaxValue) { + EXPECT_THAT(Testee("Uint32FieldMaxValue") + .ParseText(message(), R"pb(optional_uint32: 4294967295)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); + EXPECT_THAT(Testee("Uint32FieldMaxValue") + .ParseText(message(), R"pb(optional_uint32: 4294967295)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); +} + +TEST_P(TextIntegerTest, Int64FieldMaxValue) { + EXPECT_THAT( + Testee("Int64FieldMaxValue") + .ParseText(message(), R"pb(optional_int64: 9223372036854775807)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); + EXPECT_THAT( + Testee("Int64FieldMaxValue") + .ParseText(message(), R"pb(optional_int64: 9223372036854775807)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); +} + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. +TEST_P(TextIntegerTest, Int64FieldMinValue) { + EXPECT_THAT(Testee("Int64FieldMinValue") + .ParseText(message(), "optional_int64: -9223372036854775808") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_int64: -9223372036854775808")))); + EXPECT_THAT(Testee("Int64FieldMinValue") + .ParseText(message(), "optional_int64: -9223372036854775808") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_int64: -9223372036854775808")))); +} + +TEST_P(TextIntegerTest, Uint64FieldMaxValue) { + EXPECT_THAT(Testee("Uint64FieldMaxValue") + .ParseText(message(), "optional_uint64: 18446744073709551615") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); + EXPECT_THAT(Testee("Uint64FieldMaxValue") + .ParseText(message(), "optional_uint64: 18446744073709551615") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); +} + +// Extreme values in hex; the responses are compared to the decimal form. +TEST_P(TextIntegerTest, Int32FieldMaxValueHex) { + EXPECT_THAT( + Testee("Int32FieldMaxValueHex") + .ParseText(message(), R"pb(optional_int32: 0x7FFFFFFF)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); + EXPECT_THAT( + Testee("Int32FieldMaxValueHex") + .ParseText(message(), R"pb(optional_int32: 0x7FFFFFFF)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); +} + +TEST_P(TextIntegerTest, Int32FieldMinValueHex) { + EXPECT_THAT(Testee("Int32FieldMinValueHex") + .ParseText(message(), R"pb(optional_int32: -0x80000000)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); + EXPECT_THAT(Testee("Int32FieldMinValueHex") + .ParseText(message(), R"pb(optional_int32: -0x80000000)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); +} + +TEST_P(TextIntegerTest, Uint32FieldMaxValueHex) { + EXPECT_THAT(Testee("Uint32FieldMaxValueHex") + .ParseText(message(), R"pb(optional_uint32: 0xFFFFFFFF)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); + EXPECT_THAT(Testee("Uint32FieldMaxValueHex") + .ParseText(message(), R"pb(optional_uint32: 0xFFFFFFFF)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); +} + +TEST_P(TextIntegerTest, Int64FieldMaxValueHex) { + EXPECT_THAT( + Testee("Int64FieldMaxValueHex") + .ParseText(message(), R"pb(optional_int64: 0x7FFFFFFFFFFFFFFF)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); + EXPECT_THAT( + Testee("Int64FieldMaxValueHex") + .ParseText(message(), R"pb(optional_int64: 0x7FFFFFFFFFFFFFFF)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); +} + +TEST_P(TextIntegerTest, Int64FieldMinValueHex) { + EXPECT_THAT( + Testee("Int64FieldMinValueHex") + .ParseText(message(), R"pb(optional_int64: -0x8000000000000000)pb") + .SerializeBinary(), + Yields( + WhenParsed(EqualsTextProto("optional_int64: -9223372036854775808")))); + EXPECT_THAT( + Testee("Int64FieldMinValueHex") + .ParseText(message(), R"pb(optional_int64: -0x8000000000000000)pb") + .SerializeText(), + Yields( + WhenParsed(EqualsTextProto("optional_int64: -9223372036854775808")))); +} + +TEST_P(TextIntegerTest, Uint64FieldMaxValueHex) { + EXPECT_THAT( + Testee("Uint64FieldMaxValueHex") + .ParseText(message(), R"pb(optional_uint64: 0xFFFFFFFFFFFFFFFF)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); + EXPECT_THAT( + Testee("Uint64FieldMaxValueHex") + .ParseText(message(), R"pb(optional_uint64: 0xFFFFFFFFFFFFFFFF)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); +} + +// Extreme values in octal. +TEST_P(TextIntegerTest, Int32FieldMaxValueOctal) { + EXPECT_THAT( + Testee("Int32FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_int32: 017777777777)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); + EXPECT_THAT( + Testee("Int32FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_int32: 017777777777)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 2147483647)pb")))); +} + +TEST_P(TextIntegerTest, Int32FieldMinValueOctal) { + EXPECT_THAT(Testee("Int32FieldMinValueOctal") + .ParseText(message(), R"pb(optional_int32: -020000000000)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); + EXPECT_THAT(Testee("Int32FieldMinValueOctal") + .ParseText(message(), R"pb(optional_int32: -020000000000)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int32: -2147483648)pb")))); +} + +TEST_P(TextIntegerTest, Uint32FieldMaxValueOctal) { + EXPECT_THAT(Testee("Uint32FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_uint32: 037777777777)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); + EXPECT_THAT(Testee("Uint32FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_uint32: 037777777777)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_uint32: 4294967295)pb")))); +} + +TEST_P(TextIntegerTest, Int64FieldMaxValueOctal) { + EXPECT_THAT( + Testee("Int64FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_int64: 0777777777777777777777)pb") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); + EXPECT_THAT( + Testee("Int64FieldMaxValueOctal") + .ParseText(message(), R"pb(optional_int64: 0777777777777777777777)pb") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"pb(optional_int64: 9223372036854775807)pb")))); +} + +TEST_P(TextIntegerTest, Int64FieldMinValueOctal) { + EXPECT_THAT( + Testee("Int64FieldMinValueOctal") + .ParseText(message(), "optional_int64: -01000000000000000000000") + .SerializeBinary(), + Yields( + WhenParsed(EqualsTextProto("optional_int64: -9223372036854775808")))); + EXPECT_THAT( + Testee("Int64FieldMinValueOctal") + .ParseText(message(), "optional_int64: -01000000000000000000000") + .SerializeText(), + Yields( + WhenParsed(EqualsTextProto("optional_int64: -9223372036854775808")))); +} + +TEST_P(TextIntegerTest, Uint64FieldMaxValueOctal) { + EXPECT_THAT( + Testee("Uint64FieldMaxValueOctal") + .ParseText(message(), "optional_uint64: 01777777777777777777777") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); + EXPECT_THAT( + Testee("Uint64FieldMaxValueOctal") + .ParseText(message(), "optional_uint64: 01777777777777777777777") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_uint64: 18446744073709551615")))); +} + +// Parsers reject out-of-bound integer values. +TEST_P(TextIntegerTest, Int32FieldTooLarge) { + EXPECT_THAT(Testee("Int32FieldTooLarge") + .ParseText(message(), R"pb(optional_int32: 2147483648)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int32FieldTooSmall) { + EXPECT_THAT(Testee("Int32FieldTooSmall") + .ParseText(message(), R"pb(optional_int32: -2147483649)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint32FieldTooLarge) { + EXPECT_THAT(Testee("Uint32FieldTooLarge") + .ParseText(message(), R"pb(optional_uint32: 4294967296)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooLarge) { + EXPECT_THAT( + Testee("Int64FieldTooLarge") + .ParseText(message(), R"pb(optional_int64: 9223372036854775808)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooSmall) { + EXPECT_THAT(Testee("Int64FieldTooSmall") + .ParseText(message(), "optional_int64: -9223372036854775809") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint64FieldTooLarge) { + EXPECT_THAT(Testee("Uint64FieldTooLarge") + .ParseText(message(), "optional_uint64: 18446744073709551616") + .ParseOnly(), + Yields(IsParseError())); +} + +// ... in hex +TEST_P(TextIntegerTest, Int32FieldTooLargeHex) { + EXPECT_THAT(Testee("Int32FieldTooLargeHex") + .ParseText(message(), R"pb(optional_int32: 0x80000000)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int32FieldTooSmallHex) { + EXPECT_THAT(Testee("Int32FieldTooSmallHex") + .ParseText(message(), R"pb(optional_int32: -0x80000001)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint32FieldTooLargeHex) { + EXPECT_THAT(Testee("Uint32FieldTooLargeHex") + .ParseText(message(), R"pb(optional_uint32: 0x100000000)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooLargeHex) { + EXPECT_THAT( + Testee("Int64FieldTooLargeHex") + .ParseText(message(), R"pb(optional_int64: 0x8000000000000000)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooSmallHex) { + EXPECT_THAT( + Testee("Int64FieldTooSmallHex") + .ParseText(message(), R"pb(optional_int64: -0x8000000000000001)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint64FieldTooLargeHex) { + EXPECT_THAT(Testee("Uint64FieldTooLargeHex") + .ParseText(message(), "optional_uint64: 0x10000000000000000") + .ParseOnly(), + Yields(IsParseError())); +} + +// ... and in octal. +TEST_P(TextIntegerTest, Int32FieldTooLargeOctal) { + EXPECT_THAT(Testee("Int32FieldTooLargeOctal") + .ParseText(message(), R"pb(optional_int32: 020000000000)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int32FieldTooSmallOctal) { + EXPECT_THAT(Testee("Int32FieldTooSmallOctal") + .ParseText(message(), R"pb(optional_int32: -020000000001)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint32FieldTooLargeOctal) { + EXPECT_THAT(Testee("Uint32FieldTooLargeOctal") + .ParseText(message(), R"pb(optional_uint32: 040000000000)pb") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooLargeOctal) { + EXPECT_THAT( + Testee("Int64FieldTooLargeOctal") + .ParseText(message(), "optional_int64: 01000000000000000000000") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Int64FieldTooSmallOctal) { + EXPECT_THAT( + Testee("Int64FieldTooSmallOctal") + .ParseText(message(), "optional_int64: -01000000000000000000001") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextIntegerTest, Uint64FieldTooLargeOctal) { + EXPECT_THAT( + Testee("Uint64FieldTooLargeOctal") + .ParseText(message(), "optional_uint64: 02000000000000000000000") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextIntegerTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_map_test.cc b/conformance/text_map_test.cc new file mode 100644 index 0000000..b86950f --- /dev/null +++ b/conformance/text_map_test.cc
@@ -0,0 +1,141 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for map fields: entries given in key order +// round-trip, and a duplicated key keeps the last value. This replaces the +// legacy map block of TextFormatConformanceTestSuiteImpl<M>::RunAllTests(), +// which ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones. +// +// The legacy tests passed a prototype pre-filled with the map entries in +// reverse order to RunValidTextFormatTestWithMessage(); that had no effect +// (the legacy ConformanceRequestSetting only took the prototype's descriptor +// and compared against prototype.New()), so nothing replaces it here. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +using TextMapTest = MessageTypeConformanceTest; + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. +TEST_P(TextMapTest, AlphabeticallySortedMapStringKeys) { + constexpr absl::string_view kInput = R"( + map_string_string { + key: "a" + value: "value" + } + map_string_string { + key: "b" + value: "value" + } + map_string_string { + key: "c" + value: "value" + } + )"; + EXPECT_THAT(Testee("AlphabeticallySortedMapStringKeys") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AlphabeticallySortedMapStringKeys") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextMapTest, AlphabeticallySortedMapIntKeys) { + constexpr absl::string_view kInput = R"( + map_int32_int32 { + key: 1 + value: 0 + } + map_int32_int32 { + key: 2 + value: 0 + } + map_int32_int32 { + key: 3 + value: 0 + } + )"; + EXPECT_THAT(Testee("AlphabeticallySortedMapIntKeys") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AlphabeticallySortedMapIntKeys") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +TEST_P(TextMapTest, AlphabeticallySortedMapBoolKeys) { + constexpr absl::string_view kInput = R"( + map_bool_bool { + key: false + value: false + } + map_bool_bool { + key: true + value: false + } + )"; + EXPECT_THAT(Testee("AlphabeticallySortedMapBoolKeys") + .ParseText(message(), kInput) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kInput)))); + EXPECT_THAT(Testee("AlphabeticallySortedMapBoolKeys") + .ParseText(message(), kInput) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(kInput)))); +} + +// The last-specified value will be retained in a parsed map. Like the legacy +// test, this only checks the binary output. +TEST_P(TextMapTest, DuplicateMapKey) { + constexpr absl::string_view kInput = R"( + map_string_nested_message { + key: "duplicate" + value: { a: 123 } + } + map_string_nested_message { + key: "duplicate" + value: { corecursive: {} } + } + )"; + constexpr absl::string_view kExpected = R"pb( + map_string_nested_message { + key: "duplicate" + value: { corecursive: {} } + } + )pb"; + EXPECT_THAT( + Testee("DuplicateMapKey").ParseText(message(), kInput).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(kExpected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextMapTest, ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_performance_test.cc b/conformance/text_performance_test.cc new file mode 100644 index 0000000..704cbd9 --- /dev/null +++ b/conformance/text_performance_test.cc
@@ -0,0 +1,151 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format performance conformance tests: tens of thousands of occurrences +// of recursive_message, each holding one repeated field entry, that have to be +// merged into a single recursive_message. Part of the performance suite +// (see conformance.bzl). This replaces the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunTextFormatPerformanceTests() and +// its helper, which ran for the four TestAllTypes{Proto2,Proto3} message types +// (proto and editions flavours); the test names and the requests sent to the +// testee are identical to the legacy ones. + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// The number of repetitions to use for performance tests. +constexpr int kPerformanceRepeatCount = 50000; + +// The input of the legacy +// TestTextFormatPerformanceMergeMessageWithRepeatedField() test for `field`, +// one entry of a repeated field of recursive_message: "recursive_message { +// <field> }" repeated kPerformanceRepeatCount times. +std::string RepeatedRecursiveMessageInput(absl::string_view field) { + const std::string recursive_message = + absl::StrCat("recursive_message { ", field, " }"); + std::string input; + for (int i = 0; i < kPerformanceRepeatCount; ++i) { + absl::StrAppend(&input, recursive_message); + } + return input; +} + +// The expected result of parsing RepeatedRecursiveMessageInput(field): a +// single recursive_message holding all kPerformanceRepeatCount entries. +std::string MergedRecursiveMessage(absl::string_view field) { + std::string expected = "recursive_message { "; + for (int i = 0; i < kPerformanceRepeatCount; ++i) { + absl::StrAppend(&expected, field, " "); + } + absl::StrAppend(&expected, "}"); + return expected; +} + +// Parameterized over the test message type only. +class TextPerformanceTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldBool) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldBool"; + const std::string input = + RepeatedRecursiveMessageInput("repeated_bool: true"); + const std::string expected = MergedRecursiveMessage("repeated_bool: true"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldDouble) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldDouble"; + const std::string input = + RepeatedRecursiveMessageInput("repeated_double: 123"); + const std::string expected = MergedRecursiveMessage("repeated_double: 123"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +// The legacy Int32/Int64 tests actually exercise repeated_uint32 and +// repeated_uint64; the names are kept so failure-list entries keep matching. +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldInt32) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt32"; + const std::string input = + RepeatedRecursiveMessageInput("repeated_uint32: 123"); + const std::string expected = MergedRecursiveMessage("repeated_uint32: 123"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldInt64) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldInt64"; + const std::string input = + RepeatedRecursiveMessageInput("repeated_uint64: 123"); + const std::string expected = MergedRecursiveMessage("repeated_uint64: 123"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldString) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldString"; + const std::string input = + RepeatedRecursiveMessageInput(R"(repeated_string: "foo")"); + const std::string expected = + MergedRecursiveMessage(R"(repeated_string: "foo")"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +TEST_P(TextPerformanceTest, MergeMessageWithRepeatedFieldBytes) { + constexpr absl::string_view kName = + "TestTextFormatPerformanceMergeMessageWithRepeatedFieldBytes"; + const std::string input = + RepeatedRecursiveMessageInput(R"(repeated_bytes: "foo")"); + const std::string expected = + MergedRecursiveMessage(R"(repeated_bytes: "foo")"); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(expected)))); + EXPECT_THAT(Testee(kName).ParseText(message(), input).SerializeText(), + Yields(WhenParsed(EqualsTextProto(expected)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextPerformanceTest, + ValuesIn(AllTestMessageTypes()), MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_reserved_field_test.cc b/conformance/text_reserved_field_test.cc new file mode 100644 index 0000000..dac88a2 --- /dev/null +++ b/conformance/text_reserved_field_test.cc
@@ -0,0 +1,176 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for reserved field names: a field name that +// the message reserves (reserved_field in the test message types) is skipped +// by the parser, whatever the shape of its value. This replaces the legacy +// reserved-field block of TextFormatConformanceTestSuiteImpl<M>::RunAllTests(), +// which ran for the proto3-style message types only; the test names and the +// requests sent to the testee are identical to the legacy ones. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::ValuesIn; + +// The expected message of every test is the input itself, i.e. an empty +// message once the reserved field has been skipped. +using TextReservedFieldTest = MessageTypeConformanceTest; + +TEST_P(TextReservedFieldTest, Boolean) { + EXPECT_THAT( + Testee("ReservedFieldName.Boolean") + .ParseText(message(), R"pb(reserved_field: true)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: true)pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.Boolean") + .ParseText(message(), R"pb(reserved_field: true)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: true)pb")))); +} + +TEST_P(TextReservedFieldTest, Integer) { + EXPECT_THAT( + Testee("ReservedFieldName.Integer") + .ParseText(message(), R"pb(reserved_field: -123)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: -123)pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.Integer") + .ParseText(message(), R"pb(reserved_field: -123)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: -123)pb")))); +} + +TEST_P(TextReservedFieldTest, Float) { + EXPECT_THAT( + Testee("ReservedFieldName.Float") + .ParseText(message(), R"pb(reserved_field: 0.123)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: 0.123)pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.Float") + .ParseText(message(), R"pb(reserved_field: 0.123)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: 0.123)pb")))); +} + +TEST_P(TextReservedFieldTest, Enum) { + EXPECT_THAT( + Testee("ReservedFieldName.Enum") + .ParseText(message(), R"pb(reserved_field: ENUM_VALUE)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: ENUM_VALUE)pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.Enum") + .ParseText(message(), R"pb(reserved_field: ENUM_VALUE)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: ENUM_VALUE)pb")))); +} + +TEST_P(TextReservedFieldTest, String) { + EXPECT_THAT( + Testee("ReservedFieldName.String") + .ParseText(message(), R"pb(reserved_field: "hello")pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: "hello")pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.String") + .ParseText(message(), R"pb(reserved_field: "hello")pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: "hello")pb")))); +} + +TEST_P(TextReservedFieldTest, Message) { + EXPECT_THAT( + Testee("ReservedFieldName.Message") + .ParseText(message(), R"pb(reserved_field: { a: 123 })pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: { a: 123 })pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.Message") + .ParseText(message(), R"pb(reserved_field: { a: 123 })pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: { a: 123 })pb")))); +} + +TEST_P(TextReservedFieldTest, MessageAngleBrackets) { + EXPECT_THAT( + Testee("ReservedFieldName.MessageAngleBrackets") + .ParseText(message(), R"pb(reserved_field: < a: 123 >)pb") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: < a: 123 >)pb")))); + EXPECT_THAT( + Testee("ReservedFieldName.MessageAngleBrackets") + .ParseText(message(), R"pb(reserved_field: < a: 123 >)pb") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(R"pb(reserved_field: < a: 123 >)pb")))); +} + +// TODO: Write the plain-string inputs below as R"pb(...)pb" raw +// strings once the migration's request goldens are gone. The goldens pin the +// exact request bytes, and clang-format would reformat the whitespace of these +// inputs inside a pb raw string. +TEST_P(TextReservedFieldTest, RepeatedInteger) { + EXPECT_THAT( + Testee("ReservedFieldName.RepeatedInteger") + .ParseText(message(), "reserved_field: [-123, 456]") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("reserved_field: [-123, 456]")))); + EXPECT_THAT( + Testee("ReservedFieldName.RepeatedInteger") + .ParseText(message(), "reserved_field: [-123, 456]") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("reserved_field: [-123, 456]")))); +} + +// The trailing space of the legacy input is part of the request bytes and is +// kept. +TEST_P(TextReservedFieldTest, RepeatedFloat) { + EXPECT_THAT( + Testee("ReservedFieldName.RepeatedFloat") + .ParseText(message(), "reserved_field: [0.123, 1e-10] ") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("reserved_field: [0.123, 1e-10] ")))); + EXPECT_THAT( + Testee("ReservedFieldName.RepeatedFloat") + .ParseText(message(), "reserved_field: [0.123, 1e-10] ") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("reserved_field: [0.123, 1e-10] ")))); +} + +TEST_P(TextReservedFieldTest, RepeatedString) { + EXPECT_THAT(Testee("ReservedFieldName.RepeatedString") + .ParseText(message(), R"(reserved_field: ["hello", "world"])") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto(R"(reserved_field: ["hello", "world"])")))); + EXPECT_THAT(Testee("ReservedFieldName.RepeatedString") + .ParseText(message(), R"(reserved_field: ["hello", "world"])") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto(R"(reserved_field: ["hello", "world"])")))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextReservedFieldTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_string_test.cc b/conformance/text_string_test.cc new file mode 100644 index 0000000..d0b8400 --- /dev/null +++ b/conformance/text_string_test.cc
@@ -0,0 +1,293 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for string literals, run for a string and for +// a bytes field: concatenation, the escape sequences, the rejected line feeds +// and out-of-range or surrogate unicode escapes, and the invalid UTF-8 byte +// sequences that only string fields reject. This replaces the +// "String literals x {Strings, Bytes}" section of the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunAllTests(), which ran for the +// proto3-style message types only; the test names and the requests sent to +// the testee are identical to the legacy ones. + +#include <ostream> +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::ValuesIn; + +// The field a string literal is tested with. The legacy suite ran every +// literal test for the string field and for the bytes field, with "String" / +// "Bytes" appended to the test name. +struct StringField { + absl::string_view type_name; // "String" or "Bytes", as in the test names. + absl::string_view field_name; // The name of the field of that type. +}; + +constexpr StringField kStringFields[] = { + {"String", "optional_string"}, + {"Bytes", "optional_bytes"}, +}; + +std::string ParamName(const StringField& field) { + return std::string(field.type_name); +} + +// So a failing test prints the field by name instead of as raw bytes. +void PrintTo(const StringField& field, std::ostream* os) { + *os << field.field_name; +} + +// Parameterized over (test message type, string field). Reporting the +// message type through MessageUnderTest() makes the base SetUp() skip the +// editions instance when --maximum_edition doesn't cover it. TestName() and +// Input() only build values; every expectation is in the test bodies. +class TextStringLiteralTest : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, StringField>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + const StringField& field() const { return std::get<1>(GetParam()); } + + // The legacy test name: "StringLiteral<base><String|Bytes>", e.g. + // "StringLiteralConcatBytes". + std::string TestName(absl::string_view base) const { + return absl::StrCat("StringLiteral", base, field().type_name); + } + + // The text-format input setting the field to `literal`, e.g. + // "optional_bytes: 'first' \"second\"". + std::string Input(absl::string_view literal) const { + return absl::StrCat(field().field_name, ": ", literal); + } + + private: + TestPriority DefaultPriority() const override { return kP1; } + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(TextStringLiteralTest, Concat) { + const std::string input = Input("'first' \"second\"\n'third'"); + EXPECT_THAT(Testee(kP0, TestName("Concat")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(kP0, TestName("Concat")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextStringLiteralTest, BasicEscapes) { + const std::string input = Input(R"('\a\b\f\n\r\t\v\?\\\'\"')"); + EXPECT_THAT(Testee(kP0, TestName("BasicEscapes")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(kP0, TestName("BasicEscapes")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextStringLiteralTest, OctalEscapes) { + const std::string input = Input("'\\341\\210\\264'"); + EXPECT_THAT(Testee(kP0, TestName("OctalEscapes")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(kP0, TestName("OctalEscapes")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextStringLiteralTest, HexEscapes) { + const std::string input = Input("'\\xe1\\x88\\xb4'"); + EXPECT_THAT(Testee(kP0, TestName("HexEscapes")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(kP0, TestName("HexEscapes")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextStringLiteralTest, ShortUnicodeEscape) { + const std::string input = Input("'\\u1234'"); + EXPECT_THAT(Testee(TestName("ShortUnicodeEscape")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("ShortUnicodeEscape")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextStringLiteralTest, LongUnicodeEscapes) { + const std::string input = Input("'\\U00001234\\U00010437'"); + EXPECT_THAT(Testee(TestName("LongUnicodeEscapes")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("LongUnicodeEscapes")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// String literals don't include line feeds. +TEST_P(TextStringLiteralTest, IncludesLF) { + EXPECT_THAT(Testee(kP0, TestName("IncludesLF")) + .ParseText(message(), Input("'first line\nsecond line'")) + .ParseOnly(), + Yields(IsParseError())); +} + +// Unicode escapes don't include code points that lie beyond the planes +// (> 0x10ffff). +TEST_P(TextStringLiteralTest, LongUnicodeEscapeTooLarge) { + EXPECT_THAT(Testee(kP0, TestName("LongUnicodeEscapeTooLarge")) + .ParseText(message(), Input("'\\U00110000'")) + .ParseOnly(), + Yields(IsParseError())); +} + +// Unicode escapes don't include surrogates. +TEST_P(TextStringLiteralTest, ShortUnicodeEscapeSurrogatePair) { + EXPECT_THAT(Testee(TestName("ShortUnicodeEscapeSurrogatePair")) + .ParseText(message(), Input("'\\ud801\\udc37'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, ShortUnicodeEscapeSurrogateFirstOnly) { + EXPECT_THAT(Testee(TestName("ShortUnicodeEscapeSurrogateFirstOnly")) + .ParseText(message(), Input("'\\ud800'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, ShortUnicodeEscapeSurrogateSecondOnly) { + EXPECT_THAT(Testee(TestName("ShortUnicodeEscapeSurrogateSecondOnly")) + .ParseText(message(), Input("'\\udc00'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, LongUnicodeEscapeSurrogateFirstOnly) { + EXPECT_THAT(Testee(TestName("LongUnicodeEscapeSurrogateFirstOnly")) + .ParseText(message(), Input("'\\U0000d800'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, LongUnicodeEscapeSurrogateSecondOnly) { + EXPECT_THAT(Testee(TestName("LongUnicodeEscapeSurrogateSecondOnly")) + .ParseText(message(), Input("'\\U0000dc00'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, LongUnicodeEscapeSurrogatePair) { + EXPECT_THAT(Testee(TestName("LongUnicodeEscapeSurrogatePair")) + .ParseText(message(), Input("'\\U0000d801\\U00000dc37'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, UnicodeEscapeSurrogatePairLongShort) { + EXPECT_THAT(Testee(TestName("UnicodeEscapeSurrogatePairLongShort")) + .ParseText(message(), Input("'\\U0000d801\\udc37'")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringLiteralTest, UnicodeEscapeSurrogatePairShortLong) { + EXPECT_THAT(Testee(TestName("UnicodeEscapeSurrogatePairShortLong")) + .ParseText(message(), Input("'\\ud801\\U0000dc37'")) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextStringLiteralTest, + Combine(ValuesIn(Proto3TestMessageTypes()), + ValuesIn(kStringFields)), + TupleParamName<TextStringLiteralTest::ParamType>); + +// String fields reject invalid UTF-8 byte sequences; bytes fields don't. +using TextStringFieldTest = MessageTypeConformanceTest; + +TEST_P(TextStringFieldTest, BadUtf8Octal) { + EXPECT_THAT(Testee("StringFieldBadUTF8Octal") + .ParseText(message(), "optional_string: '\\300'") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextStringFieldTest, BadUtf8Hex) { + EXPECT_THAT(Testee("StringFieldBadUTF8Hex") + .ParseText(message(), "optional_string: '\\xc0'") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextStringFieldTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +using TextBytesFieldTest = MessageTypeConformanceTest; + +TEST_P(TextBytesFieldTest, BadUtf8Octal) { + EXPECT_THAT(Testee("BytesFieldBadUTF8Octal") + .ParseText(message(), "optional_bytes: '\\300'") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("optional_bytes: '\\300'")))); + EXPECT_THAT(Testee("BytesFieldBadUTF8Octal") + .ParseText(message(), "optional_bytes: '\\300'") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("optional_bytes: '\\300'")))); +} + +TEST_P(TextBytesFieldTest, BadUtf8Hex) { + EXPECT_THAT(Testee("BytesFieldBadUTF8Hex") + .ParseText(message(), "optional_bytes: '\\xc0'") + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto("optional_bytes: '\\xc0'")))); + EXPECT_THAT(Testee("BytesFieldBadUTF8Hex") + .ParseText(message(), "optional_bytes: '\\xc0'") + .SerializeText(), + Yields(WhenParsed(EqualsTextProto("optional_bytes: '\\xc0'")))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextBytesFieldTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_syntax_test.cc b/conformance/text_syntax_test.cc new file mode 100644 index 0000000..c12ff2b --- /dev/null +++ b/conformance/text_syntax_test.cc
@@ -0,0 +1,303 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for the separators of the syntax: the +// optional "," / ";" after a field, and the "," between the values of a list. +// This replaces the "Separators" block of the legacy +// TextFormatConformanceTestSuiteImpl<M>::RunAllTests(), which ran for the +// proto3-style message types only; the test names and the requests sent to +// the testee are identical to the legacy ones. + +#include <ostream> +#include <string> +#include <tuple> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "conformance/binary_test_util.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/descriptor.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::Combine; +using ::testing::Values; +using ::testing::ValuesIn; + +// A field type the separator tests run over: the legacy suite ran every +// separator test once per entry, with "_<type>" appended to the test name and +// `value` as the literal of the optional_<type> / repeated_<type> field. +struct SeparatorCase { + absl::string_view type; + absl::string_view value; + // Whether two adjacent values of the type are one value to the parser + // (string literals concatenate) rather than a missing list separator. + bool adjacent_values_concatenate; +}; + +constexpr SeparatorCase kSeparatorCases[] = { + {"string", "\"abc\"", true}, {"bytes", "\"abc\"", true}, + {"int32", "123", false}, {"bool", "true", false}, + {"double", "1.23", false}, {"fixed32", "0x123", false}, +}; + +std::string ParamName(const SeparatorCase& test_case) { + // Qualified: the overloads in this namespace hide the outer ones. + return conformance::ParamName(test_case.type); +} + +// For gtest's failure messages (the parameter is otherwise printed as bytes). +void PrintTo(const SeparatorCase& test_case, std::ostream* os) { + *os << test_case.type; +} + +// Whether the field separator tests use the optional_<type> field with a +// bare value ("Single") or the repeated_<type> field with a one-element list +// ("Repeated"); the legacy test names carry the same two words. +enum class FieldCardinality { kSingle, kRepeated }; + +std::string ParamName(FieldCardinality cardinality) { + return cardinality == FieldCardinality::kSingle ? "Single" : "Repeated"; +} + +void PrintTo(FieldCardinality cardinality, std::ostream* os) { + *os << ParamName(cardinality); +} + +// The separators after a top-level field, parameterized over (test message +// type, field type, single/repeated). Reporting the message type through +// MessageUnderTest() makes the base SetUp() skip the editions instance when +// --maximum_edition doesn't cover it. TestName() and Input() only build +// values; every expectation is in the test bodies. +class TextFieldSeparatorTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, SeparatorCase, FieldCardinality>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + const SeparatorCase& test_case() const { return std::get<1>(GetParam()); } + FieldCardinality cardinality() const { return std::get<2>(GetParam()); } + + // The legacy test name, e.g. "FieldSeparatorCommaTopLevelSingle_int32". + std::string TestName(absl::string_view base) const { + return absl::StrCat(base, ParamName(cardinality()), "_", test_case().type); + } + + // The field set to its value, followed by `separator`, e.g. + // "optional_int32: 123," or "repeated_int32: [123],". + std::string Input(absl::string_view separator) const { + const bool single = cardinality() == FieldCardinality::kSingle; + return absl::StrCat(single ? "optional_" : "repeated_", test_case().type, + ": ", single ? "" : "[", test_case().value, + single ? "" : "]", separator); + } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(TextFieldSeparatorTest, Comma) { + const std::string input = Input(","); + EXPECT_THAT(Testee(TestName("FieldSeparatorCommaTopLevel")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FieldSeparatorCommaTopLevel")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +// Note: the legacy name has "Single" in it even for the repeated variant. +TEST_P(TextFieldSeparatorTest, Semi) { + const std::string input = Input(";"); + EXPECT_THAT(Testee(TestName("FieldSeparatorSemiTopLevelSingle")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("FieldSeparatorSemiTopLevelSingle")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextFieldSeparatorTest, CommaDuplicatesFails) { + EXPECT_THAT(Testee(TestName("FieldSeparatorCommaTopLevelDuplicatesFails")) + .ParseText(message(), Input(",,")) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextFieldSeparatorTest, SemiDuplicatesFails) { + EXPECT_THAT(Testee(TestName("FieldSeparatorSemiTopLevelDuplicateFails")) + .ParseText(message(), Input(";;")) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P( + All, TextFieldSeparatorTest, + Combine(ValuesIn(Proto3TestMessageTypes()), ValuesIn(kSeparatorCases), + Values(FieldCardinality::kSingle, FieldCardinality::kRepeated)), + TupleParamName<TextFieldSeparatorTest::ParamType>); + +// The required "," between the values of a list, parameterized over (test +// message type, field type). +class TextListSeparatorTest + : public ConformanceTest, + public testing::WithParamInterface< + std::tuple<const Descriptor*, SeparatorCase>> { + protected: + const Descriptor* message() const { return std::get<0>(GetParam()); } + const SeparatorCase& test_case() const { return std::get<1>(GetParam()); } + + // The legacy test name, e.g. "ListSeparator_int32". + std::string TestName(absl::string_view base) const { + return absl::StrCat(base, "_", test_case().type); + } + + // The repeated field set to a list whose body is `list_body`, e.g. + // "repeated_int32: [123,123]". + std::string Input(absl::string_view list_body) const { + return absl::StrCat("repeated_", test_case().type, ": [", list_body, "]"); + } + + absl::string_view value() const { return test_case().value; } + + private: + const Descriptor* MessageUnderTest() const override { return message(); } +}; + +TEST_P(TextListSeparatorTest, Comma) { + const std::string input = Input(absl::StrCat(value(), ",", value())); + EXPECT_THAT(Testee(TestName("ListSeparator")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("ListSeparator")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); +} + +TEST_P(TextListSeparatorTest, SemiFails) { + EXPECT_THAT( + Testee(TestName("ListSeparatorSemiFails")) + .ParseText(message(), Input(absl::StrCat(value(), ";", value()))) + .ParseOnly(), + Yields(IsParseError())); +} + +// For string and bytes, if we skip the separator, the parser will treat the +// two values as a single value. +TEST_P(TextListSeparatorTest, Missing) { + const std::string input = Input(absl::StrCat(value(), " ", value())); + if (test_case().adjacent_values_concatenate) { + EXPECT_THAT(Testee(TestName("ListSeparatorMissingIsOneValue")) + .ParseText(message(), input) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(input)))); + EXPECT_THAT(Testee(TestName("ListSeparatorMissingIsOneValue")) + .ParseText(message(), input) + .SerializeText(), + Yields(WhenParsed(EqualsTextProto(input)))); + } else { + EXPECT_THAT(Testee(TestName("ListSeparatorMissingFails")) + .ParseText(message(), input) + .ParseOnly(), + Yields(IsParseError())); + } +} + +TEST_P(TextListSeparatorTest, DuplicateFails) { + EXPECT_THAT( + Testee(TestName("ListSeparatorDuplicateFails")) + .ParseText(message(), Input(absl::StrCat(value(), ",,", value()))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextListSeparatorTest, SingleTrailingFails) { + EXPECT_THAT(Testee(TestName("ListSeparatorSingleTrailingFails")) + .ParseText(message(), Input(absl::StrCat(value(), ","))) + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextListSeparatorTest, TwoValuesTrailingFails) { + EXPECT_THAT( + Testee(TestName("ListSeparatorTwoValuesTrailingFails")) + .ParseText(message(), Input(absl::StrCat(value(), ",", value(), ","))) + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextListSeparatorTest, + Combine(ValuesIn(Proto3TestMessageTypes()), + ValuesIn(kSeparatorCases)), + TupleParamName<TextListSeparatorTest::ParamType>); + +// The test messages don't have every type nested, so just check one data +// type for the nested field separator support. +using TextNestedSeparatorTest = MessageTypeConformanceTest; + +TEST_P(TextNestedSeparatorTest, Comma) { + EXPECT_THAT(Testee("FieldSeparatorCommaNested") + .ParseText(message(), "optional_nested_message: { a: 123, }") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_nested_message: { a: 123, }")))); + EXPECT_THAT(Testee("FieldSeparatorCommaNested") + .ParseText(message(), "optional_nested_message: { a: 123, }") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_nested_message: { a: 123, }")))); +} + +TEST_P(TextNestedSeparatorTest, Semi) { + EXPECT_THAT(Testee("FieldSeparatorSemiNested") + .ParseText(message(), "optional_nested_message: { a: 123; }") + .SerializeBinary(), + Yields(WhenParsed( + EqualsTextProto("optional_nested_message: { a: 123; }")))); + EXPECT_THAT(Testee("FieldSeparatorSemiNested") + .ParseText(message(), "optional_nested_message: { a: 123; }") + .SerializeText(), + Yields(WhenParsed( + EqualsTextProto("optional_nested_message: { a: 123; }")))); +} + +TEST_P(TextNestedSeparatorTest, CommaDuplicates) { + EXPECT_THAT(Testee("FieldSeparatorCommaNestedDuplicates") + .ParseText(message(), "optional_nested_message: { a: 123,, }") + .ParseOnly(), + Yields(IsParseError())); +} + +TEST_P(TextNestedSeparatorTest, SemiDuplicates) { + EXPECT_THAT(Testee("FieldSeparatorSemiNestedDuplicates") + .ParseText(message(), "optional_nested_message: { a: 123;; }") + .ParseOnly(), + Yields(IsParseError())); +} + +INSTANTIATE_TEST_SUITE_P(All, TextNestedSeparatorTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/text_unknown_field_test.cc b/conformance/text_unknown_field_test.cc new file mode 100644 index 0000000..6437287 --- /dev/null +++ b/conformance/text_unknown_field_test.cc
@@ -0,0 +1,132 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Text-format conformance tests for unknown fields: a binary payload whose +// fields are all unknown to the test message type is serialized as text +// format, once dropping the unknown fields ("<name>_Drop") and once printing +// them by field number ("<name>_Print"). This replaces the "Unknown Fields" +// block of the legacy TextFormatConformanceTestSuiteImpl<M>::RunAllTests() and +// its RunValidUnknownTextFormatTest() helper, which ran for the proto3-style +// message types only; the test names and the requests sent to the testee are +// identical to the legacy ones. +// +// The payloads are serializations of UnknownToTestAllTypes, whose fields have +// numbers no TestAllTypes message defines, and the testee's text output is +// decoded as that type too (see WhenParsedAs() in matchers.h): unknown +// fields printed by number become its known fields again, so the _Print leg +// expects the whole payload back and the _Drop leg an empty message. Like the +// legacy tests these are RECOMMENDED and categorized as TEXT_FORMAT_TEST +// although their input is binary. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_test_util.h" +#include "conformance/binary_wireformat.h" +#include "conformance/conformance.pb.h" +#include "conformance/matchers.h" +#include "conformance/message_type_fixtures.h" +#include "conformance/testee.h" +#include "google/protobuf/test_messages_proto2.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::proto2::UnknownToTestAllTypes; +using ::testing::ValuesIn; + +class TextUnknownFieldTest : public MessageTypeConformanceTest { + private: + TestPriority DefaultPriority() const override { return kP1; } +}; + +// Unable to print unknown Fixed32/Fixed64 fields as if they are known. +// Fixed32/Fixed64 fields are not added in the tests. +TEST_P(TextUnknownFieldTest, ScalarUnknownFields) { + UnknownToTestAllTypes unknown; + unknown.set_optional_int32(123); + unknown.set_optional_string("hello"); + unknown.set_optional_bool(true); + const Wire bytes(unknown.SerializeAsString()); + EXPECT_THAT(Testee("ScalarUnknownFields_Drop") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText(), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsTextProto("")))); + EXPECT_THAT( + Testee("ScalarUnknownFields_Print") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText({/*print_unknown_fields=*/true}), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsBinaryProto(bytes)))); +} + +TEST_P(TextUnknownFieldTest, MessageUnknownFields) { + UnknownToTestAllTypes unknown; + unknown.mutable_nested_message()->set_c(111); + const Wire bytes(unknown.SerializeAsString()); + EXPECT_THAT(Testee("MessageUnknownFields_Drop") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText(), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsTextProto("")))); + EXPECT_THAT( + Testee("MessageUnknownFields_Print") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText({/*print_unknown_fields=*/true}), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsBinaryProto(bytes)))); +} + +TEST_P(TextUnknownFieldTest, GroupUnknownFields) { + UnknownToTestAllTypes unknown; + unknown.mutable_optionalgroup()->set_a(321); + const Wire bytes(unknown.SerializeAsString()); + EXPECT_THAT(Testee("GroupUnknownFields_Drop") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText(), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsTextProto("")))); + EXPECT_THAT( + Testee("GroupUnknownFields_Print") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText({/*print_unknown_fields=*/true}), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsBinaryProto(bytes)))); +} + +// Note: the legacy test built this payload on top of the group one without +// clearing it, so the group is part of this payload too. +TEST_P(TextUnknownFieldTest, RepeatedUnknownFields) { + UnknownToTestAllTypes unknown; + unknown.mutable_optionalgroup()->set_a(321); + unknown.add_repeated_int32(1); + unknown.add_repeated_int32(2); + unknown.add_repeated_int32(3); + const Wire bytes(unknown.SerializeAsString()); + EXPECT_THAT(Testee("RepeatedUnknownFields_Drop") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText(), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsTextProto("")))); + EXPECT_THAT( + Testee("RepeatedUnknownFields_Print") + .ParseBinary(message(), bytes) + .OverrideTestCategory(::conformance::TEXT_FORMAT_TEST) + .SerializeText({/*print_unknown_fields=*/true}), + Yields(WhenParsedAs<UnknownToTestAllTypes>(EqualsBinaryProto(bytes)))); +} + +INSTANTIATE_TEST_SUITE_P(All, TextUnknownFieldTest, + ValuesIn(Proto3TestMessageTypes()), + MessageTypeParamName); + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/upb/conformance/conformance_upb_failures.txt b/upb/conformance/conformance_upb_failures.txt index d1fa6e5..9d13500 100644 --- a/upb/conformance/conformance_upb_failures.txt +++ b/upb/conformance/conformance_upb_failures.txt
@@ -1,6 +1,6 @@ -Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse or matched expected output but did not. -Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse or matched expected output but did not. +Recommended.*.JsonInput.FieldNameDuplicate # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing1 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: +Recommended.*.JsonInput.FieldNameDuplicateDifferentCasing2 # Should have failed to parse, but didn't., and Output was not equivalent to reference message: added: optional_nested_message.a: Required.*.JsonInput.Int32FieldQuotedExponentialValue.* # Failed to parse input or produce output. Required.*.JsonInput.AnyWithNoType.* # Failed to parse input or produce output. # TODO: Uncomment once conformance tests can express failures that are not expected to be fixed.