Add the main and an end-to-end test for gtest conformance suites. test_environment_main installs the ConformanceEnvironment from the command-line flags (--testee_binary, --failure_list, --maximum_edition, --fix, ...) before RUN_ALL_TESTS(), so a conformance suite links it instead of a generic gtest main. The sample suite and the integration test run that binary as a real process and check the whole flow: passing and failing tests against a failure list, unseen expected failures in full and partial runs, the --fix output, positional testee arguments and the statistics in the XML report. --enforcement_level is a TestPriority flag, written as the priority's number: 0 enforces only P0, 1 (the default) P0 and P1. PiperOrigin-RevId: 1001586140
diff --git a/conformance/BUILD b/conformance/BUILD index 11b0156..c209321 100644 --- a/conformance/BUILD +++ b/conformance/BUILD
@@ -127,6 +127,9 @@ "matchers_test", "naming_test", "recording_test_runner_test", + "sample_conformance_test", + "test_environment_flags_test", + "test_environment_integration_test", "test_environment_test", "test_fixture_test", "test_manager_test", @@ -606,6 +609,112 @@ ], ) +# Defines the global --testee_binary, --failure_list, --fix, --performance, +# ... flags of a conformance test binary. Only link it into binaries with a +# test_environment_main-style main (never into conformance_test_runner, whose +# legacy command line has flags of the same names). +cc_library( + name = "test_environment_flags", + testonly = 1, + srcs = ["test_environment_flags.cc"], + hdrs = ["test_environment_flags.h"], + deps = [ + ":test_environment", + ":testee", + "//src/google/protobuf", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/types:span", + ], +) + +cc_test( + name = "test_environment_flags_test", + srcs = ["test_environment_flags_test.cc"], + deps = [ + ":test_environment", + ":test_environment_flags", + ":testee", + "//src/google/protobuf", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/flags:reflection", + "@abseil-cpp//absl/strings:string_view", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + +# The main() for gtest-based conformance suites. Depend on this instead of a +# generic gtest main. +cc_library( + name = "test_environment_main", + testonly = 1, + srcs = ["test_environment_main.cc"], + deps = [ + ":test_environment", + ":test_environment_flags", + "@abseil-cpp//absl/flags:parse", # buildcleaner: keep + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/log:initialize", # buildcleaner: keep + "@abseil-cpp//absl/types:span", + "@googletest//:gtest", + ], + alwayslink = 1, +) + +# A tiny conformance suite exercising the framework end to end. It +# deliberately contains failing tests, which the accompanying failure lists +# account for. +cc_test( + name = "sample_conformance_test", + srcs = ["sample_conformance_test.cc"], + args = [ + "--testee_binary=$(rootpath :conformance_cpp)", + "--failure_list=$(rootpath sample_failure_list_ok.txt)", + ], + data = [ + "sample_failure_list_ok.txt", + ":conformance_cpp", + ], + deps = [ + ":binary_wireformat", + ":matchers", + ":test_environment_main", + ":test_fixture", + ":testee", + "//:test_messages_proto3_cc_proto", + "@googletest//:gtest", + ], +) + +# Runs sample_conformance_test as a subprocess. Only what needs a real test +# process is checked here; see test_environment_test for the rest. +cc_test( + name = "test_environment_integration_test", + srcs = ["test_environment_integration_test.cc"], + data = [ + "sample_failure_list_empty.txt", + "sample_failure_list_ok.txt", + "sample_failure_list_unseen.txt", + ":conformance_cpp", + ":sample_conformance_test", + ], + env = { + "SAMPLE_TEST": "$(rlocationpath :sample_conformance_test)", + "TESTEE": "$(rlocationpath :conformance_cpp)", + }, + deps = [ + "//src/google/protobuf/testing", + "//src/google/protobuf/testing:file", + "@abseil-cpp//absl/log:absl_check", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", + "@abseil-cpp//absl/types:span", + "@bazel_tools//tools/cpp/runfiles", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + cc_binary( name = "conformance_test_runner", testonly = 1,
diff --git a/conformance/sample_conformance_test.cc b/conformance/sample_conformance_test.cc new file mode 100644 index 0000000..e82a423 --- /dev/null +++ b/conformance/sample_conformance_test.cc
@@ -0,0 +1,70 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A tiny conformance suite used to exercise the gtest conformance framework end +// to end against the real C++ testee. Some of these tests fail on purpose; see +// the sample_failure_list_*.txt files and test_environment_integration_test.cc. + +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "conformance/binary_wireformat.h" +#include "conformance/matchers.h" +#include "conformance/test_fixture.h" +#include "conformance/testee.h" +#include "google/protobuf/test_messages_proto3.pb.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::protobuf_test_messages::proto3::TestAllTypesProto3; + +using SampleConformanceTest = ConformanceTest; + +TEST_F(SampleConformanceTest, SamplePass) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +// Valid input is expected to parse, so this always fails against a conforming +// testee. +TEST_F(SampleConformanceTest, SampleDeliberateFailure) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(IsParseError())); +} + +// Same as above, but only P1. +TEST_F(SampleConformanceTest, SampleP1Failure) { + EXPECT_THAT( + Testee(kP1) + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(IsParseError())); +} + +// Only runs with gtest's also_run_disabled_tests flag. Its presence must not +// make a normal run count as partial (see IsPartialRun() in +// test_environment.cc). +TEST_F(SampleConformanceTest, DISABLED_SampleDisabled) { + EXPECT_THAT( + Testee() + .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99)) + .SerializeBinary(), + Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb")))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/sample_failure_list_empty.txt b/conformance/sample_failure_list_empty.txt new file mode 100644 index 0000000..a4f32fb --- /dev/null +++ b/conformance/sample_failure_list_empty.txt
@@ -0,0 +1,2 @@ +# Intentionally empty: every deliberate failure in sample_conformance_test is +# unexpected with this list.
diff --git a/conformance/sample_failure_list_ok.txt b/conformance/sample_failure_list_ok.txt new file mode 100644 index 0000000..d2429a2 --- /dev/null +++ b/conformance/sample_failure_list_ok.txt
@@ -0,0 +1,3 @@ +# Expected failures of sample_conformance_test. These tests fail on purpose. +Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput # Should have failed to parse, but didn't.
diff --git a/conformance/sample_failure_list_unseen.txt b/conformance/sample_failure_list_unseen.txt new file mode 100644 index 0000000..1ff44a8 --- /dev/null +++ b/conformance/sample_failure_list_unseen.txt
@@ -0,0 +1,4 @@ +# Like sample_failure_list_ok.txt, plus an entry for a test that never runs. +Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput # Should have failed to parse, but didn't. +Required.Proto3.ProtobufInput.SampleNeverRuns.ProtobufOutput # This test doesn't exist.
diff --git a/conformance/test_environment.cc b/conformance/test_environment.cc index 98c69ef..eda59b2 100644 --- a/conformance/test_environment.cc +++ b/conformance/test_environment.cc
@@ -199,7 +199,7 @@ // isn't registered with gtest itself: gtest only gets a forwarding proxy, // and the real environment intentionally lives until the process exits. // Only the environment object leaks, though; TearDown() releases the testee. - // TODO: b/410122039 - register it directly once the legacy suites are gone. + // TODO: register it directly once the legacy suites are gone. auto* environment = new ConformanceEnvironment(std::move(options)); testing::AddGlobalTestEnvironment(new ForwardingEnvironment(environment)); return *environment; @@ -209,7 +209,8 @@ ABSL_CHECK(global_environment != nullptr) << "No ConformanceEnvironment has been installed. Conformance test " "binaries must call ConformanceEnvironment::Install() before " - "RUN_ALL_TESTS()."; + "RUN_ALL_TESTS(); the usual way to do that is to depend on the " + "test_environment_main library instead of a generic gtest main."; return *global_environment; }
diff --git a/conformance/test_environment.h b/conformance/test_environment.h index f66d25a..9f4c826 100644 --- a/conformance/test_environment.h +++ b/conformance/test_environment.h
@@ -10,9 +10,10 @@ // ConformanceEnvironment is the process-global state of a conformance test // binary. It owns the testee connection and the TestManager, records // statistics, and checks or regenerates the failure list at the end of the -// run. Exactly one is installed per test binary, before RUN_ALL_TESTS(); -// Install() hooks it into gtest. Nothing in this header is meant for the -// conformance suites themselves. +// run. Exactly one is installed per test binary, normally by +// test_environment_main.cc from command-line flags (see +// test_environment_flags.h). Install() hooks it into gtest. Nothing in this +// header is meant for the conformance suites themselves. // // Everything here is single-threaded. Use it only from gtest's main thread: // test bodies, fixtures and the environment hooks. The global environment is @@ -43,7 +44,9 @@ namespace conformance { namespace internal { -// Options for a ConformanceEnvironment. +// Options for a ConformanceEnvironment. Normally populated from command-line +// flags by test_environment_main.cc (see OptionsFromFlags() in +// test_environment_flags.h). struct ConformanceEnvironmentOptions { // Exactly one of `runner`, `owned_runner` or `testee_binary` must be set. // @@ -181,7 +184,7 @@ // makes it the process-global instance. Must be called exactly once, before // RUN_ALL_TESTS(): // - // ConformanceEnvironment::Install(std::move(options)); + // ConformanceEnvironment::Install(OptionsFromFlags()); // return RUN_ALL_TESTS(); // // The environment itself is never destroyed (see the class comment), but @@ -189,7 +192,7 @@ static ConformanceEnvironment& Install(ConformanceEnvironmentOptions options); // Returns the process-global instance. Check-fails if Install() hasn't been - // called. + // called, which typically means test_environment_main wasn't linked in. static ConformanceEnvironment& Get(); // Loads the failure lists. Any failure here is fatal, so no tests run.
diff --git a/conformance/test_environment_flags.cc b/conformance/test_environment_flags.cc new file mode 100644 index 0000000..90ff4ac --- /dev/null +++ b/conformance/test_environment_flags.cc
@@ -0,0 +1,85 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Defines the global --testee_binary, --testee_args, --failure_list, +// --enforcement_level, --maximum_edition, --performance, --fix and +// --fix_output_file flags. Being process-global ABSL_FLAGs with short, generic +// names, they must only be linked into binaries with a test_environment_main- +// style main (see test_environment_flags.h), never into +// conformance_test_runner, whose legacy command line parses flags of the same +// names itself. + +#include "conformance/test_environment_flags.h" + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include "absl/flags/flag.h" +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +ABSL_FLAG(std::string, testee_binary, "", + "The testee program to run conformance tests against."); +ABSL_FLAG(std::vector<std::string>, testee_args, {}, + "Comma-separated arguments to pass to --testee_binary. Positional " + "arguments are appended to these."); +ABSL_FLAG(std::vector<std::string>, failure_list, {}, + "Comma-separated list of files containing the tests that are " + "expected to fail, one test name per line. Use '#' for comments."); +ABSL_FLAG(google::protobuf::conformance::TestPriority, enforcement_level, + google::protobuf::conformance::kLowestPriority, + "The lowest priority whose failures fail the run, as its number: 0 " + "enforces only P0, 1 enforces P0 and P1. The default enforces every " + "priority. Failures of tests of a lower priority are tolerated " + "(counted, not failed) unless the test is in the failure list, which " + "is checked whatever the level."); +ABSL_FLAG(google::protobuf::Edition, maximum_edition, google::protobuf::EDITION_PROTO3, + "Only run conformance tests up to and including this edition, e.g. " + "EDITION_2023. Defaults to proto2 and proto3 only."); +// TODO: Transitional. Goes away with the in-binary +// performance mode, see ConformanceEnvironmentOptions::performance. +ABSL_FLAG(bool, performance, false, + "Run only the performance tests instead of the regular tests. A " + "performance run needs its own failure list."); +ABSL_FLAG(bool, fix, false, + "Rewrite the failure list at the end of the run to match the " + "observed results. See --fix_output_file. Refused when only some " + "of the tests run (--gtest_filter, sharding)."); +ABSL_FLAG(std::string, fix_output_file, "", + "Where --fix writes the updated failure list. Required when more " + "than one --failure_list is given; otherwise defaults to the " + "failure list itself under $BUILD_WORKSPACE_DIRECTORY, so that " + "`bazel run ... -- --fix` updates the file in your workspace."); + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +ConformanceEnvironmentOptions OptionsFromFlags( + absl::Span<char* const> positional_args) { + ConformanceEnvironmentOptions options; + options.testee_binary = absl::GetFlag(FLAGS_testee_binary); + options.testee_args = absl::GetFlag(FLAGS_testee_args); + for (const char* positional_arg : positional_args) { + options.testee_args.push_back(positional_arg); + } + options.failure_list_files = absl::GetFlag(FLAGS_failure_list); + options.enforcement_level = absl::GetFlag(FLAGS_enforcement_level); + options.maximum_edition = absl::GetFlag(FLAGS_maximum_edition); + options.performance = absl::GetFlag(FLAGS_performance); + options.fix = absl::GetFlag(FLAGS_fix); + options.fix_output_file = absl::GetFlag(FLAGS_fix_output_file); + return options; +} + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_flags.h b/conformance/test_environment_flags.h new file mode 100644 index 0000000..863f839 --- /dev/null +++ b/conformance/test_environment_flags.h
@@ -0,0 +1,58 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// The command-line flags of a conformance test binary and their translation +// into ConformanceEnvironmentOptions. test_environment_main.cc is the usual +// consumer. A binary with its own main, for example one hosting an +// in-process testee, can link this library instead to get the same flags. It +// then calls OptionsFromFlags() and sets `runner` or `owned_runner` before +// installing the environment. + +#ifndef GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__ +#define GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__ + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include "absl/flags/declare.h" +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace internal { + +// Builds the environment options from the flags declared below. +// `positional_args` are the non-flag command-line arguments, excluding +// argv[0]. They are appended to --testee_args. +// +// ConformanceEnvironment::Install(OptionsFromFlags()); +// +// This doesn't insist on --testee_binary. A main hosting an in-process testee +// leaves it unset and fills in `runner` or `owned_runner` instead. +// test_environment_main.cc checks it. +ConformanceEnvironmentOptions OptionsFromFlags( + absl::Span<char* const> positional_args = {}); + +} // namespace internal +} // namespace conformance +} // namespace protobuf +} // namespace google + +ABSL_DECLARE_FLAG(std::string, testee_binary); +ABSL_DECLARE_FLAG(std::vector<std::string>, testee_args); +ABSL_DECLARE_FLAG(std::vector<std::string>, failure_list); +ABSL_DECLARE_FLAG(google::protobuf::conformance::TestPriority, enforcement_level); +ABSL_DECLARE_FLAG(google::protobuf::Edition, maximum_edition); +ABSL_DECLARE_FLAG(bool, performance); // TODO: Transitional. +ABSL_DECLARE_FLAG(bool, fix); +ABSL_DECLARE_FLAG(std::string, fix_output_file); + +#endif // GOOGLE_PROTOBUF_CONFORMANCE_TEST_ENVIRONMENT_FLAGS_H__
diff --git a/conformance/test_environment_flags_test.cc b/conformance/test_environment_flags_test.cc new file mode 100644 index 0000000..6b1dbf2 --- /dev/null +++ b/conformance/test_environment_flags_test.cc
@@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "conformance/test_environment_flags.h" + +#include <string> +#include <vector> + +#include "google/protobuf/descriptor.pb.h" +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include "absl/flags/flag.h" +#include "absl/flags/reflection.h" +#include "absl/strings/string_view.h" +#include "conformance/test_environment.h" +#include "conformance/testee.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::google::protobuf::conformance::internal::ConformanceEnvironmentOptions; +using ::google::protobuf::conformance::internal::OptionsFromFlags; +using ::testing::ElementsAre; +using ::testing::HasSubstr; +using ::testing::IsEmpty; + +TEST(OptionsFromFlagsTest, Defaults) { + absl::FlagSaver saver; + ConformanceEnvironmentOptions options = OptionsFromFlags(); + EXPECT_THAT(options.testee_binary, IsEmpty()); + EXPECT_THAT(options.testee_args, IsEmpty()); + EXPECT_THAT(options.failure_list_files, IsEmpty()); + EXPECT_EQ(options.enforcement_level, kLowestPriority); + EXPECT_EQ(options.maximum_edition, EDITION_PROTO3); + EXPECT_FALSE(options.performance); + EXPECT_FALSE(options.fix); + EXPECT_THAT(options.fix_output_file, IsEmpty()); + EXPECT_TRUE(options.check_unseen_expected_failures); +} + +TEST(OptionsFromFlagsTest, ForwardsEveryFlag) { + absl::FlagSaver saver; + absl::SetFlag(&FLAGS_testee_binary, "testee"); + absl::SetFlag(&FLAGS_testee_args, {"--a", "--b"}); + absl::SetFlag(&FLAGS_failure_list, {"x.txt", "y.txt"}); + absl::SetFlag(&FLAGS_enforcement_level, kP0); + absl::SetFlag(&FLAGS_maximum_edition, EDITION_2023); + absl::SetFlag(&FLAGS_performance, true); + absl::SetFlag(&FLAGS_fix, true); + absl::SetFlag(&FLAGS_fix_output_file, "out.txt"); + + std::vector<std::string> positional = {"--c", "d"}; + std::vector<char*> positional_args; + for (std::string& arg : positional) positional_args.push_back(arg.data()); + + ConformanceEnvironmentOptions options = OptionsFromFlags(positional_args); + EXPECT_EQ(options.testee_binary, "testee"); + EXPECT_THAT(options.testee_args, ElementsAre("--a", "--b", "--c", "d")); + EXPECT_THAT(options.failure_list_files, ElementsAre("x.txt", "y.txt")); + EXPECT_EQ(options.enforcement_level, kP0); + EXPECT_EQ(options.maximum_edition, EDITION_2023); + EXPECT_TRUE(options.performance); + EXPECT_TRUE(options.fix); + EXPECT_EQ(options.fix_output_file, "out.txt"); +} + +TEST(EnforcementLevelFlagTest, ParsesThePriorityNumber) { + TestPriority priority = kLowestPriority; + std::string error; + EXPECT_TRUE(AbslParseFlag("0", &priority, &error)); + EXPECT_EQ(priority, kP0); + EXPECT_TRUE(AbslParseFlag("1", &priority, &error)); + EXPECT_EQ(priority, kP1); + EXPECT_EQ(AbslUnparseFlag(kP1), "1"); + + for (absl::string_view bad : {"-1", "2", "P1", ""}) { + EXPECT_FALSE(AbslParseFlag(bad, &priority, &error)) << bad; + EXPECT_THAT(error, HasSubstr("from 0 to 1")) << bad; + } +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_integration_test.cc b/conformance/test_environment_integration_test.cc new file mode 100644 index 0000000..ea4ca9c --- /dev/null +++ b/conformance/test_environment_integration_test.cc
@@ -0,0 +1,326 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// End-to-end checks of test_environment_main and ConformanceEnvironment, +// running the deliberately failing sample_conformance_test against the real +// C++ testee. Only what needs a real test process is checked here: the +// command line handling of the main, what gtest itself contributes to a run +// (partial-run detection, the XML report) and the exit code. Everything else +// is covered in-process by test_environment_test. + +#include <algorithm> +#include <cstddef> +#include <cstdio> +#include <cstdlib> +#include <initializer_list> +#include <string> +#include <vector> + +#ifndef _WIN32 +#include <sys/wait.h> // for WEXITSTATUS +#endif + +#include "tools/cpp/runfiles/runfiles.h" +#include "google/protobuf/testing/file.h" +#include "google/protobuf/testing/file.h" +#include <gmock/gmock.h> +#include "google/protobuf/testing/googletest.h" +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace google { +namespace protobuf { +namespace conformance { +namespace { + +using ::testing::AllOf; +using ::testing::HasSubstr; +using ::testing::Not; + +// The names of the sample tests, as they appear in failure lists. +constexpr absl::string_view kDeliberateFailure = + "Required.Proto3.ProtobufInput.SampleDeliberateFailure.ProtobufOutput"; +constexpr absl::string_view kP1Failure = + "Recommended.Proto3.ProtobufInput.SampleP1Failure.ProtobufOutput"; +constexpr absl::string_view kNeverRuns = + "Required.Proto3.ProtobufInput.SampleNeverRuns.ProtobufOutput"; +constexpr absl::string_view kFailureMessage = + "Should have failed to parse, but didn't."; + +// gtest spells its flags --gtest_*, Google's internal build --gunit_*. +constexpr absl::string_view kGtestFlag = "--" GTEST_FLAG_PREFIX_; + +std::string Rlocation(const std::string& path) { + static const bazel::tools::cpp::runfiles::Runfiles* const runfiles = [] { + std::string error; + bazel::tools::cpp::runfiles::Runfiles* runfiles = + bazel::tools::cpp::runfiles::Runfiles::CreateForTest(&error); + ABSL_CHECK(runfiles != nullptr) << error; + return runfiles; + }(); + return runfiles->Rlocation(path); +} + +// The BUILD rule passes the sample test and the testee as $(rlocationpath) +// in these variables. +std::string RunfilesPathFromEnvironment(const char* name) { + const char* path = std::getenv(name); + ABSL_CHECK(path != nullptr) << name << " is not set"; + return path; +} + +const std::string& SampleTest() { + static const std::string* const path = + new std::string(Rlocation(RunfilesPathFromEnvironment("SAMPLE_TEST"))); + return *path; +} + +const std::string& TesteeBinary() { + static const std::string* const path = + new std::string(Rlocation(RunfilesPathFromEnvironment("TESTEE"))); + return *path; +} + +// The --failure_list flag for sample_failure_list_<name>.txt, a data +// dependency from the sample test's package. +std::string FailureListFlag(absl::string_view name) { + std::string package = RunfilesPathFromEnvironment("SAMPLE_TEST"); + package.erase(package.rfind('/')); + return absl::StrCat( + "--failure_list=", + Rlocation(absl::StrCat(package, "/sample_failure_list_", name, ".txt"))); +} + +std::string TmpPath(absl::string_view name) { + return absl::StrCat(TestTempDir(), "/", name); +} + +std::string WriteFile(absl::string_view name, absl::string_view content) { + std::string path = TmpPath(name); + ABSL_CHECK_OK(File::SetContents(path, content, true)); + return path; +} + +// `path` is a std::string (not a string_view) for the open source +// File::GetContents() shim. +std::string ReadFile(const std::string& path) { + std::string content; + ABSL_CHECK_OK(File::GetContents(path, &content, true)); + return content; +} + +// Quotes `arg` for the shell that popen() runs commands through. +std::string ShellQuote(absl::string_view arg) { +#ifdef _WIN32 + return absl::StrCat("\"", arg, "\""); +#else + return absl::StrCat("'", absl::StrReplaceAll(arg, {{"'", "'\\''"}}), "'"); +#endif +} + +struct RunResult { + int exit_code; + std::string output; // stdout and stderr +}; + +// Runs `binary` with `args` and returns its exit code and output. +RunResult RunBinary(const std::string& binary, + absl::Span<const std::string> args) { + std::string command = ShellQuote(binary); + for (const std::string& arg : args) { + absl::StrAppend(&command, " ", ShellQuote(arg)); + } + absl::StrAppend(&command, " 2>&1"); +#ifdef _WIN32 + // cmd.exe strips the first and last quote of a command line that starts + // with one, so give it a pair to strip. + FILE* pipe = _popen(absl::StrCat("\"", command, "\"").c_str(), "r"); +#else + FILE* pipe = popen(command.c_str(), "r"); +#endif + ABSL_CHECK(pipe != nullptr) << "Failed to run " << command; + + RunResult result; + char buffer[4096]; + size_t n; + while ((n = fread(buffer, 1, sizeof(buffer), pipe)) > 0) { + result.output.append(buffer, n); + } +#ifdef _WIN32 + result.exit_code = _pclose(pipe); +#else + int status = pclose(pipe); + result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; +#endif + return result; +} + +// Runs the sample test against the C++ testee with `args`. +RunResult RunSample(std::vector<std::string> args) { + args.insert(args.begin(), absl::StrCat("--testee_binary=", TesteeBinary())); + return RunBinary(SampleTest(), args); +} + +// Returns the part of the gtest XML `report` from the element opening with +// `open` up to the first of `ends`. gtest writes an element's properties +// before its children, so the right `ends` isolate one element's properties. +std::string XmlScope(const std::string& report, absl::string_view open, + std::initializer_list<absl::string_view> ends) { + size_t begin = report.find(open); + if (begin == std::string::npos) return ""; + size_t end = report.size(); + for (absl::string_view terminator : ends) { + end = std::min(end, report.find(terminator, begin + open.size())); + } + return report.substr(begin, end - begin); +} + +std::string XmlTestCase(const std::string& report, absl::string_view name) { + return XmlScope(report, absl::StrCat("<testcase name=\"", name, "\""), + {"<testcase", "</testsuite>"}); +} + +std::string XmlTestSuite(const std::string& report, absl::string_view name) { + return XmlScope(report, absl::StrCat("<testsuite name=\"", name, "\""), + {"<testcase", "</testsuite>"}); +} + +std::string XmlTestRun(const std::string& report) { + return XmlScope(report, "<testsuites", {"<testsuite "}); +} + +class TestEnvironmentIntegrationTest : public testing::Test { + protected: + static void SetUpTestSuite() { + // The test runner configures this binary's gtest through these variables + // (its filter, sharding, XML report, ...). gtest has read them by now, + // and they must not reach the binaries the tests run. + for (const char* name : + {"TESTBRIDGE_TEST_ONLY", "TESTBRIDGE_TEST_RUNNER_FAIL_FAST", + "XML_OUTPUT_FILE", "TEST_PREMATURE_EXIT_FILE", + "TEST_WARNINGS_OUTPUT_FILE", "TEST_TOTAL_SHARDS", "TEST_SHARD_INDEX", + "TEST_SHARD_STATUS_FILE", "GTEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", + "GTEST_SHARD_STATUS_FILE"}) { +#ifdef _WIN32 + _putenv_s(name, ""); // An empty value removes the variable. +#else + unsetenv(name); +#endif + } + } +}; + +TEST_F(TestEnvironmentIntegrationTest, PassesWithAMatchingFailureList) { + RunResult run = RunSample({FailureListFlag("ok")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ OK ] SampleConformanceTest.SamplePass"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleDeliberateFailure"), + HasSubstr("[ OK ] " + "SampleConformanceTest.SampleP1Failure"))); +} + +TEST_F(TestEnvironmentIntegrationTest, FailsWithAnEmptyFailureList) { + RunResult run = RunSample({FailureListFlag("empty")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ FAILED ] " + "SampleConformanceTest.SampleDeliberateFailure"), + HasSubstr(kDeliberateFailure), HasSubstr(kFailureMessage))); +} + +TEST_F(TestEnvironmentIntegrationTest, ListsTestsWithoutATestee) { + RunResult run = + RunBinary(SampleTest(), {absl::StrCat(kGtestFlag, "list_tests")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, HasSubstr("SampleDeliberateFailure")); +} + +TEST_F(TestEnvironmentIntegrationTest, PassesPositionalArgumentsToTheTestee) { + // The testee ignores them; this checks that flag parsing lets them through. + RunResult run = + RunSample({FailureListFlag("ok"), "--", "--ignored_by_testee"}); + EXPECT_EQ(run.exit_code, 0) << run.output; +} + +// Partial runs are detected from gtest's own test counts and state (see +// IsPartialRun() in test_environment.cc), which only a real run sets up. + +TEST_F(TestEnvironmentIntegrationTest, UnseenExpectedFailuresFailAFullRun) { + // The DISABLED_ sample test doesn't run either; that mustn't make the run + // count as partial. + RunResult run = RunSample({FailureListFlag("unseen")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT( + run.output, + HasSubstr(absl::StrCat("expected failures were not seen: ", kNeverRuns))); +} + +TEST_F(TestEnvironmentIntegrationTest, UnseenExpectedFailuresPassAFilteredRun) { + RunResult run = + RunSample({FailureListFlag("unseen"), + absl::StrCat(kGtestFlag, + "filter=SampleConformanceTest.Sample*Failure")}); + EXPECT_EQ(run.exit_code, 0) << run.output; + EXPECT_THAT(run.output, Not(HasSubstr("were not seen"))); +} + +TEST_F(TestEnvironmentIntegrationTest, AnEarlyStopUnderFailFastIsAPartialRun) { + // With a list that only names the never-run test, the deliberate failure + // stops the run before the other tests, and the unseen entry isn't reported + // on top of it. + std::string list = + WriteFile("never_runs_only.txt", absl::StrCat(kNeverRuns, "\n")); + RunResult run = RunSample({absl::StrCat("--failure_list=", list), + absl::StrCat(kGtestFlag, "fail_fast")}); + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(run.output, + AllOf(HasSubstr("[ FAILED ] " + "SampleConformanceTest.SampleDeliberateFailure"), + Not(HasSubstr("were not seen")))); +} + +TEST_F(TestEnvironmentIntegrationTest, FixRewritesTheFailureListInPlace) { + std::string list = WriteFile("failure_list.txt", ""); + RunResult run = RunSample({absl::StrCat("--failure_list=", list), "--fix"}); + // The failures were unexpected in this run; the rewritten list expects them. + EXPECT_NE(run.exit_code, 0); + EXPECT_THAT(ReadFile(list), + AllOf(HasSubstr(kDeliberateFailure), HasSubstr(kP1Failure), + HasSubstr(absl::StrCat("# ", kFailureMessage)))); + run = RunSample({absl::StrCat("--failure_list=", list)}); + EXPECT_EQ(run.exit_code, 0) << run.output; +} + +TEST_F(TestEnvironmentIntegrationTest, RecordsStatisticsInTheXmlReport) { + std::string xml = TmpPath("report.xml"); + RunResult run = RunSample( + {FailureListFlag("ok"), absl::StrCat(kGtestFlag, "output=xml:", xml)}); + ASSERT_EQ(run.exit_code, 0) << run.output; + std::string report = ReadFile(xml); + // Each level gets its own numbers: the P0 and P1 failures are one expected + // failure each, two on their suite and on the run. + EXPECT_THAT(XmlTestCase(report, "SampleDeliberateFailure"), + HasSubstr("name=\"expected_failures\" value=\"1\"")); + EXPECT_THAT(XmlTestCase(report, "SamplePass"), + HasSubstr("name=\"expected_successes\" value=\"1\"")); + EXPECT_THAT(XmlTestSuite(report, "SampleConformanceTest"), + HasSubstr("name=\"expected_failures\" value=\"2\"")); + EXPECT_THAT(XmlTestRun(report), + AllOf(HasSubstr("name=\"expected_failures\" value=\"2\""), + HasSubstr("name=\"unexpected_failures\" value=\"0\""))); +} + +} // namespace +} // namespace conformance +} // namespace protobuf +} // namespace google
diff --git a/conformance/test_environment_main.cc b/conformance/test_environment_main.cc new file mode 100644 index 0000000..2c19e3e --- /dev/null +++ b/conformance/test_environment_main.cc
@@ -0,0 +1,55 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2026 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// A gtest main for conformance test binaries. It configures a +// ConformanceEnvironment from the command-line flags declared in +// test_environment_flags.h and installs it before running all tests. +// Conformance suites should depend on the test_environment_main library +// instead of a generic gtest main. +// +// Any positional arguments left after flag parsing are appended to +// --testee_args, so `conformance_test --testee_binary=foo -- --bar` passes +// `--bar` to the testee. + +#include <utility> +#include <vector> + +#include <gtest/gtest.h> +#include "absl/log/absl_check.h" +#include "absl/types/span.h" +#include "conformance/test_environment.h" +#include "conformance/test_environment_flags.h" + +#include "absl/flags/parse.h" +#include "absl/log/initialize.h" + +// TODO: A custom main only because the stock gtest main parses +// absl flags only when gtest is built with Abseil (GTEST_HAS_ABSL), which +// open-source protobuf doesn't require yet. Once it can be required, install +// the environment from a static initializer and use the stock main instead. +int main(int argc, char* argv[]) { + // In Google's internal build, InitGoogleTest() also runs InitGoogle(), which + // parses absl flags, leaves only positional arguments in argv and + // initializes logging. In open source we do that ourselves, after gtest has + // removed its own flags from argv. + testing::InitGoogleTest(&argc, argv); + std::vector<char*> positional_args(argv, argv + argc); + positional_args = absl::ParseCommandLine(argc, argv); + absl::InitializeLog(); + + // Listing tests doesn't need a testee, and shouldn't require any flags. + if (!GTEST_FLAG_GET(list_tests)) { + google::protobuf::conformance::internal::ConformanceEnvironmentOptions options = + google::protobuf::conformance::internal::OptionsFromFlags( + absl::MakeConstSpan(positional_args).subspan(1)); + ABSL_QCHECK(!options.testee_binary.empty()) + << "--testee_binary is required."; + google::protobuf::conformance::internal::ConformanceEnvironment::Install( + std::move(options)); + } + return RUN_ALL_TESTS(); +}
diff --git a/conformance/testee.cc b/conformance/testee.cc index 9ec8524..ac77e41 100644 --- a/conformance/testee.cc +++ b/conformance/testee.cc
@@ -5,6 +5,7 @@ #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "conformance/binary_wireformat.h" @@ -140,6 +141,23 @@ return "Unknown"; } +bool AbslParseFlag(absl::string_view text, TestPriority* priority, + std::string* error) { + int number; + if (!absl::SimpleAtoi(text, &number) || number < 0 || + number > static_cast<int>(kLowestPriority)) { + *error = absl::StrCat("expected a priority number from 0 to ", + static_cast<int>(kLowestPriority)); + return false; + } + *priority = static_cast<TestPriority>(number); + return true; +} + +std::string AbslUnparseFlag(TestPriority priority) { + return absl::StrCat(static_cast<int>(priority)); +} + absl::string_view PriorityLevelName(TestPriority priority) { // TODO: b/564550230 - return PriorityName() once the tests are renamed. return priority == TestPriority::kP1 ? "Recommended" : "Required";
diff --git a/conformance/testee.h b/conformance/testee.h index 2757d36..932e30e 100644 --- a/conformance/testee.h +++ b/conformance/testee.h
@@ -64,6 +64,12 @@ // The name of a priority: "P0" or "P1". absl::string_view PriorityName(TestPriority priority); +// Flag support for TestPriority, as the priority's number: "0" is kP0, "1" is +// kP1. +bool AbslParseFlag(absl::string_view text, TestPriority* priority, + std::string* error); +std::string AbslUnparseFlag(TestPriority priority); + // The level a priority is named with in test names, until the rename (see // TestPriority): "Required" for kP0, "Recommended" for kP1. absl::string_view PriorityLevelName(TestPriority priority);