conformance/test_environment_test.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2026 Google LLC.  All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd

#include "conformance/test_environment.h"

#ifdef _WIN32
#include <direct.h>  // for _chdir, _getcwd
#else
#include <unistd.h>  // for chdir, getcwd
#endif

#include <cstdlib>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "google/protobuf/testing/file.h"
#include "google/protobuf/testing/file.h"
#include "google/protobuf/testing/file.h"
#include "google/protobuf/descriptor.pb.h"
#include <gmock/gmock.h>
#include "google/protobuf/testing/googletest.h"
#include <gtest/gtest-spi.h>
#include <gtest/gtest.h>
#include "absl/log/absl_check.h"
#include "absl/status/status_matchers.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "conformance/binary_wireformat.h"
#include "conformance/conformance.pb.h"
#include "conformance/global_test_environment.h"
#include "conformance/matchers.h"
#include "conformance/mock_test_runner.h"
#include "conformance/test_environment_testing.h"
#include "conformance/test_manager.h"
#include "conformance/testee.h"
#include "google/protobuf/test_messages_proto3.pb.h"
#include "google/protobuf/text_format.h"

namespace google {
namespace protobuf {
namespace conformance {
namespace {

using ::absl_testing::IsOk;
using ::google::protobuf::conformance::internal::ConformanceEnvironment;
using ::google::protobuf::conformance::internal::ConformanceEnvironmentOptions;
using ::protobuf_test_messages::proto3::TestAllTypesProto3;
using ::testing::_;
using ::testing::HasSubstr;
using ::testing::NiceMock;
using ::testing::Not;
using ::testing::Return;

// The name the testee sees for a kP0 test named "Foo" that parses
// TestAllTypesProto3 from binary and serializes it back to binary.
constexpr absl::string_view kFooTestName =
    "Required.Proto3.ProtobufInput.Foo.ProtobufOutput";
constexpr absl::string_view kBarTestName =
    "Required.Proto3.ProtobufInput.Bar.ProtobufOutput";

std::string SerializedResponse(absl::string_view textproto) {
  ::conformance::ConformanceResponse response;
  ABSL_CHECK(TextFormat::ParseFromString(textproto, &response));
  return response.SerializeAsString();
}

// A testee that successfully round-trips `optional_int32: 99` for every test
// (by default: being a mock, a test can still set expectations on it).
class FakeTestRunner : public NiceMock<MockTestRunner> {
 public:
  FakeTestRunner() {
    ON_CALL(*this, RunTest)
        .WillByDefault(
            Return(SerializedResponse(R"pb(protobuf_payload: "\010c")pb")));
  }
};

// A FakeTestRunner that reports its destruction.
class FlaggingTestRunner : public FakeTestRunner {
 public:
  explicit FlaggingTestRunner(bool* destroyed) : destroyed_(destroyed) {}
  ~FlaggingTestRunner() override { *destroyed_ = true; }

 private:
  bool* destroyed_;
};

// Sets an environment variable for the lifetime of this object.
class ScopedEnvironmentVariable {
 public:
  ScopedEnvironmentVariable(absl::string_view name,
                            const absl::optional<std::string>& value)
      : name_(name) {
    if (const char* old = std::getenv(name_.c_str()); old != nullptr) {
      old_value_ = old;
    }
    Set(value);
  }
  ~ScopedEnvironmentVariable() { Set(old_value_); }

 private:
  void Set(const absl::optional<std::string>& value) {
#ifdef _WIN32
    // The MSVC CRT has no setenv()/unsetenv(); _putenv_s() with an empty
    // value removes the variable, so std::getenv() then returns null again.
    ABSL_CHECK_EQ(
        _putenv_s(name_.c_str(), value.has_value() ? value->c_str() : ""), 0);
#else
    if (value.has_value()) {
      ABSL_CHECK_EQ(setenv(name_.c_str(), value->c_str(), /*overwrite=*/1), 0);
    } else {
      ABSL_CHECK_EQ(unsetenv(name_.c_str()), 0);
    }
#endif
  }

  std::string name_;
  absl::optional<std::string> old_value_;
};

// Changes the working directory for the lifetime of this object.
class ScopedWorkingDirectory {
 public:
  explicit ScopedWorkingDirectory(const std::string& directory) {
    char* cwd = CurrentDirectory();
    ABSL_CHECK(cwd != nullptr);
    old_directory_ = cwd;
    std::free(cwd);
    ABSL_CHECK_EQ(ChangeDirectory(directory), 0) << directory;
  }
  ~ScopedWorkingDirectory() {
    ABSL_CHECK_EQ(ChangeDirectory(old_directory_), 0) << old_directory_;
  }

 private:
  // Both glibc/macOS (as an extension) and the MSVC CRT malloc() a buffer of
  // the right size when given a null one.
  static char* CurrentDirectory() {
#ifdef _WIN32
    return _getcwd(nullptr, 0);
#else
    return getcwd(nullptr, 0);
#endif
  }
  static int ChangeDirectory(const std::string& directory) {
#ifdef _WIN32
    return _chdir(directory.c_str());
#else
    return chdir(directory.c_str());
#endif
  }

  std::string old_directory_;
};

// Tears down `env` in a full run, expecting it to report unseen expected
// failures with a message containing `expected_message`.
void TearDownExpectingUnseenFailures(ConformanceEnvironment& env,
                                     absl::string_view expected_message) {
  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(env.TearDown(), std::string(expected_message));
}

class ConformanceEnvironmentTest : public ::testing::Test {
 protected:
  ConformanceEnvironmentTest()
      : tmp_dir_(absl::StrCat(
            TestTempDir(), "/test_environment_test/",
            testing::UnitTest::GetInstance()->current_test_info()->name())) {
    ABSL_CHECK_OK(
        File::RecursivelyCreateDir(tmp_dir_, 0777));
  }

  ~ConformanceEnvironmentTest() override {
    File::DeleteRecursively(tmp_dir_, NULL, NULL);
  }

  // Writes `content` to a file named `name` in this test's temp directory and
  // returns its path.
  std::string WriteFile(absl::string_view name, absl::string_view content) {
    std::string path = TmpPath(name);
    ABSL_CHECK_OK(File::SetContents(path, content, true));
    return path;
  }

  // `path` is a std::string (not a string_view) for the open source
  // File::GetContents() shim.
  std::string ReadFile(const std::string& path) {
    std::string content;
    ABSL_CHECK_OK(File::GetContents(path, &content, true));
    return content;
  }

  std::string TmpPath(absl::string_view name) {
    return absl::StrCat(tmp_dir_, "/", name);
  }

  std::string CreateTmpDir(absl::string_view name) {
    std::string path = TmpPath(name);
    ABSL_CHECK_OK(File::RecursivelyCreateDir(path, 0777));
    return path;
  }

  // Runs a kP0 test named `name` against the global testee.
  internal::TestResult RunP0(absl::string_view name) {
    return ConformanceEnvironment::Get()
        .testee()
        .CreateTest(name, kP0)
        .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
        .SerializeBinary();
  }

  internal::TestResult RunP1(absl::string_view name) {
    return ConformanceEnvironment::Get()
        .testee()
        .CreateTest(name, kP1)
        .ParseBinary(TestAllTypesProto3::descriptor(), VarintField(1, 99))
        .SerializeBinary();
  }

  FakeTestRunner fake_runner_;

 private:
  std::string tmp_dir_;
};

TEST_F(ConformanceEnvironmentTest, ScopedEnvironmentIsTheGlobalOne) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  EXPECT_EQ(&internal::GetGlobalTestManager(), &env->test_manager());
}

TEST_F(ConformanceEnvironmentTest, EnforcementLevelIsPassedToTestManager) {
  {
    // Every priority is enforced by default.
    internal::ScopedGlobalConformanceEnvironment env(
        {/*runner=*/&fake_runner_});
    EXPECT_THAT(env->test_manager().ReportFailure("P1.x", kP1, "boom"),
                Not(IsOk()));
  }
  {
    ConformanceEnvironmentOptions options;
    options.runner = &fake_runner_;
    options.enforcement_level = kP0;
    internal::ScopedGlobalConformanceEnvironment env(std::move(options));
    EXPECT_THAT(env->test_manager().ReportFailure("P1.x", kP1, "boom"), IsOk());
  }
}

TEST_F(ConformanceEnvironmentTest, TesteeRoutesToRunner) {
  MockTestRunner mock;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&mock});
  env->SetUp();

  EXPECT_CALL(mock, RunTest(kFooTestName, _))
      .WillOnce(Return(SerializedResponse(R"pb(protobuf_payload: "\010c")pb")));
  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));

  EXPECT_EQ(env->test_manager().expected_successes(), 1);
  EXPECT_EQ(env->test_manager().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, UnexpectedFailureFailsTheTest) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();

  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");

  EXPECT_EQ(env->test_manager().unexpected_failures(), 1);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, ExpectedFailureFromFailureListPasses) {
  std::string failure_list =
      WriteFile("failures.txt",
                absl::StrCat(kFooTestName, " # Should have failed to parse\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP0("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->test_manager().expected_failures(), 1);
  EXPECT_EQ(env->test_manager().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, LoadsMultipleFailureLists) {
  std::string first = WriteFile("first.txt", absl::StrCat(kFooTestName, "\n"));
  std::string second =
      WriteFile("second.txt", absl::StrCat(kBarTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP0("Foo"), Yields(IsParseError()));
  EXPECT_THAT(RunP0("Bar"), Yields(IsParseError()));

  EXPECT_EQ(env->test_manager().expected_failures(), 2);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, OverlappingFailureListsAreFatalInSetUp) {
  std::string first = WriteFile("first.txt", absl::StrCat(kFooTestName, "\n"));
  std::string second =
      WriteFile("second.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));

  testing::TestPartResultArray failures;
  {
    testing::ScopedFakeTestPartResultReporter reporter(
        testing::ScopedFakeTestPartResultReporter::
            INTERCEPT_ONLY_CURRENT_THREAD,
        &failures);
    env->SetUp();
  }
  ASSERT_EQ(failures.size(), 1);
  EXPECT_TRUE(failures.GetTestPartResult(0).fatally_failed());
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr(absl::StrCat("Failed to load failure list ", second)));
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr("already exists"));
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, P1FailureIsToleratedWhenNotEnforced) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.enforcement_level = kP0;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP1("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->test_manager().tolerated_failures(), 1);
  EXPECT_EQ(env->test_manager().skipped(), 0);
  EXPECT_EQ(env->test_manager().unexpected_failures(), 0);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest,
       ListedP1FailureIsExpectedEvenWhenNotEnforced) {
  std::string failure_list = WriteFile(
      "failures.txt",
      "Recommended.Proto3.ProtobufInput.Foo.ProtobufOutput # Should have "
      "failed to parse\n");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.enforcement_level = kP0;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_THAT(RunP1("Foo"), Yields(IsParseError()));

  EXPECT_EQ(env->test_manager().expected_failures(), 1);
  EXPECT_EQ(env->test_manager().tolerated_failures(), 0);
  // ...and so it isn't unseen either.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, MissingFailureListIsFatalInSetUp) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {TmpPath("does_not_exist.txt")};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));

  testing::TestPartResultArray failures;
  {
    testing::ScopedFakeTestPartResultReporter reporter(
        testing::ScopedFakeTestPartResultReporter::
            INTERCEPT_ONLY_CURRENT_THREAD,
        &failures);
    env->SetUp();
  }
  ASSERT_EQ(failures.size(), 1);
  EXPECT_TRUE(failures.GetTestPartResult(0).fatally_failed());
  EXPECT_THAT(failures.GetTestPartResult(0).message(),
              HasSubstr("Failed to load failure list"));

  // TearDown must not pile on more failures after a failed SetUp.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, TearDownFailsOnUnseenExpectedFailures) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("expected failures were not seen: ", kFooTestName,
                   "\nRemove them from the failure list, or rerun with --fix"));
}

TEST_F(ConformanceEnvironmentTest, UnseenExpectedFailuresAreFineInPartialRun) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride partial_run(true);
  env->TearDown();
  EXPECT_THAT(env->test_manager().Finalize().message(),
              HasSubstr(absl::StrCat("were not seen: ", kFooTestName)));
}

TEST_F(ConformanceEnvironmentTest, UnseenCheckCanBeDisabled) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, "\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.check_unseen_expected_failures = false;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
  EXPECT_THAT(env->test_manager().Finalize().message(),
              HasSubstr(absl::StrCat("were not seen: ", kFooTestName)));
}

TEST_F(ConformanceEnvironmentTest, FixWritesToTheOutputFile) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  std::string fixed = TmpPath("fixed.txt");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  // Foo (listed) is never run; Bar (unlisted) fails.
  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");
  // The unseen entry is still reported, but the message points at the
  // rewritten list instead of suggesting --fix.
  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("expected failures were not seen: ", kFooTestName,
                   "\nThey have been dropped from the updated failure list "
                   "written to ",
                   fixed));

  std::string content = ReadFile(fixed);
  EXPECT_THAT(content, HasSubstr(absl::StrCat(
                           kBarTestName,
                           " # Should have failed to parse, but didn't.")));
  EXPECT_THAT(content, Not(HasSubstr(kFooTestName)));
  // The input is left alone when an explicit output file is given.
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("stale"));
}

TEST_F(ConformanceEnvironmentTest, FixDefaultsToTheSingleAbsoluteFailureList) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");
  TearDownExpectingUnseenFailures(
      env.environment(),
      absl::StrCat("updated failure list written to ", failure_list));

  std::string content = ReadFile(failure_list);
  EXPECT_THAT(content, HasSubstr(kBarTestName));
  EXPECT_THAT(content, Not(HasSubstr(kFooTestName)));
}

TEST_F(ConformanceEnvironmentTest,
       FixResolvesRelativeFailureListUnderBuildWorkspaceDirectory) {
  // The relative path is loaded from the working directory (the runfiles, under
  // bazel) but rewritten under $BUILD_WORKSPACE_DIRECTORY (the source tree).
  std::string runfiles = CreateTmpDir("runfiles");
  std::string workspace = CreateTmpDir("workspace");
  std::string stale = absl::StrCat(kFooTestName, " # stale\n");
  WriteFile("runfiles/failures.txt", stale);
  WriteFile("workspace/failures.txt", stale);
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", workspace);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {"failures.txt"};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");
  TearDownExpectingUnseenFailures(
      env.environment(), absl::StrCat("updated failure list written to ",
                                      workspace, "/failures.txt"));

  EXPECT_THAT(ReadFile(TmpPath("workspace/failures.txt")),
              HasSubstr(kBarTestName));
  EXPECT_EQ(ReadFile(TmpPath("runfiles/failures.txt")), stale);
}

TEST_F(ConformanceEnvironmentTest,
       FixWithRelativeFailureListNeedsBuildWorkspaceDirectory) {
  std::string runfiles = CreateTmpDir("runfiles");
  WriteFile("runfiles/failures.txt", "");
  ScopedWorkingDirectory cwd(runfiles);
  ScopedEnvironmentVariable build_workspace_directory(
      "BUILD_WORKSPACE_DIRECTORY", absl::nullopt);

  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {"failures.txt"};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      "--fix needs either --fix_output_file or $BUILD_WORKSPACE_DIRECTORY");
  EXPECT_EQ(ReadFile(TmpPath("runfiles/failures.txt")), "");
}

TEST_F(ConformanceEnvironmentTest, FixWithMultipleListsNeedsOutputFile) {
  std::string first = WriteFile("first.txt", "");
  std::string second = WriteFile("second.txt", "");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {first, second};
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      "--fix needs --fix_output_file when more than one --failure_list");
}

TEST_F(ConformanceEnvironmentTest, FixWithNoListsNeedsOutputFile) {
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.fix = true;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(), "--fix needs --fix_output_file when no --failure_list");
}

TEST_F(ConformanceEnvironmentTest, FixIsRefusedInPartialRun) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # stale\n"));
  std::string fixed = TmpPath("fixed.txt");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");

  // Rewriting would drop Foo, whose test simply wasn't selected.  The unseen
  // check itself stays quiet in a partial run, so this is the only failure.
  internal::ScopedPartialRunOverride partial_run(true);
  EXPECT_NONFATAL_FAILURE(env->TearDown(), "--fix ignored");
  EXPECT_FALSE(File::Exists(fixed));
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("stale"));
}

TEST_F(ConformanceEnvironmentTest, FixReportsWriteFailures) {
  std::string failure_list = WriteFile("failures.txt", "");
  std::string not_a_file = CreateTmpDir("directory");
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = not_a_file;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();

  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse, but didn't.");

  internal::ScopedPartialRunOverride full_run(false);
  EXPECT_NONFATAL_FAILURE(
      env->TearDown(),
      absl::StrCat("Failed to write updated failure list to ", not_a_file));
}

TEST_F(ConformanceEnvironmentTest, StatisticsSnapshotAndDelta) {
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&fake_runner_});
  env->SetUp();
  internal::Statistics before = internal::Statistics::From(env->test_manager());

  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  EXPECT_NONFATAL_FAILURE(EXPECT_THAT(RunP0("Bar"), Yields(IsParseError())),
                          "Should have failed to parse");

  internal::Statistics delta =
      internal::Statistics::From(env->test_manager()) - before;
  EXPECT_EQ(delta.expected_successes, 1);
  EXPECT_EQ(delta.unexpected_failures, 1);
  EXPECT_EQ(delta.expected_failures, 0);
  EXPECT_EQ(delta.unexpected_successes, 0);
  EXPECT_EQ(delta.skipped_tests, 0);
  EXPECT_EQ(delta.listed_skips, 0);
  EXPECT_EQ(delta.tolerated_failures, 0);

  internal::Statistics total;
  total += delta;
  total += delta;
  EXPECT_EQ(total.expected_successes, 2);
  EXPECT_EQ(total.unexpected_failures, 2);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, StatisticsCountListedSkips) {
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # listed\n"));
  MockTestRunner mock;
  ConformanceEnvironmentOptions options;
  options.runner = &mock;
  options.failure_list_files = {failure_list};
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();
  internal::Statistics before = internal::Statistics::From(env->test_manager());

  // A listed test the testee skips fails (see matchers.h) and is both a skip
  // and a listed skip, but neither an expected nor an unexpected anything.
  EXPECT_CALL(mock, RunTest(kFooTestName, _))
      .WillOnce(Return(SerializedResponse(R"pb(skipped: "not supported")pb")));
  EXPECT_NONFATAL_FAILURE(
      EXPECT_THAT(RunP0("Foo"), Yields(IsParseError())),
      "is in the failure list but was skipped by the testee");

  internal::Statistics delta =
      internal::Statistics::From(env->test_manager()) - before;
  EXPECT_EQ(delta.skipped_tests, 1);
  EXPECT_EQ(delta.listed_skips, 1);
  EXPECT_EQ(delta.expected_failures, 0);
  EXPECT_EQ(delta.unexpected_failures, 0);
  EXPECT_EQ(delta.expected_successes, 0);
  EXPECT_EQ(delta.unexpected_successes, 0);

  internal::Statistics total;
  total += delta;
  total += delta;
  EXPECT_EQ(total.skipped_tests, 2);
  EXPECT_EQ(total.listed_skips, 2);
  // The skip counts as having seen the entry, so this doesn't report it.
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, MaximumEditionIsClampedToProto3) {
  // proto2 and proto3 tests always run, so anything older than proto3
  // (including the legacy runner's EDITION_UNKNOWN default) behaves like
  // proto3 rather than disabling everything.
  for (Edition edition : {EDITION_UNKNOWN, EDITION_LEGACY, EDITION_PROTO2}) {
    ConformanceEnvironmentOptions options;
    options.runner = &fake_runner_;
    options.maximum_edition = edition;
    internal::ScopedGlobalConformanceEnvironment env(std::move(options));
    EXPECT_EQ(env->options().maximum_edition, EDITION_PROTO3);
  }
  ConformanceEnvironmentOptions options;
  options.runner = &fake_runner_;
  options.maximum_edition = EDITION_2023;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  EXPECT_EQ(env->options().maximum_edition, EDITION_2023);
}

TEST_F(ConformanceEnvironmentTest, SpawnsForkPipeRunnerLazily) {
  // The testee binary doesn't exist, but ForkPipeRunner only spawns it on
  // first use, so constructing the environment must succeed.
  ConformanceEnvironmentOptions options;
  options.testee_binary = TmpPath("no_such_testee");
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  env->SetUp();
  env->TearDown();
}

TEST_F(ConformanceEnvironmentTest, TearDownShutsDownTheOwnedRunner) {
  bool destroyed = false;
  internal::ScopedGlobalConformanceEnvironment env(
      {/*runner=*/nullptr,
       /*owned_runner=*/std::make_unique<FlaggingTestRunner>(&destroyed)});
  env->SetUp();
  EXPECT_THAT(RunP0("Foo"),
              Yields(WhenParsed(EqualsTextProto(R"pb(optional_int32: 99)pb"))));
  EXPECT_FALSE(destroyed);

  env->TearDown();
  EXPECT_TRUE(destroyed);
  // The statistics survive the testee.
  EXPECT_EQ(env->test_manager().expected_successes(), 1);
}

TEST_F(ConformanceEnvironmentTest, TearDownWithoutSetUpOnlyReleasesTheTestee) {
  // gtest skips an environment's SetUp() and TearDown() when it selects no
  // test to run; the merged runner (conformance_test_main.cc) then calls
  // TearDown() itself so that the environment lets go of its runner.  Nothing
  // else may happen: no --fix rewrite and no unseen-entry failure, as the
  // failure list was never loaded.
  std::string failure_list =
      WriteFile("failures.txt", absl::StrCat(kFooTestName, " # never seen\n"));
  std::string fixed = TmpPath("fixed.txt");
  bool destroyed = false;
  ConformanceEnvironmentOptions options;
  options.owned_runner = std::make_unique<FlaggingTestRunner>(&destroyed);
  options.failure_list_files = {failure_list};
  options.fix = true;
  options.fix_output_file = fixed;
  internal::ScopedGlobalConformanceEnvironment env(std::move(options));
  internal::ScopedPartialRunOverride full_run(false);
  env->TearDown();
  EXPECT_TRUE(destroyed);
  EXPECT_FALSE(File::Exists(fixed));
  EXPECT_THAT(ReadFile(failure_list), HasSubstr("never seen"));
  EXPECT_EQ(env->test_manager().expected_failures(), 0);
}

TEST_F(ConformanceEnvironmentTest, OwnedRunnerDiesWithTheEnvironment) {
  bool destroyed = false;
  {
    internal::ScopedGlobalConformanceEnvironment env(
        {/*runner=*/nullptr,
         /*owned_runner=*/std::make_unique<FlaggingTestRunner>(&destroyed)});
    EXPECT_FALSE(destroyed);
  }
  EXPECT_TRUE(destroyed);
}

TEST(ConformanceEnvironmentDeathTest, GetWithoutInstall) {
  EXPECT_DEATH(ConformanceEnvironment::Get(), "Install");
}

TEST(ConformanceEnvironmentDeathTest, RequiresExactlyOneTestee) {
  FakeTestRunner runner;
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env({}),
               "Exactly one");
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env(
                   {/*runner=*/&runner, /*owned_runner=*/nullptr,
                    /*testee_binary=*/"/bin/true"}),
               "Exactly one");
  EXPECT_DEATH(internal::ScopedGlobalConformanceEnvironment env(
                   {/*runner=*/&runner,
                    /*owned_runner=*/std::make_unique<FakeTestRunner>()}),
               "Exactly one");
}

TEST(ConformanceEnvironmentDeathTest, OnlyOneGlobalEnvironmentAtATime) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  EXPECT_DEATH(
      internal::ScopedGlobalConformanceEnvironment nested({/*runner=*/&runner}),
      "already set");
  EXPECT_DEATH(ConformanceEnvironment::Install({/*runner=*/&runner}),
               "only be called once");
}

TEST(ConformanceEnvironmentDeathTest, OnlyOnePartialRunOverrideAtATime) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  internal::ScopedPartialRunOverride partial_run(true);
  EXPECT_DEATH(internal::ScopedPartialRunOverride nested(false),
               "already active");
}

TEST(ConformanceEnvironmentDeathTest, TesteeIsGoneAfterTearDown) {
  FakeTestRunner runner;
  internal::ScopedGlobalConformanceEnvironment env({/*runner=*/&runner});
  env->SetUp();
  env->TearDown();
  EXPECT_DEATH(env->testee(), "already been shut down");
}

}  // namespace
}  // namespace conformance
}  // namespace protobuf
}  // namespace google