Googletest export
gtest: Add a flag to only set up/tear down test environments once when repeating
Currently when running a test multiple times using `--gtest_repeat` the global
test environment(s) are set up and torn down for each iteration of the test.
When checking for flakes in tests that have expensive dependencies that are set
up in the test environment (subprocesses, external dependencies, etc) this can
become expensive.
To support finding flakes in tests that fit into this category, where the setup
phase is expensive but each test case is fast, allow callers to specify via
`--gtest_recreate_environments_when_repeating=false` that the test environments
should only be set up once, for the first iteration, and only torn down once, on
the last iteration. This makes running a test with `--gtest_repeat=1000` a much
faster and more pleasant experience.
PiperOrigin-RevId: 382748942
diff --git a/googletest/include/gtest/gtest.h b/googletest/include/gtest/gtest.h
index 7a5d057..7e8e83f 100644
--- a/googletest/include/gtest/gtest.h
+++ b/googletest/include/gtest/gtest.h
@@ -138,6 +138,12 @@
// is 1. If the value is -1 the tests are repeating forever.
GTEST_DECLARE_int32_(repeat);
+// This flag controls whether Google Test Environments are recreated for each
+// repeat of the tests. The default value is true. If set to false the global
+// test Environment objects are only set up once, for the first iteration, and
+// only torn down once, for the last.
+GTEST_DECLARE_bool_(recreate_environments_when_repeating);
+
// This flag controls whether Google Test includes Google Test internal
// stack frames in failure stack traces.
GTEST_DECLARE_bool_(show_internal_stack_frames);
diff --git a/googletest/src/gtest-internal-inl.h b/googletest/src/gtest-internal-inl.h
index 6d8cecb..59762c7 100644
--- a/googletest/src/gtest-internal-inl.h
+++ b/googletest/src/gtest-internal-inl.h
@@ -93,6 +93,8 @@
const char kPrintUTF8Flag[] = "print_utf8";
const char kRandomSeedFlag[] = "random_seed";
const char kRepeatFlag[] = "repeat";
+const char kRecreateEnvironmentsWhenRepeatingFlag[] =
+ "recreate_environments_when_repeating";
const char kShuffleFlag[] = "shuffle";
const char kStackTraceDepthFlag[] = "stack_trace_depth";
const char kStreamResultToFlag[] = "stream_result_to";
@@ -176,6 +178,8 @@
print_utf8_ = GTEST_FLAG(print_utf8);
random_seed_ = GTEST_FLAG(random_seed);
repeat_ = GTEST_FLAG(repeat);
+ recreate_environments_when_repeating_ =
+ GTEST_FLAG(recreate_environments_when_repeating);
shuffle_ = GTEST_FLAG(shuffle);
stack_trace_depth_ = GTEST_FLAG(stack_trace_depth);
stream_result_to_ = GTEST_FLAG(stream_result_to);
@@ -200,6 +204,8 @@
GTEST_FLAG(print_utf8) = print_utf8_;
GTEST_FLAG(random_seed) = random_seed_;
GTEST_FLAG(repeat) = repeat_;
+ GTEST_FLAG(recreate_environments_when_repeating) =
+ recreate_environments_when_repeating_;
GTEST_FLAG(shuffle) = shuffle_;
GTEST_FLAG(stack_trace_depth) = stack_trace_depth_;
GTEST_FLAG(stream_result_to) = stream_result_to_;
@@ -224,6 +230,7 @@
bool print_utf8_;
int32_t random_seed_;
int32_t repeat_;
+ bool recreate_environments_when_repeating_;
bool shuffle_;
int32_t stack_trace_depth_;
std::string stream_result_to_;
diff --git a/googletest/src/gtest.cc b/googletest/src/gtest.cc
index 21c611a..9a380eb 100644
--- a/googletest/src/gtest.cc
+++ b/googletest/src/gtest.cc
@@ -304,6 +304,17 @@
"How many times to repeat each test. Specify a negative number "
"for repeating forever. Useful for shaking out flaky tests.");
+GTEST_DEFINE_bool_(
+ recreate_environments_when_repeating,
+ internal::BoolFromGTestEnv("recreate_environments_when_repeating", true),
+ "Controls whether global test environments are recreated for each repeat "
+ "of the tests. If set to false the global test environments are only set "
+ "up once, for the first iteration, and only torn down once, for the last. "
+ "Useful for shaking out flaky tests with stable, expensive test "
+ "environments. If --gtest_repeat is set to a negative number, meaning "
+ "there is no last run, the environments will always be recreated to avoid "
+ "leaks.");
+
GTEST_DEFINE_bool_(show_internal_stack_frames, false,
"True if and only if " GTEST_NAME_
" should include internal stack frames when "
@@ -5805,8 +5816,19 @@
// How many times to repeat the tests? We don't want to repeat them
// when we are inside the subprocess of a death test.
const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat);
+
// Repeats forever if the repeat count is negative.
const bool gtest_repeat_forever = repeat < 0;
+
+ // Should test environments be set up and torn down for each repeat, or only
+ // set up on the first and torn down on the last iteration? If there is no
+ // "last" iteration because the tests will repeat forever, always recreate the
+ // environments to avoid leaks in case one of the environments is using
+ // resources that are external to this process. Without this check there would
+ // be no way to clean up those external resources automatically.
+ const bool recreate_environments_when_repeating =
+ GTEST_FLAG(recreate_environments_when_repeating) || gtest_repeat_forever;
+
for (int i = 0; gtest_repeat_forever || i != repeat; i++) {
// We want to preserve failures generated by ad-hoc test
// assertions executed before RUN_ALL_TESTS().
@@ -5828,10 +5850,13 @@
// Runs each test suite if there is at least one test to run.
if (has_tests_to_run) {
- // Sets up all environments beforehand.
- repeater->OnEnvironmentsSetUpStart(*parent_);
- ForEach(environments_, SetUpEnvironment);
- repeater->OnEnvironmentsSetUpEnd(*parent_);
+ // Sets up all environments beforehand. If test environments aren't
+ // recreated for each iteration, only do so on the first iteration.
+ if (i == 0 || recreate_environments_when_repeating) {
+ repeater->OnEnvironmentsSetUpStart(*parent_);
+ ForEach(environments_, SetUpEnvironment);
+ repeater->OnEnvironmentsSetUpEnd(*parent_);
+ }
// Runs the tests only if there was no fatal failure or skip triggered
// during global set-up.
@@ -5871,11 +5896,15 @@
}
}
- // Tears down all environments in reverse order afterwards.
- repeater->OnEnvironmentsTearDownStart(*parent_);
- std::for_each(environments_.rbegin(), environments_.rend(),
- TearDownEnvironment);
- repeater->OnEnvironmentsTearDownEnd(*parent_);
+ // Tears down all environments in reverse order afterwards. If test
+ // environments aren't recreated for each iteration, only do so on the
+ // last iteration.
+ if (i == repeat - 1 || recreate_environments_when_repeating) {
+ repeater->OnEnvironmentsTearDownStart(*parent_);
+ std::for_each(environments_.rbegin(), environments_.rend(),
+ TearDownEnvironment);
+ repeater->OnEnvironmentsTearDownEnd(*parent_);
+ }
}
elapsed_time_ = timer.Elapsed();
@@ -6437,6 +6466,10 @@
"random_seed=@Y[NUMBER]@D\n"
" Random number seed to use for shuffling test orders (between 1 and\n"
" 99999, or 0 to use a seed based on the current time).\n"
+ " @G--" GTEST_FLAG_PREFIX_
+ "recreate_environments_when_repeating@D\n"
+ " Sets up and tears down the global test environment on each repeat\n"
+ " of the test.\n"
"\n"
"Test Output:\n"
" @G--" GTEST_FLAG_PREFIX_
@@ -6519,6 +6552,8 @@
ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) ||
ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) ||
ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) ||
+ ParseBoolFlag(arg, kRecreateEnvironmentsWhenRepeatingFlag,
+ >EST_FLAG(recreate_environments_when_repeating)) ||
ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) ||
ParseInt32Flag(arg, kStackTraceDepthFlag,
>EST_FLAG(stack_trace_depth)) ||
diff --git a/googletest/test/googletest-global-environment-unittest.py b/googletest/test/googletest-global-environment-unittest.py
index 32ba628..f347559 100644
--- a/googletest/test/googletest-global-environment-unittest.py
+++ b/googletest/test/googletest-global-environment-unittest.py
@@ -35,16 +35,17 @@
googletest-global-environment-unittest_ (a program written with Google Test).
"""
+import re
import gtest_test_utils
-def RunAndReturnOutput():
+def RunAndReturnOutput(args=None):
"""Runs the test program and returns its output."""
return gtest_test_utils.Subprocess([
gtest_test_utils.GetTestExecutablePath(
'googletest-global-environment-unittest_')
- ]).output
+ ] + (args or [])).output
class GTestGlobalEnvironmentUnitTest(gtest_test_utils.TestCase):
@@ -67,6 +68,61 @@
# The test case shouldn't have been run.
self.assertNotIn('Unexpected call', txt)
+ def testEnvironmentSetUpAndTornDownForEachRepeat(self):
+ """Tests the behavior of test environments and gtest_repeat."""
+
+ txt = RunAndReturnOutput(['--gtest_repeat=2'])
+
+ # By default, with gtest_repeat=2, the global test environment should be set
+ # up and torn down for each iteration.
+ expected_pattern = ('(.|\n)*'
+ r'Repeating all tests \(iteration 1\)'
+ '(.|\n)*'
+ 'Global test environment set-up.'
+ '(.|\n)*'
+ 'SomeTest.DoesFoo'
+ '(.|\n)*'
+ 'Global test environment tear-down'
+ '(.|\n)*'
+ r'Repeating all tests \(iteration 2\)'
+ '(.|\n)*'
+ 'Global test environment set-up.'
+ '(.|\n)*'
+ 'SomeTest.DoesFoo'
+ '(.|\n)*'
+ 'Global test environment tear-down'
+ '(.|\n)*')
+ self.assertRegex(txt, expected_pattern)
+
+ def testEnvironmentSetUpAndTornDownOnce(self):
+ """Tests environment and --gtest_recreate_environments_when_repeating."""
+
+ txt = RunAndReturnOutput([
+ '--gtest_repeat=2', '--gtest_recreate_environments_when_repeating=false'
+ ])
+
+ # When --gtest_recreate_environments_when_repeating is false, the test
+ # environment should only be set up and torn down once, at the start and
+ # end of the test respectively.
+ expected_pattern = ('(.|\n)*'
+ r'Repeating all tests \(iteration 1\)'
+ '(.|\n)*'
+ 'Global test environment set-up.'
+ '(.|\n)*'
+ 'SomeTest.DoesFoo'
+ '(.|\n)*'
+ r'Repeating all tests \(iteration 2\)'
+ '(.|\n)*'
+ 'SomeTest.DoesFoo'
+ '(.|\n)*'
+ 'Global test environment tear-down'
+ '(.|\n)*')
+ self.assertRegex(txt, expected_pattern)
+
+ self.assertEqual(len(re.findall('Global test environment set-up', txt)), 1)
+ self.assertEqual(
+ len(re.findall('Global test environment tear-down', txt)), 1)
+
if __name__ == '__main__':
gtest_test_utils.Main()
diff --git a/googletest/test/gtest_unittest.cc b/googletest/test/gtest_unittest.cc
index 1730e8b..402bb6d 100644
--- a/googletest/test/gtest_unittest.cc
+++ b/googletest/test/gtest_unittest.cc
@@ -48,6 +48,7 @@
testing::GTEST_FLAG(brief) || testing::GTEST_FLAG(print_time) ||
testing::GTEST_FLAG(random_seed) ||
testing::GTEST_FLAG(repeat) > 0 ||
+ testing::GTEST_FLAG(recreate_environments_when_repeating) ||
testing::GTEST_FLAG(show_internal_stack_frames) ||
testing::GTEST_FLAG(shuffle) ||
testing::GTEST_FLAG(stack_trace_depth) > 0 ||
@@ -212,6 +213,7 @@
using testing::GTEST_FLAG(print_time);
using testing::GTEST_FLAG(random_seed);
using testing::GTEST_FLAG(repeat);
+using testing::GTEST_FLAG(recreate_environments_when_repeating);
using testing::GTEST_FLAG(show_internal_stack_frames);
using testing::GTEST_FLAG(shuffle);
using testing::GTEST_FLAG(stack_trace_depth);
@@ -1610,6 +1612,7 @@
GTEST_FLAG(print_time) = true;
GTEST_FLAG(random_seed) = 0;
GTEST_FLAG(repeat) = 1;
+ GTEST_FLAG(recreate_environments_when_repeating) = true;
GTEST_FLAG(shuffle) = false;
GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
GTEST_FLAG(stream_result_to) = "";
@@ -1639,6 +1642,7 @@
EXPECT_TRUE(GTEST_FLAG(print_time));
EXPECT_EQ(0, GTEST_FLAG(random_seed));
EXPECT_EQ(1, GTEST_FLAG(repeat));
+ EXPECT_TRUE(GTEST_FLAG(recreate_environments_when_repeating));
EXPECT_FALSE(GTEST_FLAG(shuffle));
EXPECT_EQ(kMaxStackTraceDepth, GTEST_FLAG(stack_trace_depth));
EXPECT_STREQ("", GTEST_FLAG(stream_result_to).c_str());
@@ -1657,6 +1661,7 @@
GTEST_FLAG(print_time) = false;
GTEST_FLAG(random_seed) = 1;
GTEST_FLAG(repeat) = 100;
+ GTEST_FLAG(recreate_environments_when_repeating) = false;
GTEST_FLAG(shuffle) = true;
GTEST_FLAG(stack_trace_depth) = 1;
GTEST_FLAG(stream_result_to) = "localhost:1234";
@@ -5580,6 +5585,7 @@
print_time(true),
random_seed(0),
repeat(1),
+ recreate_environments_when_repeating(true),
shuffle(false),
stack_trace_depth(kMaxStackTraceDepth),
stream_result_to(""),
@@ -5683,6 +5689,16 @@
return flags;
}
+ // Creates a Flags struct where the gtest_recreate_environments_when_repeating
+ // flag has the given value.
+ static Flags RecreateEnvironmentsWhenRepeating(
+ bool recreate_environments_when_repeating) {
+ Flags flags;
+ flags.recreate_environments_when_repeating =
+ recreate_environments_when_repeating;
+ return flags;
+ }
+
// Creates a Flags struct where the gtest_shuffle flag has the given
// value.
static Flags Shuffle(bool shuffle) {
@@ -5728,6 +5744,7 @@
bool print_time;
int32_t random_seed;
int32_t repeat;
+ bool recreate_environments_when_repeating;
bool shuffle;
int32_t stack_trace_depth;
const char* stream_result_to;
@@ -5751,6 +5768,7 @@
GTEST_FLAG(print_time) = true;
GTEST_FLAG(random_seed) = 0;
GTEST_FLAG(repeat) = 1;
+ GTEST_FLAG(recreate_environments_when_repeating) = true;
GTEST_FLAG(shuffle) = false;
GTEST_FLAG(stack_trace_depth) = kMaxStackTraceDepth;
GTEST_FLAG(stream_result_to) = "";
@@ -5783,6 +5801,8 @@
EXPECT_EQ(expected.print_time, GTEST_FLAG(print_time));
EXPECT_EQ(expected.random_seed, GTEST_FLAG(random_seed));
EXPECT_EQ(expected.repeat, GTEST_FLAG(repeat));
+ EXPECT_EQ(expected.recreate_environments_when_repeating,
+ GTEST_FLAG(recreate_environments_when_repeating));
EXPECT_EQ(expected.shuffle, GTEST_FLAG(shuffle));
EXPECT_EQ(expected.stack_trace_depth, GTEST_FLAG(stack_trace_depth));
EXPECT_STREQ(expected.stream_result_to,
@@ -6161,6 +6181,20 @@
GTEST_TEST_PARSING_FLAGS_(argv, argv2, Flags::Repeat(1000), false);
}
+// Tests parsing --gtest_recreate_environments_when_repeating
+TEST_F(ParseFlagsTest, RecreateEnvironmentsWhenRepeating) {
+ const char* argv[] = {
+ "foo.exe",
+ "--gtest_recreate_environments_when_repeating=0",
+ nullptr,
+ };
+
+ const char* argv2[] = {"foo.exe", nullptr};
+
+ GTEST_TEST_PARSING_FLAGS_(
+ argv, argv2, Flags::RecreateEnvironmentsWhenRepeating(false), false);
+}
+
// Tests having a --gtest_also_run_disabled_tests flag
TEST_F(ParseFlagsTest, AlsoRunDisabledTestsFlag) {
const char* argv[] = {"foo.exe", "--gtest_also_run_disabled_tests", nullptr};
