Support test and input skipping with fuzztest::SkipTestsOrCurrentInput. If requested at the per-fuzz-test fixture setup, all tests under the fixture will be skipped. Otherwise the current iteration will be skipped, meaning that the input will not be added to the corpus when fuzzing. This also addresses https://github.com/google/fuzztest/issues/1202 PiperOrigin-RevId: 647788696
diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index 39d317c..4465309 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc
@@ -603,6 +603,34 @@ EXPECT_THAT(status, Eq(Signal(SIGABRT))); } +TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerTest) { + auto [status, std_out, std_err] = + Run("SkippedTestFixturePerTest.SkippedTest", kDefaultTargetBinary, + /*env=*/{}, + /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); + EXPECT_THAT(std_err, + HasSubstr("Skipping SkippedTestFixturePerTest.SkippedTest")); + EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest is executed"))); + EXPECT_THAT(status, Eq(ExitCode(0))); +} + +TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerIteration) { + auto [status, std_out, std_err] = + Run("SkippedTestFixturePerIteration.SkippedTest", kDefaultTargetBinary, + /*env=*/{}, + /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); + EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest is executed"))); + EXPECT_THAT(status, Eq(ExitCode(0))); +} + +TEST_F(UnitTestModeTest, InputsAreSkippedWhenRequestedInTests) { + auto [status, std_out, std_err] = + Run("MySuite.SkipInputs", kDefaultTargetBinary, + /*env=*/{}, + /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); + EXPECT_THAT(std_err, HasSubstr("Skipped input")); +} + class GetRandomValueTest : public UnitTestModeTest { protected: int GetValueFromInnerTest( @@ -1166,6 +1194,25 @@ EXPECT_THAT(status, Eq(ExitCode(0))); } +// This tests both the command line interface and the fuzzing logic. It is under +// FuzzingModeCommandLineInterfaceTest so it can specify the command line. +TEST_F(FuzzingModeCommandLineInterfaceTest, CorpusDoesNotContainSkippedInputs) { + TempDir corpus_dir; + // Although theoretically possible, it is extreme unlikely that the test would + // find the crash without saving some corpus. + auto [producer_status, producer_std_out, producer_std_err] = + RunWith({{"fuzz", "MySuite.SkipInputs"}, {"fuzz_for", "10s"}}, + {{"FUZZTEST_TESTSUITE_OUT_DIR", corpus_dir.dirname()}}); + + ASSERT_THAT(producer_std_err, HasSubstr("Skipped input")); + + auto [replayer_status, replayer_std_out, replayer_std_err] = + RunWith({{"fuzz", "MySuite.SkipInputs"}}, + {{"FUZZTEST_REPLAY", corpus_dir.dirname()}}); + + EXPECT_THAT(replayer_std_err, Not(HasSubstr("Skipped input"))); +} + std::string CentipedePath() { const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR")); FUZZTEST_INTERNAL_CHECK_PRECONDITION( @@ -1305,6 +1352,23 @@ CountSubstrs(std_err, "<<CallCountGoogleTest::TearDownTestSuite()>>")); } +TEST_P(FuzzingModeFixtureTest, TestIsSkippedWhenRequestedInFixturePerTest) { + auto [status, std_out, std_err] = + Run("SkippedTestFixturePerTest.SkippedTest", /*iterations=*/10); + EXPECT_THAT(std_err, + HasSubstr("Skipping SkippedTestFixturePerTest.SkippedTest")); + EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest should not be run"))); + EXPECT_THAT(status, Eq(ExitCode(0))); +} + +TEST_P(FuzzingModeFixtureTest, + TestIsSkippedWhenRequestedInFixturePerIteration) { + auto [status, std_out, std_err] = + Run("SkippedTestFixturePerIteration.SkippedTest", /*iterations=*/10); + EXPECT_THAT(std_err, Not(HasSubstr("SkippedTest should not be run"))); + EXPECT_THAT(status, Eq(ExitCode(0))); +} + INSTANTIATE_TEST_SUITE_P(FuzzingModeFixtureTestWithExecutionModel, FuzzingModeFixtureTest, testing::ValuesIn(GetAvailableExecutionModels())); @@ -1705,6 +1769,14 @@ ExpectTargetAbort(status, std_err); } +TEST_P(FuzzingModeCrashFindingTest, InputsAreSkippedWhenRequestedInTests) { + auto [status, std_out, std_err] = + Run("MySuite.SkipInputs", kDefaultTargetBinary); + EXPECT_THAT(std_err, HasSubstr("Skipped input")); + EXPECT_THAT(std_err, HasSubstr("argument 0: 123456789")); + ExpectTargetAbort(status, std_err); +} + INSTANTIATE_TEST_SUITE_P(FuzzingModeCrashFindingTestWithExecutionModel, FuzzingModeCrashFindingTest, testing::ValuesIn(GetAvailableExecutionModels()));
diff --git a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc index a2dbbdd..e159bd6 100644 --- a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc +++ b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc
@@ -785,6 +785,38 @@ // 1 GiB 1ULL << 30)); +// A fuzz test that is expected to accept and skip some inputs before hitting +// the crash. +void SkipInputs(uint32_t input) { + static bool skipped_input = false; + static bool accepted_input = false; + // Crash only when `input` is 123456789. + if (input != 123456789) { + // The condition below should have enough chance to either pass or not. + // + // Note that we want the input to here be accepted at least once so that the + // fuzzing engine can learn about the branch above. + if (input % 7 % 2 == 0) { + if (!skipped_input) { + skipped_input = true; + std::cerr << "Skipped input" << std::endl; + } + fuzztest::SkipTestsOrCurrentInput(); + return; + } + if (!accepted_input) accepted_input = true; + return; + } + // This introduces statefulness which is undesired in real fuzz tests, but + // here it makes it more reliable for functional testing. + if (skipped_input && accepted_input) { + std::abort(); + } +} +// Due to the limitation of the fuzzing engine, there must be an accepted input +// when initializing the corpus for fuzzing. So we provide one. +FUZZ_TEST(MySuite, SkipInputs).WithSeeds({1}); + } // namespace int main(int argc, char** argv) {
diff --git a/e2e_tests/testdata/fuzz_tests_using_googletest.cc b/e2e_tests/testdata/fuzz_tests_using_googletest.cc index 9eee8d9..9330a0a 100644 --- a/e2e_tests/testdata/fuzz_tests_using_googletest.cc +++ b/e2e_tests/testdata/fuzz_tests_using_googletest.cc
@@ -18,6 +18,7 @@ // to show that regular FUZZ_TEST work without having to #include GoogleTest. #include <cstdio> +#include <cstdlib> #include <limits> #include "gtest/gtest.h" @@ -124,4 +125,41 @@ } FUZZ_TEST(MySuite, CrashOnFailingTestInput); +class SkippedTestFixturePerTest + : public ::fuzztest::PerFuzzTestFixtureAdapter<testing::Test> { + public: + SkippedTestFixturePerTest() { fuzztest::SkipTestsOrCurrentInput(); } + + void SkippedTest() { + fprintf(stderr, "SkippedTest is executed! Aborting\n"); + std::abort(); + } +}; +FUZZ_TEST_F(SkippedTestFixturePerTest, SkippedTest); + +class SkippedTestFixturePerIteration + : public ::fuzztest::PerIterationFixtureAdapter<testing::Test> { + public: + // For the engine limitation, there must be at least one non-skipped input + // when initializing the corpus for fuzzing. So we always accept the first + // input. + SkippedTestFixturePerIteration() { + if (!first_iteration_) fuzztest::SkipTestsOrCurrentInput(); + } + + void SkippedTest() { + if (first_iteration_) { + first_iteration_ = false; + return; + } + fprintf(stderr, "SkippedTest is executed! Aborting\n"); + std::abort(); + } + + private: + static bool first_iteration_; +}; +bool SkippedTestFixturePerIteration::first_iteration_ = true; +FUZZ_TEST_F(SkippedTestFixturePerIteration, SkippedTest); + } // namespace
diff --git a/fuzztest/fuzztest_macros.h b/fuzztest/fuzztest_macros.h index cc15835..01f9645 100644 --- a/fuzztest/fuzztest_macros.h +++ b/fuzztest/fuzztest_macros.h
@@ -153,6 +153,18 @@ return std::vector<uint8_t>(str.begin(), str.end()); } +// When called during the fixture setup (in the constructor or SetUp()), skips +// calling property functions until the matching teardown (destructor or +// TearDown()). When called in a property function, skips adding the current +// input to the corpus when fuzzing. +// +// Note that this function should not be called frequently due to engine +// limitation and efficiency reasons. Consider refining the domain definitions +// to restrict input generation if possible. +inline void SkipTestsOrCurrentInput() { + internal::Runtime::instance().SetSkippingRequested(true); +} + } // namespace fuzztest #endif // FUZZTEST_FUZZTEST_FUZZTEST_MACROS_H_
diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index 9e270d5..cca7d9e 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc
@@ -447,8 +447,10 @@ class CentipedeFixtureDriver : public UntypedFixtureDriver { public: CentipedeFixtureDriver( + Runtime& runtime, std::unique_ptr<UntypedFixtureDriver> orig_fixture_driver) - : orig_fixture_driver_(std::move(orig_fixture_driver)) {} + : runtime_(runtime), + orig_fixture_driver_(std::move(orig_fixture_driver)) {} void SetUpFuzzTest() override { orig_fixture_driver_->SetUpFuzzTest(); @@ -464,6 +466,9 @@ void TearDownIteration() override { orig_fixture_driver_->TearDownIteration(); + if (runtime_.skipping_requested()) { + CentipedeSetExecutionResult(nullptr, 0); + } if (!runner_mode) CentipedeFinalizeProcessing(); } @@ -487,6 +492,7 @@ private: const Configuration* configuration_ = nullptr; + Runtime& runtime_; const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr; std::unique_ptr<UntypedFixtureDriver> orig_fixture_driver_; }; @@ -495,7 +501,7 @@ const FuzzTest& test, std::unique_ptr<UntypedFixtureDriver> fixture_driver) : test_(test), centipede_fixture_driver_( - new CentipedeFixtureDriver(std::move(fixture_driver))), + new CentipedeFixtureDriver(runtime_, std::move(fixture_driver))), fuzzer_impl_(test_, absl::WrapUnique(centipede_fixture_driver_)) { FUZZTEST_INTERNAL_CHECK(centipede_fixture_driver_ != nullptr, "Invalid fixture driver!"); @@ -513,6 +519,7 @@ int* argc, char*** argv, const Configuration& configuration) { centipede_fixture_driver_->set_configuration(&configuration); runtime_.SetRunMode(RunMode::kFuzz); + runtime_.SetSkippingRequested(false); runtime_.SetCurrentTest(&test_, &configuration); if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr(); runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); }); @@ -527,6 +534,12 @@ // and we should not run CentipedeMain in this process. const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS"); const int result = ([&]() { + if (runtime_.skipping_requested()) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s per request from the test setup.\n", + test_.full_name()); + return 0; + } if (runner_mode) { CentipedeAdaptorRunnerCallbacks runner_callbacks(&runtime_, &fuzzer_impl_, &configuration);
diff --git a/fuzztest/internal/centipede_adaptor.h b/fuzztest/internal/centipede_adaptor.h index 8fc694b..f64bd65 100644 --- a/fuzztest/internal/centipede_adaptor.h +++ b/fuzztest/internal/centipede_adaptor.h
@@ -35,10 +35,10 @@ const Configuration& configuration) override; private: + Runtime& runtime_ = Runtime::instance(); const FuzzTest& test_; CentipedeFixtureDriver* centipede_fixture_driver_; FuzzTestFuzzerImpl fuzzer_impl_; - Runtime& runtime_ = Runtime::instance(); }; } // namespace fuzztest::internal
diff --git a/fuzztest/internal/runtime.cc b/fuzztest/internal/runtime.cc index 639d29d..cba0027 100644 --- a/fuzztest/internal/runtime.cc +++ b/fuzztest/internal/runtime.cc
@@ -827,10 +827,17 @@ } void FuzzTestFuzzerImpl::RunInUnitTestMode(const Configuration& configuration) { + runtime_.SetSkippingRequested(false); fixture_driver_->SetUpFuzzTest(); - runtime_.StartWatchdog(); - PopulateLimits(configuration, execution_coverage_); [&] { + if (runtime_.skipping_requested()) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s per request from the test setup.\n", + test_.full_name()); + return; + } + runtime_.StartWatchdog(); + PopulateLimits(configuration, execution_coverage_); runtime_.EnableReporter(&stats_, [] { return absl::Now(); }); runtime_.SetCurrentTest(&test_, &configuration); @@ -923,8 +930,11 @@ execution_coverage_->SetIsTracing(true); } + runtime_.SetSkippingRequested(false); fixture_driver_->SetUpIteration(); - fixture_driver_->Test(std::move(untyped_args)); + if (!runtime_.skipping_requested()) { + fixture_driver_->Test(std::move(untyped_args)); + } fixture_driver_->TearDownIteration(); if (execution_coverage_ != nullptr) { execution_coverage_->SetIsTracing(false); @@ -932,7 +942,7 @@ const absl::Duration run_time = absl::Now() - start; bool new_coverage = false; - if (execution_coverage_ != nullptr) { + if (execution_coverage_ != nullptr && !runtime_.skipping_requested()) { new_coverage = corpus_coverage_.Update(execution_coverage_); stats_.max_stack_used = std::max(stats_.max_stack_used, execution_coverage_->MaxStackUsed()); @@ -986,10 +996,17 @@ int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, const Configuration& configuration) { + runtime_.SetSkippingRequested(false); fixture_driver_->SetUpFuzzTest(); - runtime_.StartWatchdog(); - PopulateLimits(configuration, execution_coverage_); const int exit_code = [&] { + if (runtime_.skipping_requested()) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s per request from the test setup.\n", + test_.full_name()); + return 0; + } + runtime_.StartWatchdog(); + PopulateLimits(configuration, execution_coverage_); runtime_.SetRunMode(RunMode::kFuzz); if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr();
diff --git a/fuzztest/internal/runtime.h b/fuzztest/internal/runtime.h index f529d0c..62bed06 100644 --- a/fuzztest/internal/runtime.h +++ b/fuzztest/internal/runtime.h
@@ -126,6 +126,14 @@ return external_failure_was_detected_.load(std::memory_order_relaxed); } + void SetSkippingRequested(bool requested) { + skipping_requested_.store(requested, std::memory_order_relaxed); + } + + bool skipping_requested() const { + return skipping_requested_.load(std::memory_order_relaxed); + } + void SetShouldTerminateOnNonFatalFailure(bool v) { should_terminate_on_non_fatal_failure_ = v; } @@ -196,15 +204,20 @@ // Note: Even though failures should happen within the code under test, they // could be set from other threads at any moment. We make it an atomic to // avoid a race condition. - std::atomic<bool> external_failure_was_detected_{false}; + std::atomic<bool> external_failure_was_detected_ = false; // To support in-process minimization for non-fatal failures we signal // suppress termination until we believe minimization is complete. bool should_terminate_on_non_fatal_failure_ = true; + // If set to true in fixture setup, skips calling property functions + // utill the matching teardown is called; If set to true in a property + // function, skip adding the current input to the corpus when fuzzing. + std::atomic<bool> skipping_requested_ = false; + // If true, fuzzing should terminate as soon as possible. // Atomic because it is set from signal handlers. - std::atomic<bool> termination_requested_{false}; + std::atomic<bool> termination_requested_ = false; RunMode run_mode_ = RunMode::kUnitTest; std::atomic<bool> watchdog_thread_started = false;