No public description PiperOrigin-RevId: 985184490
diff --git a/centipede/BUILD b/centipede/BUILD index 036f7e2..a9badf0 100644 --- a/centipede/BUILD +++ b/centipede/BUILD
@@ -1940,6 +1940,7 @@ ":crash_deduplication_test_util", ":crash_summary", ":environment", + ":runner_result", ":stop", ":util", ":workdir", @@ -1953,6 +1954,8 @@ "@abseil-cpp//absl/time", "@abseil-cpp//absl/time:clock_interface", "@abseil-cpp//absl/time:simulated_clock", + "@abseil-cpp//absl/types:span", + "@com_google_fuzztest//common:defs", "@com_google_fuzztest//common:temp_dir", "@googletest//:gtest_main", ],
diff --git a/centipede/centipede.cc b/centipede/centipede.cc index 16df2e0..85911a0 100644 --- a/centipede/centipede.cc +++ b/centipede/centipede.cc
@@ -1076,44 +1076,49 @@ FUZZTEST_LOG(INFO) << log_prefix << "Executing inputs one-by-one, trying to find the reproducer"; + const size_t max_attempts = std::max<size_t>(1, env_.replay_crash_attempts); for (auto input_idx : input_idxs_to_try) { if (stop_condition_.ShouldStop()) break; const auto one_input = input_vec[input_idx]; - BatchResult one_input_batch_result; - if (!user_callbacks_.Execute(binary, {one_input}, one_input_batch_result) && - one_input_batch_result.IsInputFailure() && - one_input_batch_result.failure_signature() == - batch_result.failure_signature() && - !stop_condition_.ShouldStop()) { - auto hash = Hash(one_input); - auto crash_dir = wd_.CrashReproducerDirPaths().MyShard(); - FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir)); - std::string input_file_path = std::filesystem::path(crash_dir) / hash; - auto crash_metadata_dir = wd_.CrashMetadataDirPaths().MyShard(); - FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir)); - std::string crash_metadata_path_prefix = - std::filesystem::path(crash_metadata_dir) / hash; - FUZZTEST_LOG(INFO) - << log_prefix << "Detected crash-reproducing input:" - << "\nInput index : " << input_idx << "\nInput bytes : " - << AsPrintableString(one_input, /*max_len=*/32) - << "\nExit code : " << one_input_batch_result.exit_code() - << "\nFailure : " - << one_input_batch_result.failure_description() - << "\nSignature : " - << AsPrintableString( - AsByteSpan(one_input_batch_result.failure_signature()), - /*max_len=*/32) - << "\nSaving input to: " << input_file_path << "\nSaving crash" // - << "\nmetadata to : " << crash_metadata_path_prefix << ".*"; - FUZZTEST_CHECK_OK(RemoteFileSetContents(input_file_path, one_input)); - FUZZTEST_CHECK_OK(RemoteFileSetContents( - absl::StrCat(crash_metadata_path_prefix, ".desc"), - one_input_batch_result.failure_description())); - FUZZTEST_CHECK_OK(RemoteFileSetContents( - absl::StrCat(crash_metadata_path_prefix, ".sig"), - one_input_batch_result.failure_signature())); - return; + for (size_t attempt = 0; attempt < max_attempts; ++attempt) { + if (stop_condition_.ShouldStop()) break; + BatchResult one_input_batch_result; + if (!user_callbacks_.Execute(binary, {one_input}, + one_input_batch_result) && + one_input_batch_result.IsInputFailure() && + one_input_batch_result.failure_signature() == + batch_result.failure_signature() && + !stop_condition_.ShouldStop()) { + auto hash = Hash(one_input); + auto crash_dir = wd_.CrashReproducerDirPaths().MyShard(); + FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir)); + std::string input_file_path = std::filesystem::path(crash_dir) / hash; + auto crash_metadata_dir = wd_.CrashMetadataDirPaths().MyShard(); + FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir)); + std::string crash_metadata_path_prefix = + std::filesystem::path(crash_metadata_dir) / hash; + FUZZTEST_LOG(INFO) + << log_prefix << "Detected crash-reproducing input:" + << "\nInput index : " << input_idx << "\nInput bytes : " + << AsPrintableString(one_input, /*max_len=*/32) + << "\nExit code : " << one_input_batch_result.exit_code() + << "\nFailure : " + << one_input_batch_result.failure_description() + << "\nSignature : " + << AsPrintableString( + AsByteSpan(one_input_batch_result.failure_signature()), + /*max_len=*/32) + << "\nSaving input to: " << input_file_path << "\nSaving crash" // + << "\nmetadata to : " << crash_metadata_path_prefix << ".*"; + FUZZTEST_CHECK_OK(RemoteFileSetContents(input_file_path, one_input)); + FUZZTEST_CHECK_OK(RemoteFileSetContents( + absl::StrCat(crash_metadata_path_prefix, ".desc"), + one_input_batch_result.failure_description())); + FUZZTEST_CHECK_OK(RemoteFileSetContents( + absl::StrCat(crash_metadata_path_prefix, ".sig"), + one_input_batch_result.failure_signature())); + return; + } } }
diff --git a/centipede/centipede_flags.inc b/centipede/centipede_flags.inc index c4a246c..d838dcb 100644 --- a/centipede/centipede_flags.inc +++ b/centipede/centipede_flags.inc
@@ -526,3 +526,7 @@ CENTIPEDE_FLAG( absl::Duration, fuzztest_time_limit_per_test, absl::InfiniteDuration(), "The time limit per fuzz test for working on the corpus database.") +CENTIPEDE_FLAG( + size_t, replay_crash_attempts, 1, + "Number of attempts to replay a crash input during batch failure triage " + "or deduplication before considering it non-reproducible.")
diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index 021805e..1eae7ea 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc
@@ -956,6 +956,96 @@ EXPECT_EQ(suspect_only_mock.num_inputs_triaged(), 1); } +// Mock callback that fails on a specific attempt of an input during triage. +class FlakyCrashingInputMock : public CentipedeCallbacks { + public: + FlakyCrashingInputMock(const Environment& env, size_t crashing_input_idx, + size_t fail_on_triage_attempt) + : CentipedeCallbacks{env, internal_stop_condition_}, + crashing_input_idx_(crashing_input_idx), + fail_on_triage_attempt_(fail_on_triage_attempt) {} + + bool Execute(std::string_view binary, absl::Span<const ByteSpan> inputs, + BatchResult& batch_result) override { + batch_result.ClearAndResize(inputs.size()); + if (first_pass_) { + for (const auto& input : inputs) { + if (input[0] == crashing_input_idx_) { + first_pass_ = false; + crashing_input_ = {input.begin(), input.end()}; + batch_result.num_outputs_read() = + crashing_input_idx_ % env_.batch_size; + batch_result.exit_code() = 1; + return false; + } + } + return true; + } + // In triage + for (const auto& input : inputs) { + if (input == AsByteSpan(crashing_input_)) { + ++triage_attempts_; + if (triage_attempts_ == fail_on_triage_attempt_) { + batch_result.exit_code() = 1; + return false; + } + } + } + return true; + } + + std::vector<Mutant> Mutate(absl::Span<const MutationInputRef> inputs, + size_t num_mutants) override { + std::vector<Mutant> mutants; + mutants.reserve(num_mutants); + for (size_t i = 0; i < num_mutants; ++i) { + mutants.push_back({/*data=*/{static_cast<uint8_t>(curr_input_idx_++)}, + Mutant::kOriginNone}); + } + return mutants; + } + + ByteArray crashing_input() const { return crashing_input_; } + size_t triage_attempts() const { return triage_attempts_; } + + private: + const size_t crashing_input_idx_; + const size_t fail_on_triage_attempt_; + size_t curr_input_idx_ = 0; + size_t triage_attempts_ = 0; + ByteArray crashing_input_ = {}; + bool first_pass_ = true; + StopCondition internal_stop_condition_; +}; + +TEST(Centipede, ReportCrashRetriesWithReplayCrashAttempts) { + constexpr size_t kNumBatches = 5; + constexpr size_t kBatchSize = 10; + constexpr size_t kCrashingInputIdx = 15; + + TempDir temp_dir{test_info_->name()}; + Environment env; + env.workdir = temp_dir.path(); + env.num_runs = kBatchSize * kNumBatches; + env.batch_size = kBatchSize; + env.require_pc_table = false; + env.exit_on_crash = true; + env.batch_triage_suspect_only = true; + env.replay_crash_attempts = 3; + + FlakyCrashingInputMock mock(env, kCrashingInputIdx, + /*fail_on_triage_attempt=*/2); + NonOwningCallbacksFactory factory(mock); + CentipedeMain(env, factory); + + EXPECT_EQ(mock.triage_attempts(), 2); + const auto crashing_input_hash = Hash(mock.crashing_input()); + const auto crasher_path = + std::filesystem::path{WorkDir{env}.CrashReproducerDirPaths().MyShard()} / + crashing_input_hash; + EXPECT_TRUE(std::filesystem::exists(crasher_path)) << crasher_path; +} + TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) { Environment env; env.binary =
diff --git a/centipede/crash_deduplication.cc b/centipede/crash_deduplication.cc index 0aaddbb..3971105 100644 --- a/centipede/crash_deduplication.cc +++ b/centipede/crash_deduplication.cc
@@ -14,6 +14,7 @@ #include "./centipede/crash_deduplication.h" +#include <algorithm> #include <cstddef> #include <cstdlib> #include <filesystem> // NOLINT @@ -158,18 +159,27 @@ ByteArray input_bytes; RETURN_IF_NOT_OK(RemoteFileGetContents(input_path, input_bytes)); - BatchResult batch_result; - const bool is_reproducible = - !callbacks.Execute(env.binary, {input_bytes}, batch_result) && - batch_result.IsInputFailure(); - - if (is_reproducible) { - out_signature = batch_result.failure_signature(); - out_description = batch_result.failure_description(); - } else { - out_signature = ""; - out_description = ""; + const size_t max_attempts = std::max<size_t>(1, env.replay_crash_attempts); + for (size_t attempt = 0; attempt < max_attempts; ++attempt) { + BatchResult batch_result; + if (!callbacks.Execute(env.binary, {input_bytes}, batch_result) && + batch_result.IsInputFailure()) { + out_signature = batch_result.failure_signature(); + out_description = batch_result.failure_description(); + if (attempt > 0) { + FUZZTEST_LOG(INFO) << "Crash reproduced on attempt " << (attempt + 1) + << " of " << max_attempts << " for " << input_path; + } + return absl::OkStatus(); + } } + + if (max_attempts > 1) { + FUZZTEST_LOG(INFO) << "Crash failed to reproduce after " << max_attempts + << " attempts for " << input_path; + } + out_signature = ""; + out_description = ""; return absl::OkStatus(); }
diff --git a/centipede/crash_deduplication_test.cc b/centipede/crash_deduplication_test.cc index 6fc02bf..b65f557 100644 --- a/centipede/crash_deduplication_test.cc +++ b/centipede/crash_deduplication_test.cc
@@ -14,6 +14,7 @@ #include "./centipede/crash_deduplication.h" +#include <cstdlib> #include <filesystem> // NOLINT #include <string> #include <string_view> @@ -33,13 +34,16 @@ #include "absl/time/clock_interface.h" #include "absl/time/simulated_clock.h" #include "absl/time/time.h" +#include "absl/types/span.h" #include "./centipede/centipede_callbacks.h" #include "./centipede/crash_deduplication_test_util.h" #include "./centipede/crash_summary.h" #include "./centipede/environment.h" +#include "./centipede/runner_result.h" #include "./centipede/stop.h" #include "./centipede/util.h" #include "./centipede/workdir.h" +#include "./common/defs.h" #include "./common/temp_dir.h" namespace fuzztest::internal { @@ -1116,5 +1120,75 @@ HasSubstr("Reason: Crash expired (not reproduced for"))); } +class FlakyCrashCallbacks : public CentipedeCallbacks { + public: + FlakyCrashCallbacks(const Environment& env, int crash_on_attempt) + : CentipedeCallbacks(env, internal_stop_condition_), + crash_on_attempt_(crash_on_attempt) {} + + bool Execute(std::string_view binary, absl::Span<const ByteSpan> inputs, + BatchResult& batch_result) override { + ++execution_count_; + batch_result.ClearAndResize(inputs.size()); + if (execution_count_ == crash_on_attempt_) { + batch_result.exit_code() = EXIT_FAILURE; + batch_result.failure_signature() = "csig"; + batch_result.failure_description() = "flaky crash"; + return false; + } + return true; + } + + int execution_count() const { return execution_count_; } + + private: + int crash_on_attempt_; + int execution_count_ = 0; + StopCondition internal_stop_condition_; +}; + +TEST_F(OrganizeCrashingInputsTest, ReplaysCrashMultipleTimesUntilSuccess) { + LogCapture log_capture; + SetContentsAndGetPath(incubating_dir(), "isig1", "input1"); + + Environment test_env = env(); + test_env.replay_crash_attempts = 3; + + FlakyCrashCallbacks callbacks(test_env, /*crash_on_attempt=*/2); + NonOwningCallbacksFactory factory(callbacks); + + ASSERT_TRUE(OrganizeCrashingInputs(regression_dir(), crashing_dir(), test_env, + factory, /*new_crashes_by_signature=*/{}, + crash_summary()) + .ok()); + + EXPECT_EQ(callbacks.execution_count(), 2); + EXPECT_THAT(log_capture.FullLog(), + AllOf(HasSubstr("Crash reproduced on attempt 2 of 3 for "), + HasSubstr("Action: CleanUpIncubating for"), + HasSubstr("Reason: Input reproduced with signature 'csig' " + "and graduated from incubation"))); +} + +TEST_F(OrganizeCrashingInputsTest, ReplaysCrashUpToMaxAttemptsOnFailure) { + LogCapture log_capture; + SetContentsAndGetPath(incubating_dir(), "isig1", "input1"); + + Environment test_env = env(); + test_env.replay_crash_attempts = 3; + + FlakyCrashCallbacks callbacks(test_env, /*crash_on_attempt=*/5); + NonOwningCallbacksFactory factory(callbacks); + + ASSERT_TRUE(OrganizeCrashingInputs(regression_dir(), crashing_dir(), test_env, + factory, /*new_crashes_by_signature=*/{}, + crash_summary()) + .ok()); + + EXPECT_EQ(callbacks.execution_count(), 3); + EXPECT_THAT(log_capture.FullLog(), + HasSubstr("Crash failed to reproduce after 3 attempts for ")); +} + } // namespace } // namespace fuzztest::internal
diff --git a/centipede/environment.cc b/centipede/environment.cc index e45478b..f17f751 100644 --- a/centipede/environment.cc +++ b/centipede/environment.cc
@@ -250,6 +250,9 @@ fuzztest_execution_id = config.execution_id.value_or(""); fuzztest_replay_coverage_inputs = config.replay_coverage_inputs; fuzztest_time_limit_per_test = config.GetTimeLimitPerTest(); + if (replay_crash_attempts == Default().replay_crash_attempts) { + replay_crash_attempts = config.replay_crash_attempts; + } // Allow more crashes to be reported when running with FuzzTest. This allows // more unique crashes to collected after deduplication. But we don't want to
diff --git a/centipede/environment_test.cc b/centipede/environment_test.cc index 259a848..8589e9e 100644 --- a/centipede/environment_test.cc +++ b/centipede/environment_test.cc
@@ -220,4 +220,21 @@ EXPECT_FALSE(env.populate_binary_info); } +TEST(Environment, UpdatesReplayAttemptsFromTargetConfigWhenDefault) { + Environment env; + fuzztest::internal::Configuration config; + config.replay_crash_attempts = 5; + env.UpdateWithTargetConfig(config); + EXPECT_EQ(env.replay_crash_attempts, 5); +} + +TEST(Environment, PreservesReplayAttemptsWhenExplicitlySet) { + Environment env; + env.replay_crash_attempts = 10; + fuzztest::internal::Configuration config; + config.replay_crash_attempts = 5; + env.UpdateWithTargetConfig(config); + EXPECT_EQ(env.replay_crash_attempts, 10); +} + } // namespace fuzztest::internal
diff --git a/fuzztest/init_fuzztest.cc b/fuzztest/init_fuzztest.cc index bb94b84..9bfb384 100644 --- a/fuzztest/init_fuzztest.cc +++ b/fuzztest/init_fuzztest.cc
@@ -198,6 +198,11 @@ "regardless of the crashing inputs found, unless there is a setup failure. " "Note that reproducer tests are not affected - they always fail on crash."); +// NOLINTNEXTLINE(clang-diagnostic-pre-c++20-compat) +FUZZTEST_DEFINE_FLAG( + size_t, replay_crash_attempts, 1, + "The number of attempts to replay a crash before giving up."); + FUZZTEST_DEFINE_FLAG(bool, unguided, false, "If used together with --" FUZZTEST_FLAG_PREFIX "fuzz or --" FUZZTEST_FLAG_PREFIX @@ -416,6 +421,7 @@ num_jobs, absl::GetFlag(FUZZTEST_FLAG(internal_centipede_command)), absl::GetFlag(FUZZTEST_FLAG(internal_crashing_input_to_reproduce)), + absl::GetFlag(FUZZTEST_FLAG(replay_crash_attempts)), }; } } // namespace
diff --git a/fuzztest/internal/configuration.cc b/fuzztest/internal/configuration.cc index f3d67e2..c2de6ca 100644 --- a/fuzztest/internal/configuration.cc +++ b/fuzztest/internal/configuration.cc
@@ -204,21 +204,21 @@ std::string time_limit_str = absl::FormatDuration(time_limit); std::string time_budget_type_str = AbslUnparseFlag(time_budget_type); std::string out; - out.resize(SpaceFor(corpus_database) + SpaceFor(stats_root) + - SpaceFor(workdir_root) + SpaceFor(binary_identifier) + - SpaceFor(fuzz_tests) + SpaceFor(fuzz_tests_in_current_shard) + - SpaceFor(continue_after_crash) + - SpaceFor(reproduce_findings_as_separate_tests) + - SpaceFor(replay_coverage_inputs) + SpaceFor(only_replay) + - SpaceFor(update_corpus_database) + - SpaceFor(replay_in_single_process) + SpaceFor(execution_id) + - SpaceFor(print_subprocess_log) + - SpaceFor(subprocess_cleanup_timeout_str) + SpaceFor(stack_limit) + - SpaceFor(rss_limit) + SpaceFor(time_limit_per_input_str) + - SpaceFor(time_limit_str) + SpaceFor(time_budget_type_str) + - SpaceFor(jobs) + SpaceFor(centipede_command) + - SpaceFor(crashing_input_to_reproduce) + - SpaceFor(reproduction_command_template)); + out.resize( + SpaceFor(corpus_database) + SpaceFor(stats_root) + + SpaceFor(workdir_root) + SpaceFor(binary_identifier) + + SpaceFor(fuzz_tests) + SpaceFor(fuzz_tests_in_current_shard) + + SpaceFor(continue_after_crash) + + SpaceFor(reproduce_findings_as_separate_tests) + + SpaceFor(replay_coverage_inputs) + SpaceFor(only_replay) + + SpaceFor(update_corpus_database) + SpaceFor(replay_in_single_process) + + SpaceFor(execution_id) + SpaceFor(print_subprocess_log) + + SpaceFor(subprocess_cleanup_timeout_str) + SpaceFor(stack_limit) + + SpaceFor(rss_limit) + SpaceFor(time_limit_per_input_str) + + SpaceFor(time_limit_str) + SpaceFor(time_budget_type_str) + + SpaceFor(jobs) + SpaceFor(centipede_command) + + SpaceFor(crashing_input_to_reproduce) + SpaceFor(replay_crash_attempts) + + SpaceFor(reproduction_command_template)); size_t offset = 0; offset = WriteString(out, offset, corpus_database); offset = WriteString(out, offset, stats_root); @@ -243,6 +243,7 @@ offset = WriteIntegral(out, offset, jobs); offset = WriteOptionalString(out, offset, centipede_command); offset = WriteOptionalString(out, offset, crashing_input_to_reproduce); + offset = WriteIntegral(out, offset, replay_crash_attempts); offset = WriteOptionalString(out, offset, reproduction_command_template); FUZZTEST_CHECK_EQ(offset, out.size()); return out; @@ -277,6 +278,7 @@ ASSIGN_OR_RETURN(centipede_command, ConsumeOptionalString(serialized)); ASSIGN_OR_RETURN(crashing_input_to_reproduce, ConsumeOptionalString(serialized)); + ASSIGN_OR_RETURN(replay_crash_attempts, Consume<size_t>(serialized)); ASSIGN_OR_RETURN(reproduction_command_template, ConsumeOptionalString(serialized)); if (!serialized.empty()) { @@ -313,6 +315,7 @@ *jobs, *std::move(centipede_command), *std::move(crashing_input_to_reproduce), + *replay_crash_attempts, *std::move(reproduction_command_template)}; }(); }
diff --git a/fuzztest/internal/configuration.h b/fuzztest/internal/configuration.h index c3a96a5..0aceee9 100644 --- a/fuzztest/internal/configuration.h +++ b/fuzztest/internal/configuration.h
@@ -110,6 +110,9 @@ // When set, `FuzzTestFuzzer` replays only one input (no fuzzing is done). std::optional<std::string> crashing_input_to_reproduce; + // Number of attempts to replay a crash input. + size_t replay_crash_attempts = 1; + // A command template that could be used to replay a crashing input. // The reproduction command template must have the following place holders: // - $TEST_FILTER: for replaying only a subset of the tests in a binary.
diff --git a/fuzztest/internal/configuration_test.cc b/fuzztest/internal/configuration_test.cc index d4549bf..a2ccf21 100644 --- a/fuzztest/internal/configuration_test.cc +++ b/fuzztest/internal/configuration_test.cc
@@ -40,6 +40,7 @@ config.centipede_command == other->centipede_command && config.crashing_input_to_reproduce == other->crashing_input_to_reproduce && + config.replay_crash_attempts == other->replay_crash_attempts && config.reproduction_command_template == other->reproduction_command_template; } @@ -69,6 +70,7 @@ /*jobs=*/1, /*centipede_command=*/std::nullopt, /*crashing_input_to_reproduce=*/std::nullopt, + /*replay_crash_attempts=*/5, /*reproduction_command_template=*/std::nullopt}; EXPECT_THAT(Configuration::Deserialize(configuration.Serialize()), @@ -100,6 +102,7 @@ /*jobs=*/1, "centipede_command", "crashing_input_to_reproduce", + /*replay_crash_attempts=*/20, "reproduction_command_template"}; EXPECT_THAT(Configuration::Deserialize(configuration.Serialize()),