Dump reproducer by default. PiperOrigin-RevId: 667715190
diff --git a/fuzztest/BUILD b/fuzztest/BUILD index f1012b5..1a23958 100644 --- a/fuzztest/BUILD +++ b/fuzztest/BUILD
@@ -631,6 +631,7 @@ ":domain_core", ":runtime", ":test_protobuf_cc_proto", + "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ],
diff --git a/fuzztest/internal/runtime.cc b/fuzztest/internal/runtime.cc index 23d4aaa..7c6644a 100644 --- a/fuzztest/internal/runtime.cc +++ b/fuzztest/internal/runtime.cc
@@ -14,13 +14,6 @@ #include "./fuzztest/internal/runtime.h" -#include "./fuzztest/internal/status.h" - -#if !defined(_WIN32) && !defined(__Fuchsia__) -#define FUZZTEST_HAS_RUSAGE -#include <sys/resource.h> -#endif - #include <algorithm> #include <cerrno> #include <csignal> @@ -28,6 +21,7 @@ #include <cstdlib> #include <cstring> #include <deque> +#include <filesystem> // NOLINT #include <functional> #include <memory> #include <optional> @@ -36,6 +30,11 @@ #include <utility> #include <vector> +#if !defined(_WIN32) && !defined(__Fuchsia__) +#define FUZZTEST_HAS_RUSAGE +#include <sys/resource.h> +#endif + #include "absl/functional/bind_front.h" #include "absl/functional/function_ref.h" #include "absl/log/check.h" @@ -62,6 +61,7 @@ #include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/printer.h" #include "./fuzztest/internal/serialization.h" +#include "./fuzztest/internal/status.h" #ifdef ADDRESS_SANITIZER #include <sanitizer/asan_interface.h> @@ -78,18 +78,12 @@ void (*crash_handler_hook)(); -void Runtime::DumpReproducer(absl::string_view outdir) const { +// TODO(lszekeres): Return absl::StatusOr when WriteDataToDir returns StatusOr. +std::string Runtime::DumpReproducer(absl::string_view outdir) const { const std::string content = current_args_->domain.SerializeCorpus(current_args_->corpus_value) .ToString(); - const std::string filename = WriteDataToDir(content, outdir); - - if (filename.empty()) { - absl::FPrintF(GetStderr(), "[!] Failed to write reproducer file.\n"); - } else { - absl::FPrintF(GetStderr(), "[*] Reproducer file written to: %s\n", - filename); - } + return WriteDataToDir(content, outdir); } void Runtime::PrintFinalStats(RawSink out) const { @@ -137,6 +131,49 @@ constexpr size_t kValueMaxPrintLength = 2048; constexpr absl::string_view kTrimIndicator = " ...<value too long>"; +constexpr absl::string_view kReproducerDirName = "fuzztest_repro"; + +struct ReproducerDirectory { + std::string path; + enum class Type { kUserSpecified, kTestUndeclaredOutputs }; + Type type; +}; + +std::optional<ReproducerDirectory> GetReproducerDirectory() { + auto env = absl::NullSafeStringView(getenv("FUZZTEST_REPRODUCERS_OUT_DIR")); + if (!env.empty()) { + return ReproducerDirectory{std::string(env), + ReproducerDirectory::Type::kUserSpecified}; + } + env = absl::NullSafeStringView(getenv("TEST_UNDECLARED_OUTPUTS_DIR")); + if (!env.empty()) { + auto path = std::filesystem::path(std::string(env)) / + std::string(kReproducerDirName); + return ReproducerDirectory{ + path.string(), ReproducerDirectory::Type::kTestUndeclaredOutputs}; + } + return std::nullopt; +} + +void PrintReproductionInstructionsForUndeclaredOutputs( + RawSink out, absl::string_view reproducer_path, + absl::string_view test_name) { + absl::string_view file_name = Basename(reproducer_path); + absl::Format(out, + "Reproducer file was dumped under" + "TEST_UNDECLARED_OUTPUTS_DIR.\n"); + absl::Format(out, + "Make a copy of it with:\n\n" + "mkdir -p /tmp/%s && \\\ncp -f %s /tmp/%s/%s\n\n", + kReproducerDirName, reproducer_path, kReproducerDirName, + file_name); + absl::Format(out, + "Then replay by adding:\n\n" + "--test_filter=%s " + "--test_env=FUZZTEST_REPLAY=/tmp/%s/%s\n\n" + "after `bazel test` in your original Bazel invocation.\n", + test_name, kReproducerDirName, file_name); +} } // namespace @@ -153,14 +190,6 @@ if (crash_handler_hook) crash_handler_hook(); - // First, lets try to dump the reproducer if requested. - if (current_args_ != nullptr) { - const char* outdir = getenv("FUZZTEST_REPRODUCERS_OUT_DIR"); - if (outdir != nullptr && outdir[0]) { - DumpReproducer(outdir); - } - } - if (run_mode() != RunMode::kUnitTest) { PrintFinalStats(out); } @@ -184,10 +213,34 @@ trim ? kTrimIndicator : ""); }); + // Dump the reproducer if requested. + std::optional<ReproducerDirectory> out_dir = GetReproducerDirectory(); + if (out_dir.has_value()) { + absl::Format(out, "%s=== Reproduction\n\n", separator); + const std::string reproducer_path = DumpReproducer(out_dir->path); + if (reproducer_path.empty()) { + absl::FPrintF(GetStderr(), "[!] Failed to write reproducer file!\n"); + } else { + switch (out_dir->type) { + case ReproducerDirectory::Type::kUserSpecified: + absl::Format(out, "Reproducer file was dumped at:\n%s\n", + reproducer_path); + break; + case ReproducerDirectory::Type::kTestUndeclaredOutputs: + std::string test_name = absl::StrCat( + current_test_->suite_name(), ".", current_test_->test_name()); + PrintReproductionInstructionsForUndeclaredOutputs( + out, reproducer_path, test_name); + break; + } + } + } + // There doesn't seem to be a good way to generate a reproducer test when // the test uses a fixture (see b/241271658). if (!current_test_->uses_fixture()) { - absl::Format(out, "%s=== Reproducer test\n\n", separator); + absl::Format(out, "%s=== Regression test draft\n\n", separator); + absl::Format(out, "TEST(%1$s, %2$sRegression) {\n %2$s(\n", current_test_->suite_name(), current_test_->test_name()); printer.PrintFormattedAggregateValue( @@ -204,6 +257,13 @@ }); absl::Format(out, "\n );\n"); absl::Format(out, "}\n"); + + absl::Format(out, + "\nPlease note that the code generated above is best effort " + "and is intended\n" + "to use be used as a draft regression test.\n" + "For reproducing findings please rely on file based " + "reproduction.\n"); } if (current_configuration_ != nullptr) { const auto reproduction_command =
diff --git a/fuzztest/internal/runtime.h b/fuzztest/internal/runtime.h index 0d9cf69..7b700bc 100644 --- a/fuzztest/internal/runtime.h +++ b/fuzztest/internal/runtime.h
@@ -191,7 +191,9 @@ void CheckWatchdogLimits(); void Watchdog(); - void DumpReproducer(absl::string_view outdir) const; + // Returns the file path of the reproducer. + // Returns empty string if writing the file failed. + std::string DumpReproducer(absl::string_view outdir) const; // Some failures are not necessarily detected by signal handlers or by // sanitizers. For example, we could have test framework failures like
diff --git a/fuzztest/internal/runtime_test.cc b/fuzztest/internal/runtime_test.cc index c10ed3e..76e072b 100644 --- a/fuzztest/internal/runtime_test.cc +++ b/fuzztest/internal/runtime_test.cc
@@ -14,11 +14,11 @@ #include "./fuzztest/internal/runtime.h" -#include <memory> #include <string> #include <tuple> #include "gtest/gtest.h" +#include "absl/strings/match.h" #include "absl/time/time.h" #include "./fuzztest/domain_core.h" #include "./fuzztest/internal/test_protobuf.pb.h" @@ -47,7 +47,9 @@ Runtime::Args debug_args{generic_args, domain}; runtime.SetCurrentTest(&test, nullptr); runtime.SetCurrentArgs(&debug_args); - EXPECT_EQ(get_failure(), R"( + const std::string report = get_failure(); + + EXPECT_TRUE(absl::StartsWith(report, R"( ================================================================= === Fuzzing stats @@ -65,9 +67,11 @@ The test fails with input: argument 0: 17 argument 1: "ABC" +)")); + EXPECT_TRUE(absl::EndsWith(report, R"( ================================================================= -=== Reproducer test +=== Regression test draft TEST(SUITE_NAME, TEST_NAMERegression) { TEST_NAME( @@ -76,8 +80,12 @@ ); } +Please note that the code generated above is best effort and is intended +to use be used as a draft regression test. +For reproducing findings please rely on file based reproduction. + ================================================================= -)"); +)")); runtime.DisableReporter(); EXPECT_EQ(get_failure(), "");