Testing `CentipedeGetCoverageData()` API in the #Centipede runner interface. PiperOrigin-RevId: 590798700
diff --git a/centipede/batch_fuzz_example/BUILD b/centipede/batch_fuzz_example/BUILD index 8f125a7..41eee09 100644 --- a/centipede/batch_fuzz_example/BUILD +++ b/centipede/batch_fuzz_example/BUILD
@@ -29,6 +29,8 @@ "@com_google_fuzztest//centipede:command", "@com_google_fuzztest//centipede:defs", "@com_google_fuzztest//centipede:environment", + "@com_google_fuzztest//centipede:feature", + "@com_google_fuzztest//centipede:logging", "@com_google_fuzztest//centipede:runner_result", "@com_google_fuzztest//centipede:shared_memory_blob_sequence", "@com_google_fuzztest//centipede:util", @@ -72,6 +74,7 @@ data = [":batch_fuzz_target"], deps = [ ":customized_centipede_lib", + "@com_google_fuzztest//centipede:centipede_callbacks", "@com_google_fuzztest//centipede:defs", "@com_google_fuzztest//centipede:environment", "@com_google_fuzztest//centipede:runner_result",
diff --git a/centipede/batch_fuzz_example/customized_centipede.cc b/centipede/batch_fuzz_example/customized_centipede.cc index ed062d6..2183b6d 100644 --- a/centipede/batch_fuzz_example/customized_centipede.cc +++ b/centipede/batch_fuzz_example/customized_centipede.cc
@@ -16,8 +16,10 @@ #include <sys/types.h> +#include <algorithm> #include <cstddef> #include <cstdlib> +#include <cstring> #include <filesystem> // NOLINT #include <string> #include <vector> @@ -26,9 +28,12 @@ #include "absl/log/log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" +#include "./centipede/centipede_callbacks.h" #include "./centipede/command.h" #include "./centipede/defs.h" #include "./centipede/environment.h" +#include "./centipede/feature.h" +#include "./centipede/logging.h" #include "./centipede/runner_result.h" #include "./centipede/shared_memory_blob_sequence.h" #include "./centipede/util.h" @@ -36,21 +41,54 @@ namespace centipede { namespace { -bool UpdateBatchResult(absl::string_view output_file, - BatchResult& batch_result) { - ByteArray content; - ReadFromLocalFile(output_file, content); - if (content.empty()) { - LOG(WARNING) << "Skip updating batch result with an emtpy output file: " - << output_file; - return true; - } +void UpdateFeatures(const ByteArray& content, FeatureVec& features) { + CHECK_EQ(content.size() % sizeof(feature_t), 0) + << VV(content.size()) << VV(sizeof(feature_t)); + const size_t features_size = content.size() / sizeof(feature_t); + features.resize(features_size); + memcpy(features.data(), content.data(), content.size()); +} +void UpdateExecutionResult(ByteArray& content, + ExecutionResult& execution_result) { BlobSequence blob_seq(content.data(), content.size()); - if (batch_result.Read(blob_seq)) return true; + BatchResult local_batch_result; + local_batch_result.ClearAndResize(1); + local_batch_result.Read(blob_seq); + CHECK_EQ(local_batch_result.results().size(), 1); + CHECK_EQ(local_batch_result.num_outputs_read(), 1); + const ExecutionResult& local_execution_result = + local_batch_result.results()[0]; - LOG(ERROR) << "Failed to read blob sequence from file: " << output_file; - return false; + execution_result.metadata() = local_execution_result.metadata(); + execution_result.mutable_features() = local_execution_result.features(); +} + +void UpdateBatchResult(const bool feature_only_feedback, + std::string_view output_dir, BatchResult& batch_result) { + std::vector<std::filesystem::path> entries; + for (const auto& entry : + std::filesystem::recursive_directory_iterator(output_dir)) { + entries.push_back(entry.path()); + } + CHECK_LE(entries.size(), batch_result.results().size()); + std::sort(entries.begin(), entries.end()); + + for (size_t index = 0; index < entries.size(); ++index) { + ByteArray content; + ReadFromLocalFile(std::string(entries[index]), content); + if (content.empty()) { + LOG(WARNING) << "Skip updating batch result with an emtpy output file: " + << entries[index]; + continue; + } + ExecutionResult& execution_result = batch_result.results()[index]; + if (feature_only_feedback) { + UpdateFeatures(content, execution_result.mutable_features()); + } else { + UpdateExecutionResult(content, execution_result); + } + } } void DumpBatchResultStats(const BatchResult& batch_result) { @@ -64,19 +102,21 @@ } // namespace +CustomizedCallbacks::CustomizedCallbacks(const Environment& env, + bool feature_only_feedback) + : CentipedeCallbacks(env), feature_only_feedback_(feature_only_feedback) {} + bool CustomizedCallbacks::Execute(std::string_view binary, const std::vector<ByteArray>& inputs, BatchResult& batch_result) { const std::string temp_dir = TemporaryLocalDirPath(); - CHECK(!temp_dir.empty()); - std::filesystem::create_directory(temp_dir); + CreateLocalDirRemovedAtExit(temp_dir); std::string input_file_list; - int index = 0; - for (const auto& input : inputs) { + for (size_t index = 0; index < inputs.size(); ++index) { const std::string temp_file_path = - std::filesystem::path(temp_dir).append(absl::StrCat("input-", index++)); - WriteToLocalFile(temp_file_path, input); + std::filesystem::path(temp_dir).append(absl::StrCat("input-", index)); + WriteToLocalFile(temp_file_path, inputs[index]); absl::StrAppend(&input_file_list, temp_file_path); absl::StrAppend(&input_file_list, "\n"); } @@ -84,8 +124,9 @@ std::filesystem::path(temp_dir).append("input_file_list"); WriteToLocalFile(input_list_filepath, input_file_list); - const std::string tmp_output_filepath = - std::filesystem::path(temp_dir).append("output_execution_results"); + const std::string tmp_output_dir = + std::filesystem::path(temp_dir).append("output_data"); + std::filesystem::create_directory(tmp_output_dir); const std::string tmp_log_filepath = std::filesystem::path(temp_dir).append("tmp_log"); @@ -96,12 +137,18 @@ env = {ConstructRunnerFlags()}; } + std::vector<std::string> args = { + "--input_file", + input_list_filepath, + "--output_dir", + tmp_output_dir, + }; + if (feature_only_feedback_) { + args.push_back("--enable_feature_only_feedback"); + } + // Execute. - Command cmd{env_.binary, - {input_list_filepath, tmp_output_filepath}, - env, - tmp_log_filepath, - tmp_log_filepath}; + Command cmd{env_.binary, args, env, tmp_log_filepath, tmp_log_filepath}; const int retval = cmd.Execute(); std::string tmp_log; @@ -109,7 +156,8 @@ LOG_IF(INFO, !tmp_log.empty()) << tmp_log; batch_result.ClearAndResize(inputs.size()); - CHECK(UpdateBatchResult(tmp_output_filepath, batch_result)); + UpdateBatchResult(feature_only_feedback_, tmp_output_dir, batch_result); + DumpBatchResultStats(batch_result); return retval == 0; }
diff --git a/centipede/batch_fuzz_example/customized_centipede.h b/centipede/batch_fuzz_example/customized_centipede.h index 7a818a9..262fe50 100644 --- a/centipede/batch_fuzz_example/customized_centipede.h +++ b/centipede/batch_fuzz_example/customized_centipede.h
@@ -17,7 +17,6 @@ #include <sys/types.h> -#include <string> #include <vector> #include "absl/strings/string_view.h" @@ -36,10 +35,16 @@ class CustomizedCallbacks : public CentipedeCallbacks { public: explicit CustomizedCallbacks(const Environment& env) - : CentipedeCallbacks(env) {} + : CustomizedCallbacks(env, /*feature_only_feedback=*/false) {} + + explicit CustomizedCallbacks(const Environment& env, + bool feature_only_feedback); bool Execute(std::string_view binary, const std::vector<ByteArray>& inputs, BatchResult& batch_result) override; + + private: + const bool feature_only_feedback_; }; } // namespace centipede
diff --git a/centipede/batch_fuzz_example/customized_centipede_test.cc b/centipede/batch_fuzz_example/customized_centipede_test.cc index b85da2b..4f12671 100644 --- a/centipede/batch_fuzz_example/customized_centipede_test.cc +++ b/centipede/batch_fuzz_example/customized_centipede_test.cc
@@ -20,6 +20,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "./centipede/centipede_callbacks.h" #include "./centipede/defs.h" #include "./centipede/environment.h" #include "./centipede/runner_result.h" @@ -30,24 +31,23 @@ using ::testing::AllOf; using ::testing::Each; +using ::testing::ExplainMatchResult; using ::testing::IsEmpty; using ::testing::Property; using ::testing::SizeIs; using ::testing::UnorderedElementsAreArray; -bool RunInputsAndCollectCoverage(const Environment &env, +bool RunInputsAndCollectCoverage(CentipedeCallbacks ¢ipede_callbacks, + std::string_view binary, const std::vector<std::string> &inputs, BatchResult &batch_result) { - CustomizedCallbacks customized_callbacks(env); - // Repackage string inputs into ByteArray inputs. std::vector<ByteArray> byte_array_inputs; for (const auto &string_input : inputs) { byte_array_inputs.emplace_back(string_input.cbegin(), string_input.cend()); } // Run. - return customized_callbacks.Execute(env.binary, byte_array_inputs, - batch_result); + return centipede_callbacks.Execute(binary, byte_array_inputs, batch_result); } std::string GetTargetPath() { @@ -55,21 +55,27 @@ "centipede/batch_fuzz_example/batch_fuzz_target"); } -TEST(BatchFuzz, SucceedsToCollectCoverageForTwoInputs) { - Environment env; - env.binary = GetTargetPath(); +TEST(BatchFuzzWithExecutionResults, SucceedsToCollectCoverageForTwoInputs) { + const std::string target_path = GetTargetPath(); + Environment env = {.binary = target_path}; + CustomizedCallbacks callbacks(env, /*feature_only_feedback=*/false); + BatchResult batch_result; - ASSERT_TRUE(RunInputsAndCollectCoverage(env, {"a", "b"}, batch_result)); + ASSERT_TRUE(RunInputsAndCollectCoverage(callbacks, target_path, {"a", "b"}, + batch_result)); EXPECT_THAT(batch_result.results(), AllOf(SizeIs(2), Each(Property(&ExecutionResult::features, Not(IsEmpty()))))); } -TEST(BatchFuzz, CollectsTheSameCoverageForSameInputs) { - Environment env; - env.binary = GetTargetPath(); +TEST(BatchFuzzWithExecutionResults, CollectsTheSameCoverageForSameInputs) { + const std::string target_path = GetTargetPath(); + const Environment env = {.binary = GetTargetPath()}; + CustomizedCallbacks callbacks(env, /*feature_only_feedback=*/false); + BatchResult batch_result; - ASSERT_TRUE(RunInputsAndCollectCoverage(env, {"f", "f"}, batch_result)); + ASSERT_TRUE(RunInputsAndCollectCoverage(callbacks, target_path, {"f", "f"}, + batch_result)); ASSERT_THAT(batch_result.results(), AllOf(SizeIs(2), Each(Property(&ExecutionResult::features, Not(IsEmpty()))))); @@ -77,5 +83,41 @@ UnorderedElementsAreArray(batch_result.results()[1].features())); } +MATCHER_P(HasFeaturesOnly, features_matcher, "") { + return ExplainMatchResult(features_matcher, arg.features(), + result_listener) && + ExplainMatchResult(IsEmpty(), arg.metadata().cmp_data, + result_listener) && + ExplainMatchResult(0, arg.stats().prep_time_usec, result_listener) && + ExplainMatchResult(0, arg.stats().prep_time_usec, result_listener) && + ExplainMatchResult(0, arg.stats().prep_time_usec, result_listener); +} + +TEST(BatchFuzzWithCoverageData, SucceedsToCollectCoverageForTwoInputs) { + const std::string target_path = GetTargetPath(); + const Environment env = {.binary = GetTargetPath()}; + CustomizedCallbacks callbacks(env, /*feature_only_feedback=*/true); + + BatchResult batch_result; + ASSERT_TRUE(RunInputsAndCollectCoverage(callbacks, target_path, {"a", "b"}, + batch_result)); + EXPECT_THAT(batch_result.results(), + AllOf(SizeIs(2), Each(HasFeaturesOnly(Not(IsEmpty()))))); +} + +TEST(BatchFuzzWithCoverageData, CollectsTheSameCoverageForSameInputs) { + const std::string target_path = GetTargetPath(); + const Environment env = {.binary = target_path}; + CustomizedCallbacks callbacks(env, /*feature_only_feedback=*/true); + + BatchResult batch_result; + ASSERT_TRUE(RunInputsAndCollectCoverage(callbacks, target_path, {"f", "f"}, + batch_result)); + ASSERT_THAT(batch_result.results(), + AllOf(SizeIs(2), Each(HasFeaturesOnly(Not(IsEmpty()))))); + EXPECT_THAT(batch_result.results()[0].features(), + UnorderedElementsAreArray(batch_result.results()[1].features())); +} + } // namespace } // namespace centipede
diff --git a/centipede/batch_fuzz_example/standalone_fuzz_target_main.cc b/centipede/batch_fuzz_example/standalone_fuzz_target_main.cc index f9a68c1..257c7b4 100644 --- a/centipede/batch_fuzz_example/standalone_fuzz_target_main.cc +++ b/centipede/batch_fuzz_example/standalone_fuzz_target_main.cc
@@ -19,8 +19,11 @@ #include <cstdint> #include <cstdlib> +#include <cstring> +#include <filesystem> // NOLINT #include <fstream> #include <iostream> +#include <iterator> #include <string> #include "./centipede/runner_interface.h" @@ -32,38 +35,55 @@ } } -// - argv[1]: the path to the input file. This file should contain a list of -// file names, one per line. -// - argv[2]: the path to the output file. This file will contain the execution -// results of exercising the files listed in the input file. +// This binary takes three input flags: +// - input_file: Path to a file containing a list of file names, one per line. +// - output_dir: Path to the directory where the binary will write the execution +// results for the analyzed files. +// - enable_feature_only_feedback: Optional flag. If specified, the binary will +// output coverage features after processing each input file. Otherwise, it +// will only output the execution result. int main(int argc, char* argv[]) { - if (argc != 3) return EXIT_FAILURE; + std::string input_file_path; + std::string output_dir; + bool feature_only_feedback = false; + for (int i = 1; i < argc; ++i) { + if (strcmp(argv[i], "--input_file") == 0) { + if (i + 1 < argc) input_file_path = argv[i + 1]; + } else if (strcmp(argv[i], "--output_dir") == 0) { + if (i + 1 < argc) output_dir = argv[i + 1]; + } else if (strcmp(argv[i], "--enable_feature_only_feedback") == 0) { + feature_only_feedback = true; + } + } - std::ifstream input_file(argv[1]); + std::ifstream input_file(input_file_path); if (!input_file.is_open()) { - std::cerr << "Failed to open file arg[1]: " << argv[1] << std::endl; + std::cerr << "Failed to open --input_file: " << input_file_path + << std::endl; return EXIT_FAILURE; } - std::ofstream output_file(argv[2], std::ios::out); - if (!output_file.is_open()) { - std::cerr << "Failed to open file arg[2]: " << argv[2] << std::endl; + + if (!std::filesystem::exists(output_dir)) { + std::cerr << "Not found --output_dir: " << output_dir << std::endl; return EXIT_FAILURE; } - static constexpr int kMaxOutputLimit = 5000; + + static constexpr size_t kOutputLimit = 5000; std::string curr_filepath; + size_t index = 0; while (getline(input_file, curr_filepath)) { std::string input_data; std::ifstream curr_file(curr_filepath); - if (curr_file.is_open()) { - input_data.assign(std::istreambuf_iterator<char>(curr_file), - std::istreambuf_iterator<char>()); - } else { + if (!curr_file.is_open()) { std::cerr << "Failed to open input file: " << curr_filepath << std::endl; return EXIT_FAILURE; } + + input_data.assign(std::istreambuf_iterator<char>(curr_file), + std::istreambuf_iterator<char>()); std::string output; - output.resize(kMaxOutputLimit); + output.resize(kOutputLimit); CentipedePrepareProcessing(); @@ -71,13 +91,30 @@ CentipedeFinalizeProcessing(); - const size_t offset = CentipedeGetExecutionResult( - reinterpret_cast<uint8_t*>(output.data()), kMaxOutputLimit); - if (offset == 0) { - std::cerr << "Failed to dump output execution results."; + size_t output_data_size = 0; + if (feature_only_feedback) { + output_data_size = CentipedeGetCoverageData( + reinterpret_cast<uint8_t*>(output.data()), output.size()); + } else { + output_data_size = CentipedeGetExecutionResult( + reinterpret_cast<uint8_t*>(output.data()), output.size()); + } + + if (output_data_size == 0) { + std::cerr << "Failed to get coverage data"; return EXIT_FAILURE; } - output_file.write(output.data(), offset); + + const std::string output_filename = + std::filesystem::path(output_dir) + .append("output." + std::to_string(++index)); + std::ofstream output_file(output_filename, std::ios::out); + if (!output_file.is_open()) { + std::cerr << "Failed to open file: " << output_filename << std::endl; + return EXIT_FAILURE; + } + + output_file.write(output.data(), output_data_size); curr_file.close(); } return EXIT_SUCCESS;