| // Copyright 2022 The Centipede Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // https://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "./centipede/stats.h" |
| |
| #include <cstdint> |
| #include <filesystem> // NOLINT |
| #include <sstream> |
| #include <vector> |
| |
| #include "gmock/gmock.h" |
| #include "gtest/gtest.h" |
| #include "absl/log/log_sink.h" |
| #include "absl/log/log_sink_registry.h" |
| #include "absl/time/civil_time.h" |
| #include "absl/time/time.h" |
| #include "./centipede/logging.h" |
| #include "./centipede/test_util.h" |
| #include "./centipede/util.h" |
| |
| namespace centipede { |
| |
| namespace { |
| |
| class LogCapture : public absl::LogSink { |
| public: |
| LogCapture() { absl::AddLogSink(this); } |
| ~LogCapture() override { absl::RemoveLogSink(this); } |
| void Send(const absl::LogEntry &entry) override { |
| captured_log_ << entry.text_message() << "\n"; |
| } |
| std::string CapturedLog() const { return captured_log_.str(); } |
| |
| private: |
| std::stringstream captured_log_; |
| }; |
| |
| uint64_t CivilTimeToUnixMicros( // |
| int64_t y, int64_t m, int64_t d, int64_t hh, int64_t mm, int64_t ss) { |
| return absl::ToUnixMicros(absl::FromCivil( |
| absl::CivilSecond{y, m, d, hh, mm, ss}, absl::LocalTimeZone())); |
| } |
| |
| } // namespace |
| |
| TEST(Stats, PrintStatsToLog) { |
| std::vector<Stats> stats_vec(4); |
| stats_vec[0].num_covered_pcs = 10; |
| stats_vec[1].num_covered_pcs = 15; |
| stats_vec[2].num_covered_pcs = 25; |
| stats_vec[3].num_covered_pcs = 40; |
| |
| stats_vec[0].corpus_size = 1000; |
| stats_vec[1].corpus_size = 2000; |
| stats_vec[2].corpus_size = 3000; |
| stats_vec[3].corpus_size = 4000; |
| |
| for (size_t i = 0; i < 4; ++i) { |
| auto &stats = stats_vec[i]; |
| stats.unix_micros = CivilTimeToUnixMicros(1970, 1, 1, 0, 0, i); |
| stats.max_corpus_element_size = 2 * i + 1; |
| stats.avg_corpus_element_size = i + 1; |
| stats.num_executions = i + 100; |
| } |
| |
| std::vector<Environment> env_vec(4); |
| env_vec[0].experiment_name = "Experiment A"; |
| env_vec[0].experiment_flags = "AAA"; |
| env_vec[1].experiment_name = "Experiment B"; |
| env_vec[1].experiment_flags = "BBB"; |
| env_vec[2].experiment_name = "Experiment A"; |
| env_vec[2].experiment_flags = "AAA"; |
| env_vec[3].experiment_name = "Experiment B"; |
| env_vec[3].experiment_flags = "BBB"; |
| |
| const std::string_view kExpectedLogLines = |
| "Current stats:\n" |
| "Coverage:\n" |
| "Experiment A: min:\t10\tmax:\t25\tavg:\t17.5\t--\t10\t25\n" |
| "Experiment B: min:\t15\tmax:\t40\tavg:\t27.5\t--\t15\t40\n" |
| "Number of executions:\n" |
| "Experiment A: min:\t100\tmax:\t102\tavg:\t101.0\t--\t100\t102\n" |
| "Experiment B: min:\t101\tmax:\t103\tavg:\t102.0\t--\t101\t103\n" |
| "Corpus size:\n" |
| "Experiment A: min:\t1000\tmax:\t3000\tavg:\t2000.0\t--\t1000\t3000\n" |
| "Experiment B: min:\t2000\tmax:\t4000\tavg:\t3000.0\t--\t2000\t4000\n" |
| "Max element size:\n" |
| "Experiment A: min:\t1\tmax:\t5\tavg:\t3.0\t--\t1\t5\n" |
| "Experiment B: min:\t3\tmax:\t7\tavg:\t5.0\t--\t3\t7\n" |
| "Avg element size:\n" |
| "Experiment A: min:\t1\tmax:\t3\tavg:\t2.0\t--\t1\t3\n" |
| "Experiment B: min:\t2\tmax:\t4\tavg:\t3.0\t--\t2\t4\n" |
| "Timestamp (UNIX micros):\n" |
| "Experiment A: min:\t1970-01-01T00:00:00\tmax:\t1970-01-01T00:00:02\n" |
| "Experiment B: min:\t1970-01-01T00:00:01\tmax:\t1970-01-01T00:00:03\n" |
| "Flags:\n" |
| "Experiment A: AAA\n" |
| "Experiment B: BBB\n"; |
| |
| StatsLogger stats_logger{stats_vec, env_vec}; |
| LogCapture log_capture; |
| stats_logger.ReportCurrStats(); |
| |
| EXPECT_THAT(log_capture.CapturedLog(), testing::StrEq(kExpectedLogLines)); |
| } |
| |
| TEST(Stats, DumpStatsToCsvFile) { |
| const std::filesystem::path workdir = GetTestTempDir(test_info_->name()); |
| |
| std::vector<Stats> stats_vec(4); |
| stats_vec[0].num_covered_pcs = 10; |
| stats_vec[0].corpus_size = 1000; |
| stats_vec[1].num_covered_pcs = 15; |
| stats_vec[1].corpus_size = 2000; |
| stats_vec[2].num_covered_pcs = 25; |
| stats_vec[2].corpus_size = 3000; |
| stats_vec[3].num_covered_pcs = 40; |
| stats_vec[3].corpus_size = 4000; |
| for (size_t i = 0; i < 4; ++i) { |
| auto &stats = stats_vec[i]; |
| stats.unix_micros = i + 1000000; |
| stats.max_corpus_element_size = 2 * i + 1; |
| stats.avg_corpus_element_size = i + 1; |
| stats.num_executions = i + 100; |
| } |
| |
| std::vector<Environment> env_vec(4); |
| env_vec[0].experiment_name = "ExperimentA"; |
| env_vec[0].experiment_flags = "AAA"; |
| env_vec[1].experiment_name = "ExperimentB"; |
| env_vec[1].experiment_flags = "BBB"; |
| env_vec[2].experiment_name = "ExperimentA"; |
| env_vec[2].experiment_flags = "AAA"; |
| env_vec[3].experiment_name = "ExperimentB"; |
| env_vec[3].experiment_flags = "BBB"; |
| for (auto &env : env_vec) { |
| env.workdir = workdir; |
| } |
| |
| { |
| StatsCsvFileAppender stats_csv_appender{stats_vec, env_vec}; |
| stats_csv_appender.ReportCurrStats(); |
| |
| for (auto &stats : stats_vec) { |
| stats.unix_micros += 1; |
| stats.num_executions += 1; |
| stats.num_covered_pcs += 1; |
| stats.corpus_size += 1; |
| stats.max_corpus_element_size += 1; |
| stats.avg_corpus_element_size += 1; |
| } |
| |
| stats_csv_appender.ReportCurrStats(); |
| } |
| |
| const std::vector<std::string> kExpectedCsvs = { |
| workdir / "fuzzing-stats-.000000.ExperimentA.csv", |
| workdir / "fuzzing-stats-.000000.ExperimentB.csv", |
| }; |
| const std::vector<std::string_view> kExpectedCsvContents = { |
| R"(NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg,NumExecs_Min,NumExecs_Max,NumExecs_Avg,CorpusSize_Min,CorpusSize_Max,CorpusSize_Avg,MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg,AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg,UnixMicros_Min,UnixMicros_Max, |
| 10,25,17.5,100,102,101.0,1000,3000,2000.0,1,5,3.0,1,3,2.0,1000000,1000002, |
| 11,26,18.5,101,103,102.0,1001,3001,2001.0,2,6,4.0,2,4,3.0,1000001,1000003, |
| )", |
| R"(NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg,NumExecs_Min,NumExecs_Max,NumExecs_Avg,CorpusSize_Min,CorpusSize_Max,CorpusSize_Avg,MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg,AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg,UnixMicros_Min,UnixMicros_Max, |
| 15,40,27.5,101,103,102.0,2000,4000,3000.0,3,7,5.0,2,4,3.0,1000001,1000003, |
| 16,41,28.5,102,104,103.0,2001,4001,3001.0,4,8,6.0,3,5,4.0,1000002,1000004, |
| )", |
| }; |
| |
| for (int i = 0; i < 2; ++i) { |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsvs[i])); |
| std::string csv_contents; |
| ReadFromLocalFile(kExpectedCsvs[i], csv_contents); |
| EXPECT_EQ(csv_contents, kExpectedCsvContents[i]); |
| } |
| } |
| |
| TEST(Stats, PrintRewardValues) { |
| std::stringstream ss; |
| std::vector<Stats> stats_vec(4); |
| stats_vec[0].num_covered_pcs = 15; |
| stats_vec[1].num_covered_pcs = 10; |
| stats_vec[2].num_covered_pcs = 40; |
| stats_vec[3].num_covered_pcs = 25; |
| PrintRewardValues(stats_vec, ss); |
| LOG(INFO) << "\n" << ss.str(); |
| const char *expected = |
| "REWARD_MAX 40\n" |
| "REWARD_SECOND_MAX 25\n" |
| "REWARD_MIN 10\n" |
| "REWARD_MEDIAN 25\n" |
| "REWARD_AVERAGE 22.5\n"; |
| EXPECT_THAT(ss.str(), testing::StrEq(expected)); |
| } |
| |
| } // namespace centipede |