Skip errored runs when computing repetition statistics (#2199)
diff --git a/src/statistics.cc b/src/statistics.cc
index 2c6c858..619bb66 100644
--- a/src/statistics.cc
+++ b/src/statistics.cc
@@ -112,10 +112,15 @@
typedef BenchmarkReporter::Run Run;
std::vector<Run> results;
- auto error_count = std::count_if(reports.begin(), reports.end(),
- [](Run const& run) { return run.skipped; });
+ const auto is_successful = [](Run const& run) {
+ return run.skipped == internal::NotSkipped;
+ };
+ auto successful_run =
+ std::find_if(reports.begin(), reports.end(), is_successful);
+ const auto successful_count = static_cast<size_t>(
+ std::count_if(reports.begin(), reports.end(), is_successful));
- if (reports.size() - static_cast<size_t>(error_count) < 2) {
+ if (successful_count < 2) {
// We don't report aggregated data if there was a single run.
return results;
}
@@ -124,12 +129,12 @@
std::vector<double> real_accumulated_time_stat;
std::vector<double> cpu_accumulated_time_stat;
- real_accumulated_time_stat.reserve(reports.size());
- cpu_accumulated_time_stat.reserve(reports.size());
+ real_accumulated_time_stat.reserve(successful_count);
+ cpu_accumulated_time_stat.reserve(successful_count);
// All repetitions should be run with the same number of iterations so we
// can take this information from the first benchmark.
- const IterationCount run_iterations = reports.front().iterations;
+ const IterationCount run_iterations = successful_run->iterations;
// create stats for user counters
struct CounterStat {
Counter c;
@@ -137,6 +142,9 @@
};
std::map<std::string, CounterStat> counter_stats;
for (Run const& r : reports) {
+ if (!is_successful(r)) {
+ continue;
+ }
for (auto const& cnt : r.counters) {
auto it = counter_stats.find(cnt.first);
if (it == counter_stats.end()) {
@@ -144,7 +152,7 @@
.emplace(cnt.first,
CounterStat{cnt.second, std::vector<double>{}})
.first;
- it->second.s.reserve(reports.size());
+ it->second.s.reserve(successful_count);
} else {
BM_CHECK_EQ(it->second.c.flags, cnt.second.flags);
}
@@ -153,11 +161,11 @@
// Populate the accumulators.
for (Run const& run : reports) {
- BM_CHECK_EQ(reports[0].benchmark_name(), run.benchmark_name());
- BM_CHECK_EQ(run_iterations, run.iterations);
- if (run.skipped != 0u) {
+ BM_CHECK_EQ(successful_run->benchmark_name(), run.benchmark_name());
+ if (!is_successful(run)) {
continue;
}
+ BM_CHECK_EQ(run_iterations, run.iterations);
real_accumulated_time_stat.emplace_back(run.real_accumulated_time);
cpu_accumulated_time_stat.emplace_back(run.cpu_accumulated_time);
// user counters
@@ -169,26 +177,30 @@
}
// Only add label if it is same for all runs
- std::string report_label = reports[0].report_label;
- for (std::size_t i = 1; i < reports.size(); i++) {
- if (reports[i].report_label != report_label) {
+ std::string report_label = successful_run->report_label;
+ for (const Run& run : reports) {
+ if (!is_successful(run)) {
+ continue;
+ }
+ if (run.report_label != report_label) {
report_label = "";
break;
}
}
const double iteration_rescale_factor =
- static_cast<double>(reports.size()) / static_cast<double>(run_iterations);
+ static_cast<double>(successful_count) /
+ static_cast<double>(run_iterations);
- for (const auto& Stat : *reports[0].statistics) {
+ for (const auto& Stat : *successful_run->statistics) {
// Get the data from the accumulator to BenchmarkReporter::Run's.
Run data;
- data.run_name = reports[0].run_name;
- data.family_index = reports[0].family_index;
- data.per_family_instance_index = reports[0].per_family_instance_index;
+ data.run_name = successful_run->run_name;
+ data.family_index = successful_run->family_index;
+ data.per_family_instance_index = successful_run->per_family_instance_index;
data.run_type = BenchmarkReporter::Run::RT_Aggregate;
- data.threads = reports[0].threads;
- data.repetitions = reports[0].repetitions;
+ data.threads = successful_run->threads;
+ data.repetitions = successful_run->repetitions;
data.repetition_index = Run::no_repetition_index;
data.aggregate_name = Stat.name_;
data.aggregate_unit = Stat.unit_;
@@ -199,7 +211,7 @@
// Similarly, if there are N repetitions with 1 iterations each,
// an aggregate will be computed over N measurements, not 1.
// Thus it is best to simply use the count of separate reports.
- data.iterations = static_cast<IterationCount>(reports.size());
+ data.iterations = static_cast<IterationCount>(successful_count);
data.real_accumulated_time = Stat.compute_(real_accumulated_time_stat);
data.cpu_accumulated_time = Stat.compute_(cpu_accumulated_time_stat);
@@ -214,7 +226,7 @@
data.cpu_accumulated_time *= iteration_rescale_factor;
}
- data.time_unit = reports[0].time_unit;
+ data.time_unit = successful_run->time_unit;
// user counters
for (auto const& kv : counter_stats) {
diff --git a/test/repetitions_test.cc b/test/repetitions_test.cc
index 80216ab..0673692 100644
--- a/test/repetitions_test.cc
+++ b/test/repetitions_test.cc
@@ -209,6 +209,43 @@
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_mean\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_median\",%csv_report$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_ImplicitRepetitions_stddev\",%csv_report$"}});
+
+// ========================================================================= //
+// --------------------- Testing Skipped Repetitions ------------------------ //
+// ========================================================================= //
+
+void BM_SkippedFirstRepetition(benchmark::State& state) {
+ static int repetition_index = 0;
+ if (repetition_index++ == 0) {
+ state.SkipWithError("skipped first repetition");
+ return;
+ }
+ for (auto _ : state) {
+ }
+}
+BENCHMARK(BM_SkippedFirstRepetition)->Repetitions(3);
+
+ADD_CASES(
+ TC_ConsoleOut,
+ {{"^BM_SkippedFirstRepetition/repeats:3[ ]+ERROR OCCURRED: "
+ "'skipped first repetition'$"},
+ {"^BM_SkippedFirstRepetition/repeats:3 %console_report$", MR_Next},
+ {"^BM_SkippedFirstRepetition/repeats:3 %console_report$", MR_Next},
+ {"^BM_SkippedFirstRepetition/repeats:3_mean %console_report$", MR_Next},
+ {"^BM_SkippedFirstRepetition/repeats:3_median %console_report$", MR_Next},
+ {"^BM_SkippedFirstRepetition/repeats:3_stddev %console_report$",
+ MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_SkippedFirstRepetition/repeats:3_mean\",$"},
+ {"\"run_type\": \"aggregate\",$"},
+ {"\"repetitions\": 3,$", MR_Next},
+ {"\"threads\": 1,$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"aggregate_unit\": \"time\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next}});
+ADD_CASES(TC_CSVOut,
+ {{"^\"BM_SkippedFirstRepetition/repeats:3_mean\",2,%float,%float,"
+ "ns,,,,,$"}});
} // end namespace
// ========================================================================= //
