Add --benchmark_report_aggregates_only={true|false} flag for better summary output. (#267)
diff --git a/README.md b/README.md
index 8cf853c..0c081ff 100644
--- a/README.md
+++ b/README.md
@@ -391,6 +391,13 @@
`Repetitions` on the registered benchmark object. When a benchmark is run
more than once the mean and standard deviation of the runs will be reported.
+Additionally the `--benchmark_report_aggregates_only={true|false}` flag or
+`ReportAggregatesOnly(bool)` function can be used to change how repeated tests
+are reported. By default the result of each repeated run is reported. When this
+option is 'true' only the mean and standard deviation of the runs is reported.
+Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides
+the value of the flag for that benchmark.
+
## Fixtures
Fixture tests are created by
first defining a type that derives from ::benchmark::Fixture and then
diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h
index 1a481ac..ddfcb01 100644
--- a/include/benchmark/benchmark_api.h
+++ b/include/benchmark/benchmark_api.h
@@ -522,6 +522,11 @@
// REQUIRES: `n > 0`
Benchmark* Repetitions(int n);
+ // Specify if each repetition of the benchmark should be reported separately
+ // or if only the final statistics should be reported. If the benchmark
+ // is not repeated then the single result is always reported.
+ Benchmark* ReportAggregatesOnly(bool v = true);
+
// If a particular benchmark is I/O bound, runs multiple threads internally or
// if for some reason CPU timings are not representative, call this method. If
// called, the elapsed time will be used to control how many iterations are
diff --git a/src/benchmark.cc b/src/benchmark.cc
index 515abd7..3a199e5 100644
--- a/src/benchmark.cc
+++ b/src/benchmark.cc
@@ -66,6 +66,11 @@
"The number of runs of each benchmark. If greater than 1, the "
"mean and standard deviation of the runs will be reported.");
+DEFINE_bool(benchmark_report_aggregates_only, false,
+ "Report the result of each benchmark repetitions. When 'true' is "
+ "specified only the mean, standard deviation, and other statistics "
+ "are reported for repeated benchmarks.");
+
DEFINE_string(benchmark_format, "console",
"The format to use for console output. Valid values are "
"'console', 'json', or 'csv'.");
@@ -311,10 +316,17 @@
namespace internal {
+enum ReportMode : unsigned {
+ RM_Unspecified, // The mode has not been manually specified
+ RM_Default, // The mode is user-specified as default.
+ RM_ReportAggregatesOnly
+};
+
// Information kept per benchmark we may want to run
struct Benchmark::Instance {
std::string name;
Benchmark* benchmark;
+ ReportMode report_mode;
std::vector<int> arg;
TimeUnit time_unit;
int range_multiplier;
@@ -364,6 +376,7 @@
void RangeMultiplier(int multiplier);
void MinTime(double n);
void Repetitions(int n);
+ void ReportAggregatesOnly(bool v);
void UseRealTime();
void UseManualTime();
void Complexity(BigO complexity);
@@ -381,6 +394,7 @@
friend class BenchmarkFamilies;
std::string name_;
+ ReportMode report_mode_;
std::vector< std::vector<int> > args_; // Args for all benchmark runs
TimeUnit time_unit_;
int range_multiplier_;
@@ -443,6 +457,7 @@
Benchmark::Instance instance;
instance.name = family->name_;
instance.benchmark = bench_family.get();
+ instance.report_mode = family->report_mode_;
instance.arg = args;
instance.time_unit = family->time_unit_;
instance.range_multiplier = family->range_multiplier_;
@@ -488,7 +503,7 @@
}
BenchmarkImp::BenchmarkImp(const char* name)
- : name_(name), time_unit_(kNanosecond),
+ : name_(name), report_mode_(RM_Unspecified), time_unit_(kNanosecond),
range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0),
use_real_time_(false), use_manual_time_(false),
complexity_(oNone) {
@@ -575,6 +590,10 @@
repetitions_ = n;
}
+void BenchmarkImp::ReportAggregatesOnly(bool value) {
+ report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default;
+}
+
void BenchmarkImp::UseRealTime() {
CHECK(!use_manual_time_) << "Cannot set UseRealTime and UseManualTime simultaneously.";
use_real_time_ = true;
@@ -703,6 +722,11 @@
return this;
}
+Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
+ imp_->ReportAggregatesOnly(value);
+ return this;
+}
+
Benchmark* Benchmark::MinTime(double t) {
imp_->MinTime(t);
return this;
@@ -779,7 +803,8 @@
RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
std::vector<BenchmarkReporter::Run>* complexity_reports)
EXCLUDES(GetBenchmarkLock()) {
- std::vector<BenchmarkReporter::Run> reports; // return value
+ std::vector<BenchmarkReporter::Run> reports; // return value
+
size_t iters = 1;
std::vector<std::thread> pool;
@@ -788,6 +813,10 @@
const int repeats = b.repetitions != 0 ? b.repetitions
: FLAGS_benchmark_repetitions;
+ const bool report_aggregates_only = repeats != 1 &&
+ (b.report_mode == internal::RM_Unspecified
+ ? FLAGS_benchmark_report_aggregates_only
+ : b.report_mode == internal::RM_ReportAggregatesOnly);
for (int i = 0; i < repeats; i++) {
std::string mem;
for (;;) {
@@ -914,22 +943,21 @@
iters = static_cast<int>(next_iters + 0.5);
}
}
- std::vector<BenchmarkReporter::Run> additional_run_stats = ComputeStats(reports);
- reports.insert(reports.end(), additional_run_stats.begin(),
- additional_run_stats.end());
-
- if((b.complexity != oNone) && b.last_benchmark_instance) {
- additional_run_stats = ComputeBigO(*complexity_reports);
- reports.insert(reports.end(), additional_run_stats.begin(),
- additional_run_stats.end());
- complexity_reports->clear();
- }
-
if (b.multithreaded) {
for (std::thread& thread : pool)
thread.join();
}
+ // Calculate additional statistics
+ auto stat_reports = ComputeStats(reports);
+ if((b.complexity != oNone) && b.last_benchmark_instance) {
+ auto additional_run_stats = ComputeBigO(*complexity_reports);
+ stat_reports.insert(stat_reports.end(), additional_run_stats.begin(),
+ additional_run_stats.end());
+ complexity_reports->clear();
+ }
+ if (report_aggregates_only) reports.clear();
+ reports.insert(reports.end(), stat_reports.begin(), stat_reports.end());
return reports;
}
@@ -1117,6 +1145,7 @@
" [--benchmark_filter=<regex>]\n"
" [--benchmark_min_time=<min_time>]\n"
" [--benchmark_repetitions=<num_repetitions>]\n"
+ " [--benchmark_report_aggregates_only={true|false}\n"
" [--benchmark_format=<console|json|csv>]\n"
" [--benchmark_out=<filename>]\n"
" [--benchmark_out_format=<json|console|csv>]\n"
@@ -1137,6 +1166,8 @@
&FLAGS_benchmark_min_time) ||
ParseInt32Flag(argv[i], "benchmark_repetitions",
&FLAGS_benchmark_repetitions) ||
+ ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
+ &FLAGS_benchmark_report_aggregates_only) ||
ParseStringFlag(argv[i], "benchmark_format",
&FLAGS_benchmark_format) ||
ParseStringFlag(argv[i], "benchmark_out",
diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc
index e580008..fc71f27 100644
--- a/test/reporter_output_test.cc
+++ b/test/reporter_output_test.cc
@@ -6,6 +6,7 @@
#include <cassert>
#include <cstring>
#include <iostream>
+#include <memory>
#include <sstream>
#include <vector>
#include <utility>
@@ -18,35 +19,58 @@
enum MatchRules {
MR_Default, // Skip non-matching lines until a match is found.
- MR_Next // Match must occur on the next line.
+ MR_Next, // Match must occur on the next line.
+ MR_Not // No line between the current position and the next match matches
+ // the regex
};
struct TestCase {
- std::string regex;
+ std::string regex_str;
int match_rule;
+ std::shared_ptr<benchmark::Regex> regex;
- TestCase(std::string re, int rule = MR_Default) : regex(re), match_rule(rule) {}
-
- void Check(std::stringstream& remaining_output) const {
- benchmark::Regex r;
+ TestCase(std::string re, int rule = MR_Default)
+ : regex_str(re), match_rule(rule), regex(std::make_shared<benchmark::Regex>()) {
std::string err_str;
- r.Init(regex, &err_str);
- CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\""
+ regex->Init(regex_str, &err_str);
+ CHECK(err_str.empty()) << "Could not construct regex \"" << regex_str << "\""
<< " got Error: " << err_str;
+ }
+ void Check(std::stringstream& remaining_output,
+ std::vector<TestCase>& not_checks) const {
std::string line;
while (remaining_output.eof() == false) {
CHECK(remaining_output.good());
std::getline(remaining_output, line);
- if (r.Match(line)) return;
+ for (auto& NC : not_checks) {
+ CHECK(!NC.regex->Match(line)) << "Unexpected match for line \""
+ << line << "\" for MR_Not regex \""
+ << NC.regex_str << "\"";
+ }
+ if (regex->Match(line)) return;
CHECK(match_rule != MR_Next) << "Expected line \"" << line
- << "\" to match regex \"" << regex << "\"";
+ << "\" to match regex \"" << regex_str << "\"";
}
CHECK(remaining_output.eof() == false)
- << "End of output reached before match for regex \"" << regex
+ << "End of output reached before match for regex \"" << regex_str
<< "\" was found";
}
+
+ static void CheckCases(std::vector<TestCase> const& checks,
+ std::stringstream& output) {
+ std::vector<TestCase> not_checks;
+ for (size_t i=0; i < checks.size(); ++i) {
+ const auto& TC = checks[i];
+ if (TC.match_rule == MR_Not) {
+ not_checks.push_back(TC);
+ continue;
+ }
+ TC.Check(output, not_checks);
+ not_checks.clear();
+ }
+ }
};
std::vector<TestCase> ConsoleOutputTests;
@@ -114,8 +138,6 @@
return std::string(std::move(f)) + "[ ]+" + join(std::forward<Args>(args)...);
}
-
-
std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
} // end namespace
@@ -200,6 +222,68 @@
// ========================================================================= //
+// ----------------------- Testing Aggregate Output ------------------------ //
+// ========================================================================= //
+
+// Test that non-aggregate data is printed by default
+void BM_Repeat(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_Repeat)->Repetitions(3);
+ADD_CASES(&ConsoleOutputTests, {
+ {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+ {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+ {"^BM_Repeat/repeats:3[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+ {"^BM_Repeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+ {"^BM_Repeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+ {"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
+ {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+ {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+ {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+ {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+ {"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+ {"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
+// Test that a non-repeated test still prints non-aggregate results even when
+// only-aggregate reports have been requested
+void BM_RepeatOnce(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
+ADD_CASES(&ConsoleOutputTests, {
+ {"^BM_RepeatOnce/repeats:1[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+ {"\"name\": \"BM_RepeatOnce/repeats:1\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+ {"^\"BM_RepeatOnce/repeats:1\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
+// Test that non-aggregate data is not reported
+void BM_SummaryRepeat(benchmark::State& state) { while (state.KeepRunning()) {} }
+BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
+ADD_CASES(&ConsoleOutputTests, {
+ {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+ {"^BM_SummaryRepeat/repeats:3_mean[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"},
+ {"^BM_SummaryRepeat/repeats:3_stddev[ ]+[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+$"}
+});
+ADD_CASES(&JSONOutputTests, {
+ {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}
+});
+ADD_CASES(&CSVOutputTests, {
+ {".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+ {"^\"BM_SummaryRepeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"},
+ {"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}
+});
+
+// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= //
@@ -244,10 +328,8 @@
std::cerr << rep_test.err_stream.str();
std::cout << rep_test.out_stream.str();
- for (const auto& TC : rep_test.error_cases)
- TC.Check(rep_test.err_stream);
- for (const auto& TC : rep_test.output_cases)
- TC.Check(rep_test.out_stream);
+ TestCase::CheckCases(rep_test.error_cases, rep_test.err_stream);
+ TestCase::CheckCases(rep_test.output_cases, rep_test.out_stream);
std::cout << "\n";
}
