Report the complexity aggregates in the family's time unit (#2294)
* Report the complexity aggregates in the family's time unit
ComputeBigO left big_o.time_unit at the Run() default, so the BigO row was rendered in nanoseconds whatever unit the family declared; CSVReporter rendered the RMS row from the raw, pre-divided field, so it was smaller by the unit multiplier. Both now go through the family unit and the GetAdjusted*Time() accessors, with a millisecond family in complexity_test.cc that fails on main for both.
* complexity test: pass the unit and CSV pattern explicitly, drop a comment
* Drop the CSV reporter change
The CSV reporter is deprecated, so its RMS row is left as it is and the
test keeps the pattern it had.
* complexity test: check the magnitude of the millisecond BigO coefficient
Until now, scaling the JSON BigO coefficients by 1000 left every test
passing. Under manual time the millisecond family's real coefficient is
about 1.0004e-03, so its BigO row now has to match 1\.[0-9]+e-03, and
that scaling fails it. The skew comment is corrected: of the family's
sizes, 1024 to 65536, only 2048 has bit 11 set.
---------
Co-authored-by: Roman Lebedev <lebedev.ri@gmail.com>
diff --git a/docs/user_guide.md b/docs/user_guide.md
index 3fed926..30f3608 100644
--- a/docs/user_guide.md
+++ b/docs/user_guide.md
@@ -850,6 +850,11 @@
->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; });
```
+The `BigO` row reports the coefficient in the same time unit as the rest of the
+family, so a family declared with `->Unit(benchmark::kMicrosecond)` reports its
+coefficient in microseconds. The `RMS` row is a normalized quantity and is
+reported as a percentage, independent of the time unit.
+
<a name="custom-benchmark-name" />
## Custom Benchmark Name
diff --git a/src/complexity.cc b/src/complexity.cc
index 8fa3f07..c431a3f 100644
--- a/src/complexity.cc
+++ b/src/complexity.cc
@@ -229,6 +229,7 @@
big_o.cpu_accumulated_time = result_cpu.coef;
big_o.report_big_o = true;
big_o.complexity = result_cpu.complexity;
+ big_o.time_unit = reports[0].time_unit;
// All the time results are reported after being multiplied by the
// time unit multiplier. But since RMS is a relative quantity it
diff --git a/test/complexity_test.cc b/test/complexity_test.cc
index 64a7e72..49622dc 100644
--- a/test/complexity_test.cc
+++ b/test/complexity_test.cc
@@ -20,13 +20,16 @@
int AddComplexityTest(const std::string& test_name,
const std::string& big_o_test_name,
const std::string& rms_test_name,
- const std::string& big_o, int family_index) {
+ const std::string& big_o, int family_index,
+ const std::string& time_unit,
+ const std::string& real_coefficient) {
SetSubstitutions({{"%name", test_name},
{"%bigo_name", big_o_test_name},
{"%rms_name", rms_test_name},
{"%bigo_str", "[ ]* %float " + big_o},
{"%bigo", big_o},
- {"%rms", "[ ]*[0-9]+ %"}});
+ {"%rms", "[ ]*[0-9]+ %"},
+ {"%unit", time_unit}});
AddCases(
TC_ConsoleOut,
{{"^%bigo_name %bigo_str %bigo_str[ ]*$"},
@@ -44,9 +47,9 @@
{"\"aggregate_name\": \"BigO\",$", MR_Next},
{"\"aggregate_unit\": \"time\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next},
- {"\"real_coefficient\": %float,$", MR_Next},
+ {"\"real_coefficient\": " + real_coefficient + ",$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next},
- {"\"time_unit\": \"ns\"$", MR_Next},
+ {"\"time_unit\": \"%unit\"$", MR_Next},
{"}", MR_Next},
{"\"name\": \"%rms_name\",$"},
{"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next},
@@ -104,15 +107,15 @@
// Add enum tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
- enum_auto_big_o_1, /*family_index=*/0);
+ enum_auto_big_o_1, /*family_index=*/0, "ns", "%float");
// Add auto tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
- enum_auto_big_o_1, /*family_index=*/1);
+ enum_auto_big_o_1, /*family_index=*/1, "ns", "%float");
// Add lambda tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
- lambda_big_o_1, /*family_index=*/2);
+ lambda_big_o_1, /*family_index=*/2, "ns", "%float");
// ========================================================================= //
// --------------------------- Testing BigO O(N) --------------------------- //
@@ -161,15 +164,15 @@
// Add enum tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
- enum_auto_big_o_n, /*family_index=*/3);
+ enum_auto_big_o_n, /*family_index=*/3, "ns", "%float");
// Add auto tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
- enum_auto_big_o_n, /*family_index=*/4);
+ enum_auto_big_o_n, /*family_index=*/4, "ns", "%float");
// Add lambda tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
- lambda_big_o_n, /*family_index=*/5);
+ lambda_big_o_n, /*family_index=*/5, "ns", "%float");
// ========================================================================= //
// ------------------------- Testing BigO O(NlgN) ------------------------- //
@@ -222,17 +225,17 @@
// Add enum tests
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,
- /*family_index=*/6);
+ /*family_index=*/6, "ns", "%float");
// NOTE: auto big-o is wron.g
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,
- /*family_index=*/7);
+ /*family_index=*/7, "ns", "%float");
//// Add lambda tests
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n,
- /*family_index=*/8);
+ /*family_index=*/8, "ns", "%float");
// ========================================================================= //
// -------- Testing formatting of Complexity with captured args ------------ //
@@ -265,7 +268,36 @@
ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO",
complexity_capture_name + "_RMS", "N",
- /*family_index=*/9);
+ /*family_index=*/9, "ns", "%float");
+
+// ========================================================================= //
+// ------------------- Testing BigO with a declared unit ------------------- //
+// ========================================================================= //
+
+void BM_Complexity_O_N_ms(benchmark::State& state) {
+ for (auto _ : state) {
+ // 1us per iteration per entry, the 2048 size 50% slower so the RMS is
+ // above zero, reported in milliseconds
+ const double skew = (state.range(0) & (1 << 11)) ? 1.5 : 1.0;
+ state.SetIterationTime(static_cast<double>(state.range(0)) * 1e-6 * skew);
+ }
+ state.SetComplexityN(state.range(0));
+}
+BENCHMARK(BM_Complexity_O_N_ms)
+ ->RangeMultiplier(2)
+ ->Range(1 << 10, 1 << 16)
+ ->UseManualTime()
+ ->Unit(benchmark::kMillisecond)
+ ->Complexity(benchmark::oN);
+
+constexpr char n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time";
+constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO";
+constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS";
+
+ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name,
+ enum_auto_big_o_n, /*family_index=*/10, "ms",
+ "1\\.[0-9]+e-03");
+
} // end namespace
// ========================================================================= //
