Skip to content

Report the complexity aggregates in the family's time unit #2294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lenamonj wants to merge 3 commits into
base: main
Choose a base branch
from
+48 −13
Open

Report the complexity aggregates in the family's time unit #2294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
33523cc
Report the complexity aggregates in the family's time unit
lenamonj Sep 7, 2026
750d187
complexity test: pass the unit and CSV pattern explicitly, drop a com…
lenamonj Sep 7, 2026
9ab0153
Drop the CSV reporter change
lenamonj Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/user_guide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -850,6 +850,11 @@ BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
->Range(1<<10, 1<<18)->Complexity([](benchmark::IterationCount n)->double{return n; });
```

The `BigO` row reports the coefficient in the same time unit as the rest of the
family, so a family declared with `->Unit(benchmark::kMicrosecond)` reports its
coefficient in microseconds. The `RMS` row is a normalized quantity and is
reported as a percentage, independent of the time unit.

<a name="custom-benchmark-name" />

## Custom Benchmark Name
Expand Down
1 change: 1 addition & 0 deletions src/complexity.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ std::vector<BenchmarkReporter::Run> ComputeBigO(
big_o.cpu_accumulated_time = result_cpu.coef;
big_o.report_big_o = true;
big_o.complexity = result_cpu.complexity;
big_o.time_unit = reports[0].time_unit;

// All the time results are reported after being multiplied by the
// time unit multiplier. But since RMS is a relative quantity it
Expand Down
55 changes: 42 additions & 13 deletions test/complexity_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@ namespace {
int AddComplexityTest(const std::string& test_name,
const std::string& big_o_test_name,
const std::string& rms_test_name,
const std::string& big_o, int family_index) {
const std::string& big_o, int family_index,
const std::string& time_unit) {
SetSubstitutions({{"%name", test_name},
{"%bigo_name", big_o_test_name},
{"%rms_name", rms_test_name},
{"%bigo_str", "[ ]* %float " + big_o},
{"%bigo", big_o},
{"%rms", "[ ]*[0-9]+ %"}});
{"%rms", "[ ]*[0-9]+ %"},
{"%unit", time_unit}});
AddCases(
TC_ConsoleOut,
{{"^%bigo_name %bigo_str %bigo_str[ ]*$"},
Expand All @@ -46,7 +48,7 @@ int AddComplexityTest(const std::string& test_name,
{"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next},
{"\"time_unit\": \"%unit\"$", MR_Next},
{"}", MR_Next},
{"\"name\": \"%rms_name\",$"},
{"\"family_index\": " + std::to_string(family_index) + ",$", MR_Next},
Expand Down Expand Up @@ -104,15 +106,15 @@ constexpr char lambda_big_o_1[] = "f\\(N\\)";

// Add enum tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
enum_auto_big_o_1, /*family_index=*/0);
enum_auto_big_o_1, /*family_index=*/0, "ns");

// Add auto tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
enum_auto_big_o_1, /*family_index=*/1);
enum_auto_big_o_1, /*family_index=*/1, "ns");

// Add lambda tests
ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
lambda_big_o_1, /*family_index=*/2);
lambda_big_o_1, /*family_index=*/2, "ns");

// ========================================================================= //
// --------------------------- Testing BigO O(N) --------------------------- //
Expand Down Expand Up @@ -161,15 +163,15 @@ constexpr char lambda_big_o_n[] = "f\\(N\\)";

// Add enum tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
enum_auto_big_o_n, /*family_index=*/3);
enum_auto_big_o_n, /*family_index=*/3, "ns");

// Add auto tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
enum_auto_big_o_n, /*family_index=*/4);
enum_auto_big_o_n, /*family_index=*/4, "ns");

// Add lambda tests
ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
lambda_big_o_n, /*family_index=*/5);
lambda_big_o_n, /*family_index=*/5, "ns");

// ========================================================================= //
// ------------------------- Testing BigO O(NlgN) ------------------------- //
Expand Down Expand Up @@ -222,17 +224,17 @@ constexpr char lambda_big_o_n_lg_n[] = "f\\(N\\)";
// Add enum tests
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,
/*family_index=*/6);
/*family_index=*/6, "ns");

// NOTE: auto big-o is wron.g
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n,
/*family_index=*/7);
/*family_index=*/7, "ns");

//// Add lambda tests
ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n,
/*family_index=*/8);
/*family_index=*/8, "ns");

// ========================================================================= //
// -------- Testing formatting of Complexity with captured args ------------ //
Expand Down Expand Up @@ -265,7 +267,34 @@ const std::string complexity_capture_name =

ADD_COMPLEXITY_CASES(complexity_capture_name, complexity_capture_name + "_BigO",
complexity_capture_name + "_RMS", "N",
/*family_index=*/9);
/*family_index=*/9, "ns");
// ========================================================================= //
// ------------------- Testing BigO with a declared unit ------------------- //
// ========================================================================= //

void BM_Complexity_O_N_ms(benchmark::State& state) {
for (auto _ : state) {
// 1us per iteration per entry, half the sizes 50% slower so the RMS is
// well above zero, reported in milliseconds
const double skew = (state.range(0) & (1 << 11)) ? 1.5 : 1.0;
state.SetIterationTime(static_cast<double>(state.range(0)) * 1e-6 * skew);
}
state.SetComplexityN(state.range(0));
}
BENCHMARK(BM_Complexity_O_N_ms)
->RangeMultiplier(2)
->Range(1 << 10, 1 << 16)
->UseManualTime()
->Unit(benchmark::kMillisecond)
->Complexity(benchmark::oN);

constexpr char n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time";
constexpr char big_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_BigO";
constexpr char rms_o_n_ms_test_name[] = "BM_Complexity_O_N_ms/manual_time_RMS";

ADD_COMPLEXITY_CASES(n_ms_test_name, big_o_n_ms_test_name, rms_o_n_ms_test_name,
enum_auto_big_o_n, /*family_index=*/10, "ms");

} // end namespace

// ========================================================================= //
Expand Down