e2e_tests/benchmark_test.cc

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Benchmark test.
//
// It runs each micro-benchmark fuzz test in a child process and measures the
// time and number of iterations it takes to find the exit condition.

#include <filesystem>
#include <string>
#include <vector>

#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "absl/time/time.h"
#include "./fuzztest/internal/logging.h"
#include "./fuzztest/internal/subprocess.h"

// Mimic the open source google benchmark flags in order to work with benchmark
// driver tools.
ABSL_FLAG(std::string, benchmark_filter, "", "");
ABSL_FLAG(bool, benchmark_list_tests, false, "");
ABSL_FLAG(std::string, benchmark_format, "", "");

namespace {

struct Stats {
  uint64_t nanos;
  uint64_t runs;
  uint64_t edges_covered;
  uint64_t total_edges;
  uint64_t corpus_size;
};

struct TestResult {
  std::string test_name;
  Stats stats;
};

std::string SelfPath() {
  char buf[4096];
  ssize_t len = readlink("/proc/self/exe", buf, sizeof(buf));
  FUZZTEST_INTERNAL_CHECK(len < sizeof(buf), "Path too long!");
  return std::string(buf, len);
}
std::string MicrobenchmarksBinaryPath() {
  const std::string self_path = SelfPath();
  const std::size_t dir_path_len = self_path.find_last_of('/');
  std::string binary_path = self_path.substr(0, dir_path_len) +
                            "/testdata/fuzz_tests_for_microbenchmarking";
  FUZZTEST_INTERNAL_CHECK(std::filesystem::exists(binary_path), "Can't find ",
                          binary_path);
  return binary_path;
}

// This parses the crash report output of a fuzz test.
// We might want to pass a richer format later.
Stats ParseStats(std::string_view output) {
  std::vector<std::string> lines = absl::StrSplit(output, '\n');
  const auto get_int = [](std::string_view line) -> uint64_t {
    std::vector<std::string> p = absl::StrSplit(line, ": ");
    if (p.size() != 2) return 0;
    uint64_t i;
    if (!absl::SimpleAtoi(p.back(), &i)) return 0;
    return i;
  };
  int i = 0;
  while (i < lines.size() && !absl::StartsWith(lines[i], "Elapsed seconds"))
    ++i;
  if (i + 4 >= lines.size()) return {};
  return {get_int(lines[i]), get_int(lines[i + 1]), get_int(lines[i + 2]),
          get_int(lines[i + 3]), get_int(lines[i + 4])};
}

std::string SingleJsonResult(std::string_view name, uint64_t value) {
  constexpr absl::string_view kJsonFormat = R"(    {
      "cpu_time": %u,
      "real_time": %u,
      "iterations": 1,
      "name": "%s",
      "time_unit": "ns"
    })";
  return absl::StrFormat(kJsonFormat, value, value, name);
}

std::string AllJsonResults(const std::vector<TestResult>& test_results) {
  std::vector<std::string> json_results;

  for (const TestResult& result : test_results) {
    if (result.test_name == "Control.Iters10000") {
      json_results.push_back(SingleJsonResult(
          absl::StrCat(result.test_name, "(time)"), result.stats.nanos));
      // We also generate a synthetic control result for 10000 total edges.
      // This measures the cost of the framework on processing the edge map.
      // This way we can see if differences in a CL come from edge count
      // change or from implementation changes.
      FUZZTEST_INTERNAL_CHECK(result.stats.total_edges != 0,
                              "Total edges cannot be zero!");
      json_results.push_back(SingleJsonResult(
          "Control.Edges10000(time)",
          10000 * result.stats.nanos / result.stats.total_edges));
    } else {
      json_results.push_back(SingleJsonResult(
          absl::StrCat(result.test_name, "(runs)"), result.stats.runs));
      json_results.push_back(SingleJsonResult(
          absl::StrCat(result.test_name, "(time)"), result.stats.nanos));
    }
  }

  constexpr absl::string_view kJsonTemplate = R"(
{
  "benchmarks": [
%s
  ],
  "context": {}
})";
  return absl::StrFormat(kJsonTemplate, absl::StrJoin(json_results, ",\n"));
}

Stats RunFuzzTest(const std::string& name) {
  std::cerr << "[.] Running: " << name << '\n';
  constexpr int kTimeOutSecs = 30;
  std::vector<std::string> command_line = {MicrobenchmarksBinaryPath(),
                                           "--fuzz", name};
  auto [status, std_out, std_err] = fuzztest::internal::RunCommand(
      command_line,
      /*environment=*/{}, absl::Seconds(kTimeOutSecs));
  return ParseStats(std_err);
}

bool TestNameMatchesFilter(absl::string_view test_name,
                           absl::string_view filter) {
  return test_name == filter ||
         (test_name == "Control.Iters10000" && filter == "Control.Edges10000");
}

void RunMicrobenchmarks(const bool list_tests, const std::string& filter) {
  std::vector<TestResult> test_results;

  // Get the list of fuzz tests in the `microbenchmarks` binary.
  auto [status, std_out, std_err] = fuzztest::internal::RunCommand(
      {MicrobenchmarksBinaryPath(), "--list_fuzz_tests"});
  const std::vector<std::string> std_out_lines =
      absl::StrSplit(std_out, '\n', absl::SkipWhitespace());
  std::vector<std::string> fuzz_test_names;
  fuzz_test_names.reserve(std_out_lines.size());
  for (absl::string_view line : std_out_lines) {
    static constexpr absl::string_view kFuzzTestPrefix = "[*] Fuzz test: ";
    if (absl::ConsumePrefix(&line, kFuzzTestPrefix)) {
      fuzz_test_names.push_back(std::string(line));
    }
  }

  if (list_tests) {
    for (const std::string& name : fuzz_test_names) {
      std::cout << name << '\n';
    }
    // Synthetic control test
    std::cout << "Control.Edges10000" << '\n';
    return;
  }

  const bool has_filter = !filter.empty() && filter != "all" && filter != ".*";
  for (const std::string& name : fuzz_test_names) {
    // NOTE: This should support regex, but let's keep it simple for now.
    if (has_filter && !TestNameMatchesFilter(name, filter)) continue;

    Stats stats = RunFuzzTest(name);
    test_results.push_back({name, stats});
  }

  std::cout << AllJsonResults(test_results);
}

}  // namespace

int main(int argc, char** argv) {
#if defined(__has_feature)
#if !__has_feature(coverage_sanitizer)
  FUZZTEST_INTERNAL_CHECK(false,
                          "\n\nPlease compile with --config=fuzztest.\n");
#endif
#endif
  absl::ParseCommandLine(argc, argv);
  const bool list_tests = absl::GetFlag(FLAGS_benchmark_list_tests);
  const std::string filter = absl::GetFlag(FLAGS_benchmark_filter);
  RunMicrobenchmarks(list_tests, filter);
}