src/perf_counters.cc

// Copyright 2021 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "perf_counters.h"

#include <cstring>
#include <memory>
#include <optional>
#include <vector>

#if defined HAVE_LIBPFM
#include <dirent.h>
#include <fcntl.h>
#include <linux/perf_event.h>
#include <sys/stat.h>

#include "perfmon/pfmlib.h"
#include "perfmon/pfmlib_perf_event.h"
#endif

namespace benchmark {
namespace internal {

#if defined HAVE_LIBPFM

size_t PerfCounterValues::Read(const std::vector<int>& leaders) {
  // Create a pointer for multiple reads
  const size_t bufsize = values_.size() * sizeof(values_[0]);
  char* ptr = reinterpret_cast<char*>(values_.data());
  size_t size = bufsize;
  for (int lead : leaders) {
    auto read_bytes = ::read(lead, ptr, size);
    if (read_bytes >= ssize_t(sizeof(uint64_t))) {
      // Actual data bytes are all bytes minus initial padding
      std::size_t data_bytes =
          static_cast<std::size_t>(read_bytes) - sizeof(uint64_t);
      // This should be very cheap since it's in hot cache
      std::memmove(ptr, ptr + sizeof(uint64_t), data_bytes);
      // Increment our counters
      ptr += data_bytes;
      size -= data_bytes;
    } else {
      int err = errno;
      GetErrorLogInstance() << "Error reading lead " << lead << " errno:" << err
                            << " " << ::strerror(err) << "\n";
      return 0;
    }
  }
  return (bufsize - size) / sizeof(uint64_t);
}

const bool PerfCounters::kSupported = true;

// Initializes libpfm only on the first call.  Returns whether that single
// initialization was successful.
bool PerfCounters::Initialize() {
  // Function-scope static gets initialized only once on first call.
  static const bool success = []() {
    return pfm_initialize() == PFM_SUCCESS;
  }();
  return success;
}

bool PerfCounters::IsCounterSupported(const std::string& name) {
  Initialize();
  perf_event_attr attr;
  std::memset(&attr, 0, sizeof(attr));
  pfm_perf_encode_arg_t arg;
  std::memset(&arg, 0, sizeof(arg));
  arg.attr = &attr;
  const int mode = PFM_PLM3;  // user mode only
  int ret = pfm_get_os_event_encoding(name.c_str(), mode, PFM_OS_PERF_EVENT_EXT,
                                      &arg);
  return (ret == PFM_SUCCESS);
}

static std::optional<std::vector<uint64_t>> QueryCPUPMUTypes() {
  std::vector<uint64_t> types;
  DIR* dir = opendir("/sys/bus/event_source/devices");
  if (!dir) {
    return std::nullopt;
  }
  while (dirent* ent = readdir(dir)) {
    std::string_view name_str = ent->d_name;
    auto node_path = [&](const char* node) {
      return std::string("/sys/bus/event_source/devices/") + ent->d_name + "/" +
             node;
    };
    struct stat st;
    if (name_str == "cpu" || name_str == "cpum_cf" ||
        stat(node_path("cpus").c_str(), &st) == 0 || errno != ENOENT) {
      int type_fd = open(node_path("type").c_str(), O_RDONLY);
      if (type_fd < 0) {
        closedir(dir);
        return std::nullopt;
      }
      char type_str[32] = {};
      ssize_t res = read(type_fd, type_str, sizeof(type_str) - 1);
      close(type_fd);
      if (res < 0) {
        closedir(dir);
        return std::nullopt;
      }
      uint64_t type;
      if (sscanf(type_str, "%" PRIu64, &type) != 1) {
        closedir(dir);
        return std::nullopt;
      }
      types.push_back(type);
    }
  }
  closedir(dir);
  return types;
}

static std::vector<uint64_t> GetPMUTypesForEvent(const perf_event_attr& attr) {
  // Replicate generic hardware events on all CPU PMUs.
  if (attr.type == PERF_TYPE_HARDWARE && attr.config < PERF_COUNT_HW_MAX) {
    if (auto types = QueryCPUPMUTypes()) {
      return *types;
    }
  }
  return {0};
}

PerfCounters PerfCounters::Create(
    const std::vector<std::string>& counter_names) {
  if (!counter_names.empty()) {
    Initialize();
  }

  // Valid counters will populate these arrays but we start empty
  std::vector<std::string> valid_names;
  std::vector<int> counter_ids;
  std::vector<int> leader_ids;

  // Resize to the maximum possible
  valid_names.reserve(counter_names.size());
  counter_ids.reserve(counter_names.size());

  const int kCounterMode = PFM_PLM3;  // user mode only

  // Group leads will be assigned on demand. The idea is that once we cannot
  // create a counter descriptor, the reason is that this group has maxed out
  // so we set the group_id again to -1 and retry - giving the algorithm a
  // chance to create a new group leader to hold the next set of counters.
  int group_id = -1;

  // Loop through all performance counters
  for (size_t i = 0; i < counter_names.size(); ++i) {
    // we are about to push into the valid names vector
    // check if we did not reach the maximum
    if (valid_names.size() == PerfCounterValues::kMaxCounters) {
      // Log a message if we maxed out and stop adding
      GetErrorLogInstance()
          << counter_names.size() << " counters were requested. The maximum is "
          << PerfCounterValues::kMaxCounters << " and " << valid_names.size()
          << " were already added. All remaining counters will be ignored\n";
      // stop the loop and return what we have already
      break;
    }

    // Check if this name is empty
    const auto& name = counter_names[i];
    if (name.empty()) {
      GetErrorLogInstance()
          << "A performance counter name was the empty string\n";
      continue;
    }

    // Here first means first in group, ie the group leader
    const bool is_first = (group_id < 0);

    // This struct will be populated by libpfm from the counter string
    // and then fed into the syscall perf_event_open
    struct perf_event_attr attr {};
    attr.size = sizeof(attr);

    // This is the input struct to libpfm.
    pfm_perf_encode_arg_t arg{};
    arg.attr = &attr;
    const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode,
                                                  PFM_OS_PERF_EVENT, &arg);
    if (pfm_get != PFM_SUCCESS) {
      GetErrorLogInstance()
          << "Unknown performance counter name: " << name << "\n";
      continue;
    }

    // We then proceed to populate the remaining fields in our attribute struct
    // Note: the man page for perf_event_create suggests inherit = true and
    // read_format = PERF_FORMAT_GROUP don't work together, but that's not the
    // case.
    attr.disabled = is_first;
    attr.inherit = true;
    attr.pinned = is_first;
    attr.exclude_kernel = true;
    attr.exclude_user = false;
    attr.exclude_hv = true;

    // Read all counters in a group in one read.
    attr.read_format = PERF_FORMAT_GROUP;  //| PERF_FORMAT_TOTAL_TIME_ENABLED |
                                           // PERF_FORMAT_TOTAL_TIME_RUNNING;

    uint64_t base_config = attr.config;
    for (uint64_t pmu : GetPMUTypesForEvent(attr)) {
      attr.config = (pmu << PERF_PMU_TYPE_SHIFT) | base_config;
      int id = -1;
      while (id < 0) {
        static constexpr size_t kNrOfSyscallRetries = 5;
        // Retry syscall as it was interrupted often (b/64774091).
        for (size_t num_retries = 0; num_retries < kNrOfSyscallRetries;
             ++num_retries) {
          id = perf_event_open(&attr, 0, -1, group_id, 0);
          if (id >= 0 || errno != EINTR) {
            break;
          }
        }
        if (id < 0) {
          // If the file descriptor is negative we might have reached a limit
          // in the current group. Set the group_id to -1 and retry
          if (group_id >= 0) {
            // Create a new group
            group_id = -1;
          } else {
            // At this point we have already retried to set a new group id and
            // failed. We then give up.
            break;
          }
        }
      }

      // We failed to get a new file descriptor. We might have reached a hard
      // hardware limit that cannot be resolved even with group multiplexing
      if (id < 0) {
        GetErrorLogInstance() << "***WARNING** Failed to get a file descriptor "
                                 "for performance counter "
                              << name << ". Ignoring\n";

        // We give up on this counter but try to keep going
        // as the others would be fine
        continue;
      }
      if (group_id < 0) {
        // This is a leader, store and assign it to the current file descriptor
        leader_ids.push_back(id);
        group_id = id;
      }
      // This is a valid counter, add it to our descriptor's list
      counter_ids.push_back(id);
      valid_names.push_back(name);
    }
  }

  // Loop through all group leaders activating them
  // There is another option of starting ALL counters in a process but
  // that would be far reaching an intrusion. If the user is using PMCs
  // by themselves then this would have a side effect on them. It is
  // friendlier to loop through all groups individually.
  for (int lead : leader_ids) {
    if (ioctl(lead, PERF_EVENT_IOC_ENABLE) != 0) {
      // This should never happen but if it does, we give up on the
      // entire batch as recovery would be a mess.
      GetErrorLogInstance() << "***WARNING*** Failed to start counters. "
                               "Clearing out all counters.\n";

      // Close all performance counters
      for (int id : counter_ids) {
        ::close(id);
      }

      // Return an empty object so our internal state is still good and
      // the process can continue normally without impact
      return NoCounters();
    }
  }

  return PerfCounters(std::move(valid_names), std::move(counter_ids),
                      std::move(leader_ids));
}

void PerfCounters::CloseCounters() const {
  if (counter_ids_.empty()) {
    return;
  }
  for (int lead : leader_ids_) {
    ioctl(lead, PERF_EVENT_IOC_DISABLE);
  }
  for (int fd : counter_ids_) {
    close(fd);
  }
}
#else   // defined HAVE_LIBPFM
size_t PerfCounterValues::Read(const std::vector<int>&) { return 0; }

const bool PerfCounters::kSupported = false;

bool PerfCounters::Initialize() { return false; }

bool PerfCounters::IsCounterSupported(const std::string&) { return false; }

PerfCounters PerfCounters::Create(
    const std::vector<std::string>& counter_names) {
  if (!counter_names.empty()) {
    GetErrorLogInstance() << "Performance counters not supported.\n";
  }
  return NoCounters();
}

void PerfCounters::CloseCounters() const {}
#endif  // defined HAVE_LIBPFM

PerfCountersMeasurement::PerfCountersMeasurement(
    const std::vector<std::string>& counter_names)
    : start_values_(counter_names.size()), end_values_(counter_names.size()) {
  counters_ = PerfCounters::Create(counter_names);
}

PerfCounters& PerfCounters::operator=(PerfCounters&& other) noexcept {
  if (this != &other) {
    CloseCounters();

    counter_ids_ = std::move(other.counter_ids_);
    leader_ids_ = std::move(other.leader_ids_);
    counter_names_ = std::move(other.counter_names_);
  }
  return *this;
}
}  // namespace internal
}  // namespace benchmark