[K/N] WIP: A PoC of instrumenting allocation profiler
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
index 0a9c535..06dcb70 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
@@ -88,6 +88,9 @@
val swiftExport by booleanOption()
val genericSafeCasts by booleanOption()
+ val eventTrackerFrequency by stringOption()
+
+ val eventTrackerBacktraceDepth by stringOption()
}
open class BinaryOption<T : Any>(
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EventTrackerKind.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EventTrackerKind.kt
new file mode 100644
index 0000000..44bb72d
--- /dev/null
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EventTrackerKind.kt
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+package org.jetbrains.kotlin.backend.konan
+
+// Must match `EventKind` in profiler/ProfilerEvents.hpp
+enum class EventTrackerKind(val ord: Int, val defaultBacktraceDepth: Int) {
+ Allocation(0, 2),
+ SafePoint(1, 2),
+ SpecialRef(2, 2),
+ ;
+
+ companion object {
+ fun parse(str: String) = entries.firstOrNull {
+ it.name.equals(str, ignoreCase = true)
+ }
+ }
+}
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
index 453bf02..6f0ea6b 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
@@ -158,6 +158,54 @@
default + configured
}
+ val eventTrackerConfigured: Boolean by lazy {
+ configuration.get(BinaryOptions.eventTrackerFrequency) != null
+ }
+
+ val eventTrackerFrequency: Map<EventTrackerKind, Int> by lazy {
+ val default = EventTrackerKind.entries.associateWith { 0 }
+
+ val delimiter = "/" // FIXME consider making "," ?
+
+ val parts = configuration.get(BinaryOptions.eventTrackerFrequency)?.split(delimiter)?.toTypedArray()
+ val configured = parseKeyValuePairs(parts, configuration)?.mapNotNull { (key, value) ->
+ val event = EventTrackerKind.parse(key)
+ val frequency = value.toIntOrNull()
+ if (event == null) {
+ configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Unknown event kind '$key'.")
+ null
+ } else if (frequency == null) {
+ configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Invalid frequency '$value'.")
+ null
+ } else {
+ event to frequency
+ }
+ }?.toMap() ?: default
+ default + configured
+ }
+
+ val eventTrackerBacktraceDepth: Map<EventTrackerKind, Int> by lazy {
+ val default = EventTrackerKind.entries.associateWith { it.defaultBacktraceDepth }
+
+ val delimiter = "/" // FIXME consider making "," ?
+
+ val parts = configuration.get(BinaryOptions.eventTrackerBacktraceDepth)?.split(delimiter)?.toTypedArray()
+ val configured = parseKeyValuePairs(parts, configuration)?.mapNotNull { (key, value) ->
+ val event = EventTrackerKind.parse(key)
+ val frequency = value.toIntOrNull()
+ if (event == null) {
+ configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Unknown event kind '$key'")
+ null
+ } else if (frequency == null) {
+ configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Invalid backtrace depth '$value'")
+ null
+ } else {
+ event to frequency
+ }
+ }?.toMap() ?: return@lazy default
+ default + configured
+ }
+
val suspendFunctionsFromAnyThreadFromObjC: Boolean by lazy {
configuration.get(BinaryOptions.objcExportSuspendFunctionLaunchThreadRestriction) !=
@@ -566,6 +614,7 @@
sanitizer != null -> "with sanitizers enabled"
runtimeLogsEnabled -> "with runtime logs"
checkStateAtExternalCalls -> "with external calls state checker"
+ eventTrackerConfigured -> "with event tracker enabled"
else -> null
}
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt
index 9c8da29..85ea01a 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt
@@ -520,7 +520,7 @@
configuration: CompilerConfiguration
): Map<String, String>? = parseKeyValuePairs(arguments.overrideKonanProperties, configuration)
-private fun parseKeyValuePairs(
+fun parseKeyValuePairs(
argumentValue: Array<String>?,
configuration: CompilerConfiguration
): Map<String, String>? = argumentValue?.mapNotNull {
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
index faaeff4..06b7a73 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
@@ -3017,10 +3017,22 @@
config.runtimeLogs[it]!!.ord.let { llvm.constInt32(it) }
})
setRuntimeConstGlobal("Kotlin_runtimeLogs", runtimeLogs)
+
+ val eventTrackerFrequency = ConstArray(llvm.int32Type, EventTrackerKind.entries.sortedBy { it.ord }.map {
+ context.config.eventTrackerFrequency[it]!!.let { llvm.constInt32(it) }
+ })
+ setRuntimeConstGlobal("Kotlin_eventTrackerFrequency", eventTrackerFrequency)
+
+ val eventTrackerBacktraceDepth = ConstArray(llvm.int32Type, EventTrackerKind.entries.sortedBy { it.ord }.map {
+ context.config.eventTrackerBacktraceDepth[it]!!.let { llvm.constInt32(it) }
+ })
+ setRuntimeConstGlobal("Kotlin_eventTrackerBacktraceDepth", eventTrackerBacktraceDepth)
+
setRuntimeConstGlobal("Kotlin_freezingEnabled", llvm.constInt32(if (config.freezing.enableFreezeAtRuntime) 1 else 0))
setRuntimeConstGlobal("Kotlin_freezingChecksEnabled", llvm.constInt32(if (config.freezing.enableFreezeChecks) 1 else 0))
setRuntimeConstGlobal("Kotlin_concurrentWeakSweep", llvm.constInt32(if (context.config.concurrentWeakSweep) 1 else 0))
setRuntimeConstGlobal("Kotlin_gcMarkSingleThreaded", llvm.constInt32(if (config.gcMarkSingleThreaded) 1 else 0))
+ setRuntimeConstGlobal("Kotlin_gcMarkSingleThreaded", llvm.constInt32(if (config.gcMarkSingleThreaded) 1 else 0))
return llvmModule
}
diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
index 194deb8..98e1fbc 100644
--- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
+++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
@@ -27,6 +27,8 @@
extern "C" const int32_t Kotlin_disableMmap;
extern "C" const int32_t Kotlin_disableAllocatorOverheadEstimate;
extern "C" const int32_t Kotlin_runtimeLogs[];
+extern "C" const int32_t Kotlin_eventTrackerFrequency[];
+extern "C" const int32_t Kotlin_eventTrackerBacktraceDepth[];
extern "C" const int32_t Kotlin_concurrentWeakSweep;
extern "C" const int32_t Kotlin_gcMarkSingleThreaded;
extern "C" const int32_t Kotlin_freezingEnabled;
@@ -94,6 +96,14 @@
return Kotlin_freezingChecksEnabled != 0;
}
+ALWAYS_INLINE inline const int32_t* eventTrackerFrequency() noexcept {
+ return Kotlin_eventTrackerFrequency;
+}
+
+ALWAYS_INLINE inline const int32_t* eventTrackerBacktraceDepth() noexcept {
+ return Kotlin_eventTrackerBacktraceDepth;
+}
+
ALWAYS_INLINE inline bool concurrentWeakSweep() noexcept {
return Kotlin_concurrentWeakSweep != 0;
}
diff --git a/kotlin-native/runtime/src/main/cpp/Interop.cpp b/kotlin-native/runtime/src/main/cpp/Interop.cpp
index f4359a0..1be348c 100644
--- a/kotlin-native/runtime/src/main/cpp/Interop.cpp
+++ b/kotlin-native/runtime/src/main/cpp/Interop.cpp
@@ -28,7 +28,7 @@
extern "C" {
-KNativePtr Kotlin_Interop_createStablePointer(KRef any) {
+ALWAYS_INLINE KNativePtr Kotlin_Interop_createStablePointer(KRef any) {
KRefSharedHolder* holder = new KRefSharedHolder();
holder->init(any);
return holder;
diff --git a/kotlin-native/runtime/src/main/cpp/KString.cpp b/kotlin-native/runtime/src/main/cpp/KString.cpp
index 6ca13a3..7deee4b 100644
--- a/kotlin-native/runtime/src/main/cpp/KString.cpp
+++ b/kotlin-native/runtime/src/main/cpp/KString.cpp
@@ -479,3 +479,14 @@
} // extern "C"
+
+std::string kotlin::to_string(KConstRef kref) {
+ if (kref == nullptr) return "null";
+ RuntimeAssert(kref->type_info() == theStringTypeInfo, "A Kotlin String expected");
+ KString kstring = kref->array();
+ const KChar* utf16 = CharArrayAddressOfElementAt(kstring, 0);
+ std::string utf8;
+ utf8.reserve(kstring->count_);
+ utf8::unchecked::utf16to8(utf16, utf16 + kstring->count_, back_inserter(utf8));
+ return utf8;
+}
diff --git a/kotlin-native/runtime/src/main/cpp/KString.h b/kotlin-native/runtime/src/main/cpp/KString.h
index 17b0573..a10ee31 100644
--- a/kotlin-native/runtime/src/main/cpp/KString.h
+++ b/kotlin-native/runtime/src/main/cpp/KString.h
@@ -37,6 +37,7 @@
#ifdef __cplusplus
}
+
#endif
template <typename T>
@@ -58,4 +59,9 @@
return middle - (needle < value ? 1 : 0);
}
+
+namespace kotlin {
+std::string to_string(KConstRef kref);
+}
+
#endif // RUNTIME_KSTRING_H
diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp
index c90cc6b..2fc0e83 100644
--- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp
+++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp
@@ -148,6 +148,7 @@
delete state;
WorkerDestroyThreadDataIfNeeded(workerId);
::runtimeState = kInvalidRuntime;
+ RuntimeLogError({ logging::Tag::kLogging }, "Deinit runtime");
}
void Kotlin_deinitRuntimeCallback(void* argument) {
@@ -216,6 +217,7 @@
needsFullShutdown = Kotlin_forceCheckedShutdown() || Kotlin_memoryLeakCheckerEnabled() || Kotlin_cleanersLeakCheckerEnabled();
break;
}
+ needsFullShutdown = true; // FIXME
if (!needsFullShutdown) {
auto lastStatus = std_support::atomic_compare_swap_strong(globalRuntimeStatus, kGlobalRuntimeRunning, kGlobalRuntimeShutdown);
RuntimeAssert(lastStatus == kGlobalRuntimeRunning, "Invalid runtime status for shutdown");
@@ -269,6 +271,7 @@
}
deinitRuntime(runtime, canDestroyRuntime);
+ RuntimeLogError({ logging::Tag::kLogging }, "Shutdown runtime");
}
KInt Konan_Platform_canAccessUnaligned() {
diff --git a/kotlin-native/runtime/src/main/cpp/Utils.hpp b/kotlin-native/runtime/src/main/cpp/Utils.hpp
index 7ace717..a5f5a68 100644
--- a/kotlin-native/runtime/src/main/cpp/Utils.hpp
+++ b/kotlin-native/runtime/src/main/cpp/Utils.hpp
@@ -98,6 +98,11 @@
size_t CombineHash(size_t seed, size_t value);
+template<typename T, typename Hasher = std::hash<T>>
+std::size_t hashOf(const T& x) {
+ return Hasher{}(x);
+}
+
#define ownerOf(type, field, ref) *reinterpret_cast<type*>(reinterpret_cast<char*>(&ref) - offsetof(type, field))
// Returns `true` if the entire `span` is zeroed.
diff --git a/kotlin-native/runtime/src/main/cpp/profiler/IterableHelpers.hpp b/kotlin-native/runtime/src/main/cpp/profiler/IterableHelpers.hpp
new file mode 100644
index 0000000..b216fcc
--- /dev/null
+++ b/kotlin-native/runtime/src/main/cpp/profiler/IterableHelpers.hpp
@@ -0,0 +1,81 @@
+/*
+* Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+* that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include <unordered_map>
+#include <unordered_set>
+#include <sstream>
+
+namespace kotlin {
+
+template<typename Iterable, typename KeySelector, typename ValueTransformer>
+auto groupBy(const Iterable& iterable, KeySelector&& selectKey, ValueTransformer&& transformValue) {
+ using OriginalValue = typename Iterable::value_type;
+ using Key = typename std::result_of<KeySelector(const OriginalValue&)>::type;
+ using TransformedValue = typename std::result_of<ValueTransformer(const OriginalValue&)>::type;
+ std::unordered_map<Key, std::vector<TransformedValue>> groups;
+ for (const auto& value: iterable) {
+ auto key = selectKey(value);
+ groups[key].push_back(transformValue(value));
+ }
+ return groups;
+}
+
+template<typename Iterable, typename KeySelector>
+auto groupBy(const Iterable& iterable, KeySelector&& selectKey) {
+ return groupBy(iterable, std::forward<KeySelector>(selectKey), [](const auto& x) { return x; });
+}
+
+template<typename Iterable, typename Associator>
+auto associateWith(const Iterable& iterable, Associator&& assotiation) {
+ using OriginalValue = typename Iterable::value_type;
+ using AssociatedValue = typename std::result_of<Associator(const OriginalValue&)>::type;
+ std::unordered_map<OriginalValue, AssociatedValue> associations;
+ for (const auto& value: iterable) {
+ associations.insert_or_assign(value, assotiation(value));
+ }
+ return associations;
+}
+
+template<typename Iterable, typename Filter>
+auto filterToVector(const Iterable& iterable, Filter&& filter) {
+ using OriginalValue = typename Iterable::value_type;
+ std::vector<OriginalValue> filtered;
+ for (const auto& value: iterable) {
+ if (filter(value)) {
+ filtered.push_back(value);
+ }
+ }
+ return filtered;
+}
+
+template<typename Iterable, typename Map>
+auto mapToVector(const Iterable& iterable, Map&& map) {
+ using OriginalValue = typename Iterable::value_type;
+ using MappedValue = typename std::result_of<Map(const OriginalValue&)>::type;
+ std::vector<MappedValue> mapped;
+ for (const auto& value: iterable) {
+ mapped.push_back(map(value));
+ }
+ return mapped;
+}
+
+template<typename T, typename Ord>
+void sortBy(std::vector<T>& vector, Ord&& ord, bool descending = false) {
+ std::sort(vector.begin(), vector.end(), [ord = std::forward<Ord>(ord), descending](const T& l, const T& r) {
+ auto ordL = ord(l);
+ auto ordR = ord(r);
+ return descending ? ordL > ordR : ordL < ordR;
+ });
+}
+
+template<typename Iterable>
+auto uniq(const Iterable& iterable) {
+ std::unordered_set<typename Iterable::value_type> uniqSet(iterable.begin(), iterable.end());
+ return Iterable(uniqSet.begin(), uniqSet.end());
+}
+
+}
diff --git a/kotlin-native/runtime/src/main/cpp/profiler/ProfileReporter.hpp b/kotlin-native/runtime/src/main/cpp/profiler/ProfileReporter.hpp
new file mode 100644
index 0000000..10aee88
--- /dev/null
+++ b/kotlin-native/runtime/src/main/cpp/profiler/ProfileReporter.hpp
@@ -0,0 +1,140 @@
+/*
+* Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+* that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "Profiler.hpp"
+
+#include "Porting.h"
+#include "StackTrace.hpp"
+
+namespace kotlin::profiler {
+
+template<typename>
+class Profiler;
+
+namespace internal {
+
+template <typename EventTraits>
+class ProfileReporter {
+public:
+ using EventRecord = typename Profiler<EventTraits>::EventRecord;
+
+ explicit ProfileReporter(const Profiler<EventTraits>& profiler) : profiler_(profiler), entryPrinter_(*this) {}
+
+ void report() {
+ {
+ std::stringstream buf;
+ buf << EventTraits::kName << "s sampled with the period of " << EventTraits::samplingFrequencyPeriod()
+ << " and the backtrace depth of " << EventTraits::backtraceDepth()
+ << ".\n";
+ buf << "Total events occurred " << profiler_.totalEventCount_
+ << ", of them recorded " << profiler_.recordedEventCount_
+ << ".\n";
+ auto str = buf.str();
+ konan::consoleErrorUtf8(str.data(), str.size());
+ }
+
+ auto records = mapToVector(profiler_.eventCount_, [](const auto& p) { return p.first; });
+ printRecordEventsOnLevel(records);
+
+ konan::consoleErrorUtf8("\n", 1);
+ }
+
+private:
+ using Event = typename EventRecord::Event;
+ using ContextEntry = typename EventRecord::ContextEntry; // FIXME RecordEntry?
+
+ void printRecordEventsOnLevel(const std::vector<EventRecord>& records, std::size_t level = 0) {
+ auto entriesOnLevel = uniq(mapToVector(records, [&](const auto& r) { return r.entries()[level]; }));
+ auto recordsOfEntry = groupBy(records,
+ [&](const auto& r) { return r.entries()[level]; },
+ [](const auto& r) { return r; });
+
+ auto accumulatedHitsForEntry = associateWith(entriesOnLevel, [&](const auto& entry) {
+ auto& entryRecords = recordsOfEntry[entry];
+ auto hitCounts = mapToVector(entryRecords, [&](const auto& record) { return profiler_.eventCount_.at(record); });
+ return std::accumulate(hitCounts.begin(), hitCounts.end(), std::size_t(0));
+ });
+
+ sortBy(entriesOnLevel, [&](const auto& e) -> std::size_t { return accumulatedHitsForEntry.at(e); }, true);
+
+ for (auto& entry: entriesOnLevel) {
+ printWithSubentries(entry, recordsOfEntry, accumulatedHitsForEntry.at(entry), level);
+ }
+ }
+
+ void printWithSubentries( const ContextEntry& entry,
+ const std::unordered_map<ContextEntry, std::vector<EventRecord>>& recordsOfEntry,
+ std::size_t accumulatedHits, size_t level) {
+
+ entryPrinter_.print(accumulatedHits, entry);
+
+ const auto& subRecords = recordsOfEntry.at(entry);
+ if (level < subRecords[0].entries().size() - 1) {
+ entryPrinter_.incIdent();
+ printRecordEventsOnLevel(subRecords, level + 1);
+ entryPrinter_.decIdent();
+ }
+ }
+
+ class IndentedEntryPrinter {
+ static constexpr auto kIdentStep = 2;
+ public:
+ explicit IndentedEntryPrinter(ProfileReporter& reporter) : reporter_(reporter) {}
+
+ void incIdent() { baseIdent_ += kIdentStep; }
+ void decIdent() { baseIdent_ -= kIdentStep; }
+
+ void print(std::size_t hits, const ContextEntry& entry) {
+ auto buf = std::stringstream{};
+ auto fraction = static_cast<double>(hits) / reporter_.profiler_.recordedEventCount_;
+ buf << "[" << hits << "] (" << fraction * 100 << "%)";
+ auto hitsStr = buf.str();
+
+ buf = std::stringstream{};
+
+ appendSpaces(buf, baseIdent_);
+ buf << hitsStr << " ";
+
+ if (std::holds_alternative<Event>(entry)) {
+ buf << std::get<Event>(entry).toString();
+ } else {
+ std::array backtraceFrames = {std::get<void*>(entry)};
+ auto backtraceLines = GetStackTraceStrings(std_support::span(backtraceFrames));
+ for (std::size_t i = 0; i < backtraceLines.size(); ++i) {
+ if (i > 0) {
+ buf << "\n";
+ appendSpaces(buf, baseIdent_ + hitsStr.size() + 1);
+ }
+ buf << backtraceLines[i];
+ }
+ }
+
+ buf << "\n";
+ auto str = buf.str();
+ konan::consoleErrorUtf8(str.data(), str.size());
+ }
+
+ private:
+ void appendSpaces(std::stringstream& buf, std::size_t count) {
+ for (std::size_t i = 0; i < count; ++i) {
+ buf << " ";
+ }
+ }
+
+ ProfileReporter& reporter_;
+ std::size_t baseIdent_ = 0;
+ };
+
+
+ // FIXME padding
+
+ const Profiler<EventTraits>& profiler_;
+ IndentedEntryPrinter entryPrinter_;
+};
+
+}
+}
diff --git a/kotlin-native/runtime/src/main/cpp/profiler/Profiler.hpp b/kotlin-native/runtime/src/main/cpp/profiler/Profiler.hpp
new file mode 100644
index 0000000..a94f2e6
--- /dev/null
+++ b/kotlin-native/runtime/src/main/cpp/profiler/Profiler.hpp
@@ -0,0 +1,193 @@
+/*
+* Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+* that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include <unordered_map>
+#include <unordered_set>
+#include <sstream>
+#include <numeric>
+#include <variant>
+
+#include "IterableHelpers.hpp"
+#include "ProfileReporter.hpp"
+#include "ProfilerEvents.hpp"
+#include "Porting.h"
+#include "Utils.hpp"
+#include "StackTrace.hpp"
+#include "Logging.hpp"
+#include "KString.h"
+
+namespace kotlin::profiler::internal {
+
+template <typename EventTraits>
+struct EventRecord {
+ using Event = typename EventTraits::Event;
+ using Backtrace = StackTrace<>;
+
+ EventRecord(const Event& event, const Backtrace& backtrace) : event_(event), backtrace_(backtrace) {}
+
+ // Either the event itself or a backtrace frame
+ using ContextEntry = std::variant<Event, void*>;
+
+ auto operator==(const EventRecord& other) const noexcept {
+ return event_ == other.event_ && backtrace_ == other.backtrace_;
+ }
+ auto operator!=(const EventRecord& other) const noexcept { return !operator==(other); }
+
+ [[nodiscard]] auto entries() const noexcept {
+ std::vector<ContextEntry> elems;
+ elems.push_back(event_);
+ elems.insert(elems.end(), backtrace_.begin(), backtrace_.end());
+ return elems;
+ }
+
+ auto hash() const noexcept {
+ return CombineHash(hashOf(event_), hashOf(backtrace_));
+ }
+
+private:
+ Event event_;
+ Backtrace backtrace_;
+};
+
+} // namespace kotlin::profiler::internal
+
+template <typename EventTraits>
+struct std::hash<kotlin::profiler::internal::EventRecord<EventTraits>> {
+ std::size_t operator()(const kotlin::profiler::internal::EventRecord<EventTraits>& x) const noexcept {
+ return x.hash();
+ }
+};
+
+namespace kotlin::profiler {
+
+template <typename EventTraits>
+class Profiler : Pinned {
+ friend internal::ProfileReporter<EventTraits>;
+public:
+ using Event = typename EventTraits::Event;
+ using EventRecord = typename internal::EventRecord<EventTraits>;
+ using EventCounts = typename std::unordered_map<EventRecord, std::size_t>;
+
+ class ThreadData : Pinned {
+ friend Profiler;
+ public:
+ explicit ThreadData(Profiler& profiler) : profiler_(profiler) {}
+
+ ~ThreadData() {
+ publish();
+ }
+
+ static auto enabled() noexcept { return EventTraits::enabled(); }
+
+ NO_INLINE void hitImpl(Event event, std::size_t skipFrames = 0) {
+ RuntimeAssert(enabled(), "Registered hit with a disabled profiler");
+
+ if (totalEventCount_++ % EventTraits::samplingFrequencyPeriod() == 0) {
+ // FIXME make it collect exactly the requested amount of frames
+ EventRecord record{event, EventRecord::Backtrace::current(skipFrames + 1, EventTraits::backtraceDepth())};
+ ++eventCount_[record];
+ }
+ }
+
+ void publish() {
+ if (enabled()) {
+ profiler_.aggregate(*this);
+ eventCount_.clear();
+ totalEventCount_ = 0;
+ }
+ }
+
+ private:
+ Profiler& profiler_;
+ std::size_t totalEventCount_ = 0;
+ EventCounts eventCount_{};
+ };
+
+ ~Profiler() {
+ report();
+ }
+
+ void aggregate(const ThreadData& threadData) {
+ std::unique_lock lockGuard(aggregationMutex_);
+ totalEventCount_ += threadData.totalEventCount_;
+ for (const auto& [key, value] : threadData.eventCount_) {
+ eventCount_[key] += value;
+ recordedEventCount_ += value;
+ }
+ }
+
+ void report() const noexcept {
+ if (EventTraits::enabled()) {
+ std::unique_lock lockGuard(aggregationMutex_);
+
+ auto reporter = internal::ProfileReporter<EventTraits>{*this};
+ reporter.report();
+ }
+ }
+
+private:
+ std::size_t totalEventCount_ = 0;
+ std::size_t recordedEventCount_ = 0;
+ EventCounts eventCount_{};
+ mutable std::mutex aggregationMutex_;
+};
+
+
+class Profilers {
+public:
+ class ThreadData {
+ public:
+ explicit ThreadData(Profilers& profilers)
+ : allocationProfilerData_(profilers.allocationProfiler_)
+ , safePointProfilerData_(profilers.safePointProfiler_)
+ , specialRefProfilerData_(profilers.specialRefProfiler_)
+ {}
+
+ auto& allocation() noexcept { return allocationProfilerData_; }
+ auto& safePoint() noexcept { return safePointProfilerData_; }
+ auto& specialRef() noexcept { return specialRefProfilerData_; }
+
+ void publish() {
+ allocationProfilerData_.publish();
+ safePointProfilerData_.publish();
+ specialRefProfilerData_.publish();
+ }
+ private:
+ Profiler<AllocationEventTraits>::ThreadData allocationProfilerData_;
+ Profiler<SafePointEventTraits>::ThreadData safePointProfilerData_;
+ Profiler<SpecialRefEventTraits>::ThreadData specialRefProfilerData_;
+ };
+
+ void report() {
+ allocationProfiler_.report();
+ safePointProfiler_.report();
+ specialRefProfiler_.report();
+ }
+private:
+ Profiler<AllocationEventTraits> allocationProfiler_{};
+ Profiler<SafePointEventTraits> safePointProfiler_{};
+ Profiler<SpecialRefEventTraits> specialRefProfiler_{};
+};
+
+} // namespace kotlin::profiler
+
+#define ProfilerHit0(EventTraits, profilerThreadData, event, ...) \
+ do { \
+ if (EventTraits::enabled()) { \
+ using TDType = typename kotlin::profiler::Profiler<EventTraits>::ThreadData; \
+ static_assert(std::is_same<std::decay<decltype(profilerThreadData)>::type, TDType>::value); \
+ profilerThreadData.hitImpl(event, ##__VA_ARGS__); \
+ } \
+ } while (false)
+
+#define ProfilerHit(profilerThreadData, event, ...) \
+ do { \
+ using ThreadDataType = std::decay<decltype(profilerThreadData)>::type; \
+ if (ThreadDataType::enabled()) { \
+ profilerThreadData.hitImpl(event, ##__VA_ARGS__); \
+ } \
+ } while (false)
diff --git a/kotlin-native/runtime/src/main/cpp/profiler/ProfilerEvents.hpp b/kotlin-native/runtime/src/main/cpp/profiler/ProfilerEvents.hpp
new file mode 100644
index 0000000..d77a62a
--- /dev/null
+++ b/kotlin-native/runtime/src/main/cpp/profiler/ProfilerEvents.hpp
@@ -0,0 +1,126 @@
+/*
+* Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+* that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "KString.h"
+
+namespace kotlin::profiler {
+
+enum class EventKind : int32_t {
+ kAllocation = 0,
+ kSafePoint = 1,
+ kSpecialRef = 2,
+};
+
+namespace internal {
+template<EventKind kEventKind>
+class EventTraits {
+public:
+ ALWAYS_INLINE static bool enabled() noexcept {
+ return samplingFrequencyPeriod() != 0;
+ }
+
+ ALWAYS_INLINE static std::size_t samplingFrequencyPeriod() noexcept {
+ return compiler::eventTrackerFrequency()[static_cast<std::size_t>(kEventKind)];
+ }
+
+ ALWAYS_INLINE static std::size_t backtraceDepth() {
+ return compiler::eventTrackerBacktraceDepth()[static_cast<std::size_t>(kEventKind)];
+ }
+};
+}
+
+class AllocationEventTraits : public internal::EventTraits<EventKind::kAllocation> {
+public:
+ static constexpr auto kName = "Allocation";
+
+ struct AllocationRecord {
+ auto operator==(const AllocationRecord& other) const noexcept {
+ return typeInfo_ == other.typeInfo_ && arrayLength_ == other.arrayLength_;
+ }
+ auto operator!=(const AllocationRecord& other) const noexcept { return !operator==(other); }
+
+ auto toString() const -> std::string {
+ auto pkg = to_string(typeInfo_->packageName_);
+ auto cls = to_string(typeInfo_->relativeName_);
+ auto fqName = pkg.empty() ? cls : pkg + "." + cls;
+ if (typeInfo_->IsArray()) {
+ return fqName + "[" + std::to_string(arrayLength_) +"]";
+ }
+ return fqName;
+ }
+
+ const TypeInfo* typeInfo_;
+ std::size_t arrayLength_ = 0;
+ };
+
+ using Event = AllocationRecord;
+};
+
+class SafePointEventTraits : public internal::EventTraits<EventKind::kSafePoint> {
+public:
+ static constexpr auto kName = "SafePoint";
+
+ struct SafePointHit {
+ auto operator==(const SafePointHit&) const noexcept { return true; }
+ auto operator!=(const SafePointHit&) const noexcept { return false; }
+
+ auto toString() const -> std::string {
+ return "Safe point";
+ }
+ };
+
+ using Event = SafePointHit;
+};
+
+class SpecialRefEventTraits : public internal::EventTraits<EventKind::kSpecialRef> {
+public:
+ static constexpr auto kName = "SpecialRef";
+
+ enum class SpecialRefKind {
+ kStableRef, kWeakRef, kBackRef
+ };
+
+ struct SpecialRefCreation {
+ auto operator==(const SpecialRefCreation& other) const noexcept { return kind_ == other.kind_; }
+ auto operator!=(const SpecialRefCreation& other) const noexcept { return !operator==(other); }
+
+ auto toString() const -> std::string {
+ switch (kind_) {
+ case SpecialRefKind::kStableRef: return "StableRef";
+ case SpecialRefKind::kWeakRef: return "WeakRef";
+ case SpecialRefKind::kBackRef: return "BackRef";
+ }
+ }
+
+ SpecialRefKind kind_;
+ };
+
+ using Event = SpecialRefCreation;
+};
+
+}
+
+template<>
+struct std::hash<kotlin::profiler::AllocationEventTraits::Event> {
+ std::size_t operator()(const kotlin::profiler::AllocationEventTraits::Event& alloc) const noexcept {
+ return kotlin::CombineHash(kotlin::hashOf(alloc.typeInfo_), kotlin::hashOf(alloc.arrayLength_));
+ }
+};
+
+template<>
+struct std::hash<kotlin::profiler::SafePointEventTraits::Event> {
+ std::size_t operator()(const kotlin::profiler::SafePointEventTraits::Event&) const noexcept {
+ return 0;
+ }
+};
+
+template<>
+struct std::hash<kotlin::profiler::SpecialRefEventTraits::Event> {
+ std::size_t operator()(const kotlin::profiler::SpecialRefEventTraits::Event& e) const noexcept {
+ return static_cast<std::size_t>(e.kind_);
+ }
+};
diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
index 87bcc9e..840f37d 100644
--- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
+++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
@@ -15,6 +15,7 @@
#include "SpecialRefRegistry.hpp"
#include "ThreadRegistry.hpp"
#include "Utils.hpp"
+#include "profiler/Profiler.hpp"
namespace kotlin {
namespace mm {
@@ -35,6 +36,7 @@
alloc::Allocator& allocator() noexcept { return allocator_; }
gc::GC& gc() noexcept { return gc_; }
AppStateTracking& appStateTracking() noexcept { return appStateTracking_; }
+ profiler::Profilers& profilers() noexcept { return profilers_; }
private:
friend class ManuallyScoped<GlobalData>;
@@ -49,6 +51,7 @@
gcScheduler::GCScheduler gcScheduler_;
alloc::Allocator allocator_;
gc::GC gc_{allocator_, gcScheduler_};
+ profiler::Profilers profilers_;
};
} // namespace mm
diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp
index d95cfdd..210c9a0 100644
--- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp
@@ -112,11 +112,13 @@
// the thread registery and waits for threads to suspend or go to the native state.
AssertThreadState(state, ThreadState::kNative);
auto* node = mm::FromMemoryState(state);
+ node->Get()->profilers().publish();
if (destroyRuntime) {
ThreadStateGuard guard(state, ThreadState::kRunnable);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
// TODO: Why not just destruct `GC` object and its thread data counterpart entirely?
mm::GlobalData::Instance().gc().StopFinalizerThreadIfRunning();
+ mm::GlobalData::Instance().profilers().report();
}
if (!konan::isOnThreadExitNotSetOrAlreadyStarted()) {
// we can clear reference in advance, as Unregister function can't use it anyway
@@ -135,6 +137,7 @@
extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) {
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+ ProfilerHit(threadData->profilers().allocation(), {typeInfo});
RETURN_RESULT_OF(mm::AllocateObject, threadData, typeInfo);
}
@@ -143,6 +146,7 @@
ThrowIllegalArgumentException();
}
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+ ProfilerHit(threadData->profilers().allocation(), {typeInfo, static_cast<size_t>(elements)});
RETURN_RESULT_OF(mm::AllocateArray, threadData, typeInfo, static_cast<uint32_t>(elements));
}
@@ -500,7 +504,7 @@
return true;
}
-extern "C" RUNTIME_NOTHROW void* CreateStablePointer(ObjHeader* object) {
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void* CreateStablePointer(ObjHeader* object) {
if (!object)
return nullptr;
diff --git a/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp b/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp
index e5aa66b..a1b477e 100644
--- a/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp
@@ -19,7 +19,7 @@
obj_ = obj;
}
-void KRefSharedHolder::init(ObjHeader* obj) {
+ALWAYS_INLINE void KRefSharedHolder::init(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
ref_ = static_cast<mm::RawSpecialRef*>(mm::StableRef::create(obj));
obj_ = obj;
diff --git a/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp b/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp
index a7d5774..4eecf9e 100644
--- a/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp
@@ -127,6 +127,7 @@
ALWAYS_INLINE void mm::safePoint(std::memory_order fastPathOrder) noexcept {
AssertThreadState(ThreadState::kRunnable);
+ ProfilerHit(mm::ThreadRegistry::Instance().CurrentThreadData()->profilers().safePoint(), {});
auto action = safePointAction.load(fastPathOrder);
if (__builtin_expect(action != nullptr, false)) {
slowPath();
@@ -135,6 +136,7 @@
ALWAYS_INLINE void mm::safePoint(mm::ThreadData& threadData, std::memory_order fastPathOrder) noexcept {
AssertThreadState(&threadData, ThreadState::kRunnable);
+ ProfilerHit(threadData.profilers().safePoint(), {});
auto action = safePointAction.load(fastPathOrder);
if (__builtin_expect(action != nullptr, false)) {
slowPath(threadData);
diff --git a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.cpp b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.cpp
index 3296b90..d5445c4 100644
--- a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.cpp
@@ -8,6 +8,7 @@
#include "GlobalData.hpp"
#include "MemoryPrivate.hpp"
#include "ObjCBackRef.hpp"
+#include "profiler/Profiler.hpp"
#include "StableRef.hpp"
#include "ThreadData.hpp"
#include "ThreadState.hpp"
@@ -15,15 +16,18 @@
using namespace kotlin;
-mm::StableRef mm::SpecialRefRegistry::ThreadQueue::createStableRef(ObjHeader* object) noexcept {
+ALWAYS_INLINE mm::StableRef mm::SpecialRefRegistry::ThreadQueue::createStableRef(ObjHeader* object) noexcept {
+ ProfilerHit(mm::ThreadRegistry::Instance().CurrentThreadData()->profilers().specialRef(), {profiler::SpecialRefEventTraits::SpecialRefKind::kStableRef});
return mm::StableRef(registerNode(object, 1, true).asRaw());
}
-mm::WeakRef mm::SpecialRefRegistry::ThreadQueue::createWeakRef(ObjHeader* object) noexcept {
+ALWAYS_INLINE mm::WeakRef mm::SpecialRefRegistry::ThreadQueue::createWeakRef(ObjHeader* object) noexcept {
+ ProfilerHit(mm::ThreadRegistry::Instance().CurrentThreadData()->profilers().specialRef(), {profiler::SpecialRefEventTraits::SpecialRefKind::kWeakRef});
return mm::WeakRef(registerNode(object, 0, false).asRaw());
}
-mm::ObjCBackRef mm::SpecialRefRegistry::ThreadQueue::createObjCBackRef(ObjHeader* object) noexcept {
+ALWAYS_INLINE mm::ObjCBackRef mm::SpecialRefRegistry::ThreadQueue::createObjCBackRef(ObjHeader* object) noexcept {
+ ProfilerHit(mm::ThreadRegistry::Instance().CurrentThreadData()->profilers().specialRef(), {profiler::SpecialRefEventTraits::SpecialRefKind::kBackRef});
return mm::ObjCBackRef(registerNode(object, 1, false).asRaw());
}
diff --git a/kotlin-native/runtime/src/mm/cpp/StableRef.cpp b/kotlin-native/runtime/src/mm/cpp/StableRef.cpp
index 593dc91..8230724 100644
--- a/kotlin-native/runtime/src/mm/cpp/StableRef.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/StableRef.cpp
@@ -13,7 +13,7 @@
using namespace kotlin;
// static
-mm::StableRef mm::StableRef::create(ObjHeader* obj) noexcept {
+ALWAYS_INLINE mm::StableRef mm::StableRef::create(ObjHeader* obj) noexcept {
RuntimeAssert(obj != nullptr, "Creating StableRef for null object");
return mm::ThreadRegistry::Instance().CurrentThreadData()->specialRefRegistry().createStableRef(obj);
}
diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
index 3dee380..06356b8 100644
--- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
+++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
@@ -34,9 +34,13 @@
gcScheduler_(GlobalData::Instance().gcScheduler(), *this),
allocator_(GlobalData::Instance().allocator()),
gc_(GlobalData::Instance().gc(), *this),
- suspensionData_(ThreadState::kNative, *this) {}
+ suspensionData_(ThreadState::kNative, *this),
+ profilers_(GlobalData::Instance().profilers())
+ {}
- ~ThreadData() = default;
+ ~ThreadData() {
+ RuntimeLogError({ logging::Tag::kLogging }, "Thread deinit");
+ }
int threadId() const noexcept { return threadId_; }
@@ -62,6 +66,8 @@
ThreadSuspensionData& suspensionData() { return suspensionData_; }
+ auto& profilers() { return profilers_; }
+
void Publish() noexcept {
// TODO: These use separate locks, which is inefficient.
globalsThreadQueue_.Publish();
@@ -85,6 +91,7 @@
gc::GC::ThreadData gc_;
std::vector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
ThreadSuspensionData suspensionData_;
+ profiler::Profilers::ThreadData profilers_;
};
} // namespace mm
