[K/N] Warn gc scheduler before alloc
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
index c797fdf..ac63b04 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt
@@ -66,6 +66,8 @@
val disableMmap by booleanOption()
val disableAllocatorOverheadEstimate by booleanOption()
+
+ val gcBeforeAlloc by booleanOption()
}
open class BinaryOption<T : Any>(
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
index 9fd3db3..a0c59dc 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
@@ -209,6 +209,10 @@
configuration.get(BinaryOptions.objcDisposeOnMain) ?: true
}
+ val gcBeforeAlloc: Boolean by lazy {
+ configuration.get(BinaryOptions.gcBeforeAlloc) ?: false
+ }
+
init {
if (!platformManager.isEnabled(target)) {
error("Target ${target.visibleName} is not available on the ${HostManager.hostName} host")
diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
index 54d7a5a..54cc3c8 100644
--- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
+++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt
@@ -3046,6 +3046,7 @@
setRuntimeConstGlobal("Kotlin_freezingChecksEnabled", llvm.constInt32(if (config.freezing.enableFreezeChecks) 1 else 0))
setRuntimeConstGlobal("Kotlin_concurrentWeakSweep", llvm.constInt32(if (context.config.concurrentWeakSweep) 1 else 0))
setRuntimeConstGlobal("Kotlin_gcMarkSingleThreaded", llvm.constInt32(if (config.gcMarkSingleThreaded) 1 else 0))
+ setRuntimeConstGlobal("Kotlin_gcBeforeAlloc", llvm.constInt32(if (config.gcBeforeAlloc) 1 else 0))
return llvmModule
}
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp
index 6fc90e7..bda3f01 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/CustomAllocConstants.hpp
@@ -13,21 +13,19 @@
#include "NextFitPage.hpp"
#include "ExtraObjectPage.hpp"
-inline constexpr const size_t KiB = 1024;
-
-inline constexpr const size_t FIXED_BLOCK_PAGE_SIZE = (256 * KiB);
+inline constexpr const size_t FIXED_BLOCK_PAGE_SIZE = kotlin::alloc::FixedBlockPage::PageSize;
inline constexpr const int FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE = 128;
inline constexpr const size_t FIXED_BLOCK_PAGE_CELL_COUNT =
((FIXED_BLOCK_PAGE_SIZE - sizeof(kotlin::alloc::FixedBlockPage)) / sizeof(kotlin::alloc::FixedBlockCell));
-inline constexpr const size_t NEXT_FIT_PAGE_SIZE = (256 * KiB);
+inline constexpr const size_t NEXT_FIT_PAGE_SIZE = kotlin::alloc::NextFitPage::PageSize;
inline constexpr const size_t NEXT_FIT_PAGE_CELL_COUNT =
((NEXT_FIT_PAGE_SIZE - sizeof(kotlin::alloc::NextFitPage)) / sizeof(kotlin::alloc::Cell));
// NEXT_FIT_PAGE_CELL_COUNT minus one cell for header minus another for the 0-sized dummy block at cells_[0]
inline constexpr const size_t NEXT_FIT_PAGE_MAX_BLOCK_SIZE = (NEXT_FIT_PAGE_CELL_COUNT - 2);
-inline constexpr const size_t EXTRA_OBJECT_PAGE_SIZE = 64 * KiB;
+inline constexpr const size_t EXTRA_OBJECT_PAGE_SIZE = kotlin::alloc::ExtraObjectPage::PageSize;
inline constexpr const int EXTRA_OBJECT_COUNT =
(EXTRA_OBJECT_PAGE_SIZE - sizeof(kotlin::alloc::ExtraObjectPage)) / sizeof(kotlin::alloc::ExtraObjectCell);
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp
index 1a2cde1..8f41170 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/ExtraObjectPage.hpp
@@ -36,6 +36,7 @@
class alignas(8) ExtraObjectPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepExtraObjectsScope;
+ static inline constexpr const size_t PageSize = 64 * 1024;
static GCSweepScope currentGCSweepScope(gc::GCHandle& handle) noexcept { return handle.sweepExtraObjects(); }
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
index 173c7a0..eb23be5 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/FixedBlockPage.hpp
@@ -32,6 +32,7 @@
class alignas(8) FixedBlockPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
+ static inline constexpr const size_t PageSize = 256 * 1024;
static GCSweepScope currentGCSweepScope(gc::GCHandle& handle) noexcept { return handle.sweep(); }
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp
index 07e992c..097682b 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.cpp
@@ -10,6 +10,9 @@
#include <cstdlib>
#include <cstring>
#include <limits>
+#include "GlobalData.hpp"
+#include "SafePoint.hpp"
+#include "ThreadRegistry.hpp"
#ifndef KONAN_WINDOWS
#include <sys/mman.h>
@@ -24,6 +27,7 @@
#include "GCStatistics.hpp"
#include "KAssert.h"
#include "Memory.h"
+#include "ThreadData.hpp"
namespace {
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp
index 3fb13cf..949563e 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/NextFitPage.hpp
@@ -20,6 +20,7 @@
class alignas(8) NextFitPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
+ static inline constexpr const size_t PageSize = 256 * 1024;
static GCSweepScope currentGCSweepScope(gc::GCHandle& handle) noexcept { return handle.sweep(); }
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp
index 87050d1..77eec5d 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp
@@ -12,7 +12,9 @@
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
+#include "GCApi.hpp"
#include "GCStatistics.hpp"
+#include "Memory.h"
namespace kotlin::alloc {
@@ -46,28 +48,34 @@
T* GetPage(uint32_t cellCount, FinalizerQueue& finalizerQueue, std::atomic<std::size_t>& concurrentSweepersCount_) noexcept {
T* page;
- if ((page = ready_.Pop())) {
- used_.Push(page);
- return page;
- }
- {
- auto handle = gc::GCHandle::currentEpoch();
- ScopeGuard counterGuard(
- [&]() { ++concurrentSweepersCount_; },
- [&]() { --concurrentSweepersCount_; }
- );
+ int limit = 1;
+ for (int iter = 0 ; iter < limit ; iter++) {
+ if ((page = ready_.Pop())) {
+ used_.Push(page);
+ return page;
+ }
+ {
+ auto handle = gc::GCHandle::currentEpoch();
+ ScopeGuard counterGuard(
+ [&]() { ++concurrentSweepersCount_; },
+ [&]() { --concurrentSweepersCount_; }
+ );
- if ((page = unswept_.Pop())) {
- // If there're unswept_ pages, the GC is in progress.
- GCSweepScope sweepHandle = T::currentGCSweepScope(*handle);
- if ((page = SweepSingle(sweepHandle, page, unswept_, used_, finalizerQueue))) {
- return page;
+ if ((page = unswept_.Pop())) {
+ // If there're unswept_ pages, the GC is in progress.
+ GCSweepScope sweepHandle = T::currentGCSweepScope(*handle);
+ if ((page = SweepSingle(sweepHandle, page, unswept_, used_, finalizerQueue))) {
+ return page;
+ }
}
}
- }
- if ((page = empty_.Pop())) {
- used_.Push(page);
- return page;
+ if ((page = empty_.Pop())) {
+ used_.Push(page);
+ return page;
+ }
+ if (!used_.isEmpty() && TryGCBeforeMemoryAllocation(T::PageSize)) {
+ limit = 2;
+ }
}
return NewPage(cellCount);
}
diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp
index d1d4765..f0cbbad 100644
--- a/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp
+++ b/kotlin-native/runtime/src/alloc/custom/cpp/SingleObjectPage.hpp
@@ -19,6 +19,7 @@
class alignas(8) SingleObjectPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
+ static inline constexpr const size_t PageSize = 0; // Not used
static GCSweepScope currentGCSweepScope(gc::GCHandle& handle) noexcept { return handle.sweep(); }
diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
index eb03de0..f398dcd 100644
--- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
@@ -193,6 +193,7 @@
#endif
resumeTheWorld(gcHandle);
+ state_.resumed(epoch);
#ifndef CUSTOM_ALLOCATOR
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *extraObjectFactoryIterable);
diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
index e5eb0f5..b51f273 100644
--- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
@@ -76,6 +76,10 @@
return impl_->gc().state().schedule();
}
+void gc::GC::WaitResumed(int64_t epoch) noexcept {
+ impl_->gc().state().waitEpochResumed(epoch);
+}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
index 1451238..087b0bc 100644
--- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
@@ -76,6 +76,7 @@
// TODO: These should exist only in the scheduler.
int64_t Schedule() noexcept;
+ void WaitResumed(int64_t epoch) noexcept;
void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept;
diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
index db2648d..6ebdedb 100644
--- a/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
+++ b/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
@@ -27,6 +27,7 @@
std::unique_lock lock(mutex_);
shutdownFlag_ = true;
startedEpoch.notify();
+ resumedEpoch.notify();
finishedEpoch.notify();
scheduledEpoch.notify();
finalizedEpoch.notify();
@@ -34,10 +35,16 @@
void start(int64_t epoch) { startedEpoch.set(epoch); }
+ void resumed(int64_t epoch) { resumedEpoch.set(epoch); }
+
void finish(int64_t epoch) { finishedEpoch.set(epoch); }
void finalized(int64_t epoch) { finalizedEpoch.set(epoch); }
+ void waitEpochResumed(int64_t epoch) {
+ resumedEpoch.wait([this, epoch] { return *resumedEpoch >= epoch || shutdownFlag_; });
+ }
+
void waitEpochFinished(int64_t epoch) {
finishedEpoch.wait([this, epoch] { return *finishedEpoch >= epoch || shutdownFlag_; });
}
@@ -88,6 +95,7 @@
std::mutex mutex_;
// Use a separate conditional variable for each counter to mitigate a winpthreads bug (see KT-50948 for details).
ValueWithCondVar<int64_t> startedEpoch{0, mutex_};
+ ValueWithCondVar<int64_t> resumedEpoch{0, mutex_};
ValueWithCondVar<int64_t> finishedEpoch{0, mutex_};
ValueWithCondVar<int64_t> scheduledEpoch{0, mutex_};
ValueWithCondVar<int64_t> finalizedEpoch{0, mutex_};
diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
index 3c62fff..5eff141 100644
--- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
@@ -55,6 +55,8 @@
return 0;
}
+void gc::GC::WaitResumed(int64_t epoch) noexcept {}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {}
diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
index 0778689..8210bc9 100644
--- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
@@ -69,6 +69,10 @@
return impl_->gc().state().schedule();
}
+void gc::GC::WaitResumed(int64_t epoch) noexcept {
+ impl_->gc().state().waitEpochResumed(epoch);
+}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp
index 0604154..d845d5f 100644
--- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp
+++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp
@@ -4,6 +4,7 @@
*/
#include "GCSchedulerImpl.hpp"
+#include <cstdint>
#include "GlobalData.hpp"
#include "Memory.h"
@@ -38,6 +39,13 @@
impl().impl().schedule();
}
+void gcScheduler::GCScheduler::scheduleAndWaitResumed() noexcept {
+ RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
+ auto epoch = impl().impl().schedule();
+ NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
+ mm::GlobalData::Instance().gc().WaitResumed(epoch);
+}
+
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
@@ -52,8 +60,8 @@
mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
}
-ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
- impl().impl().setAllocatedBytes(bytes);
+ALWAYS_INLINE int64_t gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
+ return impl().impl().setAllocatedBytes(bytes);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {
diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp
index 0d8940e..6a9a9e2 100644
--- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp
+++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp
@@ -5,6 +5,7 @@
#pragma once
+#include <cstdint>
#include "GCScheduler.hpp"
#include "AppStateTracking.hpp"
@@ -67,21 +68,20 @@
timer_.restart(config_.regularGcInterval());
}
- void setAllocatedBytes(size_t bytes) noexcept {
+ int64_t setAllocatedBytes(size_t bytes) noexcept {
auto boundary = heapGrowthController_.boundaryForHeapSize(bytes);
switch (boundary) {
case HeapGrowthController::MemoryBoundary::kNone:
- return;
+ return 0;
case HeapGrowthController::MemoryBoundary::kTrigger:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
- scheduleGC_.scheduleNextEpochIfNotInProgress();
- return;
+ return scheduleGC_.scheduleNextEpochIfNotInProgress();
case HeapGrowthController::MemoryBoundary::kTarget:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress();
RuntimeLogWarning({kTagGC}, "Pausing the mutators until epoch %" PRId64 " is done", epoch);
mutatorAssists_.requestAssists(epoch);
- return;
+ return epoch;
}
}
diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp
index cb01f97..df97e35 100644
--- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp
+++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp
@@ -4,6 +4,7 @@
*/
#include "GCSchedulerImpl.hpp"
+#include <cstdint>
#include "GlobalData.hpp"
#include "Memory.h"
@@ -39,6 +40,13 @@
impl().impl().schedule();
}
+void gcScheduler::GCScheduler::scheduleAndWaitResumed() noexcept {
+ RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
+ auto epoch = impl().impl().schedule();
+ NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true);
+ mm::GlobalData::Instance().gc().WaitResumed(epoch);
+}
+
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
auto epoch = impl().impl().schedule();
@@ -53,8 +61,8 @@
mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
}
-ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
- impl().impl().setAllocatedBytes(bytes);
+ALWAYS_INLINE int64_t gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {
+ return impl().impl().setAllocatedBytes(bytes);
}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {}
diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp
index 6ef5cd5..8e4e4cc 100644
--- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp
+++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp
@@ -7,6 +7,7 @@
#include "GCScheduler.hpp"
+#include <cstdint>
#include <functional>
#include "GCSchedulerConfig.hpp"
@@ -47,32 +48,31 @@
RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized");
}
- void setAllocatedBytes(size_t bytes) noexcept {
+ int64_t setAllocatedBytes(size_t bytes) noexcept {
// Still checking allocations: with a long running loop all safepoints
// might be "met", so that's the only trigger to not run out of memory.
auto boundary = heapGrowthController_.boundaryForHeapSize(bytes);
switch (boundary) {
case HeapGrowthController::MemoryBoundary::kNone:
- safePoint();
- return;
+ return safePoint();
case HeapGrowthController::MemoryBoundary::kTrigger:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
- scheduleGC_.scheduleNextEpochIfNotInProgress();
- return;
+ return scheduleGC_.scheduleNextEpochIfNotInProgress();
case HeapGrowthController::MemoryBoundary::kTarget:
RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation");
auto epoch = scheduleGC_.scheduleNextEpochIfNotInProgress();
RuntimeLogWarning({kTagGC}, "Pausing the mutators");
mutatorAssists_.requestAssists(epoch);
- return;
+ return epoch;
}
}
- void safePoint() noexcept {
+ int64_t safePoint() noexcept {
if (safePointTracker_.registerCurrentSafePoint(1)) {
RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint");
- schedule();
+ return schedule();
}
+ return 0;
}
void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {
diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp
index 4ea9af5..c449ee0 100644
--- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp
+++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp
@@ -6,6 +6,7 @@
#pragma once
#include <cstddef>
+#include <cstdint>
#include <memory>
#include <functional>
#include <utility>
@@ -47,11 +48,14 @@
GCSchedulerConfig& config() noexcept { return config_; }
// Called by different mutator threads.
- void setAllocatedBytes(size_t bytes) noexcept;
+ int64_t setAllocatedBytes(size_t bytes) noexcept;
// Can be called by any thread.
void schedule() noexcept;
+ // Can be called by any thread
+ void scheduleAndWaitResumed() noexcept;
+
// Can be called by any thread.
void scheduleAndWaitFinished() noexcept;
diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp
index 110077d..88a7365 100644
--- a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp
+++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp
@@ -4,6 +4,7 @@
*/
#include "GCSchedulerImpl.hpp"
+#include <cstdint>
#include "CallsChecker.hpp"
#include "GC.hpp"
@@ -27,6 +28,15 @@
mm::GlobalData::Instance().gc().Schedule();
}
+void gcScheduler::GCScheduler::scheduleAndWaitResumed() noexcept {
+ RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
+ CallsCheckerIgnoreGuard guard;
+ auto& gc = mm::GlobalData::Instance().gc();
+ auto epoch = gc.Schedule();
+ NativeOrUnregisteredThreadGuard stateGuard(/* reentrant = */ true);
+ gc.WaitResumed(epoch);
+}
+
void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept {
RuntimeLogInfo({kTagGC}, "Scheduling GC manually");
CallsCheckerIgnoreGuard guard;
@@ -45,6 +55,6 @@
gc.WaitFinalizers(epoch);
}
-ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {}
+ALWAYS_INLINE int64_t gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept { return 0; }
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {}
ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {}
diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
index 421045001..adf3e4c 100644
--- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
+++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp
@@ -31,6 +31,7 @@
extern "C" const int32_t Kotlin_gcMarkSingleThreaded;
extern "C" const int32_t Kotlin_freezingEnabled;
extern "C" const int32_t Kotlin_freezingChecksEnabled;
+extern "C" const int32_t Kotlin_gcBeforeAlloc;
class SourceInfo;
@@ -103,6 +104,10 @@
}
+ALWAYS_INLINE inline bool gcBeforeAlloc() noexcept {
+ return Kotlin_gcBeforeAlloc != 0;
+}
+
WorkerExceptionHandling workerExceptionHandling() noexcept;
DestroyRuntimeMode destroyRuntimeMode() noexcept;
bool gcMutatorsCooperate() noexcept;
diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h
index ee2c113..59699ad 100644
--- a/kotlin-native/runtime/src/main/cpp/Memory.h
+++ b/kotlin-native/runtime/src/main/cpp/Memory.h
@@ -602,6 +602,9 @@
void StartFinalizerThreadIfNeeded() noexcept;
bool FinalizersThreadIsRunning() noexcept;
+// May trigger GC and wait during STW
+bool TryGCBeforeMemoryAllocation(size_t extraBytes) noexcept;
+// May trigger GC, but does not wait
void OnMemoryAllocation(size_t totalAllocatedBytes) noexcept;
void initObjectPool() noexcept;
diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp
index cc5be1f..5aea102 100644
--- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp
+++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp
@@ -4,6 +4,7 @@
*/
#include "Memory.h"
+#include "GlobalData.hpp"
#include "MemoryPrivate.hpp"
#include "Allocator.hpp"
@@ -634,6 +635,19 @@
mm::disposeRegularWeakReferenceImpl(weakRef);
}
+bool kotlin::TryGCBeforeMemoryAllocation(size_t extraBytes) noexcept {
+ if (!compiler::gcBeforeAlloc()) {
+ return false;
+ }
+ size_t totalAllocatedBytes = alloc::allocatedBytes() + extraBytes;
+ auto epoch = mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
+ if (epoch == 0) {
+ return false;
+ }
+ mm::safePoint();
+ return true;
+}
+
void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept {
mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes);
}
diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp
index b3a5ab8..813009f 100644
--- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp
+++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp
@@ -84,6 +84,7 @@
#endif
extern const int32_t Kotlin_freezingChecksEnabled = 1;
extern const int32_t Kotlin_freezingEnabled = 1;
+extern const int32_t Kotlin_gcBeforeAlloc = 0;
extern const TypeInfo* theAnyTypeInfo = theAnyTypeInfoHolder.typeInfo();
extern const TypeInfo* theArrayTypeInfo = theArrayTypeInfoHolder.typeInfo();
