[K/N] Add assist wait condition
This change lets GCs explicitly state when sweep assist is allowed.
diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
index 1a8c345..908e3d8 100644
--- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
@@ -180,6 +180,8 @@
}
allocator_.prepareForGC();
+ state_.allowAssist(epoch);
+
#ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread
// would not publish into the global state at an unexpected time.
diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
index e5eb0f5..40b5601 100644
--- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
@@ -76,6 +76,10 @@
return impl_->gc().state().schedule();
}
+void gc::GC::WaitAssist(int64_t epoch) noexcept {
+ impl_->gc().state().waitEpochAssist(epoch);
+}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
index 1451238..0092e97 100644
--- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp
@@ -76,6 +76,7 @@
// TODO: These should exist only in the scheduler.
int64_t Schedule() noexcept;
+ void WaitAssist(int64_t epoch) noexcept;
void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept;
diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
index db2648d..b08d37d 100644
--- a/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
+++ b/kotlin-native/runtime/src/gc/common/cpp/GCState.hpp
@@ -7,7 +7,6 @@
#include <condition_variable>
#include <mutex>
-#include <atomic>
#include <optional>
#include "KAssert.h"
@@ -27,6 +26,7 @@
std::unique_lock lock(mutex_);
shutdownFlag_ = true;
startedEpoch.notify();
+ assistEpoch.notify();
finishedEpoch.notify();
scheduledEpoch.notify();
finalizedEpoch.notify();
@@ -34,10 +34,16 @@
void start(int64_t epoch) { startedEpoch.set(epoch); }
+ void allowAssist(int64_t epoch) { assistEpoch.set(epoch); }
+
void finish(int64_t epoch) { finishedEpoch.set(epoch); }
void finalized(int64_t epoch) { finalizedEpoch.set(epoch); }
+ void waitEpochAssist(int64_t epoch) {
+ assistEpoch.wait([this, epoch] { return *assistEpoch >= epoch || shutdownFlag_; });
+ }
+
void waitEpochFinished(int64_t epoch) {
finishedEpoch.wait([this, epoch] { return *finishedEpoch >= epoch || shutdownFlag_; });
}
@@ -88,6 +94,7 @@
std::mutex mutex_;
// Use a separate conditional variable for each counter to mitigate a winpthreads bug (see KT-50948 for details).
ValueWithCondVar<int64_t> startedEpoch{0, mutex_};
+ ValueWithCondVar<int64_t> assistEpoch{0, mutex_};
ValueWithCondVar<int64_t> finishedEpoch{0, mutex_};
ValueWithCondVar<int64_t> scheduledEpoch{0, mutex_};
ValueWithCondVar<int64_t> finalizedEpoch{0, mutex_};
diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
index 3c62fff..d2b9369 100644
--- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
@@ -55,6 +55,8 @@
return 0;
}
+void gc::GC::WaitAssist(int64_t epoch) noexcept {}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {}
void gc::GC::WaitFinalizers(int64_t epoch) noexcept {}
diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp
index 744a5f5..178ca5a 100644
--- a/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp
@@ -76,6 +76,10 @@
return impl_->gc().state().schedule();
}
+void gc::GC::WaitAssist(int64_t epoch) noexcept {
+ impl_->gc().state().waitEpochAssist(epoch);
+}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp
index cda9871..588567c 100644
--- a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp
+++ b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp
@@ -194,6 +194,8 @@
resumeTheWorld(gcHandle);
+ state_.allowAssist(epoch);
+
#ifndef CUSTOM_ALLOCATOR
alloc::SweepExtraObjects<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *extraObjectFactoryIterable);
extraObjectFactoryIterable = std::nullopt;
diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
index 0778689..536ba6f 100644
--- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
@@ -69,6 +69,10 @@
return impl_->gc().state().schedule();
}
+void gc::GC::WaitAssist(int64_t epoch) noexcept {
+ impl_->gc().state().waitEpochAssist(epoch);
+}
+
void gc::GC::WaitFinished(int64_t epoch) noexcept {
impl_->gc().state().waitEpochFinished(epoch);
}
diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
index 03f722d..045560b 100644
--- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
@@ -78,6 +78,8 @@
}
allocator_.prepareForGC();
+ state_.allowAssist(epoch);
+
#ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread
// would not publish into the global state at an unexpected time.
diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp
index ca03f79..35287f66 100644
--- a/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp
+++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp
@@ -21,6 +21,7 @@
{
std::unique_lock guard(owner_.m_);
RuntimeLogDebug({kTagGC}, "Thread is assisting for epoch %" PRId64, epoch);
+ mm::GlobalData::Instance().gc().WaitAssist(epoch);
thread_.allocator().assistGC();
owner_.cv_.wait(guard, noNeedToWait);
RuntimeLogDebug({kTagGC}, "Thread has assisted for epoch %" PRId64, epoch);
