diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index c13c21d5ada..7ff728205a0 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -57,8 +57,6 @@ struct SweepTraits { } }; -// Global, because it's accessed on a hot path: avoid memory load from `this`. -std::atomic gNeedSafepointSlowpath = false; } // namespace ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept { @@ -71,9 +69,7 @@ ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointLoopBody() n void gc::ConcurrentMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { threadData_.gcScheduler().OnSafePointAllocation(size); - if (gNeedSafepointSlowpath.load()) { - SafePointSlowPath(); - } + mm::SuspendIfRequested(); } void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept { ThreadStateGuard guard(ThreadState::kNative); @@ -98,13 +94,7 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { threadData_.gcScheduler().OnSafePointRegular(weight); - if (gNeedSafepointSlowpath.load()) { - SafePointSlowPath(); - } -} - -NO_EXTERNAL_CALLS_CHECK NO_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointSlowPath() noexcept { - threadData_.suspensionData().suspendIfRequested(); + mm::SuspendIfRequested(); } gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep() noexcept : @@ -131,22 +121,11 @@ gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() { gcThread_.join(); } - -void gc::ConcurrentMarkAndSweep::RequestThreadsSuspension() noexcept { - gNeedSafepointSlowpath = true; - bool didSuspend = mm::RequestThreadsSuspension(); - RuntimeAssert(didSuspend, "Only GC thread can request suspension"); -} - -void gc::ConcurrentMarkAndSweep::ResumeThreads() noexcept { - mm::ResumeThreads(); - gNeedSafepointSlowpath = false; -} - bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); auto timeStartUs = konan::getTimeMicros(); - RequestThreadsSuspension(); + bool didSuspend = mm::RequestThreadsSuspension(); + RuntimeAssert(didSuspend, "Only GC thread can request suspension"); RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId()); RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "Concurrent GC must run on unregistered thread"); @@ -179,7 +158,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto objectFactoryIterable = mm::GlobalData::Instance().objectFactory().LockForIter(); - ResumeThreads(); + mm::ResumeThreads(); auto timeResumeUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 932e261fbed..32738d76694 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -67,7 +67,6 @@ public: private: void SafePointRegular(size_t weight) noexcept; - void SafePointSlowPath() noexcept; ConcurrentMarkAndSweep& gc_; mm::ThreadData& threadData_; @@ -81,8 +80,6 @@ public: private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). bool PerformFullGC(int64_t epoch) noexcept; - void RequestThreadsSuspension() noexcept; - void ResumeThreads() noexcept; uint64_t lastGCTimestampUs_ = 0; GCStateHolder state_; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index 35660d2f972..927378df0a0 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -86,6 +86,12 @@ void kotlin::mm::WaitForThreadsSuspension() noexcept { } } +ALWAYS_INLINE void kotlin::mm::SuspendIfRequested() noexcept { + if (IsThreadSuspensionRequested()) { + mm::ThreadRegistry::Instance().CurrentThreadData()->suspensionData().suspendIfRequestedSlowPath(); + } +} + void kotlin::mm::ResumeThreads() noexcept { // From the std::condition_variable docs: // Even if the shared variable is atomic, it must be modified under diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp index e453a5cb4c2..11364abf86d 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp @@ -49,6 +49,8 @@ public: } private: + friend void SuspendIfRequested() noexcept; + std::atomic state_; std::atomic suspended_; void suspendIfRequestedSlowPath() noexcept; @@ -56,6 +58,7 @@ private: bool RequestThreadsSuspension() noexcept; void WaitForThreadsSuspension() noexcept; +void SuspendIfRequested() noexcept; /** * Suspends all threads registered in ThreadRegistry except threads that are in the Native state.