diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp index 7d04cdf0cc4..676bf00a352 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp @@ -44,15 +44,11 @@ void waitForThreadsToReachCheckpoint() { mm::SafePointActivator safePointActivator; - // Disable new threads coming and going. - auto threads = mm::ThreadRegistry::Instance().LockForIter(); - // And wait for all threads to either have passed safepoint or to be in the native state. + // Wait for all threads to either have passed safepoint or to be in the native state. // Either of these mean that none of them are inside a weak reference accessing code. - while (!std::all_of(threads.begin(), threads.end(), [](mm::ThreadData& thread) noexcept { + mm::ThreadRegistry::Instance().waitAllThreads([](mm::ThreadData& thread) noexcept { return thread.gc().impl().gc().barriers().visitedCheckpoint() || thread.suspensionData().suspendedOrNative(); - })) { - std::this_thread::yield(); - } + }); } } // namespace diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 2bf5b743100..e9aa1642d61 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -162,7 +162,6 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { RuntimeAssert(didSuspend, "Only GC thread can request suspension"); gcHandle.suspensionRequested(); - RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "GC must run on unregistered thread"); WaitForThreadsReadyToMark(); gcHandle.threadsAreSuspended(); @@ -236,41 +235,16 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch); } -namespace { - bool isSuspendedOrNative(kotlin::mm::ThreadData& thread) noexcept { - auto& suspensionData = thread.suspensionData(); - return suspensionData.suspended() || suspensionData.state() == kotlin::ThreadState::kNative; - } - - template - bool allThreads(F predicate) noexcept { - auto& threadRegistry = kotlin::mm::ThreadRegistry::Instance(); - auto* currentThread = (threadRegistry.IsCurrentThreadRegistered()) ? threadRegistry.CurrentThreadData() : nullptr; - kotlin::mm::ThreadRegistry::Iterable threads = kotlin::mm::ThreadRegistry::Instance().LockForIter(); - for (auto& thread : threads) { - // Handle if suspension was initiated by the mutator thread. - if (&thread == currentThread) continue; - if (!predicate(thread)) { - return false; - } - } - return true; - } - - void yield() noexcept { - std::this_thread::yield(); - } -} // namespace - void gc::ConcurrentMarkAndSweep::SetMarkingRequested(uint64_t epoch) noexcept { markingRequested = markingBehavior_ == MarkingBehavior::kMarkOwnStack; markingEpoch = epoch; } void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept { - while(!allThreads([](kotlin::mm::ThreadData& thread) { return isSuspendedOrNative(thread) || thread.gc().impl().gc().marking_.load(); })) { - yield(); - } + RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "GC must run on unregistered thread"); + mm::ThreadRegistry::Instance().waitAllThreads([](mm::ThreadData& thread) noexcept { + return thread.suspensionData().suspendedOrNative() || thread.gc().impl().gc().marking_.load(); + }); } void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp index 1075957712e..86f05616d88 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp @@ -50,6 +50,15 @@ public: static void ClearCurrentThreadData() { currentThreadDataNode_ = nullptr; } + template + void waitAllThreads(F&& f) noexcept { + // Disable new threads coming and going. + auto iter = LockForIter(); + while (!std::all_of(iter.begin(), iter.end(), std::forward(f))) { + std::this_thread::yield(); + } + } + private: friend class GlobalData; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index af1faf84903..4a4129cc08d 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -19,26 +19,6 @@ using namespace kotlin; namespace { -template -bool allThreads(F predicate) noexcept { - auto& threadRegistry = kotlin::mm::ThreadRegistry::Instance(); - auto* currentThread = (threadRegistry.IsCurrentThreadRegistered()) ? threadRegistry.CurrentThreadData() : nullptr; - kotlin::mm::ThreadRegistry::Iterable threads = kotlin::mm::ThreadRegistry::Instance().LockForIter(); - for (auto& thread : threads) { - // Handle if suspension was initiated by the mutator thread. - if (&thread == currentThread) - continue; - if (!predicate(thread)) { - return false; - } - } - return true; -} - -void yield() noexcept { - std::this_thread::yield(); -} - [[clang::no_destroy]] thread_local std::optional gSafePointActivator = std::nullopt; [[clang::no_destroy]] std::mutex gSuspensionMutex; [[clang::no_destroy]] std::condition_variable gSuspensionCondVar; @@ -89,10 +69,12 @@ bool kotlin::mm::RequestThreadsSuspension() noexcept { } void kotlin::mm::WaitForThreadsSuspension() noexcept { + auto& threadRegistry = kotlin::mm::ThreadRegistry::Instance(); + auto* currentThread = (threadRegistry.IsCurrentThreadRegistered()) ? threadRegistry.CurrentThreadData() : nullptr; // Spin waiting for threads to suspend. Ignore Native threads. - while (!allThreads([](mm::ThreadData& thread) { return thread.suspensionData().suspendedOrNative(); })) { - yield(); - } + threadRegistry.waitAllThreads([currentThread](mm::ThreadData& thread) noexcept { + return &thread == currentThread || thread.suspensionData().suspendedOrNative(); + }); } void kotlin::mm::ResumeThreads() noexcept {