diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 366f3725b2f..a706d747a19 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -75,10 +75,6 @@ void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGCWithFinalizers gc_.state_.waitEpochFinalized(scheduled_epoch); } -void gc::ConcurrentMarkAndSweep::ThreadData::StopFinalizerThreadForTests() noexcept { - gc_.finalizerProcessor_->StopFinalizerThread(); -} - void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); ScheduleAndWaitFullGC(); @@ -110,6 +106,21 @@ gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() { gcThread_.join(); } +void gc::ConcurrentMarkAndSweep::StartFinalizerThreadIfNeeded() noexcept { + NativeOrUnregisteredThreadGuard guard(true); + finalizerProcessor_->StartFinalizerThreadIfNone(); + finalizerProcessor_->WaitFinalizerThreadInitialized(); +} + +void gc::ConcurrentMarkAndSweep::StopFinalizerThreadIfRunning() noexcept { + NativeOrUnregisteredThreadGuard guard(true); + finalizerProcessor_->StopFinalizerThread(); +} + +bool gc::ConcurrentMarkAndSweep::FinalizersThreadIsRunning() noexcept { + return finalizerProcessor_->IsRunning(); +} + bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); auto timeStartUs = konan::getTimeMicros(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 4a289adbab6..c0baa5b4f6d 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -57,7 +57,6 @@ public: void ScheduleAndWaitFullGC() noexcept; void ScheduleAndWaitFullGCWithFinalizers() noexcept; - void StopFinalizerThreadForTests() noexcept; void OnOOM(size_t size) noexcept; @@ -73,6 +72,10 @@ public: ConcurrentMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& scheduler) noexcept; ~ConcurrentMarkAndSweep(); + void StartFinalizerThreadIfNeeded() noexcept; + void StopFinalizerThreadIfRunning() noexcept; + bool FinalizersThreadIsRunning() noexcept; + private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). bool PerformFullGC(int64_t epoch) noexcept; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index e8f9db9e451..03b15336934 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -340,7 +340,6 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -360,7 +359,6 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -511,7 +509,6 @@ TEST_F(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT( Alive(threadData), diff --git a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.cpp b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.cpp index d81ef836995..74e8b68bab6 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.cpp @@ -14,6 +14,11 @@ void kotlin::gc::FinalizerProcessor::StartFinalizerThreadIfNone() noexcept { if (finalizerThread_.joinable()) return; finalizerThread_ = std::thread([this] { Kotlin_initRuntimeIfNeeded(); + { + std::unique_lock guard(initializedMutex_); + initialized_ = true; + } + initializedCondVar_.notify_all(); int64_t finalizersEpoch = 0; while (true) { std::unique_lock lock(finalizerQueueMutex_); @@ -34,6 +39,11 @@ void kotlin::gc::FinalizerProcessor::StartFinalizerThreadIfNone() noexcept { } epochDoneCallback_(finalizersEpoch); } + { + std::unique_lock guard(initializedMutex_); + initialized_ = false; + } + initializedCondVar_.notify_all(); }); } @@ -69,6 +79,11 @@ bool kotlin::gc::FinalizerProcessor::IsRunning() noexcept { return finalizerThread_.joinable(); } +void kotlin::gc::FinalizerProcessor::WaitFinalizerThreadInitialized() noexcept { + std::unique_lock guard(initializedMutex_); + initializedCondVar_.wait(guard, [this] { return initialized_; }); +} + kotlin::gc::FinalizerProcessor::~FinalizerProcessor() { StopFinalizerThread(); -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.hpp b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.hpp index f91b0da89aa..c817611a17d 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessor.hpp @@ -10,6 +10,7 @@ #include "GCState.hpp" namespace kotlin::gc { + class FinalizerProcessor : Pinned { public: using Queue = typename kotlin::mm::ObjectFactory::FinalizerQueue; @@ -19,9 +20,11 @@ public: void ScheduleTasks(Queue&& tasks, int64_t epoch) noexcept; void StopFinalizerThread() noexcept; bool IsRunning() noexcept; - ~FinalizerProcessor(); -private: void StartFinalizerThreadIfNone() noexcept; + void WaitFinalizerThreadInitialized() noexcept; + ~FinalizerProcessor(); + +private: std::thread finalizerThread_; Queue finalizerQueue_; std::condition_variable finalizerQueueCondVar_; @@ -30,5 +33,10 @@ private: int64_t finalizerQueueEpoch_ = 0; bool shutdownFlag_ = false; bool newTasksAllowed_ = true; + + std::mutex initializedMutex_; + std::condition_variable initializedCondVar_; + bool initialized_ = false; }; -} \ No newline at end of file + +} // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 3cb45f18429..c1e0b0e2e4a 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -73,5 +73,18 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { } void gc::GC::ClearForTests() noexcept { + impl_->gc().StopFinalizerThreadIfRunning(); impl_->objectFactory().ClearForTests(); } + +void gc::GC::StartFinalizerThreadIfNeeded() noexcept { + impl_->gc().StartFinalizerThreadIfNeeded(); +} + +void gc::GC::StopFinalizerThreadIfRunning() noexcept { + impl_->gc().StopFinalizerThreadIfRunning(); +} + +bool gc::GC::FinalizersThreadIsRunning() noexcept { + return impl_->gc().FinalizersThreadIsRunning(); +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 35a89dcdb81..a694fa49416 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -60,6 +60,10 @@ public: void ClearForTests() noexcept; + void StartFinalizerThreadIfNeeded() noexcept; + void StopFinalizerThreadIfRunning() noexcept; + bool FinalizersThreadIsRunning() noexcept; + private: KStdUniquePtr impl_; }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 62e64c651bf..b84854d0f5c 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -65,3 +65,11 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { void gc::GC::ClearForTests() noexcept { impl_->objectFactory().ClearForTests(); } + +void gc::GC::StartFinalizerThreadIfNeeded() noexcept {} + +void gc::GC::StopFinalizerThreadIfRunning() noexcept {} + +bool gc::GC::FinalizersThreadIsRunning() noexcept { + return false; +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index ae1d57bf1eb..d8f255e45de 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -55,7 +55,6 @@ public: ~NoOpGC() = default; GCScheduler& scheduler() noexcept { return scheduler_; } - void StopFinalizerThreadForTests() noexcept {} private: GCScheduler scheduler_; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index e3d9bcdeb0e..cd0392b6142 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -77,3 +77,11 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { void gc::GC::ClearForTests() noexcept { impl_->objectFactory().ClearForTests(); } + +void gc::GC::StartFinalizerThreadIfNeeded() noexcept {} + +void gc::GC::StopFinalizerThreadIfRunning() noexcept {} + +bool gc::GC::FinalizersThreadIsRunning() noexcept { + return false; +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 288f83a1fdb..6a24871dc18 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -75,7 +75,6 @@ public: SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept; ~SameThreadMarkAndSweep() = default; - void StopFinalizerThreadForTests() noexcept {} private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index d45d4f79f90..cfb235c9a17 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3803,3 +3803,9 @@ ALWAYS_INLINE kotlin::CalledFromNativeGuard::CalledFromNativeGuard(bool reentran } const bool kotlin::kSupportsMultipleMutators = true; + +void kotlin::StartFinalizerThreadIfNeeded() noexcept {} + +bool kotlin::FinalizersThreadIsRunning() noexcept { + return false; +} diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index f9ab6e1b16e..bdda5f37744 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -520,6 +520,9 @@ private: extern const bool kSupportsMultipleMutators; +void StartFinalizerThreadIfNeeded() noexcept; +bool FinalizersThreadIsRunning() noexcept; + } // namespace kotlin #endif // RUNTIME_MEMORY_H diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp index 06d74079389..09eacc1bc54 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp +++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp @@ -243,6 +243,10 @@ void Kotlin_shutdownRuntime() { return; } + // If we're going to need finalizers for the full shutdown, we need to start the thread before + // new runtimes are disallowed. + kotlin::StartFinalizerThreadIfNeeded(); + if (Kotlin_cleanersLeakCheckerEnabled()) { // Make sure to collect any lingering cleaners. PerformFullGC(runtime->memoryState); @@ -262,8 +266,14 @@ void Kotlin_shutdownRuntime() { // First make sure workers are gone. WaitNativeWorkersTermination(); + // Allow the current runtime. + int knownRuntimes = 1; + if (kotlin::FinalizersThreadIsRunning()) { + ++knownRuntimes; + } + // Now check for existence of any other runtimes. - auto otherRuntimesCount = atomicGet(&aliveRuntimesCount) - 1; + auto otherRuntimesCount = atomicGet(&aliveRuntimesCount) - knownRuntimes; RuntimeAssert(otherRuntimesCount >= 0, "Cannot be negative"); if (Kotlin_forceCheckedShutdown()) { if (otherRuntimesCount > 0) { diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 0cd5fa13283..62219090eef 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -111,8 +111,9 @@ extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) { auto* node = mm::FromMemoryState(state); if (destroyRuntime) { ThreadStateGuard guard(state, ThreadState::kRunnable); - node->Get()->gc().ScheduleAndWaitFullGC(); - // TODO: Also make sure that finalizers are run. + node->Get()->gc().ScheduleAndWaitFullGCWithFinalizers(); + // TODO: Why not just destruct `GC` object and its thread data counterpart entirely? + mm::GlobalData::Instance().gc().StopFinalizerThreadIfRunning(); } mm::ThreadRegistry::Instance().Unregister(node); if (destroyRuntime) { @@ -293,7 +294,7 @@ extern "C" RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) { extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().ScheduleAndWaitFullGC(); + threadData->gc().ScheduleAndWaitFullGCWithFinalizers(); } extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) { @@ -402,7 +403,7 @@ extern "C" void Kotlin_Any_share(ObjHeader* thiz) { } extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) { - memory->GetThreadData()->gc().ScheduleAndWaitFullGC(); + memory->GetThreadData()->gc().ScheduleAndWaitFullGCWithFinalizers(); } extern "C" bool TryAddHeapRef(const ObjHeader* object) { @@ -572,3 +573,11 @@ ALWAYS_INLINE kotlin::CalledFromNativeGuard::CalledFromNativeGuard(bool reentran } const bool kotlin::kSupportsMultipleMutators = kotlin::gc::kSupportsMultipleMutators; + +void kotlin::StartFinalizerThreadIfNeeded() noexcept { + mm::GlobalData::Instance().gc().StartFinalizerThreadIfNeeded(); +} + +bool kotlin::FinalizersThreadIsRunning() noexcept { + return mm::GlobalData::Instance().gc().FinalizersThreadIsRunning(); +}