diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index 8fa1efab5a7..86237e7d2dc 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -3150,8 +3150,7 @@ standaloneTest("stress_gc_allocations") { (project.testTarget != "watchos_simulator_arm64") && !isNoopGC && !isAggressiveGC && // TODO: Investigate why too slow - !runtimeAssertionsPanic && // New allocator with assertions makes this test very slow - (project.testTarget != "mingw_x64") // TODO: Fix on mingw. + !runtimeAssertionsPanic // New allocator with assertions makes this test very slow source = "runtime/memory/stress_gc_allocations.kt" flags = ['-tr', '-opt-in=kotlin.native.internal.InternalForKotlinNative'] } diff --git a/kotlin-native/backend.native/tests/runtime/memory/stress_gc_allocations.kt b/kotlin-native/backend.native/tests/runtime/memory/stress_gc_allocations.kt index 1aba0013850..45bb373d305 100644 --- a/kotlin-native/backend.native/tests/runtime/memory/stress_gc_allocations.kt +++ b/kotlin-native/backend.native/tests/runtime/memory/stress_gc_allocations.kt @@ -62,6 +62,7 @@ fun test() { if (Platform.memoryModel == MemoryModel.EXPERIMENTAL) { kotlin.native.runtime.GC.autotune = false kotlin.native.runtime.GC.targetHeapBytes = retainLimit + kotlin.native.runtime.GC.pauseOnTargetHeapOverflow = true } // On Linux, the child process might immediately commit the same amount of memory as the parent. diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 87140f7428e..408a37eb8ce 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -370,6 +370,7 @@ bitcode { headersDirs.from(files("src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) sourceSets { main {} + test {} } onlyIf { target.supportsThreads() } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index dbe8355f349..2084b8e1dde 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -59,29 +59,10 @@ struct ProcessWeaksTraits { } // namespace -void gc::ConcurrentMarkAndSweep::ThreadData::Schedule() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - gc_.state_.schedule(); -} - -void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - auto scheduled_epoch = gc_.state_.schedule(); - gc_.state_.waitEpochFinished(scheduled_epoch); -} - -void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - auto scheduled_epoch = gc_.state_.schedule(); - gc_.state_.waitEpochFinalized(scheduled_epoch); -} - void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - ScheduleAndWaitFullGC(); + // TODO: This will print the log for "manual" scheduling. Fix this. + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); } void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { @@ -173,7 +154,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #endif auto& scheduler = gcScheduler_; - scheduler.gcData().OnPerformFullGC(); + scheduler.onGCStart(); state_.start(epoch); @@ -222,7 +203,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); } #endif - scheduler.gcData().UpdateAliveSetBytes(allocatedBytes()); + scheduler.onGCFinish(epoch, allocatedBytes()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 2f85e96b490..b64e5e99039 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -78,13 +78,9 @@ public: using Allocator = AllocatorWithGC; - explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} + explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : threadData_(threadData) {} ~ThreadData() = default; - void Schedule() noexcept; - void ScheduleAndWaitFullGC() noexcept; - void ScheduleAndWaitFullGCWithFinalizers() noexcept; - void OnOOM(size_t size) noexcept; void OnSuspendForGC() noexcept; @@ -97,7 +93,6 @@ public: private: friend ConcurrentMarkAndSweep; - ConcurrentMarkAndSweep& gc_; mm::ThreadData& threadData_; std::atomic marking_; BarriersThreadData barriers_; @@ -135,8 +130,7 @@ public: alloc::Heap& heap() noexcept { return heap_; } #endif - int64_t Schedule() noexcept { return state_.schedule(); } - void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); } + GCStateHolder& state() noexcept { return state_; } private: void PerformFullGC(int64_t epoch) noexcept; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index aa52503d50e..03731c72382 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -250,7 +250,7 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) { ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(IsMarked(stack3.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -296,7 +296,7 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(IsMarked(stack3.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -320,7 +320,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) { ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(object2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -337,7 +337,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -357,7 +357,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { ASSERT_THAT(weak1.get(), object1.header()); EXPECT_CALL(finalizerHook(), Call(weak1.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -374,7 +374,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) { ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(IsMarked(weak1.header()), false); @@ -407,7 +407,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(IsMarked(object4.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -456,7 +456,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(IsMarked(object6.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -507,7 +507,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -540,7 +540,7 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(object2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); EXPECT_THAT(IsMarked(global.header()), false); @@ -584,8 +584,8 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(IsMarked(object6.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -615,7 +615,7 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) { ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(IsMarked(global2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(IsMarked(global2.header()), false); @@ -635,7 +635,7 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) { EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(IsMarked(object.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); EXPECT_THAT(IsMarked(global.header()), false); @@ -742,7 +742,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsCollect) { })); } - mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); gcDone.store(true, std::memory_order_relaxed); for (auto& future : gcFutures) { @@ -792,7 +792,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsAllCollect) { for (auto& mutator : mutators) { gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); // If GC starts before all thread executed line above, two gc will be run // So we temporary switch threads to native state and then return them back after all GC runs are done SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative); @@ -939,7 +939,7 @@ TEST_P(ConcurrentMarkAndSweepTest, CrossThreadReference) { })); } - mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); gcDone.store(true, std::memory_order_relaxed); for (auto& future : gcFutures) { @@ -1142,7 +1142,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { global1->field1 = object1_local.header(); while (weak.load() == nullptr) ; - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); ASSERT_THAT(IsMarked(global1.header()), false); @@ -1153,7 +1153,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { global1->field1 = nullptr; EXPECT_CALL(finalizerHook(), Call(weak.load()->header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); done = true; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index fb1b6b3a970..887309c7384 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Schedule() noexcept { - impl_->gc().Schedule(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { - impl_->gc().ScheduleAndWaitFullGC(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { - impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); -} - void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); @@ -144,11 +132,15 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe } int64_t gc::GC::Schedule() noexcept { - return impl_->gc().Schedule(); + return impl_->gc().state().schedule(); +} + +void gc::GC::WaitFinished(int64_t epoch) noexcept { + impl_->gc().state().waitEpochFinished(epoch); } void gc::GC::WaitFinalizers(int64_t epoch) noexcept { - impl_->gc().WaitFinalized(epoch); + impl_->gc().state().waitEpochFinalized(epoch); } bool gc::isMarked(ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 14981c77af3..3cb4c7b5151 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -6,6 +6,7 @@ #pragma once #include +#include #include "ExtraObjectData.hpp" #include "GCScheduler.hpp" @@ -34,10 +35,6 @@ public: Impl& impl() noexcept { return *impl_; } - void Schedule() noexcept; - void ScheduleAndWaitFullGC() noexcept; - void ScheduleAndWaitFullGCWithFinalizers() noexcept; - void Publish() noexcept; void ClearForTests() noexcept; @@ -73,10 +70,10 @@ public: static void processArrayInMark(void* state, ArrayHeader* array) noexcept; static void processFieldInMark(void* state, ObjHeader* field) noexcept; - // TODO: These should be moved into the scheduler. + // TODO: These should exist only in the scheduler. int64_t Schedule() noexcept; + void WaitFinished(int64_t epoch) noexcept; void WaitFinalizers(int64_t epoch) noexcept; - void ScheduleAndWaitFullGCWithFinalizers() noexcept { WaitFinalizers(Schedule()); } static const size_t objectDataSize; static bool SweepObject(void* objectData) noexcept; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 1c04cfd3bc9..2d22dd721b0 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Schedule() noexcept { - impl_->gc().Schedule(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { - impl_->gc().ScheduleAndWaitFullGC(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { - impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); -} - void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); @@ -131,6 +119,9 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe int64_t gc::GC::Schedule() noexcept { return 0; } + +void gc::GC::WaitFinished(int64_t epoch) noexcept {} + void gc::GC::WaitFinalizers(int64_t epoch) noexcept {} bool gc::isMarked(ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index d3a48790d78..60eb7f617c6 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -39,10 +39,6 @@ public: ThreadData() noexcept {} ~ThreadData() = default; - void Schedule() noexcept {} - void ScheduleAndWaitFullGC() noexcept {} - void ScheduleAndWaitFullGCWithFinalizers() noexcept {} - void OnOOM(size_t size) noexcept {} Allocator CreateAllocator() noexcept { return Allocator(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 4a8716c70a9..9384807e743 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -19,18 +19,6 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Schedule() noexcept { - impl_->gc().Schedule(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { - impl_->gc().ScheduleAndWaitFullGC(); -} - -void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { - impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); -} - void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); @@ -140,11 +128,15 @@ ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noe } int64_t gc::GC::Schedule() noexcept { - return impl_->gc().Schedule(); + return impl_->gc().state().schedule(); +} + +void gc::GC::WaitFinished(int64_t epoch) noexcept { + impl_->gc().state().waitEpochFinished(epoch); } void gc::GC::WaitFinalizers(int64_t epoch) noexcept { - impl_->gc().WaitFinalized(epoch); + impl_->gc().state().waitEpochFinalized(epoch); } bool gc::isMarked(ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index d1af83126d0..010e00420eb 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -58,29 +58,10 @@ struct ProcessWeaksTraits { } // namespace -void gc::SameThreadMarkAndSweep::ThreadData::Schedule() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - gc_.state_.schedule(); -} - -void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - auto scheduled_epoch = gc_.state_.schedule(); - gc_.state_.waitEpochFinished(scheduled_epoch); -} - -void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { - RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); - ThreadStateGuard guard(ThreadState::kNative); - auto scheduled_epoch = gc_.state_.schedule(); - gc_.state_.waitEpochFinalized(scheduled_epoch); -} - void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - ScheduleAndWaitFullGC(); + // TODO: This will print the log for "manual" scheduling. Fix this. + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); } #ifdef CUSTOM_ALLOCATOR @@ -142,7 +123,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gcHandle.threadsAreSuspended(); auto& scheduler = gcScheduler_; - scheduler.gcData().OnPerformFullGC(); + scheduler.onGCStart(); state_.start(epoch); @@ -175,7 +156,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto finalizerQueue = heap_.Sweep(gcHandle); #endif - scheduler.gcData().UpdateAliveSetBytes(allocatedBytes()); + scheduler.onGCFinish(epoch, allocatedBytes()); mm::ResumeThreads(); gcHandle.threadsAreResumed(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 2ac3f94236a..8d5436255d9 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -79,20 +79,14 @@ public: using ObjectData = SameThreadMarkAndSweep::ObjectData; using Allocator = AllocatorWithGC; - ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc) {} + ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {} ~ThreadData() = default; - void Schedule() noexcept; - void ScheduleAndWaitFullGC() noexcept; - void ScheduleAndWaitFullGCWithFinalizers() noexcept; - void OnOOM(size_t size) noexcept; Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } private: - - SameThreadMarkAndSweep& gc_; }; using Allocator = ThreadData::Allocator; @@ -118,8 +112,7 @@ public: void StopFinalizerThreadIfRunning() noexcept; bool FinalizersThreadIsRunning() noexcept; - int64_t Schedule() noexcept { return state_.schedule(); } - void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); } + GCStateHolder& state() noexcept { return state_; } #ifdef CUSTOM_ALLOCATOR alloc::Heap& heap() noexcept { return heap_; } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index b7bba4008ee..05dda65a243 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -246,7 +246,7 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) { ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(IsMarked(stack3.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -292,7 +292,7 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(IsMarked(stack3.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -316,7 +316,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) { ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(object2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -333,7 +333,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -353,7 +353,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { ASSERT_THAT(weak1.get(), object1.header()); EXPECT_CALL(finalizerHook(), Call(weak1.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -370,7 +370,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(IsMarked(weak1.header()), false); @@ -403,7 +403,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(IsMarked(object4.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -452,7 +452,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(IsMarked(object6.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -503,7 +503,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -536,7 +536,7 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(object2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); EXPECT_THAT(IsMarked(global.header()), false); @@ -580,8 +580,8 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(IsMarked(object6.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT( Alive(threadData), @@ -611,7 +611,7 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) { ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(IsMarked(global2.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(IsMarked(global2.header()), false); @@ -631,7 +631,7 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) { EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(IsMarked(object.header()), false); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); EXPECT_THAT(IsMarked(global.header()), false); @@ -738,7 +738,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) { })); } - mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); gcDone.store(true, std::memory_order_relaxed); for (auto& future : gcFutures) { @@ -788,7 +788,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAllCollect) { for (auto& mutator : mutators) { gcFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); // If GC starts before all thread executed line above, two gc will be run // So we temporary switch threads to native state and then return them back after all GC runs are done SwitchThreadState(mm::GetMemoryState(), kotlin::ThreadState::kNative); @@ -935,7 +935,7 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { })); } - mm::GlobalData::Instance().gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); gcDone.store(true, std::memory_order_relaxed); for (auto& future : gcFutures) { @@ -1138,7 +1138,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { global1->field1 = object1_local.header(); while (weak.load() == nullptr) ; - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); ASSERT_THAT(IsMarked(global1.header()), false); @@ -1149,7 +1149,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { global1->field1 = nullptr; EXPECT_CALL(finalizerHook(), Call(weak.load()->header())); - threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); done = true; diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp index f39029fb6bd..6d06f1efa51 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.cpp @@ -13,12 +13,57 @@ using namespace kotlin; -gcScheduler::GCScheduler::GCScheduler() noexcept : - gcData_(std_support::make_unique>(config_, []() noexcept { +gcScheduler::GCScheduler::ThreadData::Impl::Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept : + scheduler_(scheduler.impl().impl()), mutatorAssists_(scheduler_.mutatorAssists(), thread) {} + +gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler& scheduler, mm::ThreadData& thread) noexcept : + impl_(std_support::make_unique(scheduler, thread)) {} + +gcScheduler::GCScheduler::ThreadData::~ThreadData() = default; + +gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept : + impl_(config, []() noexcept { // This call acquires a lock, but the lock are always short-lived, // so we ignore thread state switching to avoid recursive safe points. CallsCheckerIgnoreGuard guard; - mm::GlobalData::Instance().gc().Schedule(); - })) {} + return mm::GlobalData::Instance().gc().Schedule(); + }) {} -ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept {} +gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique(config_)) {} + +gcScheduler::GCScheduler::~GCScheduler() = default; + +ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept { + impl().mutatorAssists().safePoint(); +} + +void gcScheduler::GCScheduler::schedule() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + impl().impl().schedule(); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + auto epoch = impl().impl().schedule(); + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().WaitFinished(epoch); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + auto epoch = impl().impl().schedule(); + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept { + impl().impl().setAllocatedBytes(bytes); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept { + impl().impl().onGCStart(); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept { + impl().impl().onGCFinish(epoch, aliveBytes); +} diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp index 0c0ad18de05..a80abf91660 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -12,15 +12,38 @@ #include "GlobalData.hpp" #include "HeapGrowthController.hpp" #include "Logging.hpp" +#include "MutatorAssists.hpp" #include "RegularIntervalPacer.hpp" #include "RepeatedTimer.hpp" +#include "SafePoint.hpp" +#include "ThreadData.hpp" -namespace kotlin::gcScheduler::internal { +namespace kotlin::gcScheduler { + +namespace internal { +template +class GCSchedulerDataAdaptive; +} + +class GCScheduler::ThreadData::Impl : private Pinned { +public: + Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept; + + internal::GCSchedulerDataAdaptive& scheduler() noexcept { return scheduler_; } + + internal::MutatorAssists::ThreadData& mutatorAssists() noexcept { return mutatorAssists_; } + +private: + internal::GCSchedulerDataAdaptive& scheduler_; + internal::MutatorAssists::ThreadData mutatorAssists_; +}; + +namespace internal { template -class GCSchedulerDataAdaptive : public GCSchedulerData { +class GCSchedulerDataAdaptive { public: - GCSchedulerDataAdaptive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : + GCSchedulerDataAdaptive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : config_(config), scheduleGC_(std::move(scheduleGC)), appStateTracking_(mm::GlobalData::Instance().appStateTracking()), @@ -32,34 +55,70 @@ public: } if (regularIntervalPacer_.NeedsGC()) { RuntimeLogDebug({kTagGC}, "Scheduling GC by timer"); - scheduleGC_(); + schedule(); } }) { RuntimeLogInfo({kTagGC}, "Adaptive GC scheduler initialized"); } - void OnPerformFullGC() noexcept override { - heapGrowthController_.OnPerformFullGC(); + void onGCStart() noexcept { regularIntervalPacer_.OnPerformFullGC(); timer_.restart(config_.regularGcInterval()); } - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - - void SetAllocatedBytes(size_t bytes) noexcept override { - if (heapGrowthController_.SetAllocatedBytes(bytes)) { - RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - scheduleGC_(); + void setAllocatedBytes(size_t bytes) noexcept { + auto boundary = heapGrowthController_.boundaryForHeapSize(bytes); + switch (boundary) { + case HeapGrowthController::MemoryBoundary::kNone: + return; + case HeapGrowthController::MemoryBoundary::kTrigger: + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + schedule(); + return; + case HeapGrowthController::MemoryBoundary::kTarget: + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + auto epoch = schedule(); + RuntimeLogWarning({kTagGC}, "Pausing the mutators"); + mutatorAssists_.requestAssists(epoch); + return; } } + void onGCFinish(int64_t epoch, size_t bytes) noexcept { + heapGrowthController_.updateBoundaries(bytes); + // Must wait for all mutators to be released. GC thread cannot continue. + // This is the contract between GC and mutators. With regular native state + // each mutator must check that GC is not doing something. Here GC must check + // that each mutator has done all it needs. + mutatorAssists_.completeEpoch(epoch, [](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& { + return threadData.gcScheduler().impl().mutatorAssists(); + }); + } + + int64_t schedule() noexcept { return scheduleGC_(); } + + MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; } + private: GCSchedulerConfig& config_; - std::function scheduleGC_; + std::function scheduleGC_; mm::AppStateTracking& appStateTracking_; HeapGrowthController heapGrowthController_; RegularIntervalPacer regularIntervalPacer_; RepeatedTimer timer_; + MutatorAssists mutatorAssists_; }; -} // namespace kotlin::gcScheduler::internal +} // namespace internal + +class GCScheduler::Impl : private Pinned { +public: + explicit Impl(GCSchedulerConfig& config) noexcept; + + internal::GCSchedulerDataAdaptive& impl() noexcept { return impl_; } + +private: + internal::GCSchedulerDataAdaptive impl_; +}; + +} // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp index 1a140de821f..5368d350f42 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImplTest.cpp @@ -20,18 +20,19 @@ namespace { class MutatorThread : private Pinned { public: - explicit MutatorThread(gcScheduler::GCSchedulerData& scheduler) : executor_([&scheduler] { return Context{scheduler}; }) {} + explicit MutatorThread(gcScheduler::internal::GCSchedulerDataAdaptive& scheduler) : + executor_([&scheduler] { return Context{scheduler}; }) {} std::future SetAllocatedBytes(size_t bytes) { return executor_.execute([&, bytes] { auto& context = executor_.context(); - context.scheduler.SetAllocatedBytes(bytes); + context.scheduler.setAllocatedBytes(bytes); }); } private: struct Context { - gcScheduler::GCSchedulerData& scheduler; + gcScheduler::internal::GCSchedulerDataAdaptive& scheduler; }; SingleThreadExecutor executor_; @@ -53,24 +54,26 @@ public: return mutators_[mutator]->SetAllocatedBytes(allocatedBytes); } - void OnPerformFullGC() { scheduler_.OnPerformFullGC(); } + void OnPerformFullGC() { scheduler_.onGCStart(); } - void UpdateAliveSetBytes(size_t bytes) { + void onGCFinish(int64_t epoch, size_t bytes) { allocatedBytes_.store(bytes); - scheduler_.UpdateAliveSetBytes(bytes); + scheduler_.onGCFinish(epoch, bytes); } - testing::MockFunction& scheduleGC() { return scheduleGC_; } + testing::MockFunction& scheduleGC() { return scheduleGC_; } template void advance_time(Duration duration) { test_support::manual_clock::sleep_for(duration); } + int64_t assistsRequested() noexcept { return scheduler_.mutatorAssists().assistsRequested(std::memory_order_relaxed); } + private: std::atomic allocatedBytes_ = 0; std_support::vector> mutators_; - testing::MockFunction scheduleGC_; + testing::MockFunction scheduleGC_; gcScheduler::internal::GCSchedulerDataAdaptive scheduler_; }; @@ -88,34 +91,77 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTargetHeapReached) { config.regularGcIntervalMicroseconds = 10; config.autoTune = false; config.targetHeapBytes = (mutatorsCount + 1) * 10; + config.heapTriggerCoefficient = 0.9; + config.setMutatorAssists(true); GCSchedulerDataTestApi schedulerTestApi(config); EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); std_support::vector> futures; for (int i = 0; i < mutatorsCount; ++i) { - futures.push_back(schedulerTestApi.Allocate(i, 10)); + futures.push_back(schedulerTestApi.Allocate(i, 9)); } for (auto& future : futures) { future.get(); } testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); - schedulerTestApi.Allocate(0, 10).get(); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); + schedulerTestApi.Allocate(0, 9).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + EXPECT_THAT(schedulerTestApi.assistsRequested(), 0); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); + schedulerTestApi.onGCFinish(1, 0); EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()); + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2)); schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); - testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + EXPECT_THAT(schedulerTestApi.assistsRequested(), 2); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); + schedulerTestApi.onGCFinish(2, 0); +} + +TEST_F(AdaptiveSchedulerTest, CollectOnTargetHeapReachedWithoutAssists) { + constexpr int mutatorsCount = kDefaultThreadCount; + + gcScheduler::GCSchedulerConfig config; + config.regularGcIntervalMicroseconds = 10; + config.autoTune = false; + config.targetHeapBytes = (mutatorsCount + 1) * 10; + config.heapTriggerCoefficient = 0.9; + config.setMutatorAssists(false); + GCSchedulerDataTestApi schedulerTestApi(config); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + std_support::vector> futures; + for (int i = 0; i < mutatorsCount; ++i) { + futures.push_back(schedulerTestApi.Allocate(i, 9)); + } + for (auto& future : futures) { + future.get(); + } + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(1)); + schedulerTestApi.Allocate(0, 9).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + EXPECT_THAT(schedulerTestApi.assistsRequested(), 0); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.onGCFinish(1, 0); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); + schedulerTestApi.Allocate(0, 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + + EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).WillOnce(testing::Return(2)); + schedulerTestApi.Allocate(0, mutatorsCount * 10).get(); + testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); + EXPECT_THAT(schedulerTestApi.assistsRequested(), 0); + schedulerTestApi.OnPerformFullGC(); + schedulerTestApi.onGCFinish(2, 0); } TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) { @@ -135,7 +181,7 @@ TEST_F(AdaptiveSchedulerTest, CollectOnTimeoutReached) { test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); + schedulerTestApi.onGCFinish(1, 0); } TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) { @@ -155,7 +201,7 @@ TEST_F(AdaptiveSchedulerTest, FullTimeoutAfterLastGC) { schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); + schedulerTestApi.onGCFinish(1, 0); // pending should restart to be 10us since the previous collection without scheduling another GC. EXPECT_CALL(schedulerTestApi.scheduleGC(), Call()).Times(0); @@ -176,7 +222,7 @@ TEST_F(AdaptiveSchedulerTest, DoNotTuneTargetHeap) { schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); + schedulerTestApi.onGCFinish(1, 10); EXPECT_THAT(config.targetHeapBytes.load(), 10); } @@ -197,7 +243,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(10); + schedulerTestApi.onGCFinish(1, 10); EXPECT_THAT(config.targetHeapBytes.load(), 20); @@ -206,7 +252,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { schedulerTestApi.Allocate(0, 10).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(20); + schedulerTestApi.onGCFinish(2, 20); EXPECT_THAT(config.targetHeapBytes.load(), 40); @@ -215,7 +261,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { schedulerTestApi.Allocate(0, 40).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); + schedulerTestApi.onGCFinish(3, 60); // But we will keep the 50, which means we will trigger GC every allocation, until alive set falls down EXPECT_THAT(config.targetHeapBytes.load(), 50); @@ -225,7 +271,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { schedulerTestApi.Allocate(0, 0).get(); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(60); + schedulerTestApi.onGCFinish(4, 60); EXPECT_THAT(config.targetHeapBytes.load(), 50); @@ -234,7 +280,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); // Dropping to 40 - schedulerTestApi.UpdateAliveSetBytes(40); + schedulerTestApi.onGCFinish(5, 40); EXPECT_THAT(config.targetHeapBytes.load(), 50); @@ -244,7 +290,7 @@ TEST_F(AdaptiveSchedulerTest, TuneTargetHeap) { testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); // Dropping to 1 - schedulerTestApi.UpdateAliveSetBytes(1); + schedulerTestApi.onGCFinish(6, 1); // But the minimum is set to 5. EXPECT_THAT(config.targetHeapBytes.load(), 5); @@ -283,5 +329,5 @@ TEST_F(AdaptiveSchedulerTest, DoNotCollectOnTimerInBackground) { test_support::manual_clock::waitForPending(test_support::manual_clock::now() + microseconds(10)); testing::Mock::VerifyAndClearExpectations(&schedulerTestApi.scheduleGC()); schedulerTestApi.OnPerformFullGC(); - schedulerTestApi.UpdateAliveSetBytes(0); + schedulerTestApi.onGCFinish(1, 0); } diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp index ed5393b3489..038a15814d1 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.cpp @@ -13,14 +13,56 @@ using namespace kotlin; -gcScheduler::GCScheduler::GCScheduler() noexcept : - gcData_(std_support::make_unique(config_, []() noexcept { +gcScheduler::GCScheduler::ThreadData::Impl::Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept : + scheduler_(scheduler.impl().impl()), mutatorAssists_(scheduler_.mutatorAssists(), thread) {} + +gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler& gcScheduler, mm::ThreadData& thread) noexcept : + impl_(std_support::make_unique(gcScheduler, thread)) {} + +gcScheduler::GCScheduler::ThreadData::~ThreadData() = default; + +gcScheduler::GCScheduler::Impl::Impl(gcScheduler::GCSchedulerConfig& config) noexcept : + impl_(config, []() noexcept { // This call acquires a lock, but the lock are always short-lived, // so we ignore thread state switching to avoid recursive safe points. CallsCheckerIgnoreGuard guard; - mm::GlobalData::Instance().gc().Schedule(); - })) {} + return mm::GlobalData::Instance().gc().Schedule(); + }) {} -ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept { - static_cast(gcData()).safePoint(); +gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique(config_)) {} + +gcScheduler::GCScheduler::~GCScheduler() = default; + +ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept { + impl().mutatorAssists().safePoint(); + impl().scheduler().safePoint(); +} + +void gcScheduler::GCScheduler::schedule() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + impl().impl().schedule(); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + auto epoch = impl().impl().schedule(); + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().WaitFinished(epoch); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + auto epoch = impl().impl().schedule(); + NativeOrUnregisteredThreadGuard guard(/* reentrant = */ true); + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept { + impl().impl().setAllocatedBytes(bytes); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {} + +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept { + impl().impl().onGCFinish(epoch, aliveBytes); } diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp index a9f97d74b2f..02295682c5f 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImpl.hpp @@ -12,44 +12,101 @@ #include "GCSchedulerConfig.hpp" #include "HeapGrowthController.hpp" #include "Logging.hpp" +#include "MutatorAssists.hpp" #include "SafePoint.hpp" #include "SafePointTracker.hpp" +#include "ThreadData.hpp" -namespace kotlin::gcScheduler::internal { +namespace kotlin::gcScheduler { + +namespace internal { +class GCSchedulerDataAggressive; +} + +class GCScheduler::ThreadData::Impl : private Pinned { +public: + Impl(GCScheduler& scheduler, mm::ThreadData& thread) noexcept; + + internal::GCSchedulerDataAggressive& scheduler() noexcept { return scheduler_; } + + internal::MutatorAssists::ThreadData& mutatorAssists() noexcept { return mutatorAssists_; } + +private: + internal::GCSchedulerDataAggressive& scheduler_; + internal::MutatorAssists::ThreadData mutatorAssists_; +}; + +namespace internal { // The slowpath will trigger GC if this thread didn't meet this safepoint/allocation site before. -class GCSchedulerDataAggressive : public GCSchedulerData { +class GCSchedulerDataAggressive { public: - GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : + GCSchedulerDataAggressive(GCSchedulerConfig& config, std::function scheduleGC) noexcept : scheduleGC_(std::move(scheduleGC)), heapGrowthController_(config) { RuntimeLogInfo({kTagGC}, "Aggressive GC scheduler initialized"); } - void OnPerformFullGC() noexcept override { heapGrowthController_.OnPerformFullGC(); } - void UpdateAliveSetBytes(size_t bytes) noexcept override { heapGrowthController_.UpdateAliveSetBytes(bytes); } - void SetAllocatedBytes(size_t bytes) noexcept override { + void setAllocatedBytes(size_t bytes) noexcept { // Still checking allocations: with a long running loop all safepoints // might be "met", so that's the only trigger to not run out of memory. - if (heapGrowthController_.SetAllocatedBytes(bytes)) { - RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); - scheduleGC_(); - } else { - safePoint(); + auto boundary = heapGrowthController_.boundaryForHeapSize(bytes); + switch (boundary) { + case HeapGrowthController::MemoryBoundary::kNone: + safePoint(); + return; + case HeapGrowthController::MemoryBoundary::kTrigger: + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + schedule(); + return; + case HeapGrowthController::MemoryBoundary::kTarget: + RuntimeLogDebug({kTagGC}, "Scheduling GC by allocation"); + auto epoch = schedule(); + RuntimeLogWarning({kTagGC}, "Pausing the mutators"); + mutatorAssists_.requestAssists(epoch); + return; } } void safePoint() noexcept { if (safePointTracker_.registerCurrentSafePoint(1)) { RuntimeLogDebug({kTagGC}, "Scheduling GC by safepoint"); - scheduleGC_(); + schedule(); } } + void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept { + heapGrowthController_.updateBoundaries(aliveBytes); + // Must wait for all mutators to be released. GC thread cannot continue. + // This is the contract between GC and mutators. With regular native state + // each mutator must check that GC is not doing something. Here GC must check + // that each mutator has done all it needs. + mutatorAssists_.completeEpoch(epoch, [](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& { + return threadData.gcScheduler().impl().mutatorAssists(); + }); + } + + int64_t schedule() noexcept { return scheduleGC_(); } + + MutatorAssists& mutatorAssists() noexcept { return mutatorAssists_; } + private: - std::function scheduleGC_; + std::function scheduleGC_; HeapGrowthController heapGrowthController_; SafePointTracker<> safePointTracker_; mm::SafePointActivator safePointActivator_; + MutatorAssists mutatorAssists_; }; -} // namespace kotlin::gcScheduler::internal +} // namespace internal + +class GCScheduler::Impl : private Pinned { +public: + explicit Impl(GCSchedulerConfig& config) noexcept; + + internal::GCSchedulerDataAggressive& impl() noexcept { return impl_; } + +private: + internal::GCSchedulerDataAggressive impl_; +}; + +} // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp index f0d9fff6e19..a507c382639 100644 --- a/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp +++ b/kotlin-native/runtime/src/gcScheduler/aggressive/cpp/GCSchedulerImplTest.cpp @@ -27,18 +27,18 @@ using namespace kotlin; TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) { SKIP_ON_WINDOWS(); []() OPTNONE { - testing::MockFunction scheduleGC; + testing::MockFunction scheduleGC; gcScheduler::GCSchedulerConfig config; gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); - EXPECT_CALL(scheduleGC, Call()).Times(1); + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(0)); for (int i = 0; i < 10; i++) { scheduler.safePoint(); } testing::Mock::VerifyAndClearExpectations(&scheduleGC); - EXPECT_CALL(scheduleGC, Call()).Times(1); + EXPECT_CALL(scheduleGC, Call()).WillOnce(testing::Return(1)); scheduler.safePoint(); testing::Mock::VerifyAndClearExpectations(&scheduleGC); }(); @@ -47,20 +47,34 @@ TEST(AggressiveSchedulerTest, TriggerGCOnUniqueSafePoint) { TEST(AggressiveSchedulerTest, TriggerGCOnAllocationThreshold) { SKIP_ON_WINDOWS(); []() OPTNONE { - testing::MockFunction scheduleGC; + testing::MockFunction scheduleGC; gcScheduler::GCSchedulerConfig config; config.autoTune = false; config.targetHeapBytes = 10; + config.heapTriggerCoefficient = 0.9; gcScheduler::internal::GCSchedulerDataAggressive scheduler(config, scheduleGC.AsStdFunction()); int i = 0; // We trigger GC on the first iteration, when the unique allocation point is faced, + // on the second to last iteration when weak target heap size is reached, // and on the last iteration when target heap size is reached. - EXPECT_CALL(scheduleGC, Call()).WillOnce([&i]() { EXPECT_THAT(i, 0); }).WillOnce([&i]() { EXPECT_THAT(i, 9); }); + EXPECT_CALL(scheduleGC, Call()) + .WillOnce([&i]() { + EXPECT_THAT(i, 0); + return 0; + }) + .WillOnce([&i]() { + EXPECT_THAT(i, 8); + return 1; + }) + .WillOnce([&i]() { + EXPECT_THAT(i, 9); + return 2; + }); for (; i < 10; i++) { - scheduler.SetAllocatedBytes(i + 1); + scheduler.setAllocatedBytes(i + 1); } testing::Mock::VerifyAndClearExpectations(&scheduleGC); }(); diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp index 2143703fdeb..1e427095d63 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCScheduler.hpp @@ -14,45 +14,59 @@ #include "Utils.hpp" #include "std_support/Memory.hpp" -namespace kotlin::gcScheduler { - -namespace test_support { -class GCSchedulerThreadDataTestApi; +namespace kotlin::mm { +class ThreadData; } -class GCSchedulerThreadData; - -class GCSchedulerData { -public: - virtual ~GCSchedulerData() = default; - - // The protocol is: after the scheduler schedules the GC, the GC eventually calls `OnPerformFullGC` - // when the collection has started, followed by `UpdateAliveSetBytes` when the marking has finished. - // TODO: Consider returning a sort of future from the scheduleGC, and listen to it instead. - - // Always called by the GC thread. - virtual void OnPerformFullGC() noexcept = 0; - - // Always called by the GC thread. - virtual void UpdateAliveSetBytes(size_t bytes) noexcept = 0; - - // Called by different mutator threads. - virtual void SetAllocatedBytes(size_t bytes) noexcept = 0; -}; +namespace kotlin::gcScheduler { class GCScheduler : private Pinned { public: + class Impl; + + class ThreadData : private Pinned { + public: + class Impl; + + ThreadData(GCScheduler&, mm::ThreadData&) noexcept; + ~ThreadData(); + + Impl& impl() noexcept { return *impl_; } + + void safePoint() noexcept; + + private: + std_support::unique_ptr impl_; + }; + GCScheduler() noexcept; + ~GCScheduler(); + + Impl& impl() noexcept { return *impl_; } GCSchedulerConfig& config() noexcept { return config_; } - GCSchedulerData& gcData() noexcept { return *gcData_; } - // Should be called on encountering a safepoint. - void safePoint() noexcept; + // Called by different mutator threads. + void setAllocatedBytes(size_t bytes) noexcept; + + // Can be called by any thread. + void schedule() noexcept; + + // Can be called by any thread. + void scheduleAndWaitFinished() noexcept; + + // Can be called by any thread. + void scheduleAndWaitFinalized() noexcept; + + // Always called by the GC thread. + void onGCStart() noexcept; + + // Called by the GC thread only. + void onGCFinish(int64_t epoch, size_t aliveBytes) noexcept; private: GCSchedulerConfig config_; - std_support::unique_ptr gcData_; + std_support::unique_ptr impl_; }; } // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp index 26c67d9ac98..ff06201bc12 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfig.hpp @@ -13,22 +13,63 @@ namespace kotlin::gcScheduler { // NOTE: When changing default values, reflect them in GC.kt as well. struct GCSchedulerConfig { + enum class MutatorAssists { + kDefault, + kEnable, + kDisable, + }; + std::atomic autoTune = true; // The target interval between collections when Kotlin code is idle. GC will be triggered // by timer no sooner than this value and no later than twice this value since the previous collection. std::atomic regularGcIntervalMicroseconds = 10 * 1000 * 1000; - // How many object bytes must be in the heap to trigger collection. Autotunes when autoTune is true. + // GC will try to keep object bytes under this amount. If object bytes have + // become bigger than this value, and `mutatorAssists` are enabled the GC will + // stop the world and wait until current epoch finishes. + // Adapts after each GC epoch when `autoTune = true`. std::atomic targetHeapBytes = 1024 * 1024; - // The rate at which targetHeapBytes changes when autoTune = true. Concretely: if after the collection - // N object bytes remain in the heap, the next targetHeapBytes will be N / targetHeapUtilization capped - // between minHeapBytes and maxHeapBytes. + // The rate at which `targetHeapBytes` changes when `autoTune = true`. Concretely: if after the collection + // `N` object bytes remain in the heap, the next `targetHeapBytes` will be `N / targetHeapUtilization` capped + // between `minHeapBytes` and `maxHeapBytes`. std::atomic targetHeapUtilization = 0.5; - // The minimum value of targetHeapBytes for autoTune = true + // The minimum value of `targetHeapBytes` for `autoTune = true` std::atomic minHeapBytes = 1024 * 1024; - // The maximum value of targetHeapBytes for autoTune = true + // The maximum value of `targetHeapBytes` for `autoTune = true` std::atomic maxHeapBytes = std::numeric_limits::max(); + // GC will be triggered when object bytes reach `heapTriggerCoefficient * targetHeapBytes`. + std::atomic heapTriggerCoefficient = 0.9; + // See `mutatorAssists()`. + std::atomic> mutatorAssistsImpl = + static_cast>(MutatorAssists::kDefault); std::chrono::microseconds regularGcInterval() const { return std::chrono::microseconds(regularGcIntervalMicroseconds.load()); } + + // Whether mutators should stop and wait for GC to complete when + // current object heap size is bigger than `targetHeapBytes`. + // By default on, unless `autoTune = false` or `maxHeapBytes` is set. + bool mutatorAssists() const noexcept { + switch (static_cast(mutatorAssistsImpl.load())) { + case MutatorAssists::kDisable: + return false; + case MutatorAssists::kEnable: + return true; + case MutatorAssists::kDefault: + // If after a GC epoch the alive set is more than maximum `targetHeapBytes`, the next GC will be + // scheduled instantly and when the assists are turned on, the mutators would be immediately paused. + // This will look like the program has hanged. + // So, by default, disable assisting if `targetHeapBytes` has a non-infinite limit + // (either `autoTune == false`, so `targetHeapBytes` is fixed; or `maxHeapBytes` + // is lower than infinity). + // TODO: Figure out what to do with OOMs. + return autoTune.load() && maxHeapBytes.load() == std::numeric_limits::max(); + } + } + + // See `mutatorAssists()`. + void setMutatorAssists(bool assist) noexcept { + mutatorAssistsImpl.store( + static_cast>(assist ? MutatorAssists::kEnable : MutatorAssists::kDisable)); + } }; } // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfigTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfigTest.cpp new file mode 100644 index 00000000000..d88fd1f81d7 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/GCSchedulerConfigTest.cpp @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCSchedulerConfig.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace kotlin; + +TEST(GCSchedulerConfigTest, DefaultMutatorAssists) { + gcScheduler::GCSchedulerConfig config; + EXPECT_TRUE(config.mutatorAssists()); + config.autoTune = false; + EXPECT_FALSE(config.mutatorAssists()); + config.autoTune = true; + ASSERT_TRUE(config.mutatorAssists()); + config.maxHeapBytes = 1024 * 1024 * 1024; + EXPECT_FALSE(config.mutatorAssists()); + config.maxHeapBytes = std::numeric_limits::max(); + EXPECT_TRUE(config.mutatorAssists()); +} + +TEST(GCSchedulerConfigTest, DisabledMutatorAssists) { + gcScheduler::GCSchedulerConfig config; + config.setMutatorAssists(false); + EXPECT_FALSE(config.mutatorAssists()); + config.autoTune = false; + EXPECT_FALSE(config.mutatorAssists()); + config.autoTune = true; + ASSERT_FALSE(config.mutatorAssists()); + config.maxHeapBytes = 1024 * 1024 * 1024; + EXPECT_FALSE(config.mutatorAssists()); + config.maxHeapBytes = std::numeric_limits::max(); + EXPECT_FALSE(config.mutatorAssists()); +} + +TEST(GCSchedulerConfigTest, EnabledMutatorAssists) { + gcScheduler::GCSchedulerConfig config; + config.setMutatorAssists(true); + EXPECT_TRUE(config.mutatorAssists()); + config.autoTune = false; + EXPECT_TRUE(config.mutatorAssists()); + config.autoTune = true; + ASSERT_TRUE(config.mutatorAssists()); + config.maxHeapBytes = 1024 * 1024 * 1024; + EXPECT_TRUE(config.mutatorAssists()); + config.maxHeapBytes = std::numeric_limits::max(); + EXPECT_TRUE(config.mutatorAssists()); +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp index 704e526b674..1d7f30e731c 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp @@ -17,17 +17,35 @@ namespace kotlin::gcScheduler::internal { class HeapGrowthController { public: - explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : - config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) {} + enum class MemoryBoundary { + // Memory usage is low. + kNone, + // Memory usage is high, GC should be triggered. + kTrigger, + // Memory usage is critical, GC is running behind the mutators. Mutators should pause. + kTarget, + }; - // Called by the mutators. - // Returns true if needs GC. - bool SetAllocatedBytes(size_t totalAllocatedBytes) noexcept { return totalAllocatedBytes >= targetHeapBytes_; } + explicit HeapGrowthController(GCSchedulerConfig& config) noexcept : + config_(config), targetHeapBytes_(config.targetHeapBytes.load(std::memory_order_relaxed)) { + triggerHeapBytes_ = targetHeapBytes_ * config_.heapTriggerCoefficient.load(std::memory_order_relaxed); + } + + // Can be called by any thread. + MemoryBoundary boundaryForHeapSize(size_t totalAllocatedBytes) noexcept { + if (totalAllocatedBytes >= targetHeapBytes_) { + return config_.mutatorAssists() ? MemoryBoundary::kTarget : MemoryBoundary::kTrigger; + } else if (totalAllocatedBytes >= triggerHeapBytes_) { + return MemoryBoundary::kTrigger; + } else { + return MemoryBoundary::kNone; + } + } // Called by the GC thread. - void UpdateAliveSetBytes(size_t bytes) noexcept { + void updateBoundaries(size_t aliveBytes) noexcept { if (config_.autoTune.load()) { - double targetHeapBytes = static_cast(bytes) / config_.targetHeapUtilization; + double targetHeapBytes = static_cast(aliveBytes) / config_.targetHeapUtilization; if (!std::isfinite(targetHeapBytes)) { // This shouldn't happen in practice: targetHeapUtilization is in (0, 1]. But in case it does, don't touch anything. return; @@ -35,6 +53,7 @@ public: double minHeapBytes = static_cast(config_.minHeapBytes.load(std::memory_order_relaxed)); double maxHeapBytes = static_cast(config_.maxHeapBytes.load(std::memory_order_relaxed)); targetHeapBytes = std::min(std::max(targetHeapBytes, minHeapBytes), maxHeapBytes); + triggerHeapBytes_ = static_cast(targetHeapBytes * config_.heapTriggerCoefficient.load(std::memory_order_relaxed)); config_.targetHeapBytes.store(static_cast(targetHeapBytes), std::memory_order_relaxed); targetHeapBytes_ = static_cast(targetHeapBytes); } else { @@ -42,14 +61,13 @@ public: } } - void OnPerformFullGC() noexcept { - // TODO: Need to protect against mutators that can overrun the GC thread. - targetHeapBytes_ = std::numeric_limits::max(); - } + size_t targetHeapBytes() const noexcept { return targetHeapBytes_; } + size_t triggerHeapBytes() const noexcept { return triggerHeapBytes_; } private: GCSchedulerConfig& config_; size_t targetHeapBytes_ = 0; + size_t triggerHeapBytes_ = 0; }; } // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthControllerTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthControllerTest.cpp new file mode 100644 index 00000000000..b0be2e56860 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthControllerTest.cpp @@ -0,0 +1,107 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "HeapGrowthController.hpp" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace kotlin; + +using gcScheduler::internal::HeapGrowthController; +using MemoryBoundary = HeapGrowthController::MemoryBoundary; + +TEST(HeapGrowthControllerTest, BoundariesWithAssists) { + gcScheduler::GCSchedulerConfig config; + config.targetHeapBytes = 10; + config.heapTriggerCoefficient = 0.7; + ASSERT_TRUE(config.mutatorAssists()); + + HeapGrowthController controller(config); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + for (size_t i = 0; i < 12; ++i) { + auto expected = i < 7 ? MemoryBoundary::kNone : i < 10 ? MemoryBoundary::kTrigger : MemoryBoundary::kTarget; + EXPECT_THAT(controller.boundaryForHeapSize(i), expected); + } +} + +TEST(HeapGrowthControllerTest, BoundariesWithoutAssists) { + gcScheduler::GCSchedulerConfig config; + config.targetHeapBytes = 10; + config.heapTriggerCoefficient = 0.7; + config.setMutatorAssists(false); + + HeapGrowthController controller(config); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + for (size_t i = 0; i < 12; ++i) { + auto expected = i < 7 ? MemoryBoundary::kNone : MemoryBoundary::kTrigger; + EXPECT_THAT(controller.boundaryForHeapSize(i), expected); + } +} + +TEST(HeapGrowthControllerTest, NoTune) { + gcScheduler::GCSchedulerConfig config; + config.autoTune = false; + config.targetHeapBytes = 10; + config.heapTriggerCoefficient = 0.7; + + HeapGrowthController controller(config); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + controller.updateBoundaries(0); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + controller.updateBoundaries(10); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + controller.updateBoundaries(10000); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + controller.updateBoundaries(std::numeric_limits::max()); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); +} + +TEST(HeapGrowthControllerTest, Tune) { + gcScheduler::GCSchedulerConfig config; + config.autoTune = true; + config.minHeapBytes = 10; + config.maxHeapBytes = 1000; + config.targetHeapBytes = 100; + config.heapTriggerCoefficient = 0.7; + config.targetHeapUtilization = 0.5; + + HeapGrowthController controller(config); + EXPECT_THAT(controller.targetHeapBytes(), 100); + EXPECT_THAT(controller.triggerHeapBytes(), 70); + + controller.updateBoundaries(0); + EXPECT_THAT(controller.targetHeapBytes(), 10); + EXPECT_THAT(controller.triggerHeapBytes(), 7); + + controller.updateBoundaries(10); + EXPECT_THAT(controller.targetHeapBytes(), 20); + EXPECT_THAT(controller.triggerHeapBytes(), 14); + + controller.updateBoundaries(100); + EXPECT_THAT(controller.targetHeapBytes(), 200); + EXPECT_THAT(controller.triggerHeapBytes(), 140); + + controller.updateBoundaries(10000); + EXPECT_THAT(controller.targetHeapBytes(), 1000); + EXPECT_THAT(controller.triggerHeapBytes(), 700); + + controller.updateBoundaries(std::numeric_limits::max()); + EXPECT_THAT(controller.targetHeapBytes(), 1000); + EXPECT_THAT(controller.triggerHeapBytes(), 700); +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp new file mode 100644 index 00000000000..9b5df66ed39 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.cpp @@ -0,0 +1,73 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "MutatorAssists.hpp" + +#include "CallsChecker.hpp" +#include "KAssert.h" +#include "Logging.hpp" +#include "ThreadData.hpp" + +using namespace kotlin; + +void gcScheduler::internal::MutatorAssists::ThreadData::safePoint() noexcept { + Epoch epoch = owner_.assistsEpoch_.load(std::memory_order_acquire); + auto noNeedToWait = [this, epoch] { return owner_.completedEpoch_.load(std::memory_order_acquire) >= epoch; }; + if (noNeedToWait()) return; + auto prevState = thread_.suspensionData().setStateNoSafePoint(ThreadState::kNative); + RuntimeAssert(prevState == ThreadState::kRunnable, "Expected runnable state"); + startedWaiting_.store(epoch * 2, std::memory_order_release); + { + std::unique_lock guard(owner_.m_); + RuntimeLogDebug({kTagGC}, "Thread is assisting for epoch %" PRId64, epoch); + owner_.cv_.wait(guard, noNeedToWait); + RuntimeLogDebug({kTagGC}, "Thread has assisted for epoch %" PRId64, epoch); + } + startedWaiting_.store(epoch * 2 + 1, std::memory_order_release); + // Not doing a safe point. We're a safe point. + prevState = thread_.suspensionData().setStateNoSafePoint(ThreadState::kRunnable); + RuntimeAssert(prevState == ThreadState::kNative, "Expected native state"); +} + +bool gcScheduler::internal::MutatorAssists::ThreadData::completedEpoch(Epoch epoch) const noexcept { + auto [waitingEpoch, isWaiting] = startedWaiting(std::memory_order_acquire); + if (waitingEpoch > epoch) + // Waiting for an epoch bigger than `epoch` => `epoch` is done here. + return true; + return !isWaiting; +} + +void gcScheduler::internal::MutatorAssists::requestAssists(Epoch epoch) noexcept { + RuntimeLogDebug({kTagGC}, "Requesting assists for epoch %" PRId64, epoch); + CallsCheckerIgnoreGuard guard; + std::unique_lock lockGuard(m_); + if (assistsEpoch_.load(std::memory_order_relaxed) >= epoch) { + return; + } + assistsEpoch_.store(epoch, std::memory_order_release); + if (completedEpoch_.load(std::memory_order_relaxed) >= epoch) { + return; + } + + RuntimeLogDebug({kTagGC}, "Enabling assists for epoch %" PRId64, epoch); + if (!safePointActivator_) { + safePointActivator_ = mm::SafePointActivator(); + } +} + +void gcScheduler::internal::MutatorAssists::markEpochCompleted(Epoch epoch) noexcept { + RuntimeLogDebug({kTagGC}, "Disabling assists for epoch %" PRId64, epoch); + { + std::unique_lock guard(m_); + auto previousEpoch = completedEpoch_.exchange(epoch, std::memory_order_release); + RuntimeAssert( + previousEpoch == epoch - 1, "Epochs must be increasing by 1. Previous: %" PRId64 ". Setting: %" PRId64, previousEpoch, + epoch); + if (epoch >= assistsEpoch_.load(std::memory_order_relaxed)) { + safePointActivator_ = std::nullopt; + } + } + cv_.notify_all(); +} diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.hpp new file mode 100644 index 00000000000..248e180bbca --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssists.hpp @@ -0,0 +1,107 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "SafePoint.hpp" +#include "ThreadRegistry.hpp" +#include "Utils.hpp" + +#if KONAN_WINDOWS +#include "ConditionVariable.hpp" +#else +#include +#endif + +namespace kotlin::gcScheduler::internal { + +/** + * Coordinating mutator assistance to the GC. + * + * Currently assisting only by pausing threads. So assisting means not creating + * more work for the GC thread. + * + * Threads (both mutators and any other) can call `requestAssists(epoch)` for any + * `epoch` at any time. + * + * If the current GC epoch is greater than `epoch`, the mutators should ignore + * the request to assist. + * + * Otherwise the mutators must wait in the native state + * until the GC thread calls `completeEpoch(epoch)` for epoch >= `epoch`. + * + * The GC thread shall call `completeEpoch(epoch)` once it is done with the epoch, + * and it shall wait for all mutators assisting `epoch` (or lower) to continue. + */ +class MutatorAssists : private Pinned { +public: + using Epoch = int64_t; + + class ThreadData : private Pinned { + public: + ThreadData(MutatorAssists& owner, mm::ThreadData& thread) noexcept : owner_(owner), thread_(thread) {} + + void safePoint() noexcept; + + std::pair startedWaiting(std::memory_order ordering) const noexcept { + auto value = startedWaiting_.load(ordering); + auto waitingEpoch = value / 2; + bool isWaiting = value % 2 == 0; + return {waitingEpoch, isWaiting}; + } + + private: + friend class MutatorAssists; + + bool completedEpoch(Epoch epoch) const noexcept; + + MutatorAssists& owner_; + mm::ThreadData& thread_; + // Contains epoch * 2. The lower bit is 1, if completed waiting. + std::atomic startedWaiting_ = 1; + }; + + // Request all `kRunnable` mutators to start assisting GC for epoch `epoch`. + // Can be called multiple times per `epoch`, and `epoch` may point to the past. + void requestAssists(Epoch epoch) noexcept; + + // Should be called by GC, when it completed epoch `epoch`. + // The call blocks waiting for all assisting mutators to finish assisting `epoch`. + // `f` is a map from `mm::ThreadData&` to `MutatorAssists::ThreadData&`. + // Can only be called once per `epoch`, and `epoch` must be increasing + // by exactly 1 every time. + template + void completeEpoch(Epoch epoch, F&& f) noexcept { + markEpochCompleted(epoch); + mm::ThreadRegistry::Instance().waitAllThreads( + [f = std::forward(f), epoch](mm::ThreadData& threadData) noexcept { return f(threadData).completedEpoch(epoch); }); + } + + Epoch assistsRequested(std::memory_order order) noexcept { return assistsEpoch_.load(order); } + +private: + void markEpochCompleted(Epoch epoch) noexcept; + + std::atomic assistsEpoch_ = 0; + std::atomic completedEpoch_ = 0; + std::optional safePointActivator_; + std::mutex m_; +#if KONAN_WINDOWS + // winpthreads being weird. Using this implementation of condvar means that assisting mutators will spin for the entire duration of the + // GC. This is fine: reaching assisting state should be rare and this state exists to ward of "memory leaks", and additionally assists + // can be disabled. + ConditionVariableSpin cv_; +#else + std::condition_variable cv_; +#endif +}; + +} // namespace kotlin::gcScheduler::internal diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssistsTest.cpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssistsTest.cpp new file mode 100644 index 00000000000..859e8424d38 --- /dev/null +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/MutatorAssistsTest.cpp @@ -0,0 +1,430 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "MutatorAssists.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "SafePoint.hpp" +#include "TestSupport.hpp" +#include "std_support/Map.hpp" + +using namespace kotlin; + +using gcScheduler::internal::MutatorAssists; +using Epoch = MutatorAssists::Epoch; + +class MutatorAssistsTest : public ::testing::Test { +public: + class Mutator { + public: + template + Mutator(MutatorAssistsTest& owner, F&& f) noexcept : + owner_(owner), thread_([f = std::forward(f), this]() noexcept { + ScopedMemoryInit memory; + { + std::unique_lock guard(initializedMutex_); + threadData_ = memory.memoryState()->GetThreadData(); + assists_.emplace(owner_.assists_, *threadData_); + } + owner_.registerMutator(*this); + initialized_.notify_one(); + f(*this); + }) { + std::unique_lock guard(initializedMutex_); + initialized_.wait(guard, [this] { return threadData_ && assists_.has_value(); }); + } + + ~Mutator() { + thread_.join(); + owner_.unregisterMutator(*this); + } + + mm::ThreadData& threadData() noexcept { return *threadData_; } + MutatorAssists::ThreadData& assists() noexcept { return *assists_; } + + private: + friend MutatorAssistsTest; + + MutatorAssistsTest& owner_; + std::condition_variable initialized_; + std::mutex initializedMutex_; + mm::ThreadData* threadData_; + std::optional assists_; + ScopedThread thread_; + }; + + void requestAssists(Epoch epoch) noexcept { assists_.requestAssists(epoch); } + + void completeEpoch(Epoch epoch) noexcept { + assists_.completeEpoch(epoch, [this](mm::ThreadData& threadData) noexcept -> MutatorAssists::ThreadData& { + return getMutator(threadData).assists(); + }); + } + + void safePoint() noexcept { + if (!mm::test_support::safePointsAreActive()) return; + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + getMutator(*threadData).assists().safePoint(); + } + +private: + void registerMutator(Mutator& mutator) noexcept { + std::unique_lock guard(mutatorMapMutex_); + auto [_, inserted] = mutatorMap_.insert(std::make_pair(&mutator.threadData(), &mutator)); + RuntimeAssert(inserted, "Mutator was already inserted"); + } + + void unregisterMutator(Mutator& mutator) noexcept { + std::unique_lock guard(mutatorMapMutex_); + auto count = mutatorMap_.erase(&mutator.threadData()); + RuntimeAssert(count == 1, "Mutator must be in the map"); + } + + Mutator& getMutator(mm::ThreadData& threadData) noexcept { + std::shared_lock guard(mutatorMapMutex_); + auto it = mutatorMap_.find(&threadData); + RuntimeAssert(it != mutatorMap_.end(), "Mutator must be in the map"); + return *it->second; + } + + MutatorAssists assists_; + RWSpinLock mutatorMapMutex_; + std_support::map mutatorMap_; +}; + +TEST_F(MutatorAssistsTest, EnableSafePointsWhenRequestingAssists) { + ASSERT_FALSE(mm::test_support::safePointsAreActive()); + requestAssists(1); + EXPECT_TRUE(mm::test_support::safePointsAreActive()); + completeEpoch(1); + EXPECT_FALSE(mm::test_support::safePointsAreActive()); +} + +TEST_F(MutatorAssistsTest, EnableSafePointsWithNestedRequest) { + ASSERT_FALSE(mm::test_support::safePointsAreActive()); + requestAssists(1); + ASSERT_TRUE(mm::test_support::safePointsAreActive()); + requestAssists(2); + EXPECT_TRUE(mm::test_support::safePointsAreActive()); + completeEpoch(1); + EXPECT_TRUE(mm::test_support::safePointsAreActive()); + completeEpoch(2); + EXPECT_FALSE(mm::test_support::safePointsAreActive()); +} + +TEST_F(MutatorAssistsTest, StressEnableSafePointsByMutators) { + constexpr Epoch epochsCount = 4; + std::array, epochsCount> enabled = {false}; + std::atomic canStart = false; + std::atomic canStop = false; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&, i](Mutator&) noexcept { + while (!canStart.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + requestAssists((i % epochsCount) + 1); + enabled[i % epochsCount].store(true, std::memory_order_relaxed); + while (!canStop.load(std::memory_order_relaxed)) { + safePoint(); + } + })); + } + + ASSERT_FALSE(mm::test_support::safePointsAreActive()); + canStart.store(true, std::memory_order_relaxed); + for (Epoch i = 0; i < epochsCount; ++i) { + while (!enabled[i].load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + EXPECT_TRUE(mm::test_support::safePointsAreActive()); + completeEpoch(i + 1); + } + EXPECT_FALSE(mm::test_support::safePointsAreActive()); + canStop.store(true, std::memory_order_relaxed); +} + +TEST_F(MutatorAssistsTest, Assist) { + constexpr Epoch epochsCount = 4; + std::array, epochsCount> canStart = {false}; + std::array, epochsCount> started = {0}; + std::array, epochsCount> finished = {0}; + std::atomic gcCompleted = 0; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&](Mutator&) noexcept { + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + while (!canStart[epoch].load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + started[epoch].fetch_add(1, std::memory_order_relaxed); + safePoint(); + EXPECT_THAT(gcCompleted.load(std::memory_order_relaxed), epoch); + finished[epoch].fetch_add(1, std::memory_order_relaxed); + } + })); + } + for (auto& m : mutators) { + EXPECT_THAT(m->threadData().state(), ThreadState::kRunnable); + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + requestAssists(epoch + 1); + canStart[epoch].store(true, std::memory_order_relaxed); + while (started[epoch].load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + while (!std::all_of(mutators.begin(), mutators.end(), [epoch](auto& m) noexcept { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + return waitingEpoch == epoch + 1 && waiting; + })) { + std::this_thread::yield(); + } + gcCompleted.store(epoch, std::memory_order_relaxed); + for (auto& m : mutators) { + EXPECT_THAT(m->threadData().state(), ThreadState::kNative); + // And already checked that all of them have started waiting for epoch. + } + completeEpoch(epoch + 1); + while (finished[epoch].load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + if (epoch != epochsCount - 1) { + EXPECT_THAT(m->threadData().state(), ThreadState::kRunnable); + } + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, epoch + 1); + EXPECT_FALSE(waiting); + } + } +} + +TEST_F(MutatorAssistsTest, AssistNoSync) { + constexpr Epoch epochsCount = 10000; + std::atomic canStop = false; + std::atomic finished = 0; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&](Mutator&) noexcept { + while (!canStop.load(std::memory_order_relaxed)) { + safePoint(); + std::this_thread::yield(); + } + finished.fetch_add(1, std::memory_order_relaxed); + })); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + requestAssists(epoch + 1); + completeEpoch(epoch + 1); + } + canStop.store(true, std::memory_order_relaxed); + while (finished.load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, testing::Le(epochsCount)); + EXPECT_FALSE(waiting); + } +} + +TEST_F(MutatorAssistsTest, AssistWithNativeMutators) { + constexpr Epoch epochsCount = 10000; + std::atomic canStop = false; + std::atomic finished = 0; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&, i](Mutator&) noexcept { + if (i % 2 == 0) { + ThreadStateGuard guard(ThreadState::kNative); + while (!canStop.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + } else { + while (!canStop.load(std::memory_order_relaxed)) { + safePoint(); + std::this_thread::yield(); + } + } + finished.fetch_add(1, std::memory_order_relaxed); + })); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + requestAssists(epoch + 1); + completeEpoch(epoch + 1); + } + canStop.store(true, std::memory_order_relaxed); + while (finished.load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, testing::Le(epochsCount)); + EXPECT_FALSE(waiting); + } +} + +TEST_F(MutatorAssistsTest, AssistNoRequests) { + constexpr Epoch epochsCount = 10000; + std::atomic canStart = false; + std::atomic canStop = false; + std::atomic started = 0; + std::atomic finished = 0; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&](Mutator&) noexcept { + while (!canStart.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + started.fetch_add(1, std::memory_order_relaxed); + while (!canStop.load(std::memory_order_relaxed)) { + safePoint(); + std::this_thread::yield(); + } + finished.fetch_add(1, std::memory_order_relaxed); + })); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + canStart.store(true, std::memory_order_relaxed); + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + completeEpoch(epoch + 1); + } + canStop.store(true, std::memory_order_relaxed); + while (finished.load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } +} + +TEST_F(MutatorAssistsTest, AssistRequestsByMutators) { + constexpr Epoch epochsCount = 100; + std::atomic canStart = false; + std::atomic canStop = false; + std::atomic started = 0; + std::atomic finished = 0; + std::atomic currentEpoch = 0; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&, i](Mutator&) noexcept { + while (!canStart.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + started.fetch_add(1, std::memory_order_relaxed); + while (!canStop.load(std::memory_order_relaxed)) { + if (i % 2 != 0) { + auto epoch = currentEpoch.load(std::memory_order_relaxed); + requestAssists(epoch + 1); + } + safePoint(); + std::this_thread::yield(); + } + finished.fetch_add(1, std::memory_order_relaxed); + })); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + canStart.store(true, std::memory_order_relaxed); + for (Epoch epoch = 0; epoch < epochsCount; ++epoch) { + currentEpoch.store(epoch, std::memory_order_relaxed); + completeEpoch(epoch + 1); + } + canStop.store(true, std::memory_order_relaxed); + while (finished.load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, testing::Le(epochsCount)); + EXPECT_FALSE(waiting); + } +} + +TEST_F(MutatorAssistsTest, AssistRequestsByMutatorsIntoTheFuture) { + constexpr Epoch epochsCount = 100; + std::atomic canStart = false; + std::atomic canStop = false; + std::atomic started = 0; + std::atomic finished = 0; + std::mutex mutexEpoch; + Epoch scheduledEpoch = 0; + Epoch currentEpoch = 0; + auto scheduleGC = [&]() noexcept -> Epoch { + std::unique_lock guard(mutexEpoch); + if (scheduledEpoch > currentEpoch) return scheduledEpoch; + scheduledEpoch = currentEpoch + 1; + return scheduledEpoch; + }; + std_support::vector> mutators; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators.emplace_back(std_support::make_unique(*this, [&, i](Mutator&) noexcept { + while (!canStart.load(std::memory_order_relaxed)) { + std::this_thread::yield(); + } + started.fetch_add(1, std::memory_order_relaxed); + while (!canStop.load(std::memory_order_relaxed)) { + if (i % 2 != 0) { + auto epoch = scheduleGC(); + requestAssists(epoch); + } + safePoint(); + std::this_thread::yield(); + } + finished.fetch_add(1, std::memory_order_relaxed); + })); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, 0); + EXPECT_FALSE(waiting); + } + canStart.store(true, std::memory_order_relaxed); + for (Epoch epoch = 1; epoch <= epochsCount; ++epoch) { + { + std::unique_lock guard(mutexEpoch); + currentEpoch = epoch; + EXPECT_THAT(currentEpoch, testing::Ge(scheduledEpoch)); + } + completeEpoch(epoch); + } + canStop.store(true, std::memory_order_relaxed); + completeEpoch(epochsCount + 1); // The last GC. + while (finished.load(std::memory_order_relaxed) < mutators.size()) { + std::this_thread::yield(); + } + for (auto& m : mutators) { + auto [waitingEpoch, waiting] = m->assists().startedWaiting(std::memory_order_relaxed); + EXPECT_THAT(waitingEpoch, testing::Le(epochsCount + 1)); + EXPECT_FALSE(waiting); + } +} diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp index fd34aff9283..664c0af1f96 100644 --- a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp +++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.cpp @@ -5,8 +5,46 @@ #include "GCSchedulerImpl.hpp" +#include "CallsChecker.hpp" +#include "GC.hpp" +#include "GlobalData.hpp" + using namespace kotlin; -gcScheduler::GCScheduler::GCScheduler() noexcept : gcData_(std_support::make_unique()) {} +gcScheduler::GCScheduler::ThreadData::ThreadData(gcScheduler::GCScheduler&, mm::ThreadData&) noexcept : + impl_(std_support::make_unique()) {} -ALWAYS_INLINE void gcScheduler::GCScheduler::safePoint() noexcept {} +gcScheduler::GCScheduler::ThreadData::~ThreadData() = default; + +gcScheduler::GCScheduler::GCScheduler() noexcept : impl_(std_support::make_unique()) {} + +gcScheduler::GCScheduler::~GCScheduler() = default; + +ALWAYS_INLINE void gcScheduler::GCScheduler::ThreadData::safePoint() noexcept {} + +void gcScheduler::GCScheduler::schedule() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + mm::GlobalData::Instance().gc().Schedule(); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinished() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + CallsCheckerIgnoreGuard guard; + auto& gc = mm::GlobalData::Instance().gc(); + auto epoch = gc.Schedule(); + NativeOrUnregisteredThreadGuard stateGuard(/* reentrant = */ true); + gc.WaitFinished(epoch); +} + +void gcScheduler::GCScheduler::scheduleAndWaitFinalized() noexcept { + RuntimeLogInfo({kTagGC}, "Scheduling GC manually"); + CallsCheckerIgnoreGuard guard; + auto& gc = mm::GlobalData::Instance().gc(); + auto epoch = gc.Schedule(); + NativeOrUnregisteredThreadGuard stateGuard(/* reentrant = */ true); + gc.WaitFinalizers(epoch); +} + +ALWAYS_INLINE void gcScheduler::GCScheduler::setAllocatedBytes(size_t bytes) noexcept {} +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCStart() noexcept {} +ALWAYS_INLINE void gcScheduler::GCScheduler::onGCFinish(int64_t epoch, size_t aliveBytes) noexcept {} diff --git a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp index 4f1d373b99b..9bada94c02e 100644 --- a/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/manual/cpp/GCSchedulerImpl.hpp @@ -9,15 +9,13 @@ #include "Logging.hpp" -namespace kotlin::gcScheduler::internal { +namespace kotlin::gcScheduler { -class GCSchedulerDataManual : public GCSchedulerData { +class GCScheduler::Impl : private Pinned { public: - GCSchedulerDataManual() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); } - - void OnPerformFullGC() noexcept override {} - void UpdateAliveSetBytes(size_t bytes) noexcept override {} - void SetAllocatedBytes(size_t bytes) noexcept override {} + Impl() noexcept { RuntimeLogInfo({kTagGC}, "Manual GC scheduler initialized"); } }; -} // namespace kotlin::gcScheduler::internal +class GCScheduler::ThreadData::Impl : private Pinned {}; + +} // namespace kotlin::gcScheduler diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt index 1136e2b7345..98987254792 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GC.kt @@ -177,7 +177,6 @@ public object GC { * When Kotlin code is not allocating enough to trigger GC, the GC scheduler uses timer to drive collection. * Timer-triggered collection will happen roughly in [regularGCInterval] .. 2 * [regularGCInterval] since * any previous collection. - * Unused with on-safepoints GC scheduler. * * Default: 10 seconds * @@ -193,8 +192,9 @@ public object GC { } /** - * Total amount of heap available for Kotlin objects. When Kotlin objects overflow this heap, - * the garbage collection is requested. Automatically adjusts when [autotune] is true: + * Total amount of heap available for Kotlin objects. The GC tries to schedule execution + * so that Kotlin heap doesn't overflow this heap. + * Automatically adjusts when [autotune] is true: * after each collection the [targetHeapBytes] is set to heapBytes / [targetHeapUtilization] and * capped between [minHeapBytes] and [maxHeapBytes], where heapBytes is heap usage after the garbage * is collected. @@ -265,6 +265,30 @@ public object GC { setMaxHeapBytes(value) } + /** + * The GC is scheduled when Kotlin heap overflows [heapTriggerCoefficient] * [targetHeapBytes]. + * + * Default: 0.9 + * + * @throws [IllegalArgumentException] when value is outside (0, 1] interval. + */ + var heapTriggerCoefficient: Double + get() = getHeapTriggerCoefficient() + set(value) { + require(value > 0 && value <= 1) { "heapTriggerCoefficient must be in (0, 1] interval: $value" } + setHeapTriggerCoefficient(value) + } + + /** + * If true, the GC will pause Kotlin threads when Kotlin heap overflows [targetHeapBytes] + * and will resume them only after current GC is done. + * + * Default: true, unless [autotune] is false or [maxHeapBytes] is less than [Long.MAX_VALUE]. + */ + var pauseOnTargetHeapOverflow: Boolean + get() = getPauseOnTargetHeapOverflow() + set(value) = setPauseOnTargetHeapOverflow(value) + /** * Deprecated and unused. Always returns null. * @@ -356,4 +380,16 @@ public object GC { @GCUnsafeCall("Kotlin_native_internal_GC_setMaxHeapBytes") private external fun setMaxHeapBytes(value: Long) + + @GCUnsafeCall("Kotlin_native_internal_GC_getHeapTriggerCoefficient") + private external fun getHeapTriggerCoefficient(): Double + + @GCUnsafeCall("Kotlin_native_internal_GC_setHeapTriggerCoefficient") + private external fun setHeapTriggerCoefficient(value: Double) + + @GCUnsafeCall("Kotlin_native_internal_GC_getPauseOnTargetHeapOverflow") + private external fun getPauseOnTargetHeapOverflow(): Boolean + + @GCUnsafeCall("Kotlin_native_internal_GC_setPauseOnTargetHeapOverflow") + private external fun setPauseOnTargetHeapOverflow(value: Boolean) } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 51a92f94a04..d4aa2638787 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -105,7 +105,7 @@ extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) { auto* node = mm::FromMemoryState(state); if (destroyRuntime) { ThreadStateGuard guard(state, ThreadState::kRunnable); - node->Get()->gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); // TODO: Why not just destruct `GC` object and its thread data counterpart entirely? mm::GlobalData::Instance().gc().StopFinalizerThreadIfRunning(); } @@ -300,15 +300,11 @@ extern "C" RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) { } extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) { - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - AssertThreadState(threadData, ThreadState::kRunnable); - threadData->gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); } extern "C" void Kotlin_native_internal_GC_schedule(ObjHeader*) { - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - AssertThreadState(threadData, ThreadState::kRunnable); - threadData->gc().Schedule(); + mm::GlobalData::Instance().gcScheduler().schedule(); } extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) { @@ -421,6 +417,23 @@ extern "C" void Kotlin_native_internal_GC_setMinHeapBytes(ObjHeader*, KLong valu mm::GlobalData::Instance().gcScheduler().config().minHeapBytes = value; } +extern "C" KDouble Kotlin_native_internal_GC_getHeapTriggerCoefficient(ObjHeader*) { + return mm::GlobalData::Instance().gcScheduler().config().heapTriggerCoefficient.load(); +} + +extern "C" void Kotlin_native_internal_GC_setHeapTriggerCoefficient(ObjHeader*, KDouble value) { + RuntimeAssert(value > 0 && value <= 1, "Must be handled by the caller"); + mm::GlobalData::Instance().gcScheduler().config().heapTriggerCoefficient = value; +} + +extern "C" KBoolean Kotlin_native_internal_GC_getPauseOnTargetHeapOverflow(ObjHeader*) { + return mm::GlobalData::Instance().gcScheduler().config().mutatorAssists(); +} + +extern "C" void Kotlin_native_internal_GC_setPauseOnTargetHeapOverflow(ObjHeader*, KBoolean value) { + mm::GlobalData::Instance().gcScheduler().config().setMutatorAssists(value); +} + extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) { // TODO: Remove when legacy MM is gone. RETURN_OBJ(nullptr); @@ -453,9 +466,7 @@ extern "C" void Kotlin_Any_share(ObjHeader* thiz) { } extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) { - auto* threadData = memory->GetThreadData(); - AssertThreadState(threadData, ThreadState::kRunnable); - threadData->gc().ScheduleAndWaitFullGCWithFinalizers(); + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); } extern "C" RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) { @@ -623,5 +634,5 @@ RUNTIME_NOTHROW extern "C" void DisposeRegularWeakReferenceImpl(ObjHeader* weakR } void kotlin::OnMemoryAllocation(size_t totalAllocatedBytes) noexcept { - mm::GlobalData::Instance().gcScheduler().gcData().SetAllocatedBytes(totalAllocatedBytes); + mm::GlobalData::Instance().gcScheduler().setAllocatedBytes(totalAllocatedBytes); } diff --git a/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp b/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp index 52202f17b0c..c6bc281d876 100644 --- a/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp +++ b/kotlin-native/runtime/src/mm/cpp/SafePoint.cpp @@ -26,7 +26,7 @@ void safePointActionImpl(mm::ThreadData& threadData) noexcept { RuntimeAssert(!recursion, "Recursive safepoint"); AutoReset guard(&recursion, true); - mm::GlobalData::Instance().gcScheduler().safePoint(); + threadData.gcScheduler().safePoint(); threadData.gc().safePoint(); threadData.suspensionData().suspendIfRequested(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 8a59cdebfd7..8655566dd3a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -32,6 +32,7 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), + gcScheduler_(GlobalData::Instance().gcScheduler(), *this), gc_(GlobalData::Instance().gc(), *this), suspensionData_(ThreadState::kNative, *this) {} @@ -53,6 +54,8 @@ public: std_support::vector>& initializingSingletons() noexcept { return initializingSingletons_; } + gcScheduler::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; } + gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -76,6 +79,7 @@ private: ThreadLocalStorage tls_; SpecialRefRegistry::ThreadQueue specialRefRegistry_; ShadowStack shadowStack_; + gcScheduler::GCScheduler::ThreadData gcScheduler_; gc::GC::ThreadData gc_; std_support::vector> initializingSingletons_; ThreadSuspensionData suspensionData_; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp index 375aa35ada6..63b3842c7c4 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp @@ -35,6 +35,7 @@ public: ThreadState state() noexcept { return state_; } ThreadState setState(ThreadState newState) noexcept; + ThreadState setStateNoSafePoint(ThreadState newState) noexcept { return state_.exchange(newState, std::memory_order_acq_rel); } bool suspended() noexcept { return suspended_; } bool suspendedOrNative() noexcept { return suspended() || state() == kotlin::ThreadState::kNative; }