diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp index 914966494f6..c1c7cd80fcc 100644 --- a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp @@ -51,6 +51,8 @@ public: // TODO: Move into AllocatorTestSupport.hpp void clearForTests() noexcept; + size_t estimateOverheadPerThread() noexcept; + private: std::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp index b3cebf66eb4..fc6c61b3122 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp @@ -6,6 +6,7 @@ #include "AllocatorImpl.hpp" #include "GCApi.hpp" +#include "Heap.hpp" using namespace kotlin; @@ -50,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept { impl_->heap().ClearForTests(); } +size_t alloc::Allocator::estimateOverheadPerThread() noexcept { + return impl_->heap().EstimateOverheadPerThread(); +} + void alloc::initObjectPool() noexcept {} void alloc::compactObjectPoolInCurrentThread() noexcept {} diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp index 371863e5ac0..9fc1695fd56 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp @@ -116,6 +116,16 @@ std::vector Heap::GetAllocatedObjects() noexcept { return unfinalized; } +size_t Heap::EstimateOverheadPerThread() noexcept { + size_t overHeadPerThread = 0; + if (!nextFitPages_.isEmpty()) overHeadPerThread += NEXT_FIT_PAGE_SIZE; + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + if (!fixedBlockPages_[blockSize].isEmpty()) overHeadPerThread += FIXED_BLOCK_PAGE_SIZE; + } + if (!extraObjectPages_.isEmpty()) overHeadPerThread += EXTRA_OBJECT_PAGE_SIZE; + return overHeadPerThread; +} + void Heap::ClearForTests() noexcept { for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { fixedBlockPages_[blockSize].ClearForTests(); diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp index 02087d74118..30be860272e 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp @@ -39,6 +39,7 @@ public: void AddToFinalizerQueue(FinalizerQueue queue) noexcept; FinalizerQueue ExtractFinalizerQueue() noexcept; + size_t EstimateOverheadPerThread() noexcept; // Test method std::vector GetAllocatedObjects() noexcept; diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp index 8c04500adac..87050d1f414 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp @@ -78,6 +78,13 @@ public: return page; } + bool isEmpty() noexcept { + return empty_.isEmpty() && + ready_.isEmpty() && + used_.isEmpty() && + unswept_.isEmpty(); + } + ~PageStore() noexcept { T* page; while ((page = empty_.Pop())) page->Destroy(); diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp index 7e478e8efdc..fecd9199bdb 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp @@ -51,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept { impl_->objectFactory().ClearForTests(); } +size_t alloc::Allocator::estimateOverheadPerThread() noexcept { + return 0; +} + gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept { return ObjectFactoryImpl::NodeRef::From(object).ObjectData(); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index d3e0b990866..eb03de096a8 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -176,8 +176,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // By this point all the alive heap must be marked. // All the mutations (incl. allocations) after this method will be subject for the next GC. // This should really be done by each individual thread while waiting + int threadCount = 0; for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { thread.allocator().prepareForGC(); + ++threadCount; } allocator_.prepareForGC(); @@ -206,7 +208,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { } finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index d1e235d989a..03f722d02b1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -71,8 +71,10 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); // This should really be done by each individual thread while waiting + int threadCount = 0; for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { thread.allocator().prepareForGC(); + ++threadCount; } allocator_.prepareForGC(); @@ -96,7 +98,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread()); resumeTheWorld(gcHandle); diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp index 0d8940e45de..a42173841a1 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -64,6 +64,7 @@ public: void onGCStart() noexcept { regularIntervalPacer_.OnPerformFullGC(); + heapGrowthController_.unsetBoundaries(); timer_.restart(config_.regularGcInterval()); } diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp index 1d7f30e731c..ea19ca01d2c 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp @@ -31,6 +31,10 @@ public: triggerHeapBytes_ = targetHeapBytes_ * config_.heapTriggerCoefficient.load(std::memory_order_relaxed); } + void unsetBoundaries() noexcept { + targetHeapBytes_ = triggerHeapBytes_ = std::numeric_limits::max(); + } + // Can be called by any thread. MemoryBoundary boundaryForHeapSize(size_t totalAllocatedBytes) noexcept { if (totalAllocatedBytes >= targetHeapBytes_) {