From a752d5d0003aa2e221575901211d3383c46f9a6a Mon Sep 17 00:00:00 2001 From: Troels Bjerre Lund Date: Mon, 25 Sep 2023 13:31:16 +0200 Subject: [PATCH] [K/N] Estimate allocator overhead This increases the boundary for the next GC scheduling by how much is estimated is currently being wasted by the allocator due to paging. This is necessary, since the gc scheduling boundary is set based on the size of the live set at the last GC, while the current memory use is only tracked in whole allocator pages. This patch avoids some of the situations where the GC would be triggered again immediately upon completion, which happens with many sparsely filled pages. --- .../runtime/src/alloc/common/cpp/Allocator.hpp | 2 ++ .../runtime/src/alloc/custom/cpp/AllocatorImpl.cpp | 5 +++++ kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp | 10 ++++++++++ kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp | 1 + .../runtime/src/alloc/custom/cpp/PageStore.hpp | 7 +++++++ .../runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp | 4 ++++ .../runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 4 +++- .../runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp | 4 +++- .../src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp | 1 + .../gcScheduler/common/cpp/HeapGrowthController.hpp | 4 ++++ 10 files changed, 40 insertions(+), 2 deletions(-) diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp index 914966494f6..c1c7cd80fcc 100644 --- a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp @@ -51,6 +51,8 @@ public: // TODO: Move into AllocatorTestSupport.hpp void clearForTests() noexcept; + size_t estimateOverheadPerThread() noexcept; + private: std::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp index b3cebf66eb4..fc6c61b3122 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp @@ -6,6 +6,7 @@ #include "AllocatorImpl.hpp" #include "GCApi.hpp" +#include "Heap.hpp" using namespace kotlin; @@ -50,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept { impl_->heap().ClearForTests(); } +size_t alloc::Allocator::estimateOverheadPerThread() noexcept { + return impl_->heap().EstimateOverheadPerThread(); +} + void alloc::initObjectPool() noexcept {} void alloc::compactObjectPoolInCurrentThread() noexcept {} diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp index 371863e5ac0..9fc1695fd56 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp @@ -116,6 +116,16 @@ std::vector Heap::GetAllocatedObjects() noexcept { return unfinalized; } +size_t Heap::EstimateOverheadPerThread() noexcept { + size_t overHeadPerThread = 0; + if (!nextFitPages_.isEmpty()) overHeadPerThread += NEXT_FIT_PAGE_SIZE; + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + if (!fixedBlockPages_[blockSize].isEmpty()) overHeadPerThread += FIXED_BLOCK_PAGE_SIZE; + } + if (!extraObjectPages_.isEmpty()) overHeadPerThread += EXTRA_OBJECT_PAGE_SIZE; + return overHeadPerThread; +} + void Heap::ClearForTests() noexcept { for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { fixedBlockPages_[blockSize].ClearForTests(); diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp index 02087d74118..30be860272e 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp @@ -39,6 +39,7 @@ public: void AddToFinalizerQueue(FinalizerQueue queue) noexcept; FinalizerQueue ExtractFinalizerQueue() noexcept; + size_t EstimateOverheadPerThread() noexcept; // Test method std::vector GetAllocatedObjects() noexcept; diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp index 8c04500adac..87050d1f414 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp @@ -78,6 +78,13 @@ public: return page; } + bool isEmpty() noexcept { + return empty_.isEmpty() && + ready_.isEmpty() && + used_.isEmpty() && + unswept_.isEmpty(); + } + ~PageStore() noexcept { T* page; while ((page = empty_.Pop())) page->Destroy(); diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp index 7e478e8efdc..fecd9199bdb 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp @@ -51,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept { impl_->objectFactory().ClearForTests(); } +size_t alloc::Allocator::estimateOverheadPerThread() noexcept { + return 0; +} + gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept { return ObjectFactoryImpl::NodeRef::From(object).ObjectData(); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index d3e0b990866..eb03de096a8 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -176,8 +176,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // By this point all the alive heap must be marked. // All the mutations (incl. allocations) after this method will be subject for the next GC. // This should really be done by each individual thread while waiting + int threadCount = 0; for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { thread.allocator().prepareForGC(); + ++threadCount; } allocator_.prepareForGC(); @@ -206,7 +208,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { } finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index d1e235d989a..03f722d02b1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -71,8 +71,10 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); // This should really be done by each individual thread while waiting + int threadCount = 0; for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { thread.allocator().prepareForGC(); + ++threadCount; } allocator_.prepareForGC(); @@ -96,7 +98,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes()); + scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread()); resumeTheWorld(gcHandle); diff --git a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp index 0d8940e45de..a42173841a1 100644 --- a/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp +++ b/kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp @@ -64,6 +64,7 @@ public: void onGCStart() noexcept { regularIntervalPacer_.OnPerformFullGC(); + heapGrowthController_.unsetBoundaries(); timer_.restart(config_.regularGcInterval()); } diff --git a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp index 1d7f30e731c..ea19ca01d2c 100644 --- a/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp +++ b/kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp @@ -31,6 +31,10 @@ public: triggerHeapBytes_ = targetHeapBytes_ * config_.heapTriggerCoefficient.load(std::memory_order_relaxed); } + void unsetBoundaries() noexcept { + targetHeapBytes_ = triggerHeapBytes_ = std::numeric_limits::max(); + } + // Can be called by any thread. MemoryBoundary boundaryForHeapSize(size_t totalAllocatedBytes) noexcept { if (totalAllocatedBytes >= targetHeapBytes_) {