From 906c27736fd82d30b09965b64f9c58a5ddecabd7 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Fri, 4 Aug 2023 11:19:53 +0200 Subject: [PATCH] [K/N] CustomAllocator do not lose finalizer queue --- .../runtime/src/custom_alloc/cpp/AtomicStack.hpp | 2 +- .../runtime/src/custom_alloc/cpp/CustomAllocator.cpp | 4 ++++ .../runtime/src/custom_alloc/cpp/CustomAllocator.hpp | 2 ++ kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp | 10 ++++++++++ kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp | 7 +++++++ .../runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 3 ++- .../runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp | 5 +++++ 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp index 6523709e73e..72258478ac9 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp @@ -25,7 +25,7 @@ public: AtomicStack& operator=(AtomicStack&& other) noexcept { // Not using swap idiom, because implementing swap of two atomics requires DCAS or locks. auto newHead = other.stack_.exchange(nullptr, std::memory_order_acq_rel); - stack_.store(newHead, std::memory_order_acq_rel); + stack_.store(newHead, std::memory_order_release); return *this; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 9177d7209b6..257bb68437a 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -46,6 +46,10 @@ CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); } +CustomAllocator::~CustomAllocator() { + heap_.AddToFinalizerQueue(std::move(finalizerQueue_)); +} + ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); size_t allocSize = ObjectAllocatedDataSize(typeInfo); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index c2128317821..34666648699 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -22,6 +22,8 @@ class CustomAllocator { public: explicit CustomAllocator(Heap& heap) noexcept; + ~CustomAllocator(); + ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index 593ae317297..b63385070aa 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -78,6 +78,16 @@ ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexce return extraObjectPages_.GetPage(0, finalizerQueue, concurrentSweepersCount_); } +void Heap::AddToFinalizerQueue(FinalizerQueue queue) noexcept { + std::unique_lock guard(pendingFinalizerQueueMutex_); + pendingFinalizerQueue_.TransferAllFrom(std::move(queue)); +} + +FinalizerQueue Heap::ExtractFinalizerQueue() noexcept { + std::unique_lock guard(pendingFinalizerQueueMutex_); + return std::move(pendingFinalizerQueue_); +} + std_support::vector Heap::GetAllocatedObjects() noexcept { std_support::vector allocated; for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp index 671d4944457..f3ce5fe5172 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp @@ -7,6 +7,7 @@ #define CUSTOM_ALLOC_CPP_HEAP_HPP_ #include +#include #include #include "AtomicStack.hpp" @@ -36,6 +37,9 @@ public: SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept; ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept; + void AddToFinalizerQueue(FinalizerQueue queue) noexcept; + FinalizerQueue ExtractFinalizerQueue() noexcept; + // Test method std_support::vector GetAllocatedObjects() noexcept; void ClearForTests() noexcept; @@ -46,6 +50,9 @@ private: PageStore singleObjectPages_; PageStore extraObjectPages_; + FinalizerQueue pendingFinalizerQueue_; + std::mutex pendingFinalizerQueueMutex_; + std::atomic concurrentSweepersCount_ = 0; }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index f664fc72954..20879de2a0f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -272,6 +272,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); } + finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); #endif scheduler.onGCFinish(epoch, allocatedBytes()); state_.finish(epoch); @@ -294,4 +295,4 @@ void gc::ConcurrentMarkAndSweep::reconfigure(std::size_t maxParallelism, bool mu for (std::size_t i = 0; i < auxGCThreads; ++i) { auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); })); } -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 010e00420eb..f5cbebd509d 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -153,7 +153,12 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { objectFactoryIterable = std::nullopt; kotlin::compactObjectPoolInMainThread(); #else + // also sweeps extraObjects auto finalizerQueue = heap_.Sweep(gcHandle); + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); + } + finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); #endif scheduler.onGCFinish(epoch, allocatedBytes());