[K/N] CustomAllocator do not lose finalizer queue

kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp

@@ -25,7 +25,7 @@ public:
     AtomicStack& operator=(AtomicStack&& other) noexcept {
         // Not using swap idiom, because implementing swap of two atomics requires DCAS or locks.
         auto newHead = other.stack_.exchange(nullptr, std::memory_order_acq_rel);
-        stack_.store(newHead, std::memory_order_acq_rel);
+        stack_.store(newHead, std::memory_order_release);
         return *this;
     }
 

kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp

@@ -46,6 +46,10 @@ CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage
     memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_));
 }
 
+CustomAllocator::~CustomAllocator() {
+    heap_.AddToFinalizerQueue(std::move(finalizerQueue_));
+}
+
 ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
     RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
     size_t allocSize = ObjectAllocatedDataSize(typeInfo);

kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp

@@ -22,6 +22,8 @@ class CustomAllocator {
 public:
     explicit CustomAllocator(Heap& heap) noexcept;
 
+    ~CustomAllocator();
+
     ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
 
     ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept;

kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp

@@ -78,6 +78,16 @@ ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexce
     return extraObjectPages_.GetPage(0, finalizerQueue, concurrentSweepersCount_);
 }
 
+void Heap::AddToFinalizerQueue(FinalizerQueue queue) noexcept {
+    std::unique_lock guard(pendingFinalizerQueueMutex_);
+    pendingFinalizerQueue_.TransferAllFrom(std::move(queue));
+}
+
+FinalizerQueue Heap::ExtractFinalizerQueue() noexcept {
+    std::unique_lock guard(pendingFinalizerQueueMutex_);
+    return std::move(pendingFinalizerQueue_);
+}
+
 std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
     std_support::vector<ObjHeader*> allocated;
     for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {

kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp

@@ -7,6 +7,7 @@
 #define CUSTOM_ALLOC_CPP_HEAP_HPP_
 
 #include <atomic>
+#include <mutex>
 #include <cstring>
 
 #include "AtomicStack.hpp"
@@ -36,6 +37,9 @@ public:
     SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept;
     ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept;
 
+    void AddToFinalizerQueue(FinalizerQueue queue) noexcept;
+    FinalizerQueue ExtractFinalizerQueue() noexcept;
+
     // Test method
     std_support::vector<ObjHeader*> GetAllocatedObjects() noexcept;
     void ClearForTests() noexcept;
@@ -46,6 +50,9 @@ private:
     PageStore<SingleObjectPage> singleObjectPages_;
     PageStore<ExtraObjectPage> extraObjectPages_;
 
+    FinalizerQueue pendingFinalizerQueue_;
+    std::mutex pendingFinalizerQueueMutex_;
+
     std::atomic<std::size_t> concurrentSweepersCount_ = 0;
 };
 

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -272,6 +272,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
         finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
     }
+    finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
 #endif
     scheduler.onGCFinish(epoch, allocatedBytes());
     state_.finish(epoch);
@@ -294,4 +295,4 @@ void gc::ConcurrentMarkAndSweep::reconfigure(std::size_t maxParallelism, bool mu
     for (std::size_t i = 0; i < auxGCThreads; ++i) {
         auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); }));
     }
-}
+}

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -153,7 +153,12 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     objectFactoryIterable = std::nullopt;
     kotlin::compactObjectPoolInMainThread();
 #else
+    // also sweeps extraObjects
     auto finalizerQueue = heap_.Sweep(gcHandle);
+    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
+        finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
+    }
+    finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
 #endif
 
     scheduler.onGCFinish(epoch, allocatedBytes());