[K/N] Move allocator-specific data into allocators. ^KT-60928

kotlin-native/runtime/build.gradle.kts

@@ -172,6 +172,7 @@ bitcode {
             sourceSets {
                 main {}
                 test {}
+                testFixtures {}
             }
 
             compilerArgs.add("-DCUSTOM_ALLOCATOR")
@@ -193,6 +194,7 @@ bitcode {
             sourceSets {
                 main {}
                 test {}
+                testFixtures {}
             }
         }
 
@@ -270,7 +272,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
             }
         }
 
@@ -279,7 +280,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
             }
 
             compilerArgs.add("-DCUSTOM_ALLOCATOR")
@@ -290,7 +290,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
                 test {}
             }
         }
@@ -300,7 +299,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
                 test {}
             }
 
@@ -312,7 +310,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
                 test {}
             }
         }
@@ -322,7 +319,6 @@ bitcode {
             headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp"))
             sourceSets {
                 main {}
-                testFixtures {}
                 test {}
             }
 

kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp

@@ -5,6 +5,9 @@
 
 #pragma once
 
+#include <cstdint>
+#include <memory>
+
 #include "GC.hpp"
 #include "Utils.hpp"
 
@@ -19,10 +22,37 @@ public:
     public:
         class Impl;
 
+        explicit ThreadData(Allocator& allocator) noexcept;
+        ~ThreadData();
+
+        Impl& impl() noexcept { return *impl_; }
+
+        ObjHeader* allocateObject(const TypeInfo* typeInfo) noexcept;
+        ArrayHeader* allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept;
+        mm::ExtraObjectData& allocateExtraObjectData(ObjHeader* object, const TypeInfo* typeInfo) noexcept;
+        void destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
+
+        void prepareForGC() noexcept;
+
+        // TODO: Move into AllocatorTestSupport.hpp
+        void clearForTests() noexcept;
+
     private:
+        std::unique_ptr<Impl> impl_;
     };
 
+    Allocator() noexcept;
+    ~Allocator();
+
+    Impl& impl() noexcept { return *impl_; }
+
+    void prepareForGC() noexcept;
+
+    // TODO: Move into AllocatorTestSupport.hpp
+    void clearForTests() noexcept;
+
 private:
+    std::unique_ptr<Impl> impl_;
 };
 
 void initObjectPool() noexcept;
@@ -36,4 +66,6 @@ ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept;
 size_t allocatedHeapSize(ObjHeader* object) noexcept;
 
 size_t allocatedBytes() noexcept;
+
+void destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
 }

kotlin-native/runtime/src/alloc/common/cpp/AllocatorTestSupport.hpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "Allocator.hpp"
+
+#include <vector>
+
+#include "ThreadData.hpp"
+
+namespace kotlin::alloc::test_support {
+
+void assertClear(Allocator& allocator) noexcept;
+std::vector<ObjHeader*> allocatedObjects(mm::ThreadData& threadData) noexcept;
+
+} // namespace kotlin::alloc::test_support

kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp

@@ -9,6 +9,47 @@
 
 using namespace kotlin;
 
+alloc::Allocator::ThreadData::ThreadData(Allocator& allocator) noexcept : impl_(std::make_unique<Impl>(allocator.impl())) {}
+
+alloc::Allocator::ThreadData::~ThreadData() = default;
+
+ALWAYS_INLINE ObjHeader* alloc::Allocator::ThreadData::allocateObject(const TypeInfo* typeInfo) noexcept {
+    return impl_->alloc().CreateObject(typeInfo);
+}
+
+ALWAYS_INLINE ArrayHeader* alloc::Allocator::ThreadData::allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    return impl_->alloc().CreateArray(typeInfo, elements);
+}
+
+ALWAYS_INLINE mm::ExtraObjectData& alloc::Allocator::ThreadData::allocateExtraObjectData(
+        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
+    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
+}
+
+ALWAYS_INLINE void alloc::Allocator::ThreadData::destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
+}
+
+void alloc::Allocator::ThreadData::prepareForGC() noexcept {
+    impl_->alloc().PrepareForGC();
+}
+
+void alloc::Allocator::ThreadData::clearForTests() noexcept {
+    impl_->alloc().PrepareForGC();
+}
+
+alloc::Allocator::Allocator() noexcept : impl_(std::make_unique<Impl>()) {}
+
+alloc::Allocator::~Allocator() = default;
+
+void alloc::Allocator::prepareForGC() noexcept {
+    impl_->heap().PrepareForGC();
+}
+
+void alloc::Allocator::clearForTests() noexcept {
+    impl_->heap().ClearForTests();
+}
+
 void alloc::initObjectPool() noexcept {}
 
 void alloc::compactObjectPoolInCurrentThread() noexcept {}
@@ -28,3 +69,8 @@ size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept {
 size_t alloc::allocatedBytes() noexcept {
     return GetAllocatedBytes();
 }
+
+void alloc::destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+    extraObject.ReleaseAssociatedObject();
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
+}

kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImplTestSupport.cpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "AllocatorTestSupport.hpp"
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+
+#include "AllocatorImpl.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+template <typename T>
+auto collectCopy(T& iterable) {
+    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
+    for (const auto& element : iterable) {
+        result.push_back(element);
+    }
+    return result;
+}
+
+} // namespace
+
+void alloc::test_support::assertClear(Allocator& allocator) noexcept {
+    auto objects = allocator.impl().heap().GetAllocatedObjects();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+}
+
+std::vector<ObjHeader*> alloc::test_support::allocatedObjects(mm::ThreadData& threadData) noexcept {
+    return threadData.allocator().impl().alloc().heap().GetAllocatedObjects();
+}

kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp

@@ -88,8 +88,8 @@ FinalizerQueue Heap::ExtractFinalizerQueue() noexcept {
     return std::move(pendingFinalizerQueue_);
 }
 
-std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
+std::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
-    std_support::vector<ObjHeader*> allocated;
+    std::vector<ObjHeader*> allocated;
     for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
         for (auto* page : fixedBlockPages_[blockSize].GetPages()) {
             for (auto* block : page->GetAllocatedBlocks()) {
@@ -107,7 +107,7 @@ std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
             allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
         }
     }
-    std_support::vector<ObjHeader*> unfinalized;
+    std::vector<ObjHeader*> unfinalized;
     for (auto* block: allocated) {
         if (!block->has_meta_object() || !mm::ExtraObjectData::Get(block)->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
             unfinalized.push_back(block);

kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp

@@ -41,7 +41,7 @@ public:
     FinalizerQueue ExtractFinalizerQueue() noexcept;
 
     // Test method
-    std_support::vector<ObjHeader*> GetAllocatedObjects() noexcept;
+    std::vector<ObjHeader*> GetAllocatedObjects() noexcept;
     void ClearForTests() noexcept;
 
 private:

kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp

@@ -5,8 +5,52 @@
 
 #include "AllocatorImpl.hpp"
 
+#include "ThreadData.hpp"
+
 using namespace kotlin;
 
+alloc::Allocator::ThreadData::ThreadData(Allocator& allocator) noexcept : impl_(std::make_unique<Impl>(allocator.impl())) {}
+
+alloc::Allocator::ThreadData::~ThreadData() = default;
+
+ALWAYS_INLINE ObjHeader* alloc::Allocator::ThreadData::allocateObject(const TypeInfo* typeInfo) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+}
+
+ALWAYS_INLINE ArrayHeader* alloc::Allocator::ThreadData::allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+}
+
+ALWAYS_INLINE mm::ExtraObjectData& alloc::Allocator::ThreadData::allocateExtraObjectData(
+        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
+    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+}
+
+ALWAYS_INLINE void alloc::Allocator::ThreadData::destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+}
+
+void alloc::Allocator::ThreadData::prepareForGC() noexcept {
+    impl_->extraObjectDataFactoryThreadQueue().Publish();
+    impl_->objectFactoryThreadQueue().Publish();
+}
+
+void alloc::Allocator::ThreadData::clearForTests() noexcept {
+    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
+    impl_->objectFactoryThreadQueue().ClearForTests();
+}
+
+alloc::Allocator::Allocator() noexcept : impl_(std::make_unique<Impl>()) {}
+
+alloc::Allocator::~Allocator() = default;
+
+void alloc::Allocator::prepareForGC() noexcept {}
+
+void alloc::Allocator::clearForTests() noexcept {
+    impl_->extraObjectDataFactory().ClearForTests();
+    impl_->objectFactory().ClearForTests();
+}
+
 gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept {
     return ObjectFactoryImpl::NodeRef::From(object).ObjectData();
 }
@@ -18,3 +62,9 @@ ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
 size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept {
     return ObjectFactoryImpl::GetAllocatedHeapSize(object);
 }
+
+void alloc::destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+    extraObject.Uninstall();
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->allocator().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+}

kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImplTestSupport.cpp

@@ -1,14 +1,14 @@
 /*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
  * that can be found in the LICENSE file.
  */
 
-#include "GCTestSupport.hpp"
+#include "AllocatorTestSupport.hpp"
 
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
 
-#include "GCImpl.hpp"
+#include "AllocatorImpl.hpp"
 
 using namespace kotlin;
 
@@ -34,14 +34,20 @@ auto collectPointers(T& iterable) {
 
 } // namespace
 
-void gc::AssertClear(GC& gc) noexcept {
+void alloc::test_support::assertClear(Allocator& allocator) noexcept {
-#ifdef CUSTOM_ALLOCATOR
+    auto objects = allocator.impl().objectFactory().LockForIter();
-    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
+    auto extraObjects = allocator.impl().extraObjectDataFactory().LockForIter();
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
-#else
-    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
-#endif
+}
+
+std::vector<ObjHeader*> alloc::test_support::allocatedObjects(mm::ThreadData& threadData) noexcept {
+    std::vector<ObjHeader*> objects;
+    for (auto node : threadData.allocator().impl().objectFactoryThreadQueue()) {
+        objects.push_back(node.GetObjHeader());
+    }
+    for (auto node : mm::GlobalData::Instance().allocator().impl().objectFactory().LockForIter()) {
+        objects.push_back(node.GetObjHeader());
+    }
+    return objects;
 }

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -8,6 +8,7 @@
 #include <cinttypes>
 #include <optional>
 
+#include "AllocatorImpl.hpp"
 #include "CallsChecker.hpp"
 #include "CompilerConstants.hpp"
 #include "GlobalData.hpp"
@@ -90,8 +91,7 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
 }
 
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
-        alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept
+        alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept :
-    :
     allocator_(allocator),
     gcScheduler_(gcScheduler),
     finalizerProcessor_([this](int64_t epoch) {
@@ -178,21 +178,17 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     // TODO outline as mark_.isolateMarkedHeapAndFinishMark()
     // By this point all the alive heap must be marked.
     // All the mutations (incl. allocations) after this method will be subject for the next GC.
-#ifdef CUSTOM_ALLOCATOR
     // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().impl().allocator().alloc().PrepareForGC();
+        thread.allocator().prepareForGC();
-    }
-    mm::GlobalData::Instance().gc().impl().allocator().heap().PrepareForGC();
-#else
-    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().PublishObjectFactory();
     }
+    allocator_.prepareForGC();
 
+#ifndef CUSTOM_ALLOCATOR
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter();
+    std::optional objectFactoryIterable = allocator_.impl().objectFactory().LockForIter();
-    std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter();
+    std::optional extraObjectFactoryIterable = allocator_.impl().extraObjectDataFactory().LockForIter();
 
     checkMarkCorrectness(*objectFactoryIterable);
 #endif
@@ -207,11 +203,11 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     alloc::compactObjectPoolInMainThread();
 #else
     // also sweeps extraObjects
-    auto finalizerQueue = allocator_.heap().Sweep(gcHandle);
+    auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle);
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue());
+        finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
     }
-    finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue());
+    finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
     scheduler.onGCFinish(epoch, alloc::allocatedBytes());
     state_.finish(epoch);

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -62,10 +62,7 @@ public:
     };
 
     ConcurrentMarkAndSweep(
-            alloc::Allocator::Impl& allocator,
+            alloc::Allocator& allocator, gcScheduler::GCScheduler& scheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept;
-            gcScheduler::GCScheduler& scheduler,
-            bool mutatorsCooperate,
-            std::size_t auxGCThreads) noexcept;
     ~ConcurrentMarkAndSweep();
 
     void StartFinalizerThreadIfNeeded() noexcept;
@@ -81,7 +78,7 @@ private:
     void auxiliaryGCThreadBody();
     void PerformFullGC(int64_t epoch) noexcept;
 
-    alloc::Allocator::Impl& allocator_;
+    alloc::Allocator& allocator_;
     gcScheduler::GCScheduler& gcScheduler_;
 
     GCStateHolder state_;

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -13,6 +13,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "AllocatorTestSupport.hpp"
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
 #include "GCImpl.hpp"
@@ -179,19 +180,8 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
     return test_support::Object<Payload>::FromObjHeader(holder.obj());
 }
 
-std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
+std::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
-#ifdef CUSTOM_ALLOCATOR
+    return alloc::test_support::allocatedObjects(threadData);
-    return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects();
-#else
-    std_support::vector<ObjHeader*> objects;
-    for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) {
-        objects.push_back(node.GetObjHeader());
-    }
-    for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) {
-        objects.push_back(node.GetObjHeader());
-    }
-    return objects;
-#endif
 }
 
 test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -232,6 +222,7 @@ public:
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::SpecialRefRegistry::instance().clearForTests();
         mm::GlobalData::Instance().gc().ClearForTests();
+        mm::GlobalData::Instance().allocator().clearForTests();
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }
@@ -705,7 +696,7 @@ public:
         return holderRef;
     }
 
-    std_support::vector<ObjHeader*> Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); }
+    std::vector<ObjHeader*> Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); }
 
 private:
     struct Context {
@@ -1065,7 +1056,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeakNewObj) {
             auto& extraObj = *mm::ExtraObjectData::Get(object.header());
             extraObj.ClearRegularWeakReferenceImpl();
             extraObj.Uninstall();
-            mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj);
+            alloc::destroyExtraObjectData(extraObj);
 
             while (!gcDone.load(std::memory_order_relaxed)) {
                 mm::safePoint(threadData);
@@ -1162,23 +1153,20 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
     }
 
-#ifndef CUSTOM_ALLOCATOR
     // Force mutators to publish their internal heaps
+    // Really only needed for legacy allocators.
     std_support::vector<std::future<void>> publishFutures;
     for (auto& mutator: mutators) {
-        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+        publishFutures.emplace_back(
-            threadData.gc().PublishObjectFactory();
+                mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); }));
-        }));
     }
     for (auto& mutator: newMutators) {
-        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+        publishFutures.emplace_back(
-            threadData.gc().PublishObjectFactory();
+                mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); }));
-        }));
     }
     for (auto& future : publishFutures) {
         future.wait();
     }
-#endif
 
     // All threads see the same alive objects, enough to check a single mutator.
     EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -18,59 +18,8 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::PublishObjectFactory() noexcept {
+ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept {
-#ifndef CUSTOM_ALLOCATOR
+    impl().gc().barriers().onAllocation(object);
-    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->allocator().objectFactoryThreadQueue().Publish();
-#endif
-}
-
-void gc::GC::ThreadData::ClearForTests() noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
-#else
-    impl_->allocator().alloc().PrepareForGC();
-#endif
-}
-
-ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
-    ObjHeader* obj;
-#ifndef CUSTOM_ALLOCATOR
-    obj = impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
-#else
-    obj = impl_->allocator().alloc().CreateObject(typeInfo);
-#endif
-    impl().gc().barriers().onAllocation(obj);
-    return obj;
-}
-
-ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
-    ArrayHeader* arr;
-#ifndef CUSTOM_ALLOCATOR
-    arr = impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
-#else
-    arr = impl_->allocator().alloc().CreateArray(typeInfo, elements);
-#endif
-    impl().gc().barriers().onAllocation(arr->obj());
-    return arr;
-}
-
-ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
-        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
-#else
-    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
-#endif
-}
-
-ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
-#endif
 }
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept {
@@ -85,18 +34,13 @@ void gc::GC::ThreadData::onThreadRegistration() noexcept {
     impl_->gc().onThreadRegistration();
 }
 
-gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique<Impl>(gcScheduler)) {}
+gc::GC::GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
+    impl_(std_support::make_unique<Impl>(allocator, gcScheduler)) {}
 
 gc::GC::~GC() = default;
 
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->allocator().objectFactory().ClearForTests();
-#else
-    impl_->allocator().heap().ClearForTests();
-#endif
     GCHandle::ClearForTests();
 }
 
@@ -151,18 +95,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
     return objectData.tryResetMark();
 }
 
-// static
-ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    extraObject.Uninstall();
-    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.ReleaseAssociatedObject();
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
-#endif
-}
-
 // static
 ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
     return sizeof(gc::GC::ObjectData);

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp

@@ -7,7 +7,6 @@
 
 #include "GC.hpp"
 
-#include "AllocatorImpl.hpp"
 #include "ConcurrentMarkAndSweep.hpp"
 
 namespace kotlin {
@@ -15,27 +14,23 @@ namespace gc {
 
 class GC::Impl : private Pinned {
 public:
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept :
+    Impl(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
-        gc_(allocator_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {}
+        gc_(allocator, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {}
 
-    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
     ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
 
 private:
-    alloc::Allocator::Impl allocator_;
     ConcurrentMarkAndSweep gc_;
 };
 
 class GC::ThreadData::Impl : private Pinned {
 public:
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {}
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData) {}
 
     ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
-    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
 
 private:
     ConcurrentMarkAndSweep::ThreadData gc_;
-    alloc::Allocator::ThreadData::Impl allocator_;
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp

@@ -8,7 +8,7 @@
 #include <atomic>
 #include <type_traits>
 
-#include "AllocatorImpl.hpp"
+#include "Allocator.hpp"
 #include "GC.hpp"
 #include "IntrusiveList.hpp"
 #include "KAssert.h"

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -16,6 +16,10 @@
 
 namespace kotlin {
 
+namespace alloc {
+class Allocator;
+}
+
 namespace mm {
 class ThreadData;
 }
@@ -35,20 +39,14 @@ public:
 
         Impl& impl() noexcept { return *impl_; }
 
-        void PublishObjectFactory() noexcept;
-        void ClearForTests() noexcept;
-
-        ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
-        ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept;
-        mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* object, const TypeInfo* typeInfo) noexcept;
-        void DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
-
         void OnSuspendForGC() noexcept;
 
         void safePoint() noexcept;
 
         void onThreadRegistration() noexcept;
 
+        void onAllocation(ObjHeader* object) noexcept;
+
     private:
         std_support::unique_ptr<Impl> impl_;
     };
@@ -61,7 +59,7 @@ public:
     //       the destructor is a trivial one.
     class ObjectData;
 
-    explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept;
+    GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept;
     ~GC();
 
     Impl& impl() noexcept { return *impl_; }
@@ -81,8 +79,6 @@ public:
     void WaitFinished(int64_t epoch) noexcept;
     void WaitFinalizers(int64_t epoch) noexcept;
 
-    static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
-
 private:
     std_support::unique_ptr<Impl> impl_;
 };

kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp

@@ -1,16 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#pragma once
-
-#include "GC.hpp"
-
-namespace kotlin {
-namespace gc {
-
-void AssertClear(GC& gc) noexcept;
-
-}
-} // namespace kotlin

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -149,7 +149,10 @@ public:
         return handle.getMarked();
     }
 
-    ~MarkAndSweepUtilsMarkTest() { mm::GlobalData::Instance().gc().ClearForTests(); }
+    ~MarkAndSweepUtilsMarkTest() {
+        mm::GlobalData::Instance().gc().ClearForTests();
+        mm::GlobalData::Instance().allocator().clearForTests();
+    }
 
 private:
     uint64_t epoch_ = 0;

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -5,67 +5,17 @@
 
 #include "GCImpl.hpp"
 
-#include "Common.h"
-#include "GC.hpp"
 #include "GCStatistics.hpp"
-#include "ObjectOps.hpp"
+#include "KAssert.h"
-#include "ThreadData.hpp"
+#include "Logging.hpp"
-#include "std_support/Memory.hpp"
 
 using namespace kotlin;
 
-gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique<Impl>(gc, threadData)) {}
+gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept {}
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::PublishObjectFactory() noexcept {
+ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept {}
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->allocator().objectFactoryThreadQueue().Publish();
-#endif
-}
-
-void gc::GC::ThreadData::ClearForTests() noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
-#else
-    impl_->allocator().alloc().PrepareForGC();
-#endif
-}
-
-ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
-#else
-    return impl_->allocator().alloc().CreateObject(typeInfo);
-#endif
-}
-
-ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
-#else
-    return impl_->allocator().alloc().CreateArray(typeInfo, elements);
-#endif
-}
-
-ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
-        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
-#else
-    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
-#endif
-}
-
-ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
-#endif
-}
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
@@ -73,17 +23,13 @@ void gc::GC::ThreadData::safePoint() noexcept {}
 
 void gc::GC::ThreadData::onThreadRegistration() noexcept {}
 
-gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique<Impl>()) {}
+gc::GC::GC(alloc::Allocator&, gcScheduler::GCScheduler&) noexcept {
+    RuntimeLogInfo({kTagGC}, "No-op GC initialized");
+}
 
 gc::GC::~GC() = default;
 
 void gc::GC::ClearForTests() noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->allocator().objectFactory().ClearForTests();
-#else
-    impl_->allocator().heap().ClearForTests();
-#endif
     GCHandle::ClearForTests();
 }
 
@@ -126,18 +72,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
     return true;
 }
 
-// static
-ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    extraObject.Uninstall();
-    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.ReleaseAssociatedObject();
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
-#endif
-}
-
 // static
 ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
     return 0;

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp

@@ -7,32 +7,9 @@
 
 #include "GC.hpp"
 
-#include "AllocatorImpl.hpp"
+namespace kotlin::gc {
-#include "Logging.hpp"
-#include "Utils.hpp"
 
-namespace kotlin {
+class GC::Impl {};
-namespace gc {
+class GC::ThreadData::Impl {};
 
-class GC::Impl : private Pinned {
+} // namespace kotlin::gc
-public:
-    Impl() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); }
-
-    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
-
-private:
-    alloc::Allocator::Impl allocator_;
-};
-
-class GC::ThreadData::Impl : private Pinned {
-public:
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept : allocator_(gc.impl_->allocator()) {}
-
-    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
-
-private:
-    alloc::Allocator::ThreadData::Impl allocator_;
-};
-
-} // namespace gc
-} // namespace kotlin

kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp

@@ -1,47 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#include "GCTestSupport.hpp"
-
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
-
-#include "GCImpl.hpp"
-
-using namespace kotlin;
-
-namespace {
-
-template <typename T>
-auto collectCopy(T& iterable) {
-    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
-    for (const auto& element : iterable) {
-        result.push_back(element);
-    }
-    return result;
-}
-
-template <typename T>
-auto collectPointers(T& iterable) {
-    std::vector<const std::remove_reference_t<decltype(*iterable.begin())>*> result;
-    for (const auto& element : iterable) {
-        result.push_back(&element);
-    }
-    return result;
-}
-
-} // namespace
-
-void gc::AssertClear(GC& gc) noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
-#else
-    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
-    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
-#endif
-}

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -19,54 +19,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::PublishObjectFactory() noexcept {
+ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept {}
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->allocator().objectFactoryThreadQueue().Publish();
-#endif
-}
-
-void gc::GC::ThreadData::ClearForTests() noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
-#else
-    impl_->allocator().alloc().PrepareForGC();
-#endif
-}
-
-ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
-#else
-    return impl_->allocator().alloc().CreateObject(typeInfo);
-#endif
-}
-
-ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
-#else
-    return impl_->allocator().alloc().CreateArray(typeInfo, elements);
-#endif
-}
-
-ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
-        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
-#else
-    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
-#endif
-}
-
-ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
-#endif
-}
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
@@ -74,18 +27,13 @@ void gc::GC::ThreadData::safePoint() noexcept {}
 
 void gc::GC::ThreadData::onThreadRegistration() noexcept {}
 
-gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique<Impl>(gcScheduler)) {}
+gc::GC::GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
+    impl_(std_support::make_unique<Impl>(allocator, gcScheduler)) {}
 
 gc::GC::~GC() = default;
 
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
-#ifndef CUSTOM_ALLOCATOR
-    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->allocator().objectFactory().ClearForTests();
-#else
-    impl_->allocator().heap().ClearForTests();
-#endif
     GCHandle::ClearForTests();
 }
 
@@ -140,18 +88,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
     return objectData.tryResetMark();
 }
 
-// static
-ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    extraObject.Uninstall();
-    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
-#else
-    extraObject.ReleaseAssociatedObject();
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
-#endif
-}
-
 // static
 ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
     return sizeof(gc::GC::ObjectData);

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp

@@ -7,7 +7,6 @@
 
 #include "GC.hpp"
 
-#include "AllocatorImpl.hpp"
 #include "SameThreadMarkAndSweep.hpp"
 
 namespace kotlin {
@@ -15,26 +14,22 @@ namespace gc {
 
 class GC::Impl : private Pinned {
 public:
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator_, gcScheduler) {}
+    explicit Impl(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator, gcScheduler) {}
 
-    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
     SameThreadMarkAndSweep& gc() noexcept { return gc_; }
 
 private:
-    alloc::Allocator::Impl allocator_;
     SameThreadMarkAndSweep gc_;
 };
 
 class GC::ThreadData::Impl : private Pinned {
 public:
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {}
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData) {}
 
     SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
-    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
 
 private:
     SameThreadMarkAndSweep::ThreadData gc_;
-    alloc::Allocator::ThreadData::Impl allocator_;
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp

@@ -1,47 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#include "GCTestSupport.hpp"
-
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
-
-#include "GCImpl.hpp"
-
-using namespace kotlin;
-
-namespace {
-
-template <typename T>
-auto collectCopy(T& iterable) {
-    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
-    for (const auto& element : iterable) {
-        result.push_back(element);
-    }
-    return result;
-}
-
-template <typename T>
-auto collectPointers(T& iterable) {
-    std::vector<const std::remove_reference_t<decltype(*iterable.begin())>*> result;
-    for (const auto& element : iterable) {
-        result.push_back(&element);
-    }
-    return result;
-}
-
-} // namespace
-
-void gc::AssertClear(GC& gc) noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
-#else
-    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
-    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
-#endif
-}

kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp

@@ -7,7 +7,7 @@
 
 #include <type_traits>
 
-#include "AllocatorImpl.hpp"
+#include "Allocator.hpp"
 #include "GC.hpp"
 #include "IntrusiveList.hpp"
 #include "KAssert.h"

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -22,7 +22,7 @@
 
 using namespace kotlin;
 
-gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
+gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
 
     allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
         GCHandle::getByEpoch(epoch).finalizersDone();
@@ -70,29 +70,23 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     state_.start(epoch);
 
-#ifdef CUSTOM_ALLOCATOR
-    // This should really be done by each individual thread while waiting
-    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().impl().allocator().alloc().PrepareForGC();
-    }
-    allocator_.heap().PrepareForGC();
-#endif
-
     gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [](mm::ThreadData&) { return true; });
 
     gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
 
     gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 
-#ifndef CUSTOM_ALLOCATOR
+    // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().PublishObjectFactory();
+        thread.allocator().prepareForGC();
     }
+    allocator_.prepareForGC();
 
+#ifndef CUSTOM_ALLOCATOR
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter();
+    std::optional extraObjectFactoryIterable = allocator_.impl().extraObjectDataFactory().LockForIter();
-    std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter();
+    std::optional objectFactoryIterable = allocator_.impl().objectFactory().LockForIter();
 
     alloc::SweepExtraObjects<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *extraObjectFactoryIterable);
     extraObjectFactoryIterable = std::nullopt;
@@ -101,11 +95,11 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     alloc::compactObjectPoolInMainThread();
 #else
     // also sweeps extraObjects
-    auto finalizerQueue = allocator_.heap().Sweep(gcHandle);
+    auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle);
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue());
+        finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue());
     }
-    finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue());
+    finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
 
     scheduler.onGCFinish(epoch, alloc::allocatedBytes());

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -40,7 +40,7 @@ public:
     private:
     };
 
-    SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept;
+    SameThreadMarkAndSweep(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept;
 
     ~SameThreadMarkAndSweep();
 
@@ -53,7 +53,7 @@ public:
 private:
     void PerformFullGC(int64_t epoch) noexcept;
 
-    alloc::Allocator::Impl& allocator_;
+    alloc::Allocator& allocator_;
     gcScheduler::GCScheduler& gcScheduler_;
 
     GCStateHolder state_;

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -13,6 +13,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "AllocatorTestSupport.hpp"
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
 #include "GCImpl.hpp"
@@ -179,19 +180,8 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
     return test_support::Object<Payload>::FromObjHeader(holder.obj());
 }
 
-std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
+std::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
-#ifdef CUSTOM_ALLOCATOR
+    return alloc::test_support::allocatedObjects(threadData);
-    return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects();
-#else
-    std_support::vector<ObjHeader*> objects;
-    for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) {
-        objects.push_back(node.GetObjHeader());
-    }
-    for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) {
-        objects.push_back(node.GetObjHeader());
-    }
-    return objects;
-#endif
 }
 
 test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -211,6 +201,7 @@ public:
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::SpecialRefRegistry::instance().clearForTests();
         mm::GlobalData::Instance().gc().ClearForTests();
+        mm::GlobalData::Instance().allocator().clearForTests();
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }
@@ -680,7 +671,7 @@ public:
         return holderRef;
     }
 
-    std_support::vector<ObjHeader*> Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); }
+    std::vector<ObjHeader*> Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); }
 
 private:
     struct Context {
@@ -1040,7 +1031,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeakNewObj) {
             auto& extraObj = *mm::ExtraObjectData::Get(object.header());
             extraObj.ClearRegularWeakReferenceImpl();
             extraObj.Uninstall();
-            mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj);
+            alloc::destroyExtraObjectData(extraObj);
 
             while (!gcDone.load(std::memory_order_relaxed)) {
                 mm::safePoint(threadData);
@@ -1137,23 +1128,19 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
     }
 
-#ifndef CUSTOM_ALLOCATOR
     // Force mutators to publish their internal heaps
     std_support::vector<std::future<void>> publishFutures;
     for (auto& mutator: mutators) {
-        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+        publishFutures.emplace_back(
-            threadData.gc().PublishObjectFactory();
+                mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); }));
-        }));
     }
     for (auto& mutator: newMutators) {
-        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+        publishFutures.emplace_back(
-            threadData.gc().PublishObjectFactory();
+                mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); }));
-        }));
     }
     for (auto& future : publishFutures) {
         future.wait();
     }
-#endif
 
     // All threads see the same alive objects, enough to check a single mutator.
     EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));

kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp

@@ -10,7 +10,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
-#include "AllocatorTestSupport.hpp"
+#include "StdAllocatorTestSupport.hpp"
 #include "ScopedThread.hpp"
 #include "TestSupport.hpp"
 #include "std_support/Vector.hpp"

kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp

@@ -11,7 +11,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
-#include "AllocatorTestSupport.hpp"
+#include "StdAllocatorTestSupport.hpp"
 #include "ScopedThread.hpp"
 #include "TestSupport.hpp"
 #include "std_support/Deque.hpp"

kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupportTest.cpp

@@ -3,7 +3,7 @@
  * that can be found in the LICENSE file.
  */
 
-#include "AllocatorTestSupport.hpp"
+#include "StdAllocatorTestSupport.hpp"
 
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -21,7 +21,7 @@ static_assert(sizeof(Class) > sizeof(EmptyClass));
 
 } // namespace
 
-TEST(AllocatorTestSupportTest, MockAllocate) {
+TEST(StdAllocatorTestSupportTest, MockAllocate) {
     testing::StrictMock<test_support::MockAllocatorCore> allocatorCore;
     auto allocator = test_support::MakeAllocator<Class>(allocatorCore);
 
@@ -31,7 +31,7 @@ TEST(AllocatorTestSupportTest, MockAllocate) {
     EXPECT_THAT(ptr, expectedPtr);
 }
 
-TEST(AllocatorTestSupportTest, MockDeallocate) {
+TEST(StdAllocatorTestSupportTest, MockDeallocate) {
     testing::StrictMock<test_support::MockAllocatorCore> allocatorCore;
     auto allocator = test_support::MakeAllocator<Class>(allocatorCore);
 
@@ -40,7 +40,7 @@ TEST(AllocatorTestSupportTest, MockDeallocate) {
     std::allocator_traits<decltype(allocator)>::deallocate(allocator, ptr, 2);
 }
 
-TEST(AllocatorTestSupportTest, MockAdjustType) {
+TEST(StdAllocatorTestSupportTest, MockAdjustType) {
     testing::StrictMock<test_support::MockAllocatorCore> allocatorCore;
     auto initial = test_support::MakeAllocator<EmptyClass>(allocatorCore);
 
@@ -58,7 +58,7 @@ TEST(AllocatorTestSupportTest, MockAdjustType) {
     Traits::deallocate(allocator, ptr, 2);
 }
 
-TEST(AllocatorTestSupportTest, Spy) {
+TEST(StdAllocatorTestSupportTest, Spy) {
     test_support::SpyAllocatorCore allocatorCore;
     auto allocator = test_support::MakeAllocator<Class>(allocatorCore);
 

kotlin-native/runtime/src/main/cpp/std_support/MemoryTest.cpp

@@ -8,7 +8,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
-#include "AllocatorTestSupport.hpp"
+#include "StdAllocatorTestSupport.hpp"
 #include "KAssert.h"
 #include "Utils.hpp"
 

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp

@@ -28,11 +28,11 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
 
     RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
 
-    auto& gc = mm::ThreadRegistry::Instance().CurrentThreadData()->gc();
+    auto& allocator = mm::ThreadRegistry::Instance().CurrentThreadData()->allocator();
-    auto& data = gc.CreateExtraObjectDataForObject(object, typeInfo);
+    auto& data = allocator.allocateExtraObjectData(object, typeInfo);
     if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
         // Somebody else created `mm::ExtraObjectData` for this object.
-        gc.DestroyUnattachedExtraObjectData(data);
+        allocator.destroyUnattachedExtraObjectData(data);
         return *reinterpret_cast<mm::ExtraObjectData*>(typeInfo);
     }
 

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -30,6 +30,7 @@ public:
     ~ExtraObjectDataTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().gc().ClearForTests();
+        mm::GlobalData::Instance().allocator().clearForTests();
     }
 };
 
@@ -78,7 +79,8 @@ TEST_F(ExtraObjectDataTest, ConcurrentInstall) {
             }
             auto& extraData = mm::ExtraObjectData::Install(object.header());
             actual[i] = &extraData;
-            mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->gc().PublishObjectFactory();
+            // Really only needed for legacy allocators.
+            mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->allocator().prepareForGC();
         });
     }
 

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -6,6 +6,7 @@
 #ifndef RUNTIME_MM_GLOBAL_DATA_H
 #define RUNTIME_MM_GLOBAL_DATA_H
 
+#include "Allocator.hpp"
 #include "GlobalsRegistry.hpp"
 #include "GC.hpp"
 #include "GCScheduler.hpp"
@@ -26,6 +27,7 @@ public:
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
     SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; }
     gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
+    alloc::Allocator& allocator() noexcept { return allocator_; }
     gc::GC& gc() noexcept { return gc_; }
     AppStateTracking& appStateTracking() noexcept { return appStateTracking_; }
 
@@ -41,7 +43,8 @@ private:
     GlobalsRegistry globalsRegistry_;
     SpecialRefRegistry specialRefRegistry_;
     gcScheduler::GCScheduler gcScheduler_;
-    gc::GC gc_{gcScheduler_};
+    alloc::Allocator allocator_;
+    gc::GC gc_{allocator_, gcScheduler_};
 };
 
 } // namespace mm

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -80,7 +80,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) {
 void ObjHeader::destroyMetaObject(ObjHeader* object) {
     RuntimeAssert(object->has_meta_object(), "Object must have a meta object set");
     auto &extraObject = *mm::ExtraObjectData::Get(object);
-    gc::GC::DestroyExtraObjectData(extraObject);
+    alloc::destroyExtraObjectData(extraObject);
 }
 
 ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -74,14 +74,15 @@ ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader*
 OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept {
     AssertThreadState(threadData, ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
-    auto* object = threadData->gc().CreateObject(typeInfo);
+    auto* object = threadData->allocator().allocateObject(typeInfo);
+    threadData->gc().onAllocation(object);
     RETURN_OBJ(object);
 }
 
 OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept {
     AssertThreadState(threadData, ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
-    auto* array = threadData->gc().CreateArray(typeInfo, static_cast<uint32_t>(elements));
+    auto* array = threadData->allocator().allocateArray(typeInfo, static_cast<uint32_t>(elements));
-    // `ArrayHeader` and `ObjHeader` are expected to be compatible.
+    threadData->gc().onAllocation(array->obj());
-    RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
+    RETURN_OBJ(array->obj());
 }

kotlin-native/runtime/src/mm/cpp/TestSupport.cpp

@@ -6,8 +6,8 @@
 #include "gtest/gtest.h"
 #include "gmock/gmock.h"
 
+#include "AllocatorTestSupport.hpp"
 #include "GC.hpp"
-#include "GCTestSupport.hpp"
 #include "GlobalData.hpp"
 #include "GlobalsRegistry.hpp"
 #include "TestSupport.hpp"
@@ -51,7 +51,7 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() {
     EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre());
-    gc::AssertClear(mm::GlobalData::Instance().gc());
+    alloc::test_support::assertClear(mm::GlobalData::Instance().allocator());
 }
 
 void kotlin::DeinitMemoryForTests(MemoryState* memoryState) {

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -32,6 +32,7 @@ public:
         globalsThreadQueue_(GlobalsRegistry::Instance()),
         specialRefRegistry_(SpecialRefRegistry::instance()),
         gcScheduler_(GlobalData::Instance().gcScheduler(), *this),
+        allocator_(GlobalData::Instance().allocator()),
         gc_(GlobalData::Instance().gc(), *this),
         suspensionData_(ThreadState::kNative, *this) {}
 
@@ -55,6 +56,8 @@ public:
 
     gcScheduler::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; }
 
+    alloc::Allocator::ThreadData& allocator() noexcept { return allocator_; }
+
     gc::GC::ThreadData& gc() noexcept { return gc_; }
 
     ThreadSuspensionData& suspensionData() { return suspensionData_; }
@@ -68,7 +71,7 @@ public:
     void ClearForTests() noexcept {
         globalsThreadQueue_.ClearForTests();
         specialRefRegistry_.clearForTests();
-        gc_.ClearForTests();
+        allocator_.clearForTests();
     }
 
 private:
@@ -78,6 +81,7 @@ private:
     SpecialRefRegistry::ThreadQueue specialRefRegistry_;
     ShadowStack shadowStack_;
     gcScheduler::GCScheduler::ThreadData gcScheduler_;
+    alloc::Allocator::ThreadData allocator_;
     gc::GC::ThreadData gc_;
     std_support::vector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
     ThreadSuspensionData suspensionData_;