[K/N] Move AllocatorImpl.hpp into alloc/ ^KT-60928

- Hide allocator-specific data into Allocator::*::Impl
  like with gc and gcScheduler modules.
- These *::Impl are still owned by specific GCs
- Additionally moved "stateless" allocator APIs into Allocator.hpp

kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp

@@ -5,12 +5,35 @@
 
 #pragma once
 
+#include "GC.hpp"
+#include "Utils.hpp"
+
 namespace kotlin::alloc {
 
-// TODO: Build `Allocator`, `Allocator::ThreadData` like with `gc`, `gcScheduler`,
+// TODO: Move allocator-specific data and API here.
-//       and move allocator-specific data there.
+class Allocator : private Pinned {
+public:
+    class Impl;
+
+    class ThreadData : private Pinned {
+    public:
+        class Impl;
+
+    private:
+    };
+
+private:
+};
 
 void initObjectPool() noexcept;
+// Instruct the allocator to free unused resources.
 void compactObjectPoolInCurrentThread() noexcept;
 
+gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept;
+ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept;
+
+// This does not take into account how much storage did the underlying allocator reserved.
+size_t allocatedHeapSize(ObjHeader* object) noexcept;
+
+size_t allocatedBytes() noexcept;
 }

kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp

@@ -3,10 +3,28 @@
  * that can be found in the LICENSE file.
  */
 
-#include "Allocator.hpp"
+#include "AllocatorImpl.hpp"
+
+#include "GCApi.hpp"
 
 using namespace kotlin;
 
 void alloc::initObjectPool() noexcept {}
 
 void alloc::compactObjectPoolInCurrentThread() noexcept {}
+
+gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept {
+    return HeapObjHeader::from(object).objectData();
+}
+
+ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
+    return HeapObjHeader::from(objectData).object();
+}
+
+size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept {
+    return CustomAllocator::GetAllocatedHeapSize(object);
+}
+
+size_t alloc::allocatedBytes() noexcept {
+    return GetAllocatedBytes();
+}

kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "Allocator.hpp"
+
+#include "CustomAllocator.hpp"
+#include "CustomFinalizerProcessor.hpp"
+#include "GCApi.hpp"
+#include "Heap.hpp"
+
+namespace kotlin::alloc {
+
+class Allocator::Impl : private Pinned {
+public:
+    Impl() noexcept = default;
+
+    Heap& heap() noexcept { return heap_; }
+
+private:
+    Heap heap_;
+};
+
+class Allocator::ThreadData::Impl : private Pinned {
+public:
+    explicit Impl(Allocator::Impl& allocator) noexcept : alloc_(allocator.heap()) {}
+
+    alloc::CustomAllocator& alloc() noexcept { return alloc_; }
+
+private:
+    CustomAllocator alloc_;
+};
+
+} // namespace kotlin::alloc

kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp

@@ -96,14 +96,6 @@ void Free(void* ptr, size_t size) noexcept;
 
 size_t GetAllocatedBytes() noexcept;
 
-inline gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
-    return HeapObjHeader::from(object).objectData();
-}
-
-inline ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
-    return HeapObjHeader::from(objectData).object();
-}
-
 } // namespace kotlin::alloc
 
 #endif

kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "AllocatorImpl.hpp"
+
+using namespace kotlin;
+
+gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept {
+    return ObjectFactoryImpl::NodeRef::From(object).ObjectData();
+}
+
+ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
+    return ObjectFactoryImpl::NodeRef::From(objectData)->GetObjHeader();
+}
+
+size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept {
+    return ObjectFactoryImpl::GetAllocatedHeapSize(object);
+}

kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.hpp

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "Allocator.hpp"
+
+#include "ExtraObjectDataFactory.hpp"
+#include "GC.hpp"
+#include "GlobalData.hpp"
+#include "Logging.hpp"
+#include "ObjectFactory.hpp"
+#include "ObjectFactoryAllocator.hpp"
+#include "ObjectFactorySweep.hpp"
+
+namespace kotlin::alloc {
+
+struct ObjectFactoryTraits {
+    using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
+    using ObjectData = gc::GC::ObjectData;
+
+    Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
+
+    void OnOOM(size_t size) noexcept {
+        RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
+        // TODO: This will print the log for "manual" scheduling. Fix this.
+        mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
+    }
+};
+
+using ObjectFactoryImpl = ObjectFactory<ObjectFactoryTraits>;
+
+class Allocator::Impl : private Pinned {
+public:
+    Impl() noexcept = default;
+
+    ObjectFactoryImpl& objectFactory() noexcept { return objectFactory_; }
+    ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
+
+private:
+    ObjectFactoryImpl objectFactory_;
+    ExtraObjectDataFactory extraObjectDataFactory_;
+};
+
+class Allocator::ThreadData::Impl : private Pinned {
+public:
+    explicit Impl(Allocator::Impl& allocator) noexcept :
+        objectFactoryThreadQueue_(allocator.objectFactory(), objectFactoryTraits_.CreateAllocator()),
+        extraObjectDataFactoryThreadQueue_(allocator.extraObjectDataFactory()) {}
+
+    ObjectFactoryImpl::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
+
+private:
+    [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
+    ObjectFactoryImpl::ThreadQueue objectFactoryThreadQueue_;
+    ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
+};
+
+using FinalizerQueue = ObjectFactoryImpl::FinalizerQueue;
+using FinalizerQueueTraits = ObjectFactoryImpl::FinalizerQueueTraits;
+
+} // namespace kotlin::alloc

kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp

@@ -8,19 +8,17 @@
 #include <cstddef>
 #include <type_traits>
 
+#include "Allocator.hpp"
+
 namespace kotlin::alloc {
 
 void initObjectPool() noexcept;
 void* allocateInObjectPool(size_t size) noexcept;
 void freeInObjectPool(void* ptr, size_t size) noexcept;
-// Instruct the allocator to free unused resources.
-void compactObjectPoolInCurrentThread() noexcept;
 // Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread.
 // May do nothing if the main thread is not an event loop.
 void compactObjectPoolInMainThread() noexcept;
 
-size_t allocatedBytes() noexcept;
-
 template <typename T>
 struct ObjectPoolAllocator {
     using value_type = T;

kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp

@@ -1,74 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#pragma once
-
-#ifdef CUSTOM_ALLOCATOR
-
-// TODO: Move into alloc/custom/AllocatorImpl.hpp
-
-#include "CustomAllocator.hpp"
-#include "CustomFinalizerProcessor.hpp"
-#include "GCApi.hpp"
-#include "Heap.hpp"
-
-namespace kotlin::gc {
-
-inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
-    return kotlin::alloc::objectDataForObject(object);
-}
-
-inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
-    return kotlin::alloc::objectForObjectData(objectData);
-}
-
-using FinalizerQueue = alloc::FinalizerQueue;
-using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
-
-} // namespace kotlin::gc
-
-#else
-
-// TODO: Move into alloc/legacy/AllocatorImpl.hpp
-
-#include "ExtraObjectDataFactory.hpp"
-#include "GC.hpp"
-#include "GlobalData.hpp"
-#include "Logging.hpp"
-#include "ObjectFactory.hpp"
-#include "ObjectFactoryAllocator.hpp"
-#include "ObjectFactorySweep.hpp"
-
-namespace kotlin::gc {
-
-struct ObjectFactoryTraits {
-    using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
-    using ObjectData = gc::GC::ObjectData;
-
-    Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
-
-    void OnOOM(size_t size) noexcept {
-        RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
-        // TODO: This will print the log for "manual" scheduling. Fix this.
-        mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
-    }
-};
-
-using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
-
-inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
-    return ObjectFactory::NodeRef::From(object).ObjectData();
-}
-
-inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
-    return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
-}
-
-using FinalizerQueue = ObjectFactory::FinalizerQueue;
-using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
-
-} // namespace kotlin::gc
-
-#endif

kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp

@@ -67,7 +67,7 @@ ALWAYS_INLINE void gc::BarriersThreadData::onAllocation(ObjHeader* allocated) {
         bool barriersEnabled = weakRefBarrier.load(std::memory_order_relaxed) != nullptr;
         RuntimeAssert(shouldMark == barriersEnabled, "New allocations marking must happen with and only with weak ref barriers");
         if (shouldMark) {
-            auto& objectData = objectDataForObject(allocated);
+            auto& objectData = alloc::objectDataForObject(allocated);
             bool wasUnmarked = objectData.tryMark();
             RuntimeAssert(wasUnmarked, "No one else could mark this newly allocated object before");
             markHandle_->addObject();

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -37,7 +37,7 @@ ScopedThread createGCThread(const char* name, Body&& body) {
 
 #ifndef CUSTOM_ALLOCATOR
 // TODO move to common
-[[maybe_unused]] inline void checkMarkCorrectness(gc::ObjectFactory::Iterable& heap) {
+[[maybe_unused]] inline void checkMarkCorrectness(alloc::ObjectFactoryImpl::Iterable& heap) {
     if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return;
     for (auto objRef: heap) {
         auto obj = objRef.GetObjHeader();
@@ -45,7 +45,7 @@ ScopedThread createGCThread(const char* name, Body&& body) {
         if (objData.marked()) {
             traverseReferredObjects(obj, [obj](ObjHeader* field) {
                 if (field->heap()) {
-                    auto& fieldObjData = gc::ObjectFactory::NodeRef::From(field).ObjectData();
+                    auto& fieldObjData = alloc::ObjectFactoryImpl::NodeRef::From(field).ObjectData();
                     RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj);
                 }
             });
@@ -89,20 +89,10 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
     return threadData_;
 }
 
-#ifndef CUSTOM_ALLOCATOR
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
-        ObjectFactory& objectFactory,
+        alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept
-        alloc::ExtraObjectDataFactory& extraObjectDataFactory,
+    :
-        gcScheduler::GCScheduler& gcScheduler,
+    allocator_(allocator),
-        bool mutatorsCooperate,
-        std::size_t auxGCThreads) noexcept :
-    objectFactory_(objectFactory),
-    extraObjectDataFactory_(extraObjectDataFactory),
-#else
-gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
-        gcScheduler::GCScheduler& gcScheduler,
-        bool mutatorsCooperate, std::size_t auxGCThreads) noexcept :
-#endif
     gcScheduler_(gcScheduler),
     finalizerProcessor_([this](int64_t epoch) {
         GCHandle::getByEpoch(epoch).finalizersDone();
@@ -191,10 +181,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().impl().alloc().PrepareForGC();
+        thread.gc().impl().allocator().alloc().PrepareForGC();
     }
-    auto& heap = mm::GlobalData::Instance().gc().impl().gc().heap();
+    mm::GlobalData::Instance().gc().impl().allocator().heap().PrepareForGC();
-    heap.PrepareForGC();
 #else
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
         thread.gc().PublishObjectFactory();
@@ -202,8 +191,8 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional objectFactoryIterable = objectFactory_.LockForIter();
+    std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter();
-    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
+    std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter();
 
     checkMarkCorrectness(*objectFactoryIterable);
 #endif
@@ -211,20 +200,20 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     resumeTheWorld(gcHandle);
 
 #ifndef CUSTOM_ALLOCATOR
-    alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
+    alloc::SweepExtraObjects<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *extraObjectFactoryIterable);
     extraObjectFactoryIterable = std::nullopt;
-    auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
+    auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *objectFactoryIterable);
     objectFactoryIterable = std::nullopt;
     alloc::compactObjectPoolInMainThread();
 #else
     // also sweeps extraObjects
-    auto finalizerQueue = heap_.Sweep(gcHandle);
+    auto finalizerQueue = allocator_.heap().Sweep(gcHandle);
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
+        finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue());
     }
-    finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
+    finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue());
 #endif
-    scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
+    scheduler.onGCFinish(epoch, alloc::allocatedBytes());
     state_.finish(epoch);
     gcHandle.finalizersScheduled(finalizerQueue.size());
     gcHandle.finished();

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -61,17 +61,11 @@ public:
         std::atomic<bool> published_ = false;
     };
 
-#ifdef CUSTOM_ALLOCATOR
-    explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler,
-                                    bool mutatorsCooperate, std::size_t auxGCThreads) noexcept;
-#else
     ConcurrentMarkAndSweep(
-            ObjectFactory& objectFactory,
+            alloc::Allocator::Impl& allocator,
-            alloc::ExtraObjectDataFactory& extraObjectDataFactory,
             gcScheduler::GCScheduler& scheduler,
             bool mutatorsCooperate,
             std::size_t auxGCThreads) noexcept;
-#endif
     ~ConcurrentMarkAndSweep();
 
     void StartFinalizerThreadIfNeeded() noexcept;
@@ -80,10 +74,6 @@ public:
 
     void reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, size_t auxGCThreads) noexcept;
 
-#ifdef CUSTOM_ALLOCATOR
-    alloc::Heap& heap() noexcept { return heap_; }
-#endif
-
     GCStateHolder& state() noexcept { return state_; }
 
 private:
@@ -91,16 +81,11 @@ private:
     void auxiliaryGCThreadBody();
     void PerformFullGC(int64_t epoch) noexcept;
 
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl& allocator_;
-    ObjectFactory& objectFactory_;
-    alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
-#else
-    alloc::Heap heap_;
-#endif
     gcScheduler::GCScheduler& gcScheduler_;
 
     GCStateHolder state_;
-    FinalizerProcessor<FinalizerQueue, FinalizerQueueTraits> finalizerProcessor_;
+    FinalizerProcessor<alloc::FinalizerQueue, alloc::FinalizerQueueTraits> finalizerProcessor_;
 
     mark::ParallelMark markDispatcher_;
     ScopedThread mainThread_;

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -181,13 +181,13 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
 
 std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
 #ifdef CUSTOM_ALLOCATOR
-    return threadData.gc().impl().alloc().heap().GetAllocatedObjects();
+    return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects();
 #else
     std_support::vector<ObjHeader*> objects;
-    for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
+    for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) {
         objects.push_back(node.GetObjHeader());
     }
-    for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) {
+    for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) {
         objects.push_back(node.GetObjHeader());
     }
     return objects;

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -20,26 +20,26 @@ gc::GC::ThreadData::~ThreadData() = default;
 
 void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().Publish();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->objectFactoryThreadQueue().Publish();
+    impl_->allocator().objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->objectFactoryThreadQueue().ClearForTests();
+    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
 #else
-    impl_->alloc().PrepareForGC();
+    impl_->allocator().alloc().PrepareForGC();
 #endif
 }
 
 ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
     ObjHeader* obj;
 #ifndef CUSTOM_ALLOCATOR
-    obj = impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+    obj = impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
 #else
-    obj = impl_->alloc().CreateObject(typeInfo);
+    obj = impl_->allocator().alloc().CreateObject(typeInfo);
 #endif
     impl().gc().barriers().onAllocation(obj);
     return obj;
@@ -48,9 +48,9 @@ ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeIn
 ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
     ArrayHeader* arr;
 #ifndef CUSTOM_ALLOCATOR
-    arr = impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+    arr = impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
 #else
-    arr = impl_->alloc().CreateArray(typeInfo, elements);
+    arr = impl_->allocator().alloc().CreateArray(typeInfo, elements);
 #endif
     impl().gc().barriers().onAllocation(arr->obj());
     return arr;
@@ -59,15 +59,15 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI
 ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
         ObjHeader* object, const TypeInfo* typeInfo) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
 #else
-    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
 }
 
 ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
 #endif
@@ -89,30 +89,13 @@ gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::
 
 gc::GC::~GC() = default;
 
-// static
-size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    return alloc::CustomAllocator::GetAllocatedHeapSize(object);
-#else
-    return ObjectFactory::GetAllocatedHeapSize(object);
-#endif
-}
-
-size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    return alloc::GetAllocatedBytes();
-#else
-    return alloc::allocatedBytes();
-#endif
-}
-
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactory().ClearForTests();
+    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->objectFactory().ClearForTests();
+    impl_->allocator().objectFactory().ClearForTests();
 #else
-    impl_->gc().heap().ClearForTests();
+    impl_->allocator().heap().ClearForTests();
 #endif
     GCHandle::ClearForTests();
 }
@@ -157,7 +140,7 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
 }
 
 bool gc::isMarked(ObjHeader* object) noexcept {
-    return objectDataForObject(object).marked();
+    return alloc::objectDataForObject(object).marked();
 }
 
 ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
@@ -173,7 +156,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
 #ifndef CUSTOM_ALLOCATOR
     extraObject.Uninstall();
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.ReleaseAssociatedObject();
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp

@@ -15,56 +15,27 @@ namespace gc {
 
 class GC::Impl : private Pinned {
 public:
-#ifdef CUSTOM_ALLOCATOR
+    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept :
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {}
+        gc_(allocator_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {}
-#else
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {}
-#endif
 
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
-    ObjectFactory& objectFactory() noexcept { return objectFactory_; }
-    alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
-#endif
     ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
 
 private:
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl allocator_;
-    ObjectFactory objectFactory_;
-    alloc::ExtraObjectDataFactory extraObjectDataFactory_;
-#endif
     ConcurrentMarkAndSweep gc_;
 };
 
 class GC::ThreadData::Impl : private Pinned {
 public:
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {}
-        gc_(gc.impl_->gc(), threadData),
-#ifndef CUSTOM_ALLOCATOR
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
-        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
-    }
-#else
-        alloc_(gc.impl_->gc().heap()) {
-    }
-#endif
 
     ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
-#ifdef CUSTOM_ALLOCATOR
+    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
-    alloc::CustomAllocator& alloc() noexcept { return alloc_; }
-#else
-    ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
-    alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
-#endif
 
 private:
     ConcurrentMarkAndSweep::ThreadData gc_;
-#ifdef CUSTOM_ALLOCATOR
+    alloc::Allocator::ThreadData::Impl allocator_;
-    alloc::CustomAllocator alloc_;
-#else
-    [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
-    ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
-    alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
-#endif
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp

@@ -36,11 +36,11 @@ auto collectPointers(T& iterable) {
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
-    auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
-    auto objects = gc.impl().objectFactory().LockForIter();
+    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
+    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
 #endif

kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp

@@ -85,18 +85,18 @@ public:
         static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
             auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop();
             if (obj) {
-                return objectForObjectData(*obj);
+                return alloc::objectForObjectData(*obj);
             }
             return nullptr;
         }
 
         static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
-            auto& objectData = objectDataForObject(object);
+            auto& objectData = alloc::objectDataForObject(object);
             return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData);
         }
 
         static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept {
-            auto& objectData = objectDataForObject(object);
+            auto& objectData = alloc::objectDataForObject(object);
             return objectData.tryMark();
         }
 

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -66,10 +66,6 @@ public:
 
     Impl& impl() noexcept { return *impl_; }
 
-    static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
-
-    size_t GetTotalHeapObjectsSizeBytes() const noexcept;
-
     void ClearForTests() noexcept;
 
     void StartFinalizerThreadIfNeeded() noexcept;

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp

@@ -9,6 +9,7 @@
 #include <limits>
 #include <optional>
 
+#include "Allocator.hpp"
 #include "Logging.hpp"
 #include "Mutex.hpp"
 #include "Porting.h"
@@ -135,7 +136,7 @@ GCInfo* statByEpoch(uint64_t epoch) {
 
 MemoryUsage currentHeapUsage() noexcept {
     return MemoryUsage{
-            mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes(),
+            alloc::allocatedBytes(),
     };
 }
 

kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp

@@ -1,38 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#pragma once
-
-#ifdef CUSTOM_ALLOCATOR
-
-// TODO: Move into alloc/custom/AllocatorImpl.hpp
-
-#include "CustomAllocator.hpp"
-#include "GCApi.hpp"
-#include "Heap.hpp"
-
-#else
-
-// TODO: Move into alloc/legacy/AllocatorImpl.hpp
-
-#include "ExtraObjectDataFactory.hpp"
-#include "GC.hpp"
-#include "ObjectFactory.hpp"
-#include "ObjectFactoryAllocator.hpp"
-
-namespace kotlin::gc {
-
-struct ObjectFactoryTraits {
-    using Allocator = alloc::AllocatorBasic;
-    using ObjectData = gc::GC::ObjectData;
-
-    Allocator CreateAllocator() noexcept { return Allocator(); }
-};
-
-using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
-
-} // namespace kotlin::gc
-
-#endif

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -8,7 +8,6 @@
 #include "Common.h"
 #include "GC.hpp"
 #include "GCStatistics.hpp"
-#include "NoOpGC.hpp"
 #include "ObjectOps.hpp"
 #include "ThreadData.hpp"
 #include "std_support/Memory.hpp"
@@ -21,48 +20,48 @@ gc::GC::ThreadData::~ThreadData() = default;
 
 void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().Publish();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->objectFactoryThreadQueue().Publish();
+    impl_->allocator().objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->objectFactoryThreadQueue().ClearForTests();
+    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
 #else
-    impl_->alloc().PrepareForGC();
+    impl_->allocator().alloc().PrepareForGC();
 #endif
 }
 
 ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+    return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
 #else
-    return impl_->alloc().CreateObject(typeInfo);
+    return impl_->allocator().alloc().CreateObject(typeInfo);
 #endif
 }
 
 ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+    return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
 #else
-    return impl_->alloc().CreateArray(typeInfo, elements);
+    return impl_->allocator().alloc().CreateArray(typeInfo, elements);
 #endif
 }
 
 ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
         ObjHeader* object, const TypeInfo* typeInfo) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
 #else
-    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
 }
 
 ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
 #endif
@@ -78,29 +77,12 @@ gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique<
 
 gc::GC::~GC() = default;
 
-// static
-size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return ObjectFactory::GetAllocatedHeapSize(object);
-#else
-    return alloc::CustomAllocator::GetAllocatedHeapSize(object);
-#endif
-}
-
-size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    return alloc::GetAllocatedBytes();
-#else
-    return alloc::allocatedBytes();
-#endif
-}
-
 void gc::GC::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactory().ClearForTests();
+    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->objectFactory().ClearForTests();
+    impl_->allocator().objectFactory().ClearForTests();
 #else
-    impl_->gc().heap().ClearForTests();
+    impl_->allocator().heap().ClearForTests();
 #endif
     GCHandle::ClearForTests();
 }
@@ -149,7 +131,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
 #ifndef CUSTOM_ALLOCATOR
     extraObject.Uninstall();
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.ReleaseAssociatedObject();
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp

@@ -8,56 +8,30 @@
 #include "GC.hpp"
 
 #include "AllocatorImpl.hpp"
-#include "NoOpGC.hpp"
+#include "Logging.hpp"
+#include "Utils.hpp"
 
 namespace kotlin {
 namespace gc {
 
 class GC::Impl : private Pinned {
 public:
-    Impl() noexcept = default;
+    Impl() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); }
 
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
-    ObjectFactory& objectFactory() noexcept { return objectFactory_; }
-    alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
-#endif
-    NoOpGC& gc() noexcept { return gc_; }
 
 private:
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl allocator_;
-    ObjectFactory objectFactory_;
-    alloc::ExtraObjectDataFactory extraObjectDataFactory_;
-#endif
-    NoOpGC gc_;
 };
 
 class GC::ThreadData::Impl : private Pinned {
 public:
-#ifdef CUSTOM_ALLOCATOR
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept : allocator_(gc.impl_->allocator()) {}
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {}
-#else
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
-        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {}
-#endif
 
-    NoOpGC::ThreadData& gc() noexcept { return gc_; }
+    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
-#ifdef CUSTOM_ALLOCATOR
-    alloc::CustomAllocator& alloc() noexcept { return alloc_; }
-#else
-    ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
-    alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
-#endif
 
 private:
-    NoOpGC::ThreadData gc_;
+    alloc::Allocator::ThreadData::Impl allocator_;
-#ifdef CUSTOM_ALLOCATOR
-    alloc::CustomAllocator alloc_;
-#else
-    [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
-    ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
-    alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
-#endif
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp

@@ -36,11 +36,11 @@ auto collectPointers(T& iterable) {
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
-    auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
-    auto objects = gc.impl().objectFactory().LockForIter();
+    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
+    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
 #endif

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.cpp

@@ -1,6 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#include "NoOpGC.hpp"

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp

@@ -1,52 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#ifndef RUNTIME_GC_NOOP_NOOP_GC_H
-#define RUNTIME_GC_NOOP_NOOP_GC_H
-
-#include <cstddef>
-
-#include "AllocatorImpl.hpp"
-#include "GC.hpp"
-#include "Logging.hpp"
-#include "Utils.hpp"
-
-namespace kotlin {
-
-namespace mm {
-class ThreadData;
-}
-
-namespace gc {
-
-// No-op GC is a GC that does not free memory.
-// TODO: It can be made more efficient.
-class NoOpGC : private Pinned {
-public:
-    class ThreadData : private Pinned {
-    public:
-        ThreadData() noexcept {}
-        ~ThreadData() = default;
-
-    private:
-    };
-
-    NoOpGC() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); }
-    ~NoOpGC() = default;
-
-#ifdef CUSTOM_ALLOCATOR
-    alloc::Heap& heap() noexcept { return heap_; }
-#endif
-
-private:
-#ifdef CUSTOM_ALLOCATOR
-    alloc::Heap heap_;
-#endif
-};
-
-} // namespace gc
-} // namespace kotlin
-
-#endif // RUNTIME_GC_NOOP_NOOP_GC_H

kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp

@@ -1,74 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#pragma once
-
-#ifdef CUSTOM_ALLOCATOR
-
-// TODO: Move into alloc/custom/AllocatorImpl.hpp
-
-#include "CustomAllocator.hpp"
-#include "CustomFinalizerProcessor.hpp"
-#include "GCApi.hpp"
-#include "Heap.hpp"
-
-namespace kotlin::gc {
-
-inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
-    return kotlin::alloc::objectDataForObject(object);
-}
-
-inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
-    return kotlin::alloc::objectForObjectData(objectData);
-}
-
-using FinalizerQueue = alloc::FinalizerQueue;
-using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
-
-} // namespace kotlin::gc
-
-#else
-
-// TODO: Move into alloc/legacy/AllocatorImpl.hpp
-
-#include "ExtraObjectDataFactory.hpp"
-#include "GC.hpp"
-#include "GlobalData.hpp"
-#include "Logging.hpp"
-#include "ObjectFactory.hpp"
-#include "ObjectFactoryAllocator.hpp"
-#include "ObjectFactorySweep.hpp"
-
-namespace kotlin::gc {
-
-struct ObjectFactoryTraits {
-    using Allocator = alloc::AllocatorWithGC<alloc::AllocatorBasic, ObjectFactoryTraits>;
-    using ObjectData = gc::GC::ObjectData;
-
-    Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); }
-
-    void OnOOM(size_t size) noexcept {
-        RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
-        // TODO: This will print the log for "manual" scheduling. Fix this.
-        mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
-    }
-};
-
-using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
-
-inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
-    return ObjectFactory::NodeRef::From(object).ObjectData();
-}
-
-inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
-    return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
-}
-
-using FinalizerQueue = ObjectFactory::FinalizerQueue;
-using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
-
-} // namespace kotlin::gc
-
-#endif

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -21,48 +21,48 @@ gc::GC::ThreadData::~ThreadData() = default;
 
 void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().Publish();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().Publish();
-    impl_->objectFactoryThreadQueue().Publish();
+    impl_->allocator().objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
+    impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests();
-    impl_->objectFactoryThreadQueue().ClearForTests();
+    impl_->allocator().objectFactoryThreadQueue().ClearForTests();
 #else
-    impl_->alloc().PrepareForGC();
+    impl_->allocator().alloc().PrepareForGC();
 #endif
 }
 
 ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+    return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo);
 #else
-    return impl_->alloc().CreateObject(typeInfo);
+    return impl_->allocator().alloc().CreateObject(typeInfo);
 #endif
 }
 
 ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+    return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements);
 #else
-    return impl_->alloc().CreateArray(typeInfo, elements);
+    return impl_->allocator().alloc().CreateArray(typeInfo, elements);
 #endif
 }
 
 ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
         ObjHeader* object, const TypeInfo* typeInfo) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
 #else
-    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
+    return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
 }
 
 ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
 #endif
@@ -78,30 +78,13 @@ gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::
 
 gc::GC::~GC() = default;
 
-// static
-size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    return alloc::CustomAllocator::GetAllocatedHeapSize(object);
-#else
-    return ObjectFactory::GetAllocatedHeapSize(object);
-#endif
-}
-
-size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
-#ifdef CUSTOM_ALLOCATOR
-    return alloc::GetAllocatedBytes();
-#else
-    return alloc::allocatedBytes();
-#endif
-}
-
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
 #ifndef CUSTOM_ALLOCATOR
-    impl_->extraObjectDataFactory().ClearForTests();
+    impl_->allocator().extraObjectDataFactory().ClearForTests();
-    impl_->objectFactory().ClearForTests();
+    impl_->allocator().objectFactory().ClearForTests();
 #else
-    impl_->gc().heap().ClearForTests();
+    impl_->allocator().heap().ClearForTests();
 #endif
     GCHandle::ClearForTests();
 }
@@ -146,7 +129,7 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
 }
 
 bool gc::isMarked(ObjHeader* object) noexcept {
-    return objectDataForObject(object).marked();
+    return alloc::objectDataForObject(object).marked();
 }
 
 ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
@@ -162,7 +145,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
 #ifndef CUSTOM_ALLOCATOR
     extraObject.Uninstall();
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+    threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
 #else
     extraObject.ReleaseAssociatedObject();
     extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp

@@ -15,54 +15,26 @@ namespace gc {
 
 class GC::Impl : private Pinned {
 public:
-#ifdef CUSTOM_ALLOCATOR
+    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator_, gcScheduler) {}
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {}
-#else
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
 
-    ObjectFactory& objectFactory() noexcept { return objectFactory_; }
+    alloc::Allocator::Impl& allocator() noexcept { return allocator_; }
-    alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
-#endif
     SameThreadMarkAndSweep& gc() noexcept { return gc_; }
 
 private:
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl allocator_;
-    ObjectFactory objectFactory_;
-    alloc::ExtraObjectDataFactory extraObjectDataFactory_;
-#endif
     SameThreadMarkAndSweep gc_;
 };
 
 class GC::ThreadData::Impl : private Pinned {
 public:
-    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {}
-        gc_(gc.impl_->gc(), threadData),
-#ifdef CUSTOM_ALLOCATOR
-        alloc_(gc.impl_->gc().heap()) {
-    }
-#else
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
-        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
-    }
-#endif
 
     SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; }
-    ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
-    alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
-#else
-    alloc::CustomAllocator& alloc() noexcept { return alloc_; }
-#endif
 
 private:
     SameThreadMarkAndSweep::ThreadData gc_;
-#ifdef CUSTOM_ALLOCATOR
+    alloc::Allocator::ThreadData::Impl allocator_;
-    alloc::CustomAllocator alloc_;
-#else
-    [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
-    ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
-    alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
-#endif
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp

@@ -36,11 +36,11 @@ auto collectPointers(T& iterable) {
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
-    auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    auto objects = gc.impl().allocator().heap().GetAllocatedObjects();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
-    auto objects = gc.impl().objectFactory().LockForIter();
+    auto objects = gc.impl().allocator().objectFactory().LockForIter();
-    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
+    auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
 #endif

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -22,17 +22,9 @@
 
 using namespace kotlin;
 
-#ifdef CUSTOM_ALLOCATOR
+gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept :
-gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
-        gcScheduler::GCScheduler& gcScheduler) noexcept :
-#else
-gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
-        ObjectFactory& objectFactory, alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
 
-    objectFactory_(objectFactory),
+    allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
-    extraObjectDataFactory_(extraObjectDataFactory),
-#endif
-    gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
         GCHandle::getByEpoch(epoch).finalizersDone();
         state_.finalized(epoch);
     }) {
@@ -81,9 +73,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().impl().alloc().PrepareForGC();
+        thread.gc().impl().allocator().alloc().PrepareForGC();
     }
-    heap_.PrepareForGC();
+    allocator_.heap().PrepareForGC();
 #endif
 
     gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [](mm::ThreadData&) { return true; });
@@ -99,24 +91,24 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
+    std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter();
-    std::optional objectFactoryIterable = objectFactory_.LockForIter();
+    std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter();
 
-    alloc::SweepExtraObjects<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
+    alloc::SweepExtraObjects<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *extraObjectFactoryIterable);
     extraObjectFactoryIterable = std::nullopt;
-    auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
+    auto finalizerQueue = alloc::Sweep<alloc::DefaultSweepTraits<alloc::ObjectFactoryImpl>>(gcHandle, *objectFactoryIterable);
     objectFactoryIterable = std::nullopt;
     alloc::compactObjectPoolInMainThread();
 #else
     // also sweeps extraObjects
-    auto finalizerQueue = heap_.Sweep(gcHandle);
+    auto finalizerQueue = allocator_.heap().Sweep(gcHandle);
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
+        finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue());
     }
-    finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue());
+    finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue());
 #endif
 
-    scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes());
+    scheduler.onGCFinish(epoch, alloc::allocatedBytes());
 
     resumeTheWorld(gcHandle);
 

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -40,14 +40,7 @@ public:
     private:
     };
 
-#ifdef CUSTOM_ALLOCATOR
+    SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept;
-    SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept;
-#else
-    SameThreadMarkAndSweep(
-            ObjectFactory& objectFactory,
-            alloc::ExtraObjectDataFactory& extraObjectDataFactory,
-            gcScheduler::GCScheduler& gcScheduler) noexcept;
-#endif
 
     ~SameThreadMarkAndSweep();
 
@@ -57,24 +50,15 @@ public:
 
     GCStateHolder& state() noexcept { return state_; }
 
-#ifdef CUSTOM_ALLOCATOR
-    alloc::Heap& heap() noexcept { return heap_; }
-#endif
-
 private:
     void PerformFullGC(int64_t epoch) noexcept;
 
-#ifndef CUSTOM_ALLOCATOR
+    alloc::Allocator::Impl& allocator_;
-    ObjectFactory& objectFactory_;
-    alloc::ExtraObjectDataFactory& extraObjectDataFactory_;
-#else
-    alloc::Heap heap_;
-#endif
     gcScheduler::GCScheduler& gcScheduler_;
 
     GCStateHolder state_;
     ScopedThread gcThread_;
-    FinalizerProcessor<FinalizerQueue, FinalizerQueueTraits> finalizerProcessor_;
+    FinalizerProcessor<alloc::FinalizerQueue, alloc::FinalizerQueueTraits> finalizerProcessor_;
 
     MarkQueue markQueue_;
 };
@@ -88,14 +72,16 @@ struct MarkTraits {
 
     static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
         if (auto* top = queue.try_pop_front()) {
-            return objectForObjectData(*top);
+            return alloc::objectForObjectData(*top);
         }
         return nullptr;
     }
 
-    static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { return queue.try_push_front(objectDataForObject(object)); }
+    static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+        return queue.try_push_front(alloc::objectDataForObject(object));
+    }
 
-    static bool tryMark(ObjHeader* object) noexcept { return objectDataForObject(object).tryMark(); }
+    static bool tryMark(ObjHeader* object) noexcept { return alloc::objectDataForObject(object).tryMark(); }
 
     static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
         auto process = object->type_info()->processObjectInMark;

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -181,13 +181,13 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
 
 std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
 #ifdef CUSTOM_ALLOCATOR
-    return threadData.gc().impl().alloc().heap().GetAllocatedObjects();
+    return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects();
 #else
     std_support::vector<ObjHeader*> objects;
-    for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
+    for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) {
         objects.push_back(node.GetObjHeader());
     }
-    for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) {
+    for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) {
         objects.push_back(node.GetObjHeader());
     }
     return objects;

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -85,7 +85,3 @@ OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo,
     // `ArrayHeader` and `ObjHeader` are expected to be compatible.
     RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
 }
-
-size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept {
-    return gc::GC::GetAllocatedHeapSize(object);
-}

kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp

@@ -32,9 +32,6 @@ OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept;
 OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept;
 OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;
 
-// This does not take into account how much storage did the underlying allocator (malloc/mimalloc) reserved.
-size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
-
 } // namespace mm
 } // namespace kotlin