[K/N] Move CUSTOM_ALLOCATOR macro to gc modules only ^KT-55364

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -108,8 +108,11 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
 
 #ifndef CUSTOM_ALLOCATOR
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
-        mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
+        mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory,
+        mm::ExtraObjectDataFactory& extraObjectDataFactory,
+        gcScheduler::GCScheduler& gcScheduler) noexcept :
     objectFactory_(objectFactory),
+    extraObjectDataFactory_(extraObjectDataFactory),
 #else
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept :
 #endif
@@ -168,7 +171,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().Allocator().PrepareForGC();
+        thread.gc().impl().alloc().PrepareForGC();
     }
     heap_.PrepareForGC();
 #endif
@@ -190,7 +193,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifndef CUSTOM_ALLOCATOR
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
+    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
     std::optional objectFactoryIterable = objectFactory_.LockForIter();
 #endif
 
@@ -222,7 +225,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     // also sweeps extraObjects
     auto finalizerQueue = heap_.Sweep(gcHandle);
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        finalizerQueue.TransferAllFrom(thread.gc().Allocator().ExtractFinalizerQueue());
+        finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
     }
 #endif
     state_.finish(epoch);

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -10,6 +10,7 @@
 
 #include "Allocator.hpp"
 #include "Barriers.hpp"
+#include "ExtraObjectDataFactory.hpp"
 #include "FinalizerProcessor.hpp"
 #include "GCScheduler.hpp"
 #include "GCState.hpp"
@@ -120,7 +121,10 @@ public:
 #ifdef CUSTOM_ALLOCATOR
     explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler) noexcept;
 #else
-    ConcurrentMarkAndSweep(mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, gcScheduler::GCScheduler& scheduler) noexcept;
+    ConcurrentMarkAndSweep(
+            mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory,
+            mm::ExtraObjectDataFactory& extraObjectDataFactory,
+            gcScheduler::GCScheduler& scheduler) noexcept;
 #endif
     ~ConcurrentMarkAndSweep();
 
@@ -144,6 +148,7 @@ private:
 
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
+    mm::ExtraObjectDataFactory& extraObjectDataFactory_;
 #else
     alloc::Heap heap_;
 #endif

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -219,9 +219,6 @@ public:
     ~ConcurrentMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::SpecialRefRegistry::instance().clearForTests();
-#ifndef CUSTOM_ALLOCATOR
-        mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-#endif
         mm::GlobalData::Instance().gc().ClearForTests();
     }
 

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -34,12 +34,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
 
 void gc::GC::ThreadData::Publish() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
     impl_->objectFactoryThreadQueue().ClearForTests();
 #else
     impl_->alloc().PrepareForGC();
@@ -62,11 +64,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI
 #endif
 }
 
-#ifdef CUSTOM_ALLOCATOR
-alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
-    return impl_->alloc();
-}
+ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
+        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+#else
+    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
+#endif
+}
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept {
     impl_->gc().OnSuspendForGC();
@@ -96,6 +109,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactory().ClearForTests();
     impl_->objectFactory().ClearForTests();
 #else
     impl_->gc().heap().ClearForTests();
@@ -154,3 +168,15 @@ const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData)
 ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
     return reinterpret_cast<ConcurrentMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
 }
+
+// static
+ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    extraObject.Uninstall();
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.ReleaseAssociatedObject();
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
+#endif
+}

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp

@@ -11,6 +11,8 @@
 
 #ifdef CUSTOM_ALLOCATOR
 #include "CustomAllocator.hpp"
+#else
+#include "ExtraObjectDataFactory.hpp"
 #endif
 
 namespace kotlin {
@@ -23,17 +25,19 @@ public:
 #ifdef CUSTOM_ALLOCATOR
     explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {}
 #else
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {}
+    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
 #endif
 
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
 #endif
     GCImpl& gc() noexcept { return gc_; }
 
 private:
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    mm::ExtraObjectDataFactory extraObjectDataFactory_;
 #endif
     GCImpl gc_;
 };
@@ -43,7 +47,9 @@ public:
     Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
         gc_(gc.impl_->gc(), threadData, gcScheduler),
 #ifndef CUSTOM_ALLOCATOR
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
+        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
+    }
 #else
         alloc_(gc.impl_->gc().heap(), gcScheduler) {
     }
@@ -54,6 +60,7 @@ public:
     alloc::CustomAllocator& alloc() noexcept { return alloc_; }
 #else
     mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
 #endif
 
 private:
@@ -62,6 +69,7 @@ private:
     alloc::CustomAllocator alloc_;
 #else
     mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+    mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
 #endif
 };
 

kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp

@@ -23,13 +23,25 @@ auto collectCopy(T& iterable) {
     return result;
 }
 
+template <typename T>
+auto collectPointers(T& iterable) {
+    std::vector<const std::remove_reference_t<decltype(*iterable.begin())>*> result;
+    for (const auto& element : iterable) {
+        result.push_back(&element);
+    }
+    return result;
+}
+
 } // namespace
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
     auto objects = gc.impl().objectFactory().LockForIter();
-#endif
+    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
+#endif
 }

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -7,15 +7,12 @@
 
 #include <atomic>
 
+#include "ExtraObjectData.hpp"
 #include "GCScheduler.hpp"
 #include "Memory.h"
 #include "Utils.hpp"
 #include "std_support/Memory.hpp"
 
-#ifdef CUSTOM_ALLOCATOR
-#include "CustomAllocator.hpp"
-#endif
-
 namespace kotlin {
 
 namespace mm {
@@ -46,10 +43,8 @@ public:
 
         ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
         ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept;
-
-#ifdef CUSTOM_ALLOCATOR
-        alloc::CustomAllocator& Allocator() noexcept;
-#endif
+        mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* object, const TypeInfo* typeInfo) noexcept;
+        void DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
 
         void OnSuspendForGC() noexcept;
 
@@ -86,6 +81,8 @@ public:
     static const size_t objectDataSize;
     static bool SweepObject(void* objectData) noexcept;
 
+    static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
+
 private:
     std_support::unique_ptr<Impl> impl_;
 };

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -168,7 +168,7 @@ public:
         auto deallocExtraObject = [this](ObjHeader* obj) {
             auto *extraObject = mm::ExtraObjectData::Get(obj);
             extraObject->Uninstall();
-            extraObjectFactory_.DestroyExtraObjectData(extraObjectFactoryThreadQueue_, *extraObject);
+            extraObjectFactoryThreadQueue_.DestroyExtraObjectData(*extraObject);
             extraObjectFactoryThreadQueue_.Publish();
         };
         for (auto& finalizerQueue : finalizers_) {
@@ -240,7 +240,7 @@ public:
     }
 
     mm::ExtraObjectData& InstallExtraData(ObjHeader *objHeader) {
-        auto& extraObjectData = extraObjectFactory_.CreateExtraObjectDataForObject(extraObjectFactoryThreadQueue_, objHeader, objHeader->type_info());
+        auto& extraObjectData = extraObjectFactoryThreadQueue_.CreateExtraObjectDataForObject(objHeader, objHeader->type_info());
         extraObjectFactoryThreadQueue_.Publish();
         objHeader->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(&extraObjectData);
         return *mm::ExtraObjectData::Get(objHeader);

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -8,6 +8,7 @@
 #include "Common.h"
 #include "GC.hpp"
 #include "NoOpGC.hpp"
+#include "ThreadData.hpp"
 #include "std_support/Memory.hpp"
 #include "GCStatistics.hpp"
 #include "ObjectOps.hpp"
@@ -37,12 +38,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
 
 void gc::GC::ThreadData::Publish() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
     impl_->objectFactoryThreadQueue().ClearForTests();
 #else
     impl_->alloc().PrepareForGC();
@@ -65,11 +68,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI
 #endif
 }
 
-#ifdef CUSTOM_ALLOCATOR
-alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
-    return impl_->alloc();
-}
+ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
+        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+#else
+    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
+#endif
+}
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
@@ -94,6 +108,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
 
 void gc::GC::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactory().ClearForTests();
     impl_->objectFactory().ClearForTests();
 #else
     impl_->gc().heap().ClearForTests();
@@ -139,3 +154,15 @@ const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no
 ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
     return true;
 }
+
+// static
+ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    extraObject.Uninstall();
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.ReleaseAssociatedObject();
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
+#endif
+}

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp

@@ -12,7 +12,8 @@
 
 #ifdef CUSTOM_ALLOCATOR
 #include "CustomAllocator.hpp"
-#include "GCScheduler.hpp"
+#else
+#include "ExtraObjectDataFactory.hpp"
 #endif
 
 namespace kotlin {
@@ -26,12 +27,14 @@ public:
 
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
 #endif
     GCImpl& gc() noexcept { return gc_; }
 
 private:
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    mm::ExtraObjectDataFactory extraObjectDataFactory_;
 #endif
     GCImpl gc_;
 };
@@ -43,7 +46,8 @@ public:
         alloc_(gc.impl_->gc().heap(), gcScheduler) {}
 #else
     Impl(GC& gc, mm::ThreadData& threadData) noexcept :
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
+        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {}
 #endif
 
     GCImpl::ThreadData& gc() noexcept { return gc_; }
@@ -51,15 +55,16 @@ public:
     alloc::CustomAllocator& alloc() noexcept { return alloc_; }
 #else
     mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
 #endif
 
 private:
     GCImpl::ThreadData gc_;
 #ifdef CUSTOM_ALLOCATOR
     alloc::CustomAllocator alloc_;
-    /* gcScheduler::GCSchedulerThreadData; */
 #else
     mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+    mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
 #endif
 };
 

kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp

@@ -23,13 +23,25 @@ auto collectCopy(T& iterable) {
     return result;
 }
 
+template <typename T>
+auto collectPointers(T& iterable) {
+    std::vector<const std::remove_reference_t<decltype(*iterable.begin())>*> result;
+    for (const auto& element : iterable) {
+        result.push_back(&element);
+    }
+    return result;
+}
+
 } // namespace
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
     auto objects = gc.impl().objectFactory().LockForIter();
-#endif
+    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
+#endif
 }

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -34,12 +34,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
 
 void gc::GC::ThreadData::Publish() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
 #endif
 }
 
 void gc::GC::ThreadData::ClearForTests() noexcept {
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().ClearForTests();
     impl_->objectFactoryThreadQueue().ClearForTests();
 #else
     impl_->alloc().PrepareForGC();
@@ -62,11 +64,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI
 #endif
 }
 
-#ifdef CUSTOM_ALLOCATOR
-alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
-    return impl_->alloc();
-}
+ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
+        ObjHeader* object, const TypeInfo* typeInfo) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo);
+#else
+    return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo);
 #endif
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
+#endif
+}
 
 void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
@@ -92,6 +105,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
 #ifndef CUSTOM_ALLOCATOR
+    impl_->extraObjectDataFactory().ClearForTests();
     impl_->objectFactory().ClearForTests();
 #else
     impl_->gc().heap().ClearForTests();
@@ -150,3 +164,15 @@ const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData)
 ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
     return reinterpret_cast<SameThreadMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
 }
+
+// static
+ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept {
+#ifndef CUSTOM_ALLOCATOR
+    extraObject.Uninstall();
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject);
+#else
+    extraObject.ReleaseAssociatedObject();
+    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
+#endif
+}

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp

@@ -9,6 +9,12 @@
 
 #include "SameThreadMarkAndSweep.hpp"
 
+#ifdef CUSTOM_ALLOCATOR
+#include "CustomAllocator.hpp"
+#else
+#include "ExtraObjectDataFactory.hpp"
+#endif
+
 namespace kotlin {
 namespace gc {
 
@@ -19,15 +25,17 @@ public:
 #ifdef CUSTOM_ALLOCATOR
     explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {}
 #else
-    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {}
+    explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
 
     mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
 #endif
     GCImpl& gc() noexcept { return gc_; }
 
 private:
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    mm::ExtraObjectDataFactory extraObjectDataFactory_;
 #endif
     GCImpl gc_;
 };
@@ -39,12 +47,15 @@ public:
 #ifdef CUSTOM_ALLOCATOR
         alloc_(gc.impl_->gc().heap(), gcScheduler) {}
 #else
-        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
+        extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
+    }
 #endif
 
     GCImpl::ThreadData& gc() noexcept { return gc_; }
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
 #else
     alloc::CustomAllocator& alloc() noexcept { return alloc_; }
 #endif
@@ -55,6 +66,7 @@ private:
     alloc::CustomAllocator alloc_;
 #else
     mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+    mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
 #endif
 };
 

kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp

@@ -23,13 +23,25 @@ auto collectCopy(T& iterable) {
     return result;
 }
 
+template <typename T>
+auto collectPointers(T& iterable) {
+    std::vector<const std::remove_reference_t<decltype(*iterable.begin())>*> result;
+    for (const auto& element : iterable) {
+        result.push_back(&element);
+    }
+    return result;
+}
+
 } // namespace
 
 void gc::AssertClear(GC& gc) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     auto objects = gc.impl().gc().heap().GetAllocatedObjects();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
 #else
     auto objects = gc.impl().objectFactory().LockForIter();
-#endif
+    auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter();
     EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
+#endif
 }

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -9,6 +9,7 @@
 
 #include "CompilerConstants.hpp"
 #include "GlobalData.hpp"
+#include "GCImpl.hpp"
 #include "GCStatistics.hpp"
 #include "Logging.hpp"
 #include "MarkAndSweepUtils.hpp"
@@ -92,9 +93,11 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
 #else
 gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
         mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory,
+        mm::ExtraObjectDataFactory& extraObjectDataFactory,
         gcScheduler::GCScheduler& gcScheduler) noexcept :
 
     objectFactory_(objectFactory),
+    extraObjectDataFactory_(extraObjectDataFactory),
 #endif
     gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
         GCHandle::getByEpoch(epoch).finalizersDone();
@@ -150,7 +153,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifdef CUSTOM_ALLOCATOR
     // This should really be done by each individual thread while waiting
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().Allocator().PrepareForGC();
+        thread.gc().impl().alloc().PrepareForGC();
     }
     heap_.PrepareForGC();
 #endif
@@ -166,7 +169,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 #ifndef CUSTOM_ALLOCATOR
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
-    std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
+    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
     std::optional objectFactoryIterable = objectFactory_.LockForIter();
 
     gc::SweepExtraObjects<SweepTraits>(gcHandle, *extraObjectFactoryIterable);

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -9,6 +9,7 @@
 #include <cstddef>
 
 #include "Allocator.hpp"
+#include "ExtraObjectDataFactory.hpp"
 #include "FinalizerProcessor.hpp"
 #include "GCScheduler.hpp"
 #include "GCState.hpp"
@@ -109,7 +110,10 @@ public:
     using FinalizerQueue = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueue;
     using FinalizerQueueTraits = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueueTraits;
 
-    SameThreadMarkAndSweep(mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept;
+    SameThreadMarkAndSweep(
+            mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory,
+            mm::ExtraObjectDataFactory& extraObjectDataFactory,
+            gcScheduler::GCScheduler& gcScheduler) noexcept;
 #endif
 
     ~SameThreadMarkAndSweep();
@@ -130,6 +134,7 @@ private:
 
 #ifndef CUSTOM_ALLOCATOR
     mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
+    mm::ExtraObjectDataFactory& extraObjectDataFactory_;
 #else
     alloc::Heap heap_;
 #endif

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -215,12 +215,6 @@ public:
     ~SameThreadMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::SpecialRefRegistry::instance().clearForTests();
-#ifndef CUSTOM_ALLOCATOR
-        mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-        mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests();
-#else
-        mm::GlobalData::Instance().gc().impl().gc().heap().ClearForTests();
-#endif
         mm::GlobalData::Instance().gc().ClearForTests();
     }
 

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp

@@ -13,10 +13,6 @@
 #include "ObjCMMAPI.h"
 #endif
 
-#ifdef CUSTOM_ALLOCATOR
-#include "CustomAllocator.hpp"
-#endif
-
 using namespace kotlin;
 
 // static
@@ -32,20 +28,11 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
 
     RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
 
-    auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-#ifdef CUSTOM_ALLOCATOR
-    auto& data = threadData->gc().Allocator().CreateExtraObjectDataForObject(object, typeInfo);
-
+    auto& gc = mm::ThreadRegistry::Instance().CurrentThreadData()->gc();
+    auto& data = gc.CreateExtraObjectDataForObject(object, typeInfo);
     if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
         // Somebody else created `mm::ExtraObjectData` for this object.
-        data.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
-#else
-    auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo);
-
-    if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
-        // Somebody else created `mm::ExtraObjectData` for this object
-        mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, data);
-#endif
+        gc.DestroyUnattachedExtraObjectData(data);
         return *reinterpret_cast<mm::ExtraObjectData*>(typeInfo);
     }
 

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp

@@ -10,36 +10,14 @@
 
 using namespace kotlin;
 
-#ifndef CUSTOM_ALLOCATOR
-// static
-mm::ExtraObjectDataFactory& mm::ExtraObjectDataFactory::Instance() noexcept {
-    return GlobalData::Instance().extraObjectDataFactory();
+mm::ExtraObjectData& mm::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject(
+        ObjHeader* baseObject, const TypeInfo* info) noexcept {
+    return **Emplace(baseObject, info);
 }
 
-mm::ExtraObjectData& mm::ExtraObjectDataFactory::CreateExtraObjectDataForObject(
-        mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info
-        ) noexcept {
-    return CreateExtraObjectDataForObject(threadData->extraObjectDataThreadQueue(), baseObject, info);
-}
-
-void mm::ExtraObjectDataFactory::DestroyExtraObjectData(mm::ThreadData* threadData, ExtraObjectData& data) noexcept {
-        DestroyExtraObjectData(threadData->extraObjectDataThreadQueue(), data);
-}
-
-mm::ExtraObjectData& mm::ExtraObjectDataFactory::CreateExtraObjectDataForObject(
-        ThreadQueue& threadQueue, ObjHeader* baseObject, const TypeInfo* info
-        ) noexcept {
-    return **threadQueue.Emplace(baseObject, info);
-}
-
-void mm::ExtraObjectDataFactory::DestroyExtraObjectData(ThreadQueue& threadQueue, ExtraObjectData& data) noexcept {
-    threadQueue.Erase(&Queue::Node::fromValue(data));
-}
-
-void mm::ExtraObjectDataFactory::ProcessThread(mm::ThreadData* threadData) noexcept {
-    threadData->extraObjectDataThreadQueue().Publish();
+void mm::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(ExtraObjectData& data) noexcept {
+    Erase(&Queue::Node::fromValue(data));
 }
 
 mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
 mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
-#endif

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp

@@ -23,6 +23,15 @@ public:
     class ThreadQueue : public Queue::Producer {
     public:
         explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {}
+
+        ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept;
+
+        void DestroyExtraObjectData(ExtraObjectData& data) noexcept;
+
+        // Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread
+        // when it's asked by GC to stop.
+        using Producer::Publish;
+
         // Do not add fields as this is just a wrapper and Producer does not have virtual destructor.
     };
 
@@ -32,17 +41,6 @@ public:
     ExtraObjectDataFactory();
     ~ExtraObjectDataFactory();
 
-    static ExtraObjectDataFactory& Instance() noexcept;
-
-    ExtraObjectData& CreateExtraObjectDataForObject(mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept;
-    ExtraObjectData& CreateExtraObjectDataForObject(ThreadQueue& threadQueue, ObjHeader* baseObject, const TypeInfo* info) noexcept;
-
-    void DestroyExtraObjectData(mm::ThreadData* threadData, ExtraObjectData& data) noexcept;
-    void DestroyExtraObjectData(ThreadQueue& threadQueue, ExtraObjectData& data) noexcept;
-
-    // Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread
-    // when it's asked by GC to stop.
-    void ProcessThread(mm::ThreadData* threadData) noexcept;
 
     // Lock registry for safe iteration.
     Iterable LockForIter() noexcept { return extraObjects_.LockForIter(); }

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -29,9 +29,6 @@ public:
 
     ~ExtraObjectDataTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
-#ifndef CUSTOM_ALLOCATOR
-        mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-#endif
         mm::GlobalData::Instance().gc().ClearForTests();
     }
 };

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -13,7 +13,6 @@
 #include "SpecialRefRegistry.hpp"
 #include "ThreadRegistry.hpp"
 #include "Utils.hpp"
-#include "ExtraObjectDataFactory.hpp"
 #include "AppStateTracking.hpp"
 
 namespace kotlin {
@@ -27,9 +26,6 @@ public:
     ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; }
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
     SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; }
-#ifndef CUSTOM_ALLOCATOR
-    ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
-#endif
     gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
     gc::GC& gc() noexcept { return gc_; }
     AppStateTracking& appStateTracking() noexcept { return appStateTracking_; }
@@ -47,12 +43,6 @@ private:
     SpecialRefRegistry specialRefRegistry_;
     gcScheduler::GCScheduler gcScheduler_;
     gc::GC gc_{gcScheduler_};
-#ifndef CUSTOM_ALLOCATOR
-    // by being last, ommiting it will not affect the offsets of the other
-    // members, and we avoid having to have _custom versions of the gcScheduler
-    // modules.
-    ExtraObjectDataFactory extraObjectDataFactory_;
-#endif
 };
 
 } // namespace mm

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -79,14 +79,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) {
 void ObjHeader::destroyMetaObject(ObjHeader* object) {
     RuntimeAssert(object->has_meta_object(), "Object must have a meta object set");
     auto &extraObject = *mm::ExtraObjectData::Get(object);
-#ifdef CUSTOM_ALLOCATOR
-    extraObject.ReleaseAssociatedObject();
-    extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
-#else
-    extraObject.Uninstall();
-    auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, extraObject);
-#endif
+    gc::GC::DestroyExtraObjectData(extraObject);
 }
 
 ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -22,14 +22,17 @@
 #include "std_support/Vector.hpp"
 
 // ObjectFactory is not used by custom allocator
-#ifndef CUSTOM_ALLOCATOR
 using namespace kotlin;
 
 using testing::_;
 
 namespace {
 
-using SimpleAllocator = gc::Allocator;
+class SimpleAllocator {
+public:
+    void* Alloc(size_t size) noexcept { return std_support::calloc(1, size); }
+    static void Free(void* instance, size_t size) noexcept { std_support::free(instance); }
+};
 
 struct DataSizeProvider {
     static size_t GetDataSize(void* data) noexcept { return 0; }
@@ -790,9 +793,9 @@ public:
     MOCK_METHOD(void*, Alloc, (size_t));
     MOCK_METHOD(void, Free, (void*, size_t));
 
-    void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); }
+    void* DefaultAlloc(size_t size) { return std_support::calloc(1, size); }
 
-    void DefaultFree(void* instance, size_t size) { freeInObjectPool(instance, size); }
+    void DefaultFree(void* instance, size_t size) { std_support::free(instance); }
 };
 
 class GlobalMockAllocator {
@@ -1132,4 +1135,3 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
     EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
     EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount);
 }
-#endif

kotlin-native/runtime/src/mm/cpp/TestSupport.cpp

@@ -48,11 +48,6 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() {
     auto specialRefs = mm::SpecialRefRegistry::instance().lockForIter();
     auto threads = mm::ThreadRegistry::Instance().LockForIter();
 
-#ifndef CUSTOM_ALLOCATOR
-    auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
-    EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
-#endif
-
     EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre());

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -13,7 +13,6 @@
 #include "GC.hpp"
 #include "GCScheduler.hpp"
 #include "ObjectFactory.hpp"
-#include "ExtraObjectDataFactory.hpp"
 #include "ShadowStack.hpp"
 #include "SpecialRefRegistry.hpp"
 #include "ThreadLocalStorage.hpp"
@@ -34,9 +33,6 @@ public:
         threadId_(threadId),
         globalsThreadQueue_(GlobalsRegistry::Instance()),
         specialRefRegistry_(SpecialRefRegistry::instance()),
-#ifndef CUSTOM_ALLOCATOR
-        extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()),
-#endif
         gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()),
         gc_(GlobalData::Instance().gc(), gcScheduler_, *this),
         suspensionData_(ThreadState::kNative, *this) {}
@@ -51,10 +47,6 @@ public:
 
     SpecialRefRegistry::ThreadQueue& specialRefRegistry() noexcept { return specialRefRegistry_; }
 
-#ifndef CUSTOM_ALLOCATOR
-    ExtraObjectDataFactory::ThreadQueue& extraObjectDataThreadQueue() noexcept { return extraObjectDataThreadQueue_; }
-#endif
-
     ThreadState state() noexcept { return suspensionData_.state(); }
 
     ThreadState setState(ThreadState state) noexcept { return suspensionData_.setState(state); }
@@ -73,18 +65,12 @@ public:
         // TODO: These use separate locks, which is inefficient.
         globalsThreadQueue_.Publish();
         specialRefRegistry_.publish();
-#ifndef CUSTOM_ALLOCATOR
-        extraObjectDataThreadQueue_.Publish();
-#endif
         gc_.Publish();
     }
 
     void ClearForTests() noexcept {
         globalsThreadQueue_.ClearForTests();
         specialRefRegistry_.clearForTests();
-#ifndef CUSTOM_ALLOCATOR
-        extraObjectDataThreadQueue_.ClearForTests();
-#endif
         gc_.ClearForTests();
     }
 
@@ -93,9 +79,6 @@ private:
     GlobalsRegistry::ThreadQueue globalsThreadQueue_;
     ThreadLocalStorage tls_;
     SpecialRefRegistry::ThreadQueue specialRefRegistry_;
-#ifndef CUSTOM_ALLOCATOR
-    ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_;
-#endif
     ShadowStack shadowStack_;
     gcScheduler::GCSchedulerThreadData gcScheduler_;
     gc::GC::ThreadData gc_;