From 0a726d4ebfa7d28553740c4cb63ce2be0e83a5e7 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Wed, 26 Jul 2023 14:54:11 +0200 Subject: [PATCH] [K/N] Move CUSTOM_ALLOCATOR macro to gc modules only ^KT-55364 --- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 11 +++--- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 7 +++- .../gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp | 3 -- .../runtime/src/gc/cms/cpp/GCImpl.cpp | 34 +++++++++++++++--- .../runtime/src/gc/cms/cpp/GCImpl.hpp | 12 +++++-- .../src/gc/cms/cpp/GCImplTestSupport.cpp | 14 +++++++- .../runtime/src/gc/common/cpp/GC.hpp | 13 +++---- .../common/cpp/MarkAndSweepUtilsSweepTest.cpp | 4 +-- .../runtime/src/gc/noop/cpp/GCImpl.cpp | 35 ++++++++++++++++--- .../runtime/src/gc/noop/cpp/GCImpl.hpp | 11 ++++-- .../src/gc/noop/cpp/GCImplTestSupport.cpp | 14 +++++++- .../runtime/src/gc/stms/cpp/GCImpl.cpp | 34 +++++++++++++++--- .../runtime/src/gc/stms/cpp/GCImpl.hpp | 16 +++++++-- .../src/gc/stms/cpp/GCImplTestSupport.cpp | 14 +++++++- .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 7 ++-- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 7 +++- .../stms/cpp/SameThreadMarkAndSweepTest.cpp | 6 ---- .../runtime/src/mm/cpp/ExtraObjectData.cpp | 19 ++-------- .../src/mm/cpp/ExtraObjectDataFactory.cpp | 32 +++-------------- .../src/mm/cpp/ExtraObjectDataFactory.hpp | 20 +++++------ .../src/mm/cpp/ExtraObjectDataTest.cpp | 3 -- .../runtime/src/mm/cpp/GlobalData.hpp | 10 ------ kotlin-native/runtime/src/mm/cpp/Memory.cpp | 9 +---- .../runtime/src/mm/cpp/ObjectFactoryTest.cpp | 12 ++++--- .../runtime/src/mm/cpp/TestSupport.cpp | 5 --- .../runtime/src/mm/cpp/ThreadData.hpp | 17 --------- 26 files changed, 218 insertions(+), 151 deletions(-) diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index e9aa1642d61..090b8768cb2 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -108,8 +108,11 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { #ifndef CUSTOM_ALLOCATOR gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : + mm::ObjectFactory& objectFactory, + mm::ExtraObjectDataFactory& extraObjectDataFactory, + gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), + extraObjectDataFactory_(extraObjectDataFactory), #else gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept : #endif @@ -168,7 +171,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifdef CUSTOM_ALLOCATOR // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().Allocator().PrepareForGC(); + thread.gc().impl().alloc().PrepareForGC(); } heap_.PrepareForGC(); #endif @@ -190,7 +193,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); + std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); std::optional objectFactoryIterable = objectFactory_.LockForIter(); #endif @@ -222,7 +225,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // also sweeps extraObjects auto finalizerQueue = heap_.Sweep(gcHandle); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().Allocator().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); } #endif state_.finish(epoch); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 33b14a7f1fe..1297177d3d3 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -10,6 +10,7 @@ #include "Allocator.hpp" #include "Barriers.hpp" +#include "ExtraObjectDataFactory.hpp" #include "FinalizerProcessor.hpp" #include "GCScheduler.hpp" #include "GCState.hpp" @@ -120,7 +121,10 @@ public: #ifdef CUSTOM_ALLOCATOR explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler) noexcept; #else - ConcurrentMarkAndSweep(mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& scheduler) noexcept; + ConcurrentMarkAndSweep( + mm::ObjectFactory& objectFactory, + mm::ExtraObjectDataFactory& extraObjectDataFactory, + gcScheduler::GCScheduler& scheduler) noexcept; #endif ~ConcurrentMarkAndSweep(); @@ -144,6 +148,7 @@ private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory_; + mm::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; #endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 9f6c8cdc01a..aa52503d50e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -219,9 +219,6 @@ public: ~ConcurrentMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); -#ifndef CUSTOM_ALLOCATOR - mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); -#endif mm::GlobalData::Instance().gc().ClearForTests(); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 14cf6bf6f58..480c1ebabcb 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -34,12 +34,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); impl_->objectFactoryThreadQueue().ClearForTests(); #else impl_->alloc().PrepareForGC(); @@ -62,11 +64,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI #endif } -#ifdef CUSTOM_ALLOCATOR -alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { - return impl_->alloc(); -} +ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( + ObjHeader* object, const TypeInfo* typeInfo) noexcept { +#ifndef CUSTOM_ALLOCATOR + return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); +#else + return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif +} + +ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); +#endif +} void gc::GC::ThreadData::OnSuspendForGC() noexcept { impl_->gc().OnSuspendForGC(); @@ -96,6 +109,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactory().ClearForTests(); impl_->objectFactory().ClearForTests(); #else impl_->gc().heap().ClearForTests(); @@ -154,3 +168,15 @@ const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData) ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { return reinterpret_cast(objectData)->tryResetMark(); } + +// static +ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + extraObject.Uninstall(); + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.ReleaseAssociatedObject(); + extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); +#endif +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 68c94572caa..086dcf6be15 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -11,6 +11,8 @@ #ifdef CUSTOM_ALLOCATOR #include "CustomAllocator.hpp" +#else +#include "ExtraObjectDataFactory.hpp" #endif namespace kotlin { @@ -23,17 +25,19 @@ public: #ifdef CUSTOM_ALLOCATOR explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {} #else - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} #endif #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif GCImpl& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; + mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif GCImpl gc_; }; @@ -43,7 +47,9 @@ public: Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData, gcScheduler), #ifndef CUSTOM_ALLOCATOR - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { + } #else alloc_(gc.impl_->gc().heap(), gcScheduler) { } @@ -54,6 +60,7 @@ public: alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: @@ -62,6 +69,7 @@ private: alloc::CustomAllocator alloc_; #else mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp index 131801a41e1..9cc05533b19 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp @@ -23,13 +23,25 @@ auto collectCopy(T& iterable) { return result; } +template +auto collectPointers(T& iterable) { + std::vector*> result; + for (const auto& element : iterable) { + result.push_back(&element); + } + return result; +} + } // namespace void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else auto objects = gc.impl().objectFactory().LockForIter(); -#endif + auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); + EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); +#endif } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index b55223b2fbb..e00d36f8140 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -7,15 +7,12 @@ #include +#include "ExtraObjectData.hpp" #include "GCScheduler.hpp" #include "Memory.h" #include "Utils.hpp" #include "std_support/Memory.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#endif - namespace kotlin { namespace mm { @@ -46,10 +43,8 @@ public: ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; - -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator& Allocator() noexcept; -#endif + mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* object, const TypeInfo* typeInfo) noexcept; + void DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; void OnSuspendForGC() noexcept; @@ -86,6 +81,8 @@ public: static const size_t objectDataSize; static bool SweepObject(void* objectData) noexcept; + static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; + private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index d59137c2eb2..2fd398e08f4 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -168,7 +168,7 @@ public: auto deallocExtraObject = [this](ObjHeader* obj) { auto *extraObject = mm::ExtraObjectData::Get(obj); extraObject->Uninstall(); - extraObjectFactory_.DestroyExtraObjectData(extraObjectFactoryThreadQueue_, *extraObject); + extraObjectFactoryThreadQueue_.DestroyExtraObjectData(*extraObject); extraObjectFactoryThreadQueue_.Publish(); }; for (auto& finalizerQueue : finalizers_) { @@ -240,7 +240,7 @@ public: } mm::ExtraObjectData& InstallExtraData(ObjHeader *objHeader) { - auto& extraObjectData = extraObjectFactory_.CreateExtraObjectDataForObject(extraObjectFactoryThreadQueue_, objHeader, objHeader->type_info()); + auto& extraObjectData = extraObjectFactoryThreadQueue_.CreateExtraObjectDataForObject(objHeader, objHeader->type_info()); extraObjectFactoryThreadQueue_.Publish(); objHeader->typeInfoOrMeta_ = reinterpret_cast(&extraObjectData); return *mm::ExtraObjectData::Get(objHeader); diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 4321d72ee1c..53ae4c6bf06 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -8,6 +8,7 @@ #include "Common.h" #include "GC.hpp" #include "NoOpGC.hpp" +#include "ThreadData.hpp" #include "std_support/Memory.hpp" #include "GCStatistics.hpp" #include "ObjectOps.hpp" @@ -37,12 +38,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); impl_->objectFactoryThreadQueue().ClearForTests(); #else impl_->alloc().PrepareForGC(); @@ -65,11 +68,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI #endif } -#ifdef CUSTOM_ALLOCATOR -alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { - return impl_->alloc(); -} +ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( + ObjHeader* object, const TypeInfo* typeInfo) noexcept { +#ifndef CUSTOM_ALLOCATOR + return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); +#else + return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif +} + +ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); +#endif +} void gc::GC::ThreadData::OnSuspendForGC() noexcept { } @@ -94,6 +108,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { void gc::GC::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactory().ClearForTests(); impl_->objectFactory().ClearForTests(); #else impl_->gc().heap().ClearForTests(); @@ -139,3 +154,15 @@ const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { return true; } + +// static +ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + extraObject.Uninstall(); + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.ReleaseAssociatedObject(); + extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); +#endif +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index d988b973b28..2e9d1fe3857 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -12,7 +12,8 @@ #ifdef CUSTOM_ALLOCATOR #include "CustomAllocator.hpp" -#include "GCScheduler.hpp" +#else +#include "ExtraObjectDataFactory.hpp" #endif namespace kotlin { @@ -26,12 +27,14 @@ public: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif GCImpl& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; + mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif GCImpl gc_; }; @@ -43,7 +46,8 @@ public: alloc_(gc.impl_->gc().heap(), gcScheduler) {} #else Impl(GC& gc, mm::ThreadData& threadData) noexcept : - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {} #endif GCImpl::ThreadData& gc() noexcept { return gc_; } @@ -51,15 +55,16 @@ public: alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: GCImpl::ThreadData gc_; #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator alloc_; - /* gcScheduler::GCSchedulerThreadData; */ #else mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp index 131801a41e1..9cc05533b19 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp @@ -23,13 +23,25 @@ auto collectCopy(T& iterable) { return result; } +template +auto collectPointers(T& iterable) { + std::vector*> result; + for (const auto& element : iterable) { + result.push_back(&element); + } + return result; +} + } // namespace void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else auto objects = gc.impl().objectFactory().LockForIter(); -#endif + auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); + EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); +#endif } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 76608237883..c6adf7df860 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -34,12 +34,14 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { void gc::GC::ThreadData::Publish() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); impl_->objectFactoryThreadQueue().ClearForTests(); #else impl_->alloc().PrepareForGC(); @@ -62,11 +64,22 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI #endif } -#ifdef CUSTOM_ALLOCATOR -alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { - return impl_->alloc(); -} +ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( + ObjHeader* object, const TypeInfo* typeInfo) noexcept { +#ifndef CUSTOM_ALLOCATOR + return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); +#else + return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif +} + +ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); +#endif +} void gc::GC::ThreadData::OnSuspendForGC() noexcept { } @@ -92,6 +105,7 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR + impl_->extraObjectDataFactory().ClearForTests(); impl_->objectFactory().ClearForTests(); #else impl_->gc().heap().ClearForTests(); @@ -150,3 +164,15 @@ const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData) ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { return reinterpret_cast(objectData)->tryResetMark(); } + +// static +ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { +#ifndef CUSTOM_ALLOCATOR + extraObject.Uninstall(); + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +#else + extraObject.ReleaseAssociatedObject(); + extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); +#endif +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index b8148ffef6a..cfbfb2e1d93 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -9,6 +9,12 @@ #include "SameThreadMarkAndSweep.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "CustomAllocator.hpp" +#else +#include "ExtraObjectDataFactory.hpp" +#endif + namespace kotlin { namespace gc { @@ -19,15 +25,17 @@ public: #ifdef CUSTOM_ALLOCATOR explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {} #else - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif GCImpl& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; + mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif GCImpl gc_; }; @@ -39,12 +47,15 @@ public: #ifdef CUSTOM_ALLOCATOR alloc_(gc.impl_->gc().heap(), gcScheduler) {} #else - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { + } #endif GCImpl::ThreadData& gc() noexcept { return gc_; } #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #else alloc::CustomAllocator& alloc() noexcept { return alloc_; } #endif @@ -55,6 +66,7 @@ private: alloc::CustomAllocator alloc_; #else mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp index 131801a41e1..9cc05533b19 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp @@ -23,13 +23,25 @@ auto collectCopy(T& iterable) { return result; } +template +auto collectPointers(T& iterable) { + std::vector*> result; + for (const auto& element : iterable) { + result.push_back(&element); + } + return result; +} + } // namespace void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else auto objects = gc.impl().objectFactory().LockForIter(); -#endif + auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); + EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); +#endif } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 3b1702960d0..47d3111a3a5 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -9,6 +9,7 @@ #include "CompilerConstants.hpp" #include "GlobalData.hpp" +#include "GCImpl.hpp" #include "GCStatistics.hpp" #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" @@ -92,9 +93,11 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( #else gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( mm::ObjectFactory& objectFactory, + mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), + extraObjectDataFactory_(extraObjectDataFactory), #endif gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { GCHandle::getByEpoch(epoch).finalizersDone(); @@ -150,7 +153,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifdef CUSTOM_ALLOCATOR // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().Allocator().PrepareForGC(); + thread.gc().impl().alloc().PrepareForGC(); } heap_.PrepareForGC(); #endif @@ -166,7 +169,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); + std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); std::optional objectFactoryIterable = objectFactory_.LockForIter(); gc::SweepExtraObjects(gcHandle, *extraObjectFactoryIterable); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 45ede2edcc7..90d10f850b3 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -9,6 +9,7 @@ #include #include "Allocator.hpp" +#include "ExtraObjectDataFactory.hpp" #include "FinalizerProcessor.hpp" #include "GCScheduler.hpp" #include "GCState.hpp" @@ -109,7 +110,10 @@ public: using FinalizerQueue = mm::ObjectFactory::FinalizerQueue; using FinalizerQueueTraits = mm::ObjectFactory::FinalizerQueueTraits; - SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept; + SameThreadMarkAndSweep( + mm::ObjectFactory& objectFactory, + mm::ExtraObjectDataFactory& extraObjectDataFactory, + gcScheduler::GCScheduler& gcScheduler) noexcept; #endif ~SameThreadMarkAndSweep(); @@ -130,6 +134,7 @@ private: #ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory_; + mm::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; #endif diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 3c3da0d6bb7..b7bba4008ee 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -215,12 +215,6 @@ public: ~SameThreadMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); -#ifndef CUSTOM_ALLOCATOR - mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); - mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests(); -#else - mm::GlobalData::Instance().gc().impl().gc().heap().ClearForTests(); -#endif mm::GlobalData::Instance().gc().ClearForTests(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index 89b829d9166..3facaa97852 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -13,10 +13,6 @@ #include "ObjCMMAPI.h" #endif -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#endif - using namespace kotlin; // static @@ -32,20 +28,11 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged"); - auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); -#ifdef CUSTOM_ALLOCATOR - auto& data = threadData->gc().Allocator().CreateExtraObjectDataForObject(object, typeInfo); - + auto& gc = mm::ThreadRegistry::Instance().CurrentThreadData()->gc(); + auto& data = gc.CreateExtraObjectDataForObject(object, typeInfo); if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { // Somebody else created `mm::ExtraObjectData` for this object. - data.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); -#else - auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo); - - if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { - // Somebody else created `mm::ExtraObjectData` for this object - mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, data); -#endif + gc.DestroyUnattachedExtraObjectData(data); return *reinterpret_cast(typeInfo); } diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp index dd7d1462ff1..884b7ac7040 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp @@ -10,36 +10,14 @@ using namespace kotlin; -#ifndef CUSTOM_ALLOCATOR -// static -mm::ExtraObjectDataFactory& mm::ExtraObjectDataFactory::Instance() noexcept { - return GlobalData::Instance().extraObjectDataFactory(); +mm::ExtraObjectData& mm::ExtraObjectDataFactory::ThreadQueue::CreateExtraObjectDataForObject( + ObjHeader* baseObject, const TypeInfo* info) noexcept { + return **Emplace(baseObject, info); } -mm::ExtraObjectData& mm::ExtraObjectDataFactory::CreateExtraObjectDataForObject( - mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info - ) noexcept { - return CreateExtraObjectDataForObject(threadData->extraObjectDataThreadQueue(), baseObject, info); -} - -void mm::ExtraObjectDataFactory::DestroyExtraObjectData(mm::ThreadData* threadData, ExtraObjectData& data) noexcept { - DestroyExtraObjectData(threadData->extraObjectDataThreadQueue(), data); -} - -mm::ExtraObjectData& mm::ExtraObjectDataFactory::CreateExtraObjectDataForObject( - ThreadQueue& threadQueue, ObjHeader* baseObject, const TypeInfo* info - ) noexcept { - return **threadQueue.Emplace(baseObject, info); -} - -void mm::ExtraObjectDataFactory::DestroyExtraObjectData(ThreadQueue& threadQueue, ExtraObjectData& data) noexcept { - threadQueue.Erase(&Queue::Node::fromValue(data)); -} - -void mm::ExtraObjectDataFactory::ProcessThread(mm::ThreadData* threadData) noexcept { - threadData->extraObjectDataThreadQueue().Publish(); +void mm::ExtraObjectDataFactory::ThreadQueue::DestroyExtraObjectData(ExtraObjectData& data) noexcept { + Erase(&Queue::Node::fromValue(data)); } mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default; mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default; -#endif diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp index 50079455565..dc1a5e41118 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp @@ -23,6 +23,15 @@ public: class ThreadQueue : public Queue::Producer { public: explicit ThreadQueue(ExtraObjectDataFactory& registry) : Producer(registry.extraObjects_) {} + + ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* baseObject, const TypeInfo* info) noexcept; + + void DestroyExtraObjectData(ExtraObjectData& data) noexcept; + + // Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread + // when it's asked by GC to stop. + using Producer::Publish; + // Do not add fields as this is just a wrapper and Producer does not have virtual destructor. }; @@ -32,17 +41,6 @@ public: ExtraObjectDataFactory(); ~ExtraObjectDataFactory(); - static ExtraObjectDataFactory& Instance() noexcept; - - ExtraObjectData& CreateExtraObjectDataForObject(mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept; - ExtraObjectData& CreateExtraObjectDataForObject(ThreadQueue& threadQueue, ObjHeader* baseObject, const TypeInfo* info) noexcept; - - void DestroyExtraObjectData(mm::ThreadData* threadData, ExtraObjectData& data) noexcept; - void DestroyExtraObjectData(ThreadQueue& threadQueue, ExtraObjectData& data) noexcept; - - // Collect extra data objects from thread corresponding to `threadData`. Must be called by the thread - // when it's asked by GC to stop. - void ProcessThread(mm::ThreadData* threadData) noexcept; // Lock registry for safe iteration. Iterable LockForIter() noexcept { return extraObjects_.LockForIter(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index 00532077eb3..25b2c42aaf1 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -29,9 +29,6 @@ public: ~ExtraObjectDataTest() { mm::GlobalsRegistry::Instance().ClearForTests(); -#ifndef CUSTOM_ALLOCATOR - mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); -#endif mm::GlobalData::Instance().gc().ClearForTests(); } }; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index f96e2fc7b41..cc4504c27b1 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -13,7 +13,6 @@ #include "SpecialRefRegistry.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" -#include "ExtraObjectDataFactory.hpp" #include "AppStateTracking.hpp" namespace kotlin { @@ -27,9 +26,6 @@ public: ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; } GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; } -#ifndef CUSTOM_ALLOCATOR - ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } -#endif gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } gc::GC& gc() noexcept { return gc_; } AppStateTracking& appStateTracking() noexcept { return appStateTracking_; } @@ -47,12 +43,6 @@ private: SpecialRefRegistry specialRefRegistry_; gcScheduler::GCScheduler gcScheduler_; gc::GC gc_{gcScheduler_}; -#ifndef CUSTOM_ALLOCATOR - // by being last, ommiting it will not affect the offsets of the other - // members, and we avoid having to have _custom versions of the gcScheduler - // modules. - ExtraObjectDataFactory extraObjectDataFactory_; -#endif }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 89a7524de25..f00d0971288 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -79,14 +79,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { void ObjHeader::destroyMetaObject(ObjHeader* object) { RuntimeAssert(object->has_meta_object(), "Object must have a meta object set"); auto &extraObject = *mm::ExtraObjectData::Get(object); -#ifdef CUSTOM_ALLOCATOR - extraObject.ReleaseAssociatedObject(); - extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); -#else - extraObject.Uninstall(); - auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, extraObject); -#endif + gc::GC::DestroyExtraObjectData(extraObject); } ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index a795beba1d5..3bc0ec6ccb9 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -22,14 +22,17 @@ #include "std_support/Vector.hpp" // ObjectFactory is not used by custom allocator -#ifndef CUSTOM_ALLOCATOR using namespace kotlin; using testing::_; namespace { -using SimpleAllocator = gc::Allocator; +class SimpleAllocator { +public: + void* Alloc(size_t size) noexcept { return std_support::calloc(1, size); } + static void Free(void* instance, size_t size) noexcept { std_support::free(instance); } +}; struct DataSizeProvider { static size_t GetDataSize(void* data) noexcept { return 0; } @@ -790,9 +793,9 @@ public: MOCK_METHOD(void*, Alloc, (size_t)); MOCK_METHOD(void, Free, (void*, size_t)); - void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); } + void* DefaultAlloc(size_t size) { return std_support::calloc(1, size); } - void DefaultFree(void* instance, size_t size) { freeInObjectPool(instance, size); } + void DefaultFree(void* instance, size_t size) { std_support::free(instance); } }; class GlobalMockAllocator { @@ -1132,4 +1135,3 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount); } -#endif diff --git a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp index 2c8ebbfbf90..1fcdb59c5bb 100644 --- a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp +++ b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp @@ -48,11 +48,6 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() { auto specialRefs = mm::SpecialRefRegistry::instance().lockForIter(); auto threads = mm::ThreadRegistry::Instance().LockForIter(); -#ifndef CUSTOM_ALLOCATOR - auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); - EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); -#endif - EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre()); diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 792a5b98d07..5a158d81713 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -13,7 +13,6 @@ #include "GC.hpp" #include "GCScheduler.hpp" #include "ObjectFactory.hpp" -#include "ExtraObjectDataFactory.hpp" #include "ShadowStack.hpp" #include "SpecialRefRegistry.hpp" #include "ThreadLocalStorage.hpp" @@ -34,9 +33,6 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), -#ifndef CUSTOM_ALLOCATOR - extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()), -#endif gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), gc_(GlobalData::Instance().gc(), gcScheduler_, *this), suspensionData_(ThreadState::kNative, *this) {} @@ -51,10 +47,6 @@ public: SpecialRefRegistry::ThreadQueue& specialRefRegistry() noexcept { return specialRefRegistry_; } -#ifndef CUSTOM_ALLOCATOR - ExtraObjectDataFactory::ThreadQueue& extraObjectDataThreadQueue() noexcept { return extraObjectDataThreadQueue_; } -#endif - ThreadState state() noexcept { return suspensionData_.state(); } ThreadState setState(ThreadState state) noexcept { return suspensionData_.setState(state); } @@ -73,18 +65,12 @@ public: // TODO: These use separate locks, which is inefficient. globalsThreadQueue_.Publish(); specialRefRegistry_.publish(); -#ifndef CUSTOM_ALLOCATOR - extraObjectDataThreadQueue_.Publish(); -#endif gc_.Publish(); } void ClearForTests() noexcept { globalsThreadQueue_.ClearForTests(); specialRefRegistry_.clearForTests(); -#ifndef CUSTOM_ALLOCATOR - extraObjectDataThreadQueue_.ClearForTests(); -#endif gc_.ClearForTests(); } @@ -93,9 +79,6 @@ private: GlobalsRegistry::ThreadQueue globalsThreadQueue_; ThreadLocalStorage tls_; SpecialRefRegistry::ThreadQueue specialRefRegistry_; -#ifndef CUSTOM_ALLOCATOR - ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_; -#endif ShadowStack shadowStack_; gcScheduler::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_;