diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp index 1cc7c58c5c8..ad11be79944 100644 --- a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp @@ -5,12 +5,35 @@ #pragma once +#include "GC.hpp" +#include "Utils.hpp" + namespace kotlin::alloc { -// TODO: Build `Allocator`, `Allocator::ThreadData` like with `gc`, `gcScheduler`, -// and move allocator-specific data there. +// TODO: Move allocator-specific data and API here. +class Allocator : private Pinned { +public: + class Impl; + + class ThreadData : private Pinned { + public: + class Impl; + + private: + }; + +private: +}; void initObjectPool() noexcept; +// Instruct the allocator to free unused resources. void compactObjectPoolInCurrentThread() noexcept; +gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept; +ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept; + +// This does not take into account how much storage did the underlying allocator reserved. +size_t allocatedHeapSize(ObjHeader* object) noexcept; + +size_t allocatedBytes() noexcept; } diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp index 1bc03350b2b..a63590c5d51 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp @@ -3,10 +3,28 @@ * that can be found in the LICENSE file. */ -#include "Allocator.hpp" +#include "AllocatorImpl.hpp" + +#include "GCApi.hpp" using namespace kotlin; void alloc::initObjectPool() noexcept {} void alloc::compactObjectPoolInCurrentThread() noexcept {} + +gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept { + return HeapObjHeader::from(object).objectData(); +} + +ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept { + return HeapObjHeader::from(objectData).object(); +} + +size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept { + return CustomAllocator::GetAllocatedHeapSize(object); +} + +size_t alloc::allocatedBytes() noexcept { + return GetAllocatedBytes(); +} diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.hpp new file mode 100644 index 00000000000..80ebeb94d62 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.hpp @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Allocator.hpp" + +#include "CustomAllocator.hpp" +#include "CustomFinalizerProcessor.hpp" +#include "GCApi.hpp" +#include "Heap.hpp" + +namespace kotlin::alloc { + +class Allocator::Impl : private Pinned { +public: + Impl() noexcept = default; + + Heap& heap() noexcept { return heap_; } + +private: + Heap heap_; +}; + +class Allocator::ThreadData::Impl : private Pinned { +public: + explicit Impl(Allocator::Impl& allocator) noexcept : alloc_(allocator.heap()) {} + + alloc::CustomAllocator& alloc() noexcept { return alloc_; } + +private: + CustomAllocator alloc_; +}; + +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp index 3df7d21f6d7..967e85beb60 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/GCApi.hpp @@ -96,14 +96,6 @@ void Free(void* ptr, size_t size) noexcept; size_t GetAllocatedBytes() noexcept; -inline gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { - return HeapObjHeader::from(object).objectData(); -} - -inline ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept { - return HeapObjHeader::from(objectData).object(); -} - } // namespace kotlin::alloc #endif diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp new file mode 100644 index 00000000000..ebe15e0dc65 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp @@ -0,0 +1,20 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "AllocatorImpl.hpp" + +using namespace kotlin; + +gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept { + return ObjectFactoryImpl::NodeRef::From(object).ObjectData(); +} + +ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept { + return ObjectFactoryImpl::NodeRef::From(objectData)->GetObjHeader(); +} + +size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept { + return ObjectFactoryImpl::GetAllocatedHeapSize(object); +} diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.hpp new file mode 100644 index 00000000000..cba96724817 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.hpp @@ -0,0 +1,65 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Allocator.hpp" + +#include "ExtraObjectDataFactory.hpp" +#include "GC.hpp" +#include "GlobalData.hpp" +#include "Logging.hpp" +#include "ObjectFactory.hpp" +#include "ObjectFactoryAllocator.hpp" +#include "ObjectFactorySweep.hpp" + +namespace kotlin::alloc { + +struct ObjectFactoryTraits { + using Allocator = alloc::AllocatorWithGC; + using ObjectData = gc::GC::ObjectData; + + Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); } + + void OnOOM(size_t size) noexcept { + RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); + // TODO: This will print the log for "manual" scheduling. Fix this. + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); + } +}; + +using ObjectFactoryImpl = ObjectFactory; + +class Allocator::Impl : private Pinned { +public: + Impl() noexcept = default; + + ObjectFactoryImpl& objectFactory() noexcept { return objectFactory_; } + ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } + +private: + ObjectFactoryImpl objectFactory_; + ExtraObjectDataFactory extraObjectDataFactory_; +}; + +class Allocator::ThreadData::Impl : private Pinned { +public: + explicit Impl(Allocator::Impl& allocator) noexcept : + objectFactoryThreadQueue_(allocator.objectFactory(), objectFactoryTraits_.CreateAllocator()), + extraObjectDataFactoryThreadQueue_(allocator.extraObjectDataFactory()) {} + + ObjectFactoryImpl::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } + +private: + [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; + ObjectFactoryImpl::ThreadQueue objectFactoryThreadQueue_; + ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; +}; + +using FinalizerQueue = ObjectFactoryImpl::FinalizerQueue; +using FinalizerQueueTraits = ObjectFactoryImpl::FinalizerQueueTraits; + +} // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp index 4c813319b58..374b415817a 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectAlloc.hpp @@ -8,19 +8,17 @@ #include #include +#include "Allocator.hpp" + namespace kotlin::alloc { void initObjectPool() noexcept; void* allocateInObjectPool(size_t size) noexcept; void freeInObjectPool(void* ptr, size_t size) noexcept; -// Instruct the allocator to free unused resources. -void compactObjectPoolInCurrentThread() noexcept; // Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread. // May do nothing if the main thread is not an event loop. void compactObjectPoolInMainThread() noexcept; -size_t allocatedBytes() noexcept; - template struct ObjectPoolAllocator { using value_type = T; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp deleted file mode 100644 index a5b33896534..00000000000 --- a/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#ifdef CUSTOM_ALLOCATOR - -// TODO: Move into alloc/custom/AllocatorImpl.hpp - -#include "CustomAllocator.hpp" -#include "CustomFinalizerProcessor.hpp" -#include "GCApi.hpp" -#include "Heap.hpp" - -namespace kotlin::gc { - -inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { - return kotlin::alloc::objectDataForObject(object); -} - -inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { - return kotlin::alloc::objectForObjectData(objectData); -} - -using FinalizerQueue = alloc::FinalizerQueue; -using FinalizerQueueTraits = alloc::FinalizerQueueTraits; - -} // namespace kotlin::gc - -#else - -// TODO: Move into alloc/legacy/AllocatorImpl.hpp - -#include "ExtraObjectDataFactory.hpp" -#include "GC.hpp" -#include "GlobalData.hpp" -#include "Logging.hpp" -#include "ObjectFactory.hpp" -#include "ObjectFactoryAllocator.hpp" -#include "ObjectFactorySweep.hpp" - -namespace kotlin::gc { - -struct ObjectFactoryTraits { - using Allocator = alloc::AllocatorWithGC; - using ObjectData = gc::GC::ObjectData; - - Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); } - - void OnOOM(size_t size) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - // TODO: This will print the log for "manual" scheduling. Fix this. - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); - } -}; - -using ObjectFactory = alloc::ObjectFactory; - -inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { - return ObjectFactory::NodeRef::From(object).ObjectData(); -} - -inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { - return ObjectFactory::NodeRef::From(objectData)->GetObjHeader(); -} - -using FinalizerQueue = ObjectFactory::FinalizerQueue; -using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits; - -} // namespace kotlin::gc - -#endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp index 4a913188c88..7c40dfac8c0 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp @@ -67,7 +67,7 @@ ALWAYS_INLINE void gc::BarriersThreadData::onAllocation(ObjHeader* allocated) { bool barriersEnabled = weakRefBarrier.load(std::memory_order_relaxed) != nullptr; RuntimeAssert(shouldMark == barriersEnabled, "New allocations marking must happen with and only with weak ref barriers"); if (shouldMark) { - auto& objectData = objectDataForObject(allocated); + auto& objectData = alloc::objectDataForObject(allocated); bool wasUnmarked = objectData.tryMark(); RuntimeAssert(wasUnmarked, "No one else could mark this newly allocated object before"); markHandle_->addObject(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index a025b35d705..e5b5278d4be 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -37,7 +37,7 @@ ScopedThread createGCThread(const char* name, Body&& body) { #ifndef CUSTOM_ALLOCATOR // TODO move to common -[[maybe_unused]] inline void checkMarkCorrectness(gc::ObjectFactory::Iterable& heap) { +[[maybe_unused]] inline void checkMarkCorrectness(alloc::ObjectFactoryImpl::Iterable& heap) { if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return; for (auto objRef: heap) { auto obj = objRef.GetObjHeader(); @@ -45,7 +45,7 @@ ScopedThread createGCThread(const char* name, Body&& body) { if (objData.marked()) { traverseReferredObjects(obj, [obj](ObjHeader* field) { if (field->heap()) { - auto& fieldObjData = gc::ObjectFactory::NodeRef::From(field).ObjectData(); + auto& fieldObjData = alloc::ObjectFactoryImpl::NodeRef::From(field).ObjectData(); RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj); } }); @@ -89,20 +89,10 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const return threadData_; } -#ifndef CUSTOM_ALLOCATOR gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - ObjectFactory& objectFactory, - alloc::ExtraObjectDataFactory& extraObjectDataFactory, - gcScheduler::GCScheduler& gcScheduler, - bool mutatorsCooperate, - std::size_t auxGCThreads) noexcept : - objectFactory_(objectFactory), - extraObjectDataFactory_(extraObjectDataFactory), -#else -gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - gcScheduler::GCScheduler& gcScheduler, - bool mutatorsCooperate, std::size_t auxGCThreads) noexcept : -#endif + alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept + : + allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) { GCHandle::getByEpoch(epoch).finalizersDone(); @@ -191,10 +181,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifdef CUSTOM_ALLOCATOR // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().alloc().PrepareForGC(); + thread.gc().impl().allocator().alloc().PrepareForGC(); } - auto& heap = mm::GlobalData::Instance().gc().impl().gc().heap(); - heap.PrepareForGC(); + mm::GlobalData::Instance().gc().impl().allocator().heap().PrepareForGC(); #else for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { thread.gc().PublishObjectFactory(); @@ -202,8 +191,8 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional objectFactoryIterable = objectFactory_.LockForIter(); - std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); + std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter(); + std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter(); checkMarkCorrectness(*objectFactoryIterable); #endif @@ -211,20 +200,20 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { resumeTheWorld(gcHandle); #ifndef CUSTOM_ALLOCATOR - alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); + alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); + auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects - auto finalizerQueue = heap_.Sweep(gcHandle); + auto finalizerQueue = allocator_.heap().Sweep(gcHandle); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue()); } - finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes()); + scheduler.onGCFinish(epoch, alloc::allocatedBytes()); state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); gcHandle.finished(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 91ab8a55a9b..ada4557bd6f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -61,17 +61,11 @@ public: std::atomic published_ = false; }; -#ifdef CUSTOM_ALLOCATOR - explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler, - bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; -#else ConcurrentMarkAndSweep( - ObjectFactory& objectFactory, - alloc::ExtraObjectDataFactory& extraObjectDataFactory, + alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& scheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; -#endif ~ConcurrentMarkAndSweep(); void StartFinalizerThreadIfNeeded() noexcept; @@ -80,10 +74,6 @@ public: void reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, size_t auxGCThreads) noexcept; -#ifdef CUSTOM_ALLOCATOR - alloc::Heap& heap() noexcept { return heap_; } -#endif - GCStateHolder& state() noexcept { return state_; } private: @@ -91,16 +81,11 @@ private: void auxiliaryGCThreadBody(); void PerformFullGC(int64_t epoch) noexcept; -#ifndef CUSTOM_ALLOCATOR - ObjectFactory& objectFactory_; - alloc::ExtraObjectDataFactory& extraObjectDataFactory_; -#else - alloc::Heap heap_; -#endif + alloc::Allocator::Impl& allocator_; gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; - FinalizerProcessor finalizerProcessor_; + FinalizerProcessor finalizerProcessor_; mark::ParallelMark markDispatcher_; ScopedThread mainThread_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 93163c89d65..c0e3eedb62c 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -181,13 +181,13 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa std_support::vector Alive(mm::ThreadData& threadData) { #ifdef CUSTOM_ALLOCATOR - return threadData.gc().impl().alloc().heap().GetAllocatedObjects(); + return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects(); #else std_support::vector objects; - for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { + for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) { objects.push_back(node.GetObjHeader()); } - for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) { + for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) { objects.push_back(node.GetObjHeader()); } return objects; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 475952e07e2..d1db1d948be 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -20,26 +20,26 @@ gc::GC::ThreadData::~ThreadData() = default; void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().Publish(); - impl_->objectFactoryThreadQueue().Publish(); + impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); + impl_->allocator().objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->objectFactoryThreadQueue().ClearForTests(); + impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); + impl_->allocator().objectFactoryThreadQueue().ClearForTests(); #else - impl_->alloc().PrepareForGC(); + impl_->allocator().alloc().PrepareForGC(); #endif } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { ObjHeader* obj; #ifndef CUSTOM_ALLOCATOR - obj = impl_->objectFactoryThreadQueue().CreateObject(typeInfo); + obj = impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); #else - obj = impl_->alloc().CreateObject(typeInfo); + obj = impl_->allocator().alloc().CreateObject(typeInfo); #endif impl().gc().barriers().onAllocation(obj); return obj; @@ -48,9 +48,9 @@ ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeIn ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { ArrayHeader* arr; #ifndef CUSTOM_ALLOCATOR - arr = impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); + arr = impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); #else - arr = impl_->alloc().CreateArray(typeInfo, elements); + arr = impl_->allocator().alloc().CreateArray(typeInfo, elements); #endif impl().gc().barriers().onAllocation(arr->obj()); return arr; @@ -59,15 +59,15 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( ObjHeader* object, const TypeInfo* typeInfo) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); #else - return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif } ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); #endif @@ -89,30 +89,13 @@ gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support:: gc::GC::~GC() = default; -// static -size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { -#ifdef CUSTOM_ALLOCATOR - return alloc::CustomAllocator::GetAllocatedHeapSize(object); -#else - return ObjectFactory::GetAllocatedHeapSize(object); -#endif -} - -size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { -#ifdef CUSTOM_ALLOCATOR - return alloc::GetAllocatedBytes(); -#else - return alloc::allocatedBytes(); -#endif -} - void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactory().ClearForTests(); - impl_->objectFactory().ClearForTests(); + impl_->allocator().extraObjectDataFactory().ClearForTests(); + impl_->allocator().objectFactory().ClearForTests(); #else - impl_->gc().heap().ClearForTests(); + impl_->allocator().heap().ClearForTests(); #endif GCHandle::ClearForTests(); } @@ -157,7 +140,7 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept { } bool gc::isMarked(ObjHeader* object) noexcept { - return objectDataForObject(object).marked(); + return alloc::objectDataForObject(object).marked(); } ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { @@ -173,7 +156,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje #ifndef CUSTOM_ALLOCATOR extraObject.Uninstall(); auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.ReleaseAssociatedObject(); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index a197708d6b2..976a5d69d49 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -15,56 +15,27 @@ namespace gc { class GC::Impl : private Pinned { public: -#ifdef CUSTOM_ALLOCATOR - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {} -#else - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {} -#endif + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : + gc_(allocator_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {} -#ifndef CUSTOM_ALLOCATOR - ObjectFactory& objectFactory() noexcept { return objectFactory_; } - alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } -#endif + alloc::Allocator::Impl& allocator() noexcept { return allocator_; } ConcurrentMarkAndSweep& gc() noexcept { return gc_; } private: -#ifndef CUSTOM_ALLOCATOR - ObjectFactory objectFactory_; - alloc::ExtraObjectDataFactory extraObjectDataFactory_; -#endif + alloc::Allocator::Impl allocator_; ConcurrentMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : - gc_(gc.impl_->gc(), threadData), -#ifndef CUSTOM_ALLOCATOR - objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), - extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { - } -#else - alloc_(gc.impl_->gc().heap()) { - } -#endif + Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {} ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; } -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator& alloc() noexcept { return alloc_; } -#else - ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } -#endif + alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } private: ConcurrentMarkAndSweep::ThreadData gc_; -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator alloc_; -#else - [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; -#endif + alloc::Allocator::ThreadData::Impl allocator_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp index 9cc05533b19..f75fd949b74 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp @@ -36,11 +36,11 @@ auto collectPointers(T& iterable) { void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else - auto objects = gc.impl().objectFactory().LockForIter(); - auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); + auto objects = gc.impl().allocator().objectFactory().LockForIter(); + auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); #endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp index 2a1241d169c..a2310a18a5c 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp @@ -85,18 +85,18 @@ public: static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept { auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop(); if (obj) { - return objectForObjectData(*obj); + return alloc::objectForObjectData(*obj); } return nullptr; } static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { - auto& objectData = objectDataForObject(object); + auto& objectData = alloc::objectDataForObject(object); return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData); } static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept { - auto& objectData = objectDataForObject(object); + auto& objectData = alloc::objectDataForObject(object); return objectData.tryMark(); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 385ac6b3c5f..5f5c29920ff 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -66,10 +66,6 @@ public: Impl& impl() noexcept { return *impl_; } - static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; - - size_t GetTotalHeapObjectsSizeBytes() const noexcept; - void ClearForTests() noexcept; void StartFinalizerThreadIfNeeded() noexcept; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index 90914a029dc..702e9d018c6 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -9,6 +9,7 @@ #include #include +#include "Allocator.hpp" #include "Logging.hpp" #include "Mutex.hpp" #include "Porting.h" @@ -135,7 +136,7 @@ GCInfo* statByEpoch(uint64_t epoch) { MemoryUsage currentHeapUsage() noexcept { return MemoryUsage{ - mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes(), + alloc::allocatedBytes(), }; } diff --git a/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp deleted file mode 100644 index 01f5544e460..00000000000 --- a/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#ifdef CUSTOM_ALLOCATOR - -// TODO: Move into alloc/custom/AllocatorImpl.hpp - -#include "CustomAllocator.hpp" -#include "GCApi.hpp" -#include "Heap.hpp" - -#else - -// TODO: Move into alloc/legacy/AllocatorImpl.hpp - -#include "ExtraObjectDataFactory.hpp" -#include "GC.hpp" -#include "ObjectFactory.hpp" -#include "ObjectFactoryAllocator.hpp" - -namespace kotlin::gc { - -struct ObjectFactoryTraits { - using Allocator = alloc::AllocatorBasic; - using ObjectData = gc::GC::ObjectData; - - Allocator CreateAllocator() noexcept { return Allocator(); } -}; - -using ObjectFactory = alloc::ObjectFactory; - -} // namespace kotlin::gc - -#endif diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index d316c279423..22e466266df 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -8,7 +8,6 @@ #include "Common.h" #include "GC.hpp" #include "GCStatistics.hpp" -#include "NoOpGC.hpp" #include "ObjectOps.hpp" #include "ThreadData.hpp" #include "std_support/Memory.hpp" @@ -21,48 +20,48 @@ gc::GC::ThreadData::~ThreadData() = default; void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().Publish(); - impl_->objectFactoryThreadQueue().Publish(); + impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); + impl_->allocator().objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->objectFactoryThreadQueue().ClearForTests(); + impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); + impl_->allocator().objectFactoryThreadQueue().ClearForTests(); #else - impl_->alloc().PrepareForGC(); + impl_->allocator().alloc().PrepareForGC(); #endif } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); + return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); #else - return impl_->alloc().CreateObject(typeInfo); + return impl_->allocator().alloc().CreateObject(typeInfo); #endif } ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); + return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); #else - return impl_->alloc().CreateArray(typeInfo, elements); + return impl_->allocator().alloc().CreateArray(typeInfo, elements); #endif } ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( ObjHeader* object, const TypeInfo* typeInfo) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); #else - return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif } ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); #endif @@ -78,29 +77,12 @@ gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique< gc::GC::~GC() = default; -// static -size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { -#ifndef CUSTOM_ALLOCATOR - return ObjectFactory::GetAllocatedHeapSize(object); -#else - return alloc::CustomAllocator::GetAllocatedHeapSize(object); -#endif -} - -size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { -#ifdef CUSTOM_ALLOCATOR - return alloc::GetAllocatedBytes(); -#else - return alloc::allocatedBytes(); -#endif -} - void gc::GC::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactory().ClearForTests(); - impl_->objectFactory().ClearForTests(); + impl_->allocator().extraObjectDataFactory().ClearForTests(); + impl_->allocator().objectFactory().ClearForTests(); #else - impl_->gc().heap().ClearForTests(); + impl_->allocator().heap().ClearForTests(); #endif GCHandle::ClearForTests(); } @@ -149,7 +131,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje #ifndef CUSTOM_ALLOCATOR extraObject.Uninstall(); auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.ReleaseAssociatedObject(); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 7ffbbf35995..d748ca0ee41 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -8,56 +8,30 @@ #include "GC.hpp" #include "AllocatorImpl.hpp" -#include "NoOpGC.hpp" +#include "Logging.hpp" +#include "Utils.hpp" namespace kotlin { namespace gc { class GC::Impl : private Pinned { public: - Impl() noexcept = default; + Impl() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); } -#ifndef CUSTOM_ALLOCATOR - ObjectFactory& objectFactory() noexcept { return objectFactory_; } - alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } -#endif - NoOpGC& gc() noexcept { return gc_; } + alloc::Allocator::Impl& allocator() noexcept { return allocator_; } private: -#ifndef CUSTOM_ALLOCATOR - ObjectFactory objectFactory_; - alloc::ExtraObjectDataFactory extraObjectDataFactory_; -#endif - NoOpGC gc_; + alloc::Allocator::Impl allocator_; }; class GC::ThreadData::Impl : private Pinned { public: -#ifdef CUSTOM_ALLOCATOR - Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {} -#else - Impl(GC& gc, mm::ThreadData& threadData) noexcept : - objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), - extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {} -#endif + Impl(GC& gc, mm::ThreadData& threadData) noexcept : allocator_(gc.impl_->allocator()) {} - NoOpGC::ThreadData& gc() noexcept { return gc_; } -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator& alloc() noexcept { return alloc_; } -#else - ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } -#endif + alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } private: - NoOpGC::ThreadData gc_; -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator alloc_; -#else - [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; -#endif + alloc::Allocator::ThreadData::Impl allocator_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp index 9cc05533b19..f75fd949b74 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp @@ -36,11 +36,11 @@ auto collectPointers(T& iterable) { void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else - auto objects = gc.impl().objectFactory().LockForIter(); - auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); + auto objects = gc.impl().allocator().objectFactory().LockForIter(); + auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); #endif diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.cpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.cpp deleted file mode 100644 index 0b0f25487af..00000000000 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.cpp +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "NoOpGC.hpp" diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp deleted file mode 100644 index 4e0c5ab463e..00000000000 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#ifndef RUNTIME_GC_NOOP_NOOP_GC_H -#define RUNTIME_GC_NOOP_NOOP_GC_H - -#include - -#include "AllocatorImpl.hpp" -#include "GC.hpp" -#include "Logging.hpp" -#include "Utils.hpp" - -namespace kotlin { - -namespace mm { -class ThreadData; -} - -namespace gc { - -// No-op GC is a GC that does not free memory. -// TODO: It can be made more efficient. -class NoOpGC : private Pinned { -public: - class ThreadData : private Pinned { - public: - ThreadData() noexcept {} - ~ThreadData() = default; - - private: - }; - - NoOpGC() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); } - ~NoOpGC() = default; - -#ifdef CUSTOM_ALLOCATOR - alloc::Heap& heap() noexcept { return heap_; } -#endif - -private: -#ifdef CUSTOM_ALLOCATOR - alloc::Heap heap_; -#endif -}; - -} // namespace gc -} // namespace kotlin - -#endif // RUNTIME_GC_NOOP_NOOP_GC_H diff --git a/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp deleted file mode 100644 index a5b33896534..00000000000 --- a/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#ifdef CUSTOM_ALLOCATOR - -// TODO: Move into alloc/custom/AllocatorImpl.hpp - -#include "CustomAllocator.hpp" -#include "CustomFinalizerProcessor.hpp" -#include "GCApi.hpp" -#include "Heap.hpp" - -namespace kotlin::gc { - -inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { - return kotlin::alloc::objectDataForObject(object); -} - -inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { - return kotlin::alloc::objectForObjectData(objectData); -} - -using FinalizerQueue = alloc::FinalizerQueue; -using FinalizerQueueTraits = alloc::FinalizerQueueTraits; - -} // namespace kotlin::gc - -#else - -// TODO: Move into alloc/legacy/AllocatorImpl.hpp - -#include "ExtraObjectDataFactory.hpp" -#include "GC.hpp" -#include "GlobalData.hpp" -#include "Logging.hpp" -#include "ObjectFactory.hpp" -#include "ObjectFactoryAllocator.hpp" -#include "ObjectFactorySweep.hpp" - -namespace kotlin::gc { - -struct ObjectFactoryTraits { - using Allocator = alloc::AllocatorWithGC; - using ObjectData = gc::GC::ObjectData; - - Allocator CreateAllocator() noexcept { return Allocator(alloc::AllocatorBasic(), *this); } - - void OnOOM(size_t size) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - // TODO: This will print the log for "manual" scheduling. Fix this. - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); - } -}; - -using ObjectFactory = alloc::ObjectFactory; - -inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { - return ObjectFactory::NodeRef::From(object).ObjectData(); -} - -inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { - return ObjectFactory::NodeRef::From(objectData)->GetObjHeader(); -} - -using FinalizerQueue = ObjectFactory::FinalizerQueue; -using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits; - -} // namespace kotlin::gc - -#endif diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 1227758ed8c..4ddca9e1948 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -21,48 +21,48 @@ gc::GC::ThreadData::~ThreadData() = default; void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().Publish(); - impl_->objectFactoryThreadQueue().Publish(); + impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); + impl_->allocator().objectFactoryThreadQueue().Publish(); #endif } void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->objectFactoryThreadQueue().ClearForTests(); + impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); + impl_->allocator().objectFactoryThreadQueue().ClearForTests(); #else - impl_->alloc().PrepareForGC(); + impl_->allocator().alloc().PrepareForGC(); #endif } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); + return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); #else - return impl_->alloc().CreateObject(typeInfo); + return impl_->allocator().alloc().CreateObject(typeInfo); #endif } ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); + return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); #else - return impl_->alloc().CreateArray(typeInfo, elements); + return impl_->allocator().alloc().CreateArray(typeInfo, elements); #endif } ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( ObjHeader* object, const TypeInfo* typeInfo) noexcept { #ifndef CUSTOM_ALLOCATOR - return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); #else - return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); + return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); #endif } ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); #endif @@ -78,30 +78,13 @@ gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support:: gc::GC::~GC() = default; -// static -size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { -#ifdef CUSTOM_ALLOCATOR - return alloc::CustomAllocator::GetAllocatedHeapSize(object); -#else - return ObjectFactory::GetAllocatedHeapSize(object); -#endif -} - -size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { -#ifdef CUSTOM_ALLOCATOR - return alloc::GetAllocatedBytes(); -#else - return alloc::allocatedBytes(); -#endif -} - void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR - impl_->extraObjectDataFactory().ClearForTests(); - impl_->objectFactory().ClearForTests(); + impl_->allocator().extraObjectDataFactory().ClearForTests(); + impl_->allocator().objectFactory().ClearForTests(); #else - impl_->gc().heap().ClearForTests(); + impl_->allocator().heap().ClearForTests(); #endif GCHandle::ClearForTests(); } @@ -146,7 +129,7 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept { } bool gc::isMarked(ObjHeader* object) noexcept { - return objectDataForObject(object).marked(); + return alloc::objectDataForObject(object).marked(); } ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { @@ -162,7 +145,7 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje #ifndef CUSTOM_ALLOCATOR extraObject.Uninstall(); auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); + threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); #else extraObject.ReleaseAssociatedObject(); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index 638db32f43a..be53c942a1c 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -15,54 +15,26 @@ namespace gc { class GC::Impl : private Pinned { public: -#ifdef CUSTOM_ALLOCATOR - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {} -#else - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator_, gcScheduler) {} - ObjectFactory& objectFactory() noexcept { return objectFactory_; } - alloc::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } -#endif + alloc::Allocator::Impl& allocator() noexcept { return allocator_; } SameThreadMarkAndSweep& gc() noexcept { return gc_; } private: -#ifndef CUSTOM_ALLOCATOR - ObjectFactory objectFactory_; - alloc::ExtraObjectDataFactory extraObjectDataFactory_; -#endif + alloc::Allocator::Impl allocator_; SameThreadMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : - gc_(gc.impl_->gc(), threadData), -#ifdef CUSTOM_ALLOCATOR - alloc_(gc.impl_->gc().heap()) { - } -#else - objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), - extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { - } -#endif + Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {} SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; } -#ifndef CUSTOM_ALLOCATOR - ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - alloc::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } -#else - alloc::CustomAllocator& alloc() noexcept { return alloc_; } -#endif + alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } private: SameThreadMarkAndSweep::ThreadData gc_; -#ifdef CUSTOM_ALLOCATOR - alloc::CustomAllocator alloc_; -#else - [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_; - alloc::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; -#endif + alloc::Allocator::ThreadData::Impl allocator_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp index 9cc05533b19..f75fd949b74 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp @@ -36,11 +36,11 @@ auto collectPointers(T& iterable) { void gc::AssertClear(GC& gc) noexcept { #ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().gc().heap().GetAllocatedObjects(); + auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); #else - auto objects = gc.impl().objectFactory().LockForIter(); - auto extraObjects = gc.impl().extraObjectDataFactory().LockForIter(); + auto objects = gc.impl().allocator().objectFactory().LockForIter(); + auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); #endif diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 913003a465d..b70e85b4c5e 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -22,17 +22,9 @@ using namespace kotlin; -#ifdef CUSTOM_ALLOCATOR -gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - gcScheduler::GCScheduler& gcScheduler) noexcept : -#else -gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - ObjectFactory& objectFactory, alloc::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : - objectFactory_(objectFactory), - extraObjectDataFactory_(extraObjectDataFactory), -#endif - gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { + allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { GCHandle::getByEpoch(epoch).finalizersDone(); state_.finalized(epoch); }) { @@ -81,9 +73,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #ifdef CUSTOM_ALLOCATOR // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().alloc().PrepareForGC(); + thread.gc().impl().allocator().alloc().PrepareForGC(); } - heap_.PrepareForGC(); + allocator_.heap().PrepareForGC(); #endif gc::collectRootSet(gcHandle, markQueue_, [](mm::ThreadData&) { return true; }); @@ -99,24 +91,24 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); - std::optional objectFactoryIterable = objectFactory_.LockForIter(); + std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter(); + std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter(); - alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); + alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); + auto finalizerQueue = alloc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects - auto finalizerQueue = heap_.Sweep(gcHandle); + auto finalizerQueue = allocator_.heap().Sweep(gcHandle); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue()); } - finalizerQueue.TransferAllFrom(heap_.ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue()); #endif - scheduler.onGCFinish(epoch, mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes()); + scheduler.onGCFinish(epoch, alloc::allocatedBytes()); resumeTheWorld(gcHandle); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 761201e6ab3..2b4fb425a23 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -40,14 +40,7 @@ public: private: }; -#ifdef CUSTOM_ALLOCATOR - SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept; -#else - SameThreadMarkAndSweep( - ObjectFactory& objectFactory, - alloc::ExtraObjectDataFactory& extraObjectDataFactory, - gcScheduler::GCScheduler& gcScheduler) noexcept; -#endif + SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept; ~SameThreadMarkAndSweep(); @@ -57,24 +50,15 @@ public: GCStateHolder& state() noexcept { return state_; } -#ifdef CUSTOM_ALLOCATOR - alloc::Heap& heap() noexcept { return heap_; } -#endif - private: void PerformFullGC(int64_t epoch) noexcept; -#ifndef CUSTOM_ALLOCATOR - ObjectFactory& objectFactory_; - alloc::ExtraObjectDataFactory& extraObjectDataFactory_; -#else - alloc::Heap heap_; -#endif + alloc::Allocator::Impl& allocator_; gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; ScopedThread gcThread_; - FinalizerProcessor finalizerProcessor_; + FinalizerProcessor finalizerProcessor_; MarkQueue markQueue_; }; @@ -88,14 +72,16 @@ struct MarkTraits { static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { if (auto* top = queue.try_pop_front()) { - return objectForObjectData(*top); + return alloc::objectForObjectData(*top); } return nullptr; } - static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { return queue.try_push_front(objectDataForObject(object)); } + static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { + return queue.try_push_front(alloc::objectDataForObject(object)); + } - static bool tryMark(ObjHeader* object) noexcept { return objectDataForObject(object).tryMark(); } + static bool tryMark(ObjHeader* object) noexcept { return alloc::objectDataForObject(object).tryMark(); } static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { auto process = object->type_info()->processObjectInMark; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 990e8390c3a..20510635df1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -181,13 +181,13 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa std_support::vector Alive(mm::ThreadData& threadData) { #ifdef CUSTOM_ALLOCATOR - return threadData.gc().impl().alloc().heap().GetAllocatedObjects(); + return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects(); #else std_support::vector objects; - for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { + for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) { objects.push_back(node.GetObjHeader()); } - for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) { + for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) { objects.push_back(node.GetObjHeader()); } return objects; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index 7444497ff62..2f9b25c0ccc 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -85,7 +85,3 @@ OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, // `ArrayHeader` and `ObjHeader` are expected to be compatible. RETURN_OBJ(reinterpret_cast(array)); } - -size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept { - return gc::GC::GetAllocatedHeapSize(object); -} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp index a4075be7aac..e26fc66220e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -32,9 +32,6 @@ OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept; OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept; -// This does not take into account how much storage did the underlying allocator (malloc/mimalloc) reserved. -size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; - } // namespace mm } // namespace kotlin