From 83a70ddf8b0ca3be5f130251e1943eac8b0a8dfd Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Fri, 28 Jul 2023 15:08:15 +0200 Subject: [PATCH] [K/N] Decouple ObjectFactory from concrete ObjectData ^KT-60928 * Use type_layout to declaratively express heap object headers in both custom allocator and ObjectFactory. * Invoke constructor (w/o invoking Kotin constructors) for created objects and arrays from both custom allocator and ObjectFactory. Previously: - custom allocator only checked body for nullability (now this is performed in body constructor) - ObjectFactory only constructed ObjectData * In each GC have a AllocatorImpl.hpp and ObjectData.hpp headers the first encapsulating allocator-specific types, the second containing specific ObjectData implementation. * In each GC have a separate ObjectFactoryTraits that does not actually depend on the specific GC anymore. * Each GC now expose ObjectData (as undefined type) and its descriptor, the latter being used by the custom allocator and ObjectFactory. * Descriptors for ObjectBody and ArrayBody now live in Memory.h and the code calculating size is now shared. Their constructors check that the memory is zeroed (Kotlin constructors will expect this). --- .../src/custom_alloc/cpp/CustomAllocator.cpp | 38 +--- .../runtime/src/custom_alloc/cpp/GCApi.cpp | 9 +- .../runtime/src/custom_alloc/cpp/GCApi.hpp | 75 +++++- .../runtime/src/custom_alloc/cpp/Heap.cpp | 6 +- .../runtime/src/gc/cms/cpp/AllocatorImpl.hpp | 69 ++++++ .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 55 +---- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 44 +--- .../gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp | 213 +++++++++--------- .../runtime/src/gc/cms/cpp/GCImpl.cpp | 31 ++- .../runtime/src/gc/cms/cpp/GCImpl.hpp | 28 +-- .../cms/cpp/{MarkStack.hpp => ObjectData.hpp} | 37 ++- .../runtime/src/gc/cms/cpp/ParallelMark.cpp | 1 - .../runtime/src/gc/cms/cpp/ParallelMark.hpp | 18 +- .../src/gc/common/cpp/FinalizerProcessor.hpp | 1 - .../runtime/src/gc/common/cpp/GC.hpp | 28 ++- .../src/gc/common/cpp/MarkAndSweepUtils.hpp | 19 ++ .../runtime/src/gc/noop/cpp/AllocatorImpl.hpp | 33 +++ .../runtime/src/gc/noop/cpp/GCImpl.cpp | 29 ++- .../runtime/src/gc/noop/cpp/GCImpl.hpp | 29 +-- .../runtime/src/gc/noop/cpp/NoOpGC.hpp | 17 +- .../runtime/src/gc/stms/cpp/AllocatorImpl.hpp | 69 ++++++ .../runtime/src/gc/stms/cpp/GCImpl.cpp | 35 ++- .../runtime/src/gc/stms/cpp/GCImpl.hpp | 28 +-- .../runtime/src/gc/stms/cpp/ObjectData.hpp | 50 ++++ .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 53 +---- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 84 +------ .../stms/cpp/SameThreadMarkAndSweepTest.cpp | 213 +++++++++--------- kotlin-native/runtime/src/main/cpp/Memory.h | 58 +++++ kotlin-native/runtime/src/main/cpp/Utils.cpp | 11 +- kotlin-native/runtime/src/main/cpp/Utils.hpp | 5 + .../runtime/src/main/cpp/UtilsTest.cpp | 24 ++ .../runtime/src/mm/cpp/GlobalData.hpp | 1 - .../runtime/src/mm/cpp/ObjectFactory.hpp | 154 +++++++------ .../runtime/src/mm/cpp/ObjectFactoryTest.cpp | 6 +- .../runtime/src/mm/cpp/ThreadData.hpp | 1 - .../runtime/src/mm/cpp/ThreadRegistryTest.cpp | 1 + .../runtime/src/mm/cpp/ThreadSuspension.cpp | 1 + 37 files changed, 901 insertions(+), 673 deletions(-) create mode 100644 kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp rename kotlin-native/runtime/src/gc/cms/cpp/{MarkStack.hpp => ObjectData.hpp} (59%) create mode 100644 kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp create mode 100644 kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp create mode 100644 kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 257bb68437a..489c6ce266e 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -28,19 +28,6 @@ namespace kotlin::alloc { -size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { - size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); - return AlignUp(heapObjectHeaderSize + membersSize, kObjectAlignment); -} - -uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { - // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped - // at about half of uint64_t max. - uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; - // Note: array body is aligned, but for size computation it is enough to align the sum. - return AlignUp(heapArrayHeaderSize + membersSize, kObjectAlignment); -} - CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) { CustomAllocInfo("CustomAllocator::CustomAllocator(heap)"); memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); @@ -52,9 +39,9 @@ CustomAllocator::~CustomAllocator() { ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - size_t allocSize = ObjectAllocatedDataSize(typeInfo); - uint8_t* heapObject = Allocate(allocSize); - auto* object = reinterpret_cast(heapObject + gcDataSize); + auto descriptor = HeapObject::make_descriptor(typeInfo); + auto& heapObject = *descriptor.construct(Allocate(descriptor.size())); + ObjHeader* object = heapObject.header(descriptor).object(); if (typeInfo->flags_ & TF_HAS_FINALIZER) { auto* extraObject = CreateExtraObject(); object->typeInfoOrMeta_ = reinterpret_cast(new (extraObject) mm::ExtraObjectData(object, typeInfo)); @@ -69,9 +56,9 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count); RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - auto allocSize = ArrayAllocatedDataSize(typeInfo, count); - uint8_t* heapArray = Allocate(allocSize); - auto* array = reinterpret_cast(heapArray + gcDataSize); + auto descriptor = HeapArray::make_descriptor(typeInfo, count); + auto& heapArray = *descriptor.construct(Allocate(descriptor.size())); + ArrayHeader* array = heapArray.header(descriptor).array(); array->typeInfoOrMeta_ = const_cast(typeInfo); array->count_ = count; return array; @@ -121,9 +108,9 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept { RuntimeAssert(object->heap(), "Object must be a heap object"); const auto* typeInfo = object->type_info(); if (typeInfo->IsArray()) { - return ArrayAllocatedDataSize(typeInfo, object->array()->count_); + return HeapArray::make_descriptor(typeInfo, object->array()->count_).size(); } else { - return ObjectAllocatedDataSize(typeInfo); + return HeapObject::make_descriptor(typeInfo).size(); } } @@ -131,16 +118,13 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes"); CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size); uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell); - uint8_t* ptr; if (cellCount <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE) { - ptr = AllocateInFixedBlockPage(cellCount); + return AllocateInFixedBlockPage(cellCount); } else if (cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE) { - ptr = AllocateInSingleObjectPage(cellCount); + return AllocateInSingleObjectPage(cellCount); } else { - ptr = AllocateInNextFitPage(cellCount); + return AllocateInNextFitPage(cellCount); } - RuntimeAssert(ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0, "CustomAllocator::Allocate: memory not zero!"); - return ptr; } uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 2ded2becfaa..43d66b24f9e 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -23,7 +23,6 @@ #include "GCStatistics.hpp" #include "KAssert.h" #include "Memory.h" -#include "ObjectFactory.hpp" namespace { @@ -33,14 +32,14 @@ std::atomic allocatedBytesCounter; namespace kotlin::alloc { -bool SweepObject(uint8_t* heapObjHeader, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { - if (gc::GC::SweepObject(heapObjHeader)) { +bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { + auto* heapObjHeader = reinterpret_cast(object); + if (gc::tryResetMark(heapObjHeader->objectData())) { CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader); gcHandle.addKeptObject(); return true; } - auto* objHeader = reinterpret_cast(heapObjHeader + gcDataSize); - auto* extraObject = mm::ExtraObjectData::Get(objHeader); + auto* extraObject = mm::ExtraObjectData::Get(heapObjHeader->object()); if (extraObject) { if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index 70fafee62f2..3df7d21f6d7 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -15,15 +15,74 @@ #include "AtomicStack.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" +#include "GC.hpp" #include "GCStatistics.hpp" #include "Memory.h" -#include "GC.hpp" +#include "TypeLayout.hpp" namespace kotlin::alloc { -const size_t gcDataSize = AlignUp(gc::GC::objectDataSize, kObjectAlignment); -const size_t heapObjectHeaderSize = AlignUp(gcDataSize + sizeof(ObjHeader), kObjectAlignment); -const size_t heapArrayHeaderSize = AlignUp(gcDataSize + sizeof(ArrayHeader), kObjectAlignment); +struct HeapObjHeader { + using descriptor = type_layout::Composite; + + static HeapObjHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); } + + static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().fromField<1>(object); } + + gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; } + + ObjHeader* object() noexcept { return descriptor().field<1>(this).second; } + +private: + HeapObjHeader() = delete; + ~HeapObjHeader() = delete; +}; + +struct HeapObject { + using descriptor = type_layout::Composite; + + static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept { + return descriptor{{}, type_layout::descriptor_t{typeInfo}}; + } + + HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; } + +private: + HeapObject() = delete; + ~HeapObject() = delete; +}; + +// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible +// with `ObjHeader`: the former can always be casted to the other. +struct HeapArrayHeader { + using descriptor = type_layout::Composite; + + static HeapArrayHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); } + + static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().fromField<1>(array); } + + gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; } + + ArrayHeader* array() noexcept { return descriptor().field<1>(this).second; } + +private: + HeapArrayHeader() = delete; + ~HeapArrayHeader() = delete; +}; + +struct HeapArray { + using descriptor = type_layout::Composite; + + static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept { + return descriptor{{}, type_layout::descriptor_t{typeInfo, size}}; + } + + HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; } + +private: + HeapArray() = delete; + ~HeapArray() = delete; +}; // Returns `true` if the `object` must be kept alive still. bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept; @@ -37,6 +96,14 @@ void Free(void* ptr, size_t size) noexcept; size_t GetAllocatedBytes() noexcept; +inline gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { + return HeapObjHeader::from(object).objectData(); +} + +inline ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept { + return HeapObjHeader::from(objectData).object(); +} + } // namespace kotlin::alloc #endif diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index b63385070aa..e49116dd288 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -93,18 +93,18 @@ std_support::vector Heap::GetAllocatedObjects() noexcept { for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { for (auto* page : fixedBlockPages_[blockSize].GetPages()) { for (auto* block : page->GetAllocatedBlocks()) { - allocated.push_back(reinterpret_cast(block + gcDataSize)); + allocated.push_back(reinterpret_cast(block)->object()); } } } for (auto* page : nextFitPages_.GetPages()) { for (auto* block : page->GetAllocatedBlocks()) { - allocated.push_back(reinterpret_cast(block + gcDataSize)); + allocated.push_back(reinterpret_cast(block)->object()); } } for (auto* page : singleObjectPages_.GetPages()) { for (auto* block : page->GetAllocatedBlocks()) { - allocated.push_back(reinterpret_cast(block + gcDataSize)); + allocated.push_back(reinterpret_cast(block)->object()); } } std_support::vector unfinalized; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp new file mode 100644 index 00000000000..e8660e8027d --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp @@ -0,0 +1,69 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#ifdef CUSTOM_ALLOCATOR + +#include "CustomAllocator.hpp" +#include "CustomFinalizerProcessor.hpp" +#include "GCApi.hpp" +#include "Heap.hpp" + +namespace kotlin::gc { + +inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { + return kotlin::alloc::objectDataForObject(object); +} + +inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { + return kotlin::alloc::objectForObjectData(objectData); +} + +using FinalizerQueue = alloc::FinalizerQueue; +using FinalizerQueueTraits = alloc::FinalizerQueueTraits; + +} // namespace kotlin::gc + +#else + +#include "Allocator.hpp" +#include "ExtraObjectDataFactory.hpp" +#include "GC.hpp" +#include "GlobalData.hpp" +#include "Logging.hpp" +#include "ObjectFactory.hpp" + +namespace kotlin::gc { + +struct ObjectFactoryTraits { + using Allocator = AllocatorWithGC; + using ObjectData = gc::GC::ObjectData; + + Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } + + void OnOOM(size_t size) noexcept { + RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); + // TODO: This will print the log for "manual" scheduling. Fix this. + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); + } +}; + +using ObjectFactory = mm::ObjectFactory; + +inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { + return ObjectFactory::NodeRef::From(object).ObjectData(); +} + +inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { + return ObjectFactory::NodeRef::From(objectData)->GetObjHeader(); +} + +using FinalizerQueue = ObjectFactory::FinalizerQueue; +using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits; + +} // namespace kotlin::gc + +#endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 20879de2a0f..4207e004731 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -20,40 +20,12 @@ #include "GCState.hpp" #include "GCStatistics.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "Heap.hpp" -#endif - using namespace kotlin; namespace { [[clang::no_destroy]] std::mutex gcMutex; -struct SweepTraits { - using ObjectFactory = mm::ObjectFactory; - using ExtraObjectsFactory = mm::ExtraObjectDataFactory; - - static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { - auto *baseObject = object.GetBaseObject(); - if (!baseObject->heap()) return true; - auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).ObjectData(); - return objectData.marked(); - } - - static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { - auto& objectData = node.ObjectData(); - return objectData.tryResetMark(); - } -}; - -struct ProcessWeaksTraits { - static bool IsMarked(ObjHeader* obj) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(obj).ObjectData(); - return objectData.marked(); - } -}; - template ScopedThread createGCThread(const char* name, Body&& body) { return ScopedThread(ScopedThread::attributes().name(name), [name, body] { @@ -63,8 +35,9 @@ ScopedThread createGCThread(const char* name, Body&& body) { }); } +#ifndef CUSTOM_ALLOCATOR // TODO move to common -[[maybe_unused]] inline void checkMarkCorrectness(mm::ObjectFactory::Iterable& heap) { +[[maybe_unused]] inline void checkMarkCorrectness(gc::ObjectFactory::Iterable& heap) { if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return; for (auto objRef: heap) { auto obj = objRef.GetObjHeader(); @@ -72,23 +45,17 @@ ScopedThread createGCThread(const char* name, Body&& body) { if (objData.marked()) { traverseReferredObjects(obj, [obj](ObjHeader* field) { if (field->heap()) { - auto& fieldObjData = - mm::ObjectFactory::NodeRef::From(field).ObjectData(); + auto& fieldObjData = gc::ObjectFactory::NodeRef::From(field).ObjectData(); RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj); } }); } } } +#endif } // namespace -void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - // TODO: This will print the log for "manual" scheduling. Fix this. - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); -} - void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { CallsCheckerIgnoreGuard guard; @@ -133,10 +100,11 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const #ifndef CUSTOM_ALLOCATOR gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - mm::ObjectFactory& objectFactory, + ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler, - bool mutatorsCooperate, std::size_t auxGCThreads) noexcept : + bool mutatorsCooperate, + std::size_t auxGCThreads) noexcept : objectFactory_(objectFactory), extraObjectDataFactory_(extraObjectDataFactory), #else @@ -150,8 +118,7 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( state_.finalized(epoch); }), markDispatcher_(mutatorsCooperate), - mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) -{ + mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) { for (std::size_t i = 0; i < auxGCThreads; ++i) { auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); })); } @@ -250,7 +217,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gcHandle.threadsAreResumed(); } - gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); + gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); if (compiler::concurrentWeakSweep()) { // Expected to happen outside STW. @@ -261,9 +228,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { } #ifndef CUSTOM_ALLOCATOR - gc::SweepExtraObjects(gcHandle, *extraObjectFactoryIterable); + gc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = gc::Sweep(gcHandle, *objectFactoryIterable); + auto finalizerQueue = gc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; kotlin::compactObjectPoolInMainThread(); #else diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 854e48871d4..6913da5cac3 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -8,29 +8,21 @@ #include #include -#include "Allocator.hpp" +#include "AllocatorImpl.hpp" #include "Barriers.hpp" -#include "ExtraObjectDataFactory.hpp" #include "FinalizerProcessor.hpp" #include "GCScheduler.hpp" #include "GCState.hpp" #include "GCStatistics.hpp" #include "IntrusiveList.hpp" #include "MarkAndSweepUtils.hpp" -#include "ObjectFactory.hpp" +#include "ObjectData.hpp" +#include "ParallelMark.hpp" #include "ScopedThread.hpp" #include "ThreadData.hpp" #include "Types.h" #include "Utils.hpp" #include "std_support/Memory.hpp" -#include "MarkStack.hpp" -#include "ParallelMark.hpp" - -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#include "CustomFinalizerProcessor.hpp" -#include "Heap.hpp" -#endif namespace kotlin { namespace gc { @@ -39,25 +31,15 @@ namespace gc { // TODO: Also make marking run concurrently with Kotlin threads. class ConcurrentMarkAndSweep : private Pinned { public: - class ThreadData : private Pinned { public: - using ObjectData = mark::ObjectData; - using Allocator = AllocatorWithGC; - - explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept - : gc_(gc), threadData_(threadData) {} - + explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} ~ThreadData() = default; - void OnOOM(size_t size) noexcept; - void OnSuspendForGC() noexcept; void safePoint() noexcept { barriers_.onCheckpoint(); } - Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } - BarriersThreadData& barriers() noexcept { return barriers_; } bool tryLockRootSet(); @@ -80,26 +62,16 @@ public: std::atomic cooperative_ = false; }; - using ObjectData = ThreadData::ObjectData; - using Allocator = ThreadData::Allocator; - -#ifndef CUSTOM_ALLOCATOR - using FinalizerQueue = mm::ObjectFactory::FinalizerQueue; - using FinalizerQueueTraits = mm::ObjectFactory::FinalizerQueueTraits; -#else - using FinalizerQueue = alloc::FinalizerQueue; - using FinalizerQueueTraits = alloc::FinalizerQueueTraits; -#endif - #ifdef CUSTOM_ALLOCATOR explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; #else ConcurrentMarkAndSweep( - mm::ObjectFactory& objectFactory, + ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& scheduler, - bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; + bool mutatorsCooperate, + std::size_t auxGCThreads) noexcept; #endif ~ConcurrentMarkAndSweep(); @@ -121,7 +93,7 @@ private: void PerformFullGC(int64_t epoch) noexcept; #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory& objectFactory_; + ObjectFactory& objectFactory_; mm::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 379f6d783e3..f0d3e08d426 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -193,11 +193,6 @@ std_support::vector Alive(mm::ThreadData& threadData) { #endif } -bool IsMarked(ObjHeader* objHeader) { - auto nodeRef = mm::ObjectFactory::NodeRef::From(objHeader); - return nodeRef.ObjectData().marked(); -} - test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); @@ -263,12 +258,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(global2.header()), false); - ASSERT_THAT(IsMarked(global3.header()), false); - ASSERT_THAT(IsMarked(stack1.header()), false); - ASSERT_THAT(IsMarked(stack2.header()), false); - ASSERT_THAT(IsMarked(stack3.header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(global2.header()), false); + ASSERT_THAT(gc::isMarked(global3.header()), false); + ASSERT_THAT(gc::isMarked(stack1.header()), false); + ASSERT_THAT(gc::isMarked(stack2.header()), false); + ASSERT_THAT(gc::isMarked(stack3.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -276,12 +271,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(IsMarked(global1.header()), false); - EXPECT_THAT(IsMarked(global2.header()), false); - EXPECT_THAT(IsMarked(global3.header()), false); - EXPECT_THAT(IsMarked(stack1.header()), false); - EXPECT_THAT(IsMarked(stack2.header()), false); - EXPECT_THAT(IsMarked(stack3.header()), false); + EXPECT_THAT(gc::isMarked(global1.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global3.header()), false); + EXPECT_THAT(gc::isMarked(stack1.header()), false); + EXPECT_THAT(gc::isMarked(stack2.header()), false); + EXPECT_THAT(gc::isMarked(stack3.header()), false); }); } @@ -309,12 +304,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(global2.header()), false); - ASSERT_THAT(IsMarked(global3.header()), false); - ASSERT_THAT(IsMarked(stack1.header()), false); - ASSERT_THAT(IsMarked(stack2.header()), false); - ASSERT_THAT(IsMarked(stack3.header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(global2.header()), false); + ASSERT_THAT(gc::isMarked(global3.header()), false); + ASSERT_THAT(gc::isMarked(stack1.header()), false); + ASSERT_THAT(gc::isMarked(stack2.header()), false); + ASSERT_THAT(gc::isMarked(stack3.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -322,12 +317,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(IsMarked(global1.header()), false); - EXPECT_THAT(IsMarked(global2.header()), false); - EXPECT_THAT(IsMarked(global3.header()), false); - EXPECT_THAT(IsMarked(stack1.header()), false); - EXPECT_THAT(IsMarked(stack2.header()), false); - EXPECT_THAT(IsMarked(stack3.header()), false); + EXPECT_THAT(gc::isMarked(global1.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global3.header()), false); + EXPECT_THAT(gc::isMarked(stack1.header()), false); + EXPECT_THAT(gc::isMarked(stack2.header()), false); + EXPECT_THAT(gc::isMarked(stack3.header()), false); }); } @@ -337,8 +332,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) { auto& object2 = AllocateObject(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -352,8 +347,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { auto& object2 = AllocateObjectWithFinalizer(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); @@ -372,8 +367,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(weak1.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); EXPECT_CALL(finalizerHook(), Call(weak1.header())); @@ -390,14 +385,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) { auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(weak1.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(IsMarked(weak1.header()), false); + EXPECT_THAT(gc::isMarked(weak1.header()), false); EXPECT_THAT(weak1.get(), nullptr); }); } @@ -420,12 +415,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -433,12 +428,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -467,14 +462,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -482,12 +477,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -516,14 +511,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); @@ -533,12 +528,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -555,18 +550,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { object2->field1 = stack.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); }); } @@ -595,14 +590,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -611,12 +606,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -633,12 +628,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) { global2->field1 = global1.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); }); } @@ -652,14 +647,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) { ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(object.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(object.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(object.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(object.header()), false); }); } @@ -1165,9 +1160,9 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(object1_local.header()), false); - ASSERT_THAT(IsMarked(weak.load()->header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(object1_local.header()), false); + ASSERT_THAT(gc::isMarked(weak.load()->header()), false); ASSERT_THAT(weak.load()->get(), object1_local.header()); global1->field1 = nullptr; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index bef983babee..2a84ca89cd0 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -9,9 +9,8 @@ #include "GC.hpp" #include "GCStatistics.hpp" #include "MarkAndSweepUtils.hpp" +#include "ObjectAlloc.hpp" #include "ObjectOps.hpp" -#include "ThreadSuspension.hpp" -#include "MarkStack.hpp" #include "std_support/Memory.hpp" using namespace kotlin; @@ -86,7 +85,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { #ifdef CUSTOM_ALLOCATOR return alloc::CustomAllocator::GetAllocatedHeapSize(object); #else - return mm::ObjectFactory::GetAllocatedHeapSize(object); + return ObjectFactory::GetAllocatedHeapSize(object); #endif } @@ -145,20 +144,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept { } bool gc::isMarked(ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - return objectData.marked(); + return objectDataForObject(object).marked(); } ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_RESULT_OF(gc::WeakRefRead, object); } -// static -const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData); - -// static -ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { - return reinterpret_cast(objectData)->tryResetMark(); +ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { + return objectData.tryResetMark(); } // static @@ -172,3 +166,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); #endif } + +// static +ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { + return sizeof(gc::GC::ObjectData); +} + +// static +ALWAYS_INLINE size_t type_layout::descriptor::type::alignment() noexcept { + return alignof(gc::GC::ObjectData); +} + +// static +ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor::type::construct(uint8_t* ptr) noexcept { + return new (ptr) gc::GC::ObjectData(); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 375fff5f617..887d4007a8e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -7,19 +7,12 @@ #include "GC.hpp" +#include "AllocatorImpl.hpp" #include "ConcurrentMarkAndSweep.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#else -#include "ExtraObjectDataFactory.hpp" -#endif - namespace kotlin { namespace gc { -using GCImpl = ConcurrentMarkAndSweep; - class GC::Impl : private Pinned { public: #ifdef CUSTOM_ALLOCATOR @@ -29,17 +22,17 @@ public: #endif #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + ObjectFactory& objectFactory() noexcept { return objectFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif - GCImpl& gc() noexcept { return gc_; } + ConcurrentMarkAndSweep& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory objectFactory_; + ObjectFactory objectFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif - GCImpl gc_; + ConcurrentMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { @@ -47,7 +40,7 @@ public: Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), #ifndef CUSTOM_ALLOCATOR - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { } #else @@ -55,20 +48,21 @@ public: } #endif - GCImpl::ThreadData& gc() noexcept { return gc_; } + ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; } #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else - mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: - GCImpl::ThreadData gc_; + ConcurrentMarkAndSweep::ThreadData gc_; #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator alloc_; #else - mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/MarkStack.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp similarity index 59% rename from kotlin-native/runtime/src/gc/cms/cpp/MarkStack.hpp rename to kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp index e73fe15c203..346106ec0c4 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/MarkStack.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp @@ -1,28 +1,23 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + #pragma once #include -#include +#include -#include "Allocator.hpp" +#include "AllocatorImpl.hpp" +#include "GC.hpp" #include "IntrusiveList.hpp" -#include "ObjectFactory.hpp" -#include "Types.h" -#include "Utils.hpp" -#include "std_support/Memory.hpp" +#include "KAssert.h" -namespace kotlin::gc::mark { +namespace kotlin::gc { -class ObjectData { - struct ObjectFactoryTraits { - using ObjectData = ObjectData; - class Allocator; - }; +class GC::ObjectData { public: - using ObjectFactory = mm::ObjectFactory; - - bool tryMark() noexcept { - return trySetNext(reinterpret_cast(1)); - } + bool tryMark() noexcept { return trySetNext(reinterpret_cast(1)); } bool marked() const noexcept { return next() != nullptr; } @@ -32,10 +27,6 @@ public: return true; } - ObjHeader* objHeader() noexcept { // FIXME const - return ObjectFactory::NodeRef::From(*this).GetObjHeader(); - } - private: friend struct DefaultIntrusiveForwardListTraits; @@ -52,6 +43,6 @@ private: std::atomic next_ = nullptr; }; +static_assert(std::is_trivially_destructible_v); - -} // namespace kotlin::gc::mark \ No newline at end of file +} // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp index f302b851095..80c81d2793d 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp @@ -1,6 +1,5 @@ #include "ParallelMark.hpp" -#include "MarkStack.hpp" #include "MarkAndSweepUtils.hpp" #include "GCStatistics.hpp" #include "Utils.hpp" diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp index 8568ad5f8d0..621d3d4f79e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp @@ -4,12 +4,12 @@ #include #include "GCStatistics.hpp" -#include "MarkStack.hpp" -#include "std_support/Vector.hpp" +#include "ManuallyScoped.hpp" +#include "ObjectData.hpp" +#include "ParallelProcessor.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" -#include "ParallelProcessor.hpp" -#include "ManuallyScoped.hpp" +#include "std_support/Vector.hpp" namespace kotlin::gc::mark { @@ -70,14 +70,13 @@ private: * Mark workers are able to balance work between each other through sharing/stealing. */ class ParallelMark : private Pinned { - using MarkStackImpl = intrusive_forward_list; + using MarkStackImpl = intrusive_forward_list; // work balancing parameters were chosen pretty arbitrary using ParallelProcessor = ParallelProcessor; public: class MarkTraits { public: using MarkQueue = ParallelProcessor::Worker; - using ObjectFactory = ObjectData::ObjectFactory; static void clear(MarkQueue& queue) noexcept { RuntimeAssert(queue.localEmpty(), "Mark queue must be empty"); @@ -86,19 +85,18 @@ public: static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept { auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop(); if (obj) { - auto node = ObjectFactory::NodeRef::From(*obj); - return node->GetObjHeader(); + return objectForObjectData(*obj); } return nullptr; } static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { - auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData(); + auto& objectData = objectDataForObject(object); return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData); } static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept { - auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData(); + auto& objectData = objectDataForObject(object); return objectData.tryMark(); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp b/kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp index 324a6c49a18..363ba249163 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp @@ -13,7 +13,6 @@ #include "KAssert.h" #include "Memory.h" -#include "ObjectFactory.hpp" #include "Runtime.h" #include "ScopedThread.hpp" #include "Utils.hpp" diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 3cb4c7b5151..af39b14c44f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -51,6 +51,14 @@ public: std_support::unique_ptr impl_; }; + // Header to be placed before each heap object. GC will use this to keep its data if needed. + // This is used via `type_layout::descriptor_t`, which is specialized below. + // If GC doesn't need any data, it can make `size()` return 0 and `alignment()` + // return 1. + // Note: GC does not deinitialize `ObjectData`, so the implementations must ensure that + // the destructor is a trivial one. + class ObjectData; + explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept; ~GC(); @@ -75,9 +83,6 @@ public: void WaitFinished(int64_t epoch) noexcept; void WaitFinalizers(int64_t epoch) noexcept; - static const size_t objectDataSize; - static bool SweepObject(void* objectData) noexcept; - static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; private: @@ -87,7 +92,24 @@ private: bool isMarked(ObjHeader* object) noexcept; OBJ_GETTER(tryRef, std::atomic& object) noexcept; +// This will drop the mark bit if it was set and return `true`. +// If the mark bit was unset, this will return `false`. +bool tryResetMark(GC::ObjectData& objectData) noexcept; + inline constexpr bool kSupportsMultipleMutators = true; } // namespace gc + +template <> +struct type_layout::descriptor { + struct type { + using value_type = gc::GC::ObjectData; + + static uint64_t size() noexcept; + static size_t alignment() noexcept; + + static value_type* construct(uint8_t* ptr) noexcept; + }; +}; + } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 953f81fbf91..a7d4e4c676d 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -7,6 +7,7 @@ #define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H #include "ExtraObjectData.hpp" +#include "ExtraObjectDataFactory.hpp" #include "FinalizerHooks.hpp" #include "GlobalData.hpp" #include "GCStatistics.hpp" @@ -164,6 +165,20 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T return Sweep(handle, iter); } +template +struct DefaultSweepTraits { + using ObjectFactory = T; + using ExtraObjectsFactory = mm::ExtraObjectDataFactory; + + static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept { + auto* baseObject = object.GetBaseObject(); + if (!baseObject->heap()) return true; + return gc::isMarked(baseObject); + } + + static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); } +}; + template void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { auto handle = gcHandle.collectThreadRoots(thread); @@ -235,6 +250,10 @@ void processWeaks(GCHandle gcHandle, mm::SpecialRefRegistry& registry) noexcept } } +struct DefaultProcessWeaksTraits { + static bool IsMarked(ObjHeader* obj) noexcept { return gc::isMarked(obj); } +}; + } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp new file mode 100644 index 00000000000..ab53ce6c561 --- /dev/null +++ b/kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#ifdef CUSTOM_ALLOCATOR + +#include "CustomAllocator.hpp" +#include "Heap.hpp" + +#else + +#include "Allocator.hpp" +#include "ExtraObjectDataFactory.hpp" +#include "GC.hpp" +#include "ObjectFactory.hpp" + +namespace kotlin::gc { + +struct ObjectFactoryTraits { + using Allocator = Allocator; + using ObjectData = gc::GC::ObjectData; + + Allocator CreateAllocator() noexcept { return Allocator(); } +}; + +using ObjectFactory = mm::ObjectFactory; + +} // namespace kotlin::gc + +#endif diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 2d22dd721b0..081b49adc41 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -7,11 +7,12 @@ #include "Common.h" #include "GC.hpp" +#include "GCStatistics.hpp" #include "NoOpGC.hpp" +#include "ObjectAlloc.hpp" +#include "ObjectOps.hpp" #include "ThreadData.hpp" #include "std_support/Memory.hpp" -#include "GCStatistics.hpp" -#include "ObjectOps.hpp" using namespace kotlin; @@ -79,7 +80,7 @@ gc::GC::~GC() = default; // static size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { #ifndef CUSTOM_ALLOCATOR - return mm::ObjectFactory::GetAllocatedHeapSize(object); + return ObjectFactory::GetAllocatedHeapSize(object); #else return alloc::CustomAllocator::GetAllocatedHeapSize(object); #endif @@ -133,11 +134,8 @@ ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_OBJ(object.load(std::memory_order_relaxed)); } -// static -const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no_unique_address]] - -// static -ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { +ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { + RuntimeAssert(false, "Should not reach here"); return true; } @@ -152,3 +150,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); #endif } + +// static +ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { + return 0; +} + +// static +ALWAYS_INLINE size_t type_layout::descriptor::type::alignment() noexcept { + return 1; +} + +// static +ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor::type::construct(uint8_t* ptr) noexcept { + return reinterpret_cast(ptr); +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 1d3e3e93ad1..64f441f6968 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -7,36 +7,28 @@ #include "GC.hpp" +#include "AllocatorImpl.hpp" #include "NoOpGC.hpp" -#include "ObjectFactory.hpp" - -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#else -#include "ExtraObjectDataFactory.hpp" -#endif namespace kotlin { namespace gc { -using GCImpl = NoOpGC; - class GC::Impl : private Pinned { public: Impl() noexcept = default; #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + ObjectFactory& objectFactory() noexcept { return objectFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif - GCImpl& gc() noexcept { return gc_; } + NoOpGC& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory objectFactory_; + ObjectFactory objectFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif - GCImpl gc_; + NoOpGC gc_; }; class GC::ThreadData::Impl : private Pinned { @@ -45,24 +37,25 @@ public: Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {} #else Impl(GC& gc, mm::ThreadData& threadData) noexcept : - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {} #endif - GCImpl::ThreadData& gc() noexcept { return gc_; } + NoOpGC::ThreadData& gc() noexcept { return gc_; } #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator& alloc() noexcept { return alloc_; } #else - mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #endif private: - GCImpl::ThreadData gc_; + NoOpGC::ThreadData gc_; #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator alloc_; #else - mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 60eb7f617c6..4e0c5ab463e 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -8,14 +8,11 @@ #include -#include "Allocator.hpp" +#include "AllocatorImpl.hpp" +#include "GC.hpp" #include "Logging.hpp" #include "Utils.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "Heap.hpp" -#endif - namespace kotlin { namespace mm { @@ -28,21 +25,11 @@ namespace gc { // TODO: It can be made more efficient. class NoOpGC : private Pinned { public: - class ObjectData {}; - - using Allocator = gc::Allocator; - class ThreadData : private Pinned { public: - using ObjectData = NoOpGC::ObjectData; - ThreadData() noexcept {} ~ThreadData() = default; - void OnOOM(size_t size) noexcept {} - - Allocator CreateAllocator() noexcept { return Allocator(); } - private: }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp new file mode 100644 index 00000000000..e8660e8027d --- /dev/null +++ b/kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp @@ -0,0 +1,69 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#ifdef CUSTOM_ALLOCATOR + +#include "CustomAllocator.hpp" +#include "CustomFinalizerProcessor.hpp" +#include "GCApi.hpp" +#include "Heap.hpp" + +namespace kotlin::gc { + +inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { + return kotlin::alloc::objectDataForObject(object); +} + +inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { + return kotlin::alloc::objectForObjectData(objectData); +} + +using FinalizerQueue = alloc::FinalizerQueue; +using FinalizerQueueTraits = alloc::FinalizerQueueTraits; + +} // namespace kotlin::gc + +#else + +#include "Allocator.hpp" +#include "ExtraObjectDataFactory.hpp" +#include "GC.hpp" +#include "GlobalData.hpp" +#include "Logging.hpp" +#include "ObjectFactory.hpp" + +namespace kotlin::gc { + +struct ObjectFactoryTraits { + using Allocator = AllocatorWithGC; + using ObjectData = gc::GC::ObjectData; + + Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } + + void OnOOM(size_t size) noexcept { + RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); + // TODO: This will print the log for "manual" scheduling. Fix this. + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); + } +}; + +using ObjectFactory = mm::ObjectFactory; + +inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept { + return ObjectFactory::NodeRef::From(object).ObjectData(); +} + +inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept { + return ObjectFactory::NodeRef::From(objectData)->GetObjHeader(); +} + +using FinalizerQueue = ObjectFactory::FinalizerQueue; +using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits; + +} // namespace kotlin::gc + +#endif diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 9384807e743..3b54c9bc996 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -6,12 +6,13 @@ #include "GCImpl.hpp" #include "GC.hpp" +#include "GCStatistics.hpp" +#include "GlobalData.hpp" #include "MarkAndSweepUtils.hpp" +#include "ObjectAlloc.hpp" +#include "ObjectOps.hpp" #include "SameThreadMarkAndSweep.hpp" #include "std_support/Memory.hpp" -#include "GlobalData.hpp" -#include "GCStatistics.hpp" -#include "ObjectOps.hpp" using namespace kotlin; @@ -81,7 +82,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { #ifdef CUSTOM_ALLOCATOR return alloc::CustomAllocator::GetAllocatedHeapSize(object); #else - return mm::ObjectFactory::GetAllocatedHeapSize(object); + return ObjectFactory::GetAllocatedHeapSize(object); #endif } @@ -140,20 +141,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept { } bool gc::isMarked(ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - return objectData.marked(); + return objectDataForObject(object).marked(); } ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_OBJ(object.load(std::memory_order_relaxed)); } -// static -const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData); - -// static -ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { - return reinterpret_cast(objectData)->tryResetMark(); +ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { + return objectData.tryResetMark(); } // static @@ -167,3 +163,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); #endif } + +// static +ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { + return sizeof(gc::GC::ObjectData); +} + +// static +ALWAYS_INLINE size_t type_layout::descriptor::type::alignment() noexcept { + return alignof(gc::GC::ObjectData); +} + +// static +ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor::type::construct(uint8_t* ptr) noexcept { + return new (ptr) gc::GC::ObjectData(); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index afed207b219..9e54d8e38b6 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -7,19 +7,12 @@ #include "GC.hpp" +#include "AllocatorImpl.hpp" #include "SameThreadMarkAndSweep.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#else -#include "ExtraObjectDataFactory.hpp" -#endif - namespace kotlin { namespace gc { -using GCImpl = SameThreadMarkAndSweep; - class GC::Impl : private Pinned { public: #ifdef CUSTOM_ALLOCATOR @@ -27,17 +20,17 @@ public: #else explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} - mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + ObjectFactory& objectFactory() noexcept { return objectFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } #endif - GCImpl& gc() noexcept { return gc_; } + SameThreadMarkAndSweep& gc() noexcept { return gc_; } private: #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory objectFactory_; + ObjectFactory objectFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_; #endif - GCImpl gc_; + SameThreadMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { @@ -48,25 +41,26 @@ public: alloc_(gc.impl_->gc().heap()) { } #else - objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()), extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { } #endif - GCImpl::ThreadData& gc() noexcept { return gc_; } + SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; } #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } #else alloc::CustomAllocator& alloc() noexcept { return alloc_; } #endif private: - GCImpl::ThreadData gc_; + SameThreadMarkAndSweep::ThreadData gc_; #ifdef CUSTOM_ALLOCATOR alloc::CustomAllocator alloc_; #else - mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; + [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; #endif }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp b/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp new file mode 100644 index 00000000000..a372497969f --- /dev/null +++ b/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp @@ -0,0 +1,50 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include + +#include "AllocatorImpl.hpp" +#include "GC.hpp" +#include "IntrusiveList.hpp" +#include "KAssert.h" + +namespace kotlin::gc { + +class GC::ObjectData { +public: + bool tryMark() noexcept { return trySetNext(reinterpret_cast(1)); } + + bool marked() const noexcept { return next_ != nullptr; } + + bool tryResetMark() noexcept { + if (next_ == nullptr) return false; + next_ = nullptr; + return true; + } + +private: + friend struct DefaultIntrusiveForwardListTraits; + + ObjectData* next() const noexcept { return next_; } + void setNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + next_ = next; + } + bool trySetNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + if (next_ != nullptr) { + return false; + } + next_ = next; + return true; + } + + ObjectData* next_ = nullptr; +}; +static_assert(std::is_trivially_destructible_v); + +} // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index f5cbebd509d..31855aceec2 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -14,64 +14,21 @@ #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" #include "Memory.h" +#include "ObjectAlloc.hpp" #include "RootSet.hpp" #include "Runtime.h" #include "ThreadData.hpp" #include "ThreadRegistry.hpp" #include "ThreadSuspension.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "CustomFinalizerProcessor.hpp" -#endif - using namespace kotlin; -namespace { - -struct SweepTraits { - using ObjectFactory = mm::ObjectFactory; - using ExtraObjectsFactory = mm::ExtraObjectDataFactory; - - static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { - auto *baseObject = object.GetBaseObject(); - if (!baseObject->heap()) return true; - auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).ObjectData(); - return objectData.marked(); - } - - static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { - auto& objectData = node.ObjectData(); - return objectData.tryResetMark(); - } -}; - -struct FinalizeTraits { - using ObjectFactory = mm::ObjectFactory; -}; - -struct ProcessWeaksTraits { - static bool IsMarked(ObjHeader* obj) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(obj).ObjectData(); - return objectData.marked(); - } -}; - -} // namespace - -void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); - // TODO: This will print the log for "manual" scheduling. Fix this. - mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished(); -} - #ifdef CUSTOM_ALLOCATOR gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( gcScheduler::GCScheduler& gcScheduler) noexcept : #else gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - mm::ObjectFactory& objectFactory, - mm::ExtraObjectDataFactory& extraObjectDataFactory, - gcScheduler::GCScheduler& gcScheduler) noexcept : + ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), extraObjectDataFactory_(extraObjectDataFactory), @@ -139,7 +96,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::Mark(gcHandle, markQueue_); - gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); + gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); #ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread @@ -147,9 +104,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); std::optional objectFactoryIterable = objectFactory_.LockForIter(); - gc::SweepExtraObjects(gcHandle, *extraObjectFactoryIterable); + gc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; - auto finalizerQueue = gc::Sweep(gcHandle, *objectFactoryIterable); + auto finalizerQueue = gc::Sweep>(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; kotlin::compactObjectPoolInMainThread(); #else diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 8d5436255d9..03755be9550 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -8,24 +8,17 @@ #include -#include "Allocator.hpp" -#include "ExtraObjectDataFactory.hpp" +#include "AllocatorImpl.hpp" #include "FinalizerProcessor.hpp" +#include "GC.hpp" #include "GCScheduler.hpp" #include "GCState.hpp" +#include "GlobalData.hpp" #include "IntrusiveList.hpp" -#include "ObjectFactory.hpp" +#include "ObjectData.hpp" #include "Types.h" #include "Utils.hpp" -#ifdef CUSTOM_ALLOCATOR -#include "CustomAllocator.hpp" -#include "CustomFinalizerProcessor.hpp" -#include "ExtraObjectPage.hpp" -#include "GCApi.hpp" -#include "Heap.hpp" -#endif - namespace kotlin { namespace mm { @@ -38,70 +31,20 @@ namespace gc { // TODO: Rename to StopTheWorldMarkAndSweep. class SameThreadMarkAndSweep : private Pinned { public: - class ObjectData { - public: - bool tryMark() noexcept { - return trySetNext(reinterpret_cast(1)); - } - - bool marked() const noexcept { return next_ != nullptr; } - - bool tryResetMark() noexcept { - if (next_ == nullptr) return false; - next_ = nullptr; - return true; - } - - private: - friend struct DefaultIntrusiveForwardListTraits; - - ObjectData* next() const noexcept { return next_; } - void setNext(ObjectData* next) noexcept { - RuntimeAssert(next, "next cannot be nullptr"); - next_ = next; - } - bool trySetNext(ObjectData* next) noexcept { - RuntimeAssert(next, "next cannot be nullptr"); - if (next_ != nullptr) { - return false; - } - next_ = next; - return true; - } - - ObjectData* next_ = nullptr; - }; - - using MarkQueue = intrusive_forward_list; + using MarkQueue = intrusive_forward_list; class ThreadData : private Pinned { public: - using ObjectData = SameThreadMarkAndSweep::ObjectData; - using Allocator = AllocatorWithGC; - ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {} ~ThreadData() = default; - - void OnOOM(size_t size) noexcept; - - Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } - private: }; - using Allocator = ThreadData::Allocator; - #ifdef CUSTOM_ALLOCATOR - using FinalizerQueue = alloc::FinalizerQueue; - using FinalizerQueueTraits = alloc::FinalizerQueueTraits; - SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept; #else - using FinalizerQueue = mm::ObjectFactory::FinalizerQueue; - using FinalizerQueueTraits = mm::ObjectFactory::FinalizerQueueTraits; - SameThreadMarkAndSweep( - mm::ObjectFactory& objectFactory, + ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept; #endif @@ -122,7 +65,7 @@ private: void PerformFullGC(int64_t epoch) noexcept; #ifndef CUSTOM_ALLOCATOR - mm::ObjectFactory& objectFactory_; + ObjectFactory& objectFactory_; mm::ExtraObjectDataFactory& extraObjectDataFactory_; #else alloc::Heap heap_; @@ -145,21 +88,14 @@ struct MarkTraits { static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { if (auto* top = queue.try_pop_front()) { - auto node = mm::ObjectFactory::NodeRef::From(*top); - return node->GetObjHeader(); + return objectForObjectData(*top); } return nullptr; } - static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - return queue.try_push_front(objectData); - } + static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { return queue.try_push_front(objectDataForObject(object)); } - static bool tryMark(ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - return objectData.tryMark(); - } + static bool tryMark(ObjHeader* object) noexcept { return objectDataForObject(object).tryMark(); } static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { auto process = object->type_info()->processObjectInMark; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 05dda65a243..5d38e22df44 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -194,11 +194,6 @@ std_support::vector Alive(mm::ThreadData& threadData) { #endif } -bool IsMarked(ObjHeader* objHeader) { - auto nodeRef = mm::ObjectFactory::NodeRef::From(objHeader); - return nodeRef.ObjectData().marked(); -} - test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); @@ -239,12 +234,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(global2.header()), false); - ASSERT_THAT(IsMarked(global3.header()), false); - ASSERT_THAT(IsMarked(stack1.header()), false); - ASSERT_THAT(IsMarked(stack2.header()), false); - ASSERT_THAT(IsMarked(stack3.header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(global2.header()), false); + ASSERT_THAT(gc::isMarked(global3.header()), false); + ASSERT_THAT(gc::isMarked(stack1.header()), false); + ASSERT_THAT(gc::isMarked(stack2.header()), false); + ASSERT_THAT(gc::isMarked(stack3.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -252,12 +247,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(IsMarked(global1.header()), false); - EXPECT_THAT(IsMarked(global2.header()), false); - EXPECT_THAT(IsMarked(global3.header()), false); - EXPECT_THAT(IsMarked(stack1.header()), false); - EXPECT_THAT(IsMarked(stack2.header()), false); - EXPECT_THAT(IsMarked(stack3.header()), false); + EXPECT_THAT(gc::isMarked(global1.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global3.header()), false); + EXPECT_THAT(gc::isMarked(stack1.header()), false); + EXPECT_THAT(gc::isMarked(stack2.header()), false); + EXPECT_THAT(gc::isMarked(stack3.header()), false); }); } @@ -285,12 +280,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(global2.header()), false); - ASSERT_THAT(IsMarked(global3.header()), false); - ASSERT_THAT(IsMarked(stack1.header()), false); - ASSERT_THAT(IsMarked(stack2.header()), false); - ASSERT_THAT(IsMarked(stack3.header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(global2.header()), false); + ASSERT_THAT(gc::isMarked(global3.header()), false); + ASSERT_THAT(gc::isMarked(stack1.header()), false); + ASSERT_THAT(gc::isMarked(stack2.header()), false); + ASSERT_THAT(gc::isMarked(stack3.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -298,12 +293,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(IsMarked(global1.header()), false); - EXPECT_THAT(IsMarked(global2.header()), false); - EXPECT_THAT(IsMarked(global3.header()), false); - EXPECT_THAT(IsMarked(stack1.header()), false); - EXPECT_THAT(IsMarked(stack2.header()), false); - EXPECT_THAT(IsMarked(stack3.header()), false); + EXPECT_THAT(gc::isMarked(global1.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global3.header()), false); + EXPECT_THAT(gc::isMarked(stack1.header()), false); + EXPECT_THAT(gc::isMarked(stack2.header()), false); + EXPECT_THAT(gc::isMarked(stack3.header()), false); }); } @@ -313,8 +308,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) { auto& object2 = AllocateObject(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -328,8 +323,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { auto& object2 = AllocateObjectWithFinalizer(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); @@ -348,8 +343,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(weak1.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); EXPECT_CALL(finalizerHook(), Call(weak1.header())); @@ -366,14 +361,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(weak1.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(weak1.header()), false); ASSERT_THAT(weak1.get(), object1.header()); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(IsMarked(weak1.header()), false); + EXPECT_THAT(gc::isMarked(weak1.header()), false); EXPECT_THAT(weak1.get(), nullptr); }); } @@ -396,12 +391,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -409,12 +404,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -443,14 +438,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -458,12 +453,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -492,14 +487,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); @@ -509,12 +504,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -531,18 +526,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { object2->field1 = stack.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); }); } @@ -571,14 +566,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(IsMarked(global.header()), false); - ASSERT_THAT(IsMarked(stack.header()), false); - ASSERT_THAT(IsMarked(object1.header()), false); - ASSERT_THAT(IsMarked(object2.header()), false); - ASSERT_THAT(IsMarked(object3.header()), false); - ASSERT_THAT(IsMarked(object4.header()), false); - ASSERT_THAT(IsMarked(object5.header()), false); - ASSERT_THAT(IsMarked(object6.header()), false); + ASSERT_THAT(gc::isMarked(global.header()), false); + ASSERT_THAT(gc::isMarked(stack.header()), false); + ASSERT_THAT(gc::isMarked(object1.header()), false); + ASSERT_THAT(gc::isMarked(object2.header()), false); + ASSERT_THAT(gc::isMarked(object3.header()), false); + ASSERT_THAT(gc::isMarked(object4.header()), false); + ASSERT_THAT(gc::isMarked(object5.header()), false); + ASSERT_THAT(gc::isMarked(object6.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); @@ -587,12 +582,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(stack.header()), false); - EXPECT_THAT(IsMarked(object1.header()), false); - EXPECT_THAT(IsMarked(object2.header()), false); - EXPECT_THAT(IsMarked(object3.header()), false); - EXPECT_THAT(IsMarked(object4.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(stack.header()), false); + EXPECT_THAT(gc::isMarked(object1.header()), false); + EXPECT_THAT(gc::isMarked(object2.header()), false); + EXPECT_THAT(gc::isMarked(object3.header()), false); + EXPECT_THAT(gc::isMarked(object4.header()), false); }); } @@ -609,12 +604,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) { global2->field1 = global1.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(gc::isMarked(global2.header()), false); }); } @@ -628,14 +623,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) { ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(object.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(object.header()), false); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(IsMarked(global.header()), false); - EXPECT_THAT(IsMarked(object.header()), false); + EXPECT_THAT(gc::isMarked(global.header()), false); + EXPECT_THAT(gc::isMarked(object.header()), false); }); } @@ -1141,9 +1136,9 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(IsMarked(global1.header()), false); - ASSERT_THAT(IsMarked(object1_local.header()), false); - ASSERT_THAT(IsMarked(weak.load()->header()), false); + ASSERT_THAT(gc::isMarked(global1.header()), false); + ASSERT_THAT(gc::isMarked(object1_local.header()), false); + ASSERT_THAT(gc::isMarked(weak.load()->header()), false); ASSERT_THAT(weak.load()->get(), object1_local.header()); global1->field1 = nullptr; diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index d04aa0b2289..b45177c5881 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -19,9 +19,11 @@ #include +#include "Alignment.hpp" #include "KAssert.h" #include "Common.h" #include "TypeInfo.h" +#include "TypeLayout.hpp" #include "Atomic.h" #include "PointerBits.h" #include "Utils.hpp" @@ -125,6 +127,7 @@ struct ObjHeader { static MetaObjHeader* createMetaObject(ObjHeader* object); static void destroyMetaObject(ObjHeader* object); }; +static_assert(alignof(ObjHeader) <= kotlin::kObjectAlignment); // Header of value type array objects. Keep layout in sync with that of object header. struct ArrayHeader { @@ -140,6 +143,61 @@ struct ArrayHeader { // Elements count. Element size is stored in instanceSize_ field of TypeInfo, negated. uint32_t count_; }; +static_assert(alignof(ArrayHeader) <= kotlin::kObjectAlignment); + +#ifndef KONAN_WASM +namespace kotlin { + +struct ObjectBody; +struct ArrayBody; + +template <> +struct type_layout::descriptor { + class type { + public: + using value_type = ObjectBody; + + explicit type(const TypeInfo* typeInfo) noexcept : size_(typeInfo->instanceSize_ - sizeof(ObjHeader)) {} + + static constexpr size_t alignment() noexcept { return kObjectAlignment; } + uint64_t size() const noexcept { return size_; } + + value_type* construct(uint8_t* ptr) noexcept { + RuntimeAssert(isZeroed(std_support::span(ptr, size_)), "ObjectBodyDescriptor::construct@%p memory is not zeroed", ptr); + return reinterpret_cast(ptr); + } + + private: + uint64_t size_; + }; +}; + +template <> +struct type_layout::descriptor { + class type { + public: + using value_type = ArrayBody; + + explicit type(const TypeInfo* typeInfo, uint32_t count) noexcept : + // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped + // at about half of uint64_t max. + size_(static_cast(-typeInfo->instanceSize_) * count) {} + + static constexpr size_t alignment() noexcept { return kObjectAlignment; } + uint64_t size() const noexcept { return size_; } + + value_type* construct(uint8_t* ptr) noexcept { + RuntimeAssert(isZeroed(std_support::span(ptr, size_)), "ArrayBodyDescriptor::construct@%p memory is not zeroed", ptr); + return reinterpret_cast(ptr); + } + + private: + uint64_t size_; + }; +}; + +} // namespace kotlin +#endif ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj); ALWAYS_INLINE bool isShareable(const ObjHeader* obj); diff --git a/kotlin-native/runtime/src/main/cpp/Utils.cpp b/kotlin-native/runtime/src/main/cpp/Utils.cpp index 9dacec6b375..f9c580b29f0 100644 --- a/kotlin-native/runtime/src/main/cpp/Utils.cpp +++ b/kotlin-native/runtime/src/main/cpp/Utils.cpp @@ -7,6 +7,9 @@ #include #include +#include + +using namespace kotlin; namespace { @@ -68,4 +71,10 @@ struct HashCompineImpl<64> { size_t kotlin::CombineHash(size_t seed, size_t value) { return HashCompineImpl::fn(seed, value); -} \ No newline at end of file +} + +bool kotlin::isZeroed(std_support::span span) noexcept { + if (span.size() == 0) return true; + if (span[0] != 0) return false; + return memcmp(span.data(), span.data() + 1, span.size() - 1) == 0; +} diff --git a/kotlin-native/runtime/src/main/cpp/Utils.hpp b/kotlin-native/runtime/src/main/cpp/Utils.hpp index ac47b6e08df..7ace7176be3 100644 --- a/kotlin-native/runtime/src/main/cpp/Utils.hpp +++ b/kotlin-native/runtime/src/main/cpp/Utils.hpp @@ -10,6 +10,8 @@ #include +#include "std_support/Span.hpp" + namespace kotlin { // A helper for implementing classes with disabled copy constructor and copy assignment. @@ -98,6 +100,9 @@ size_t CombineHash(size_t seed, size_t value); #define ownerOf(type, field, ref) *reinterpret_cast(reinterpret_cast(&ref) - offsetof(type, field)) +// Returns `true` if the entire `span` is zeroed. +bool isZeroed(std_support::span span) noexcept; + } // namespace kotlin #endif // RUNTIME_UTILS_H diff --git a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp index 279455862f2..527f2840aa5 100644 --- a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp @@ -5,6 +5,7 @@ #include "Utils.hpp" +#include #include #include "gmock/gmock.h" @@ -73,3 +74,26 @@ TEST(UtilsTest, OwnerOf) { EXPECT_THAT(&c, &Container::fromX(c.x())); EXPECT_THAT(&c, &Container::fromY(c.y())); } + +TEST(UtilsTest, IsZeroed) { + std::array empty; + EXPECT_TRUE(isZeroed(empty)); + + std::array zeroed1 = {0}; + EXPECT_TRUE(isZeroed(zeroed1)); + + std::array notZeroed1 = {1}; + EXPECT_FALSE(isZeroed(notZeroed1)); + + std::array zeroed3 = {0, 0, 0}; + EXPECT_TRUE(isZeroed(zeroed3)); + + std::array notZeroed3_0 = {1, 0, 0}; + EXPECT_FALSE(isZeroed(notZeroed3_0)); + + std::array notZeroed3_1 = {0, 1, 0}; + EXPECT_FALSE(isZeroed(notZeroed3_1)); + + std::array notZeroed3_2 = {0, 0, 1}; + EXPECT_FALSE(isZeroed(notZeroed3_2)); +} diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index cc4504c27b1..660665d85c5 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -6,7 +6,6 @@ #ifndef RUNTIME_MM_GLOBAL_DATA_H #define RUNTIME_MM_GLOBAL_DATA_H -#include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" #include "GCScheduler.hpp" diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index bc1d6f5364b..abbe60ef68d 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -17,6 +17,7 @@ #include "Memory.h" #include "Mutex.hpp" #include "Porting.h" +#include "TypeLayout.hpp" #include "Types.h" #include "Utils.hpp" @@ -69,9 +70,9 @@ public: } // Note: This can only be trivially destructible data, as nobody can invoke its destructor. - void* Data() noexcept { + uint8_t* Data() noexcept { constexpr size_t kDataOffset = DataOffset(); - void* ptr = reinterpret_cast(this) + kDataOffset; + auto* ptr = reinterpret_cast(this) + kDataOffset; RuntimeAssert(IsAligned(ptr, DataAlignment), "Data=%p is not aligned to %zu", ptr, DataAlignment); return ptr; } @@ -79,7 +80,7 @@ public: // It's a caller responsibility to know if the underlying data is `T`. template T& Data() noexcept { - return *static_cast(Data()); + return *reinterpret_cast(Data()); } size_t GetAllocatedSize() noexcept { @@ -438,46 +439,85 @@ private: template class ObjectFactory : private Pinned { - using ObjectData = typename Traits::ObjectData; using Allocator = typename Traits::Allocator; + using ObjectData = typename Traits::ObjectData; struct HeapObjHeader { - [[no_unique_address]] // to account for GCs with empty ObjectData - ObjectData gcData; - alignas(kObjectAlignment) ObjHeader object; + using descriptor = type_layout::Composite; + + static HeapObjHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); } + + static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().template fromField<1>(object); } + + ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; } + + ObjHeader* object() noexcept { return descriptor().template field<1>(this).second; } + + private: + HeapObjHeader() = delete; + ~HeapObjHeader() = delete; + }; + + struct HeapObject { + using descriptor = type_layout::Composite; + + static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept { + return descriptor{{}, type_layout::descriptor_t{typeInfo}}; + } + + HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; } + + private: + HeapObject() = delete; + ~HeapObject() = delete; }; // Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible // with `ObjHeader`: the former can always be casted to the other. struct HeapArrayHeader { - [[no_unique_address]] - ObjectData gcData; - alignas(kObjectAlignment) ArrayHeader array; + using descriptor = type_layout::Composite; + + static HeapArrayHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); } + + static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().template fromField<1>(array); } + + ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; } + + ArrayHeader* array() noexcept { return descriptor().template field<1>(this).second; } + + private: + HeapArrayHeader() = delete; + ~HeapArrayHeader() = delete; }; - static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { - size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); - return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); - } + struct HeapArray { + using descriptor = type_layout::Composite; - static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { - // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped - // at about half of uint64_t max. - uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; - // Note: array body is aligned, but for size computation it is enough to align the sum. - return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept { + return descriptor{{}, type_layout::descriptor_t{typeInfo, size}}; + } + + HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; } + + private: + HeapArray() = delete; + ~HeapArray() = delete; + }; + + // Only used for already allocated objects. Cannot overflow size_t. + static size_t GetDataSizeForAllocated(ObjHeader* object) noexcept { + RuntimeAssert(object->heap(), "Object must be a heap object"); + const auto* typeInfo = object->type_info(); + if (typeInfo->IsArray()) { + return HeapArray::make_descriptor(typeInfo, object->array()->count_).size(); + } else { + return HeapObject::make_descriptor(typeInfo).size(); + } } struct DataSizeProvider { - static size_t GetDataSize(void* data) noexcept { - ObjHeader* object = &static_cast(data)->object; - RuntimeAssert(object->heap(), "Object must be a heap object"); - const auto* typeInfo = object->type_info(); - if (typeInfo->IsArray()) { - return ArrayAllocatedDataSize(typeInfo, object->array()->count_); - } else { - return ObjectAllocatedDataSize(typeInfo); - } + static size_t GetDataSize(uint8_t* data) noexcept { + return GetDataSizeForAllocated(reinterpret_cast(data)->object()); } }; @@ -490,32 +530,23 @@ public: static NodeRef From(ObjHeader* object) noexcept { RuntimeAssert(object->heap(), "Must be a heap object"); - auto& heapObject = ownerOf(HeapObjHeader, object, *object); - return NodeRef(Storage::Node::FromData(&heapObject)); + return NodeRef(Storage::Node::FromData(reinterpret_cast(&HeapObjHeader::from(object)))); } static NodeRef From(ArrayHeader* array) noexcept { RuntimeAssert(array->obj()->heap(), "Must be a heap object"); - auto& heapArray = ownerOf(HeapArrayHeader, array, *array); - return NodeRef(Storage::Node::FromData(&heapArray)); + return NodeRef(Storage::Node::FromData(reinterpret_cast(&HeapArrayHeader::from(array)))); } static NodeRef From(ObjectData& objectData) noexcept { - auto& heapObject = ownerOf(HeapObjHeader, gcData, objectData); - return NodeRef(Storage::Node::FromData(&heapObject)); + return NodeRef(Storage::Node::FromData(reinterpret_cast(&HeapObjHeader::from(objectData)))); } NodeRef* operator->() noexcept { return this; } - ObjectData& ObjectData() noexcept { - // `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can - // be always casted to the other. - return static_cast(node_.Data())->gcData; - } + ObjectData& ObjectData() noexcept { return reinterpret_cast(node_.Data())->objectData(); } - ObjHeader* GetObjHeader() noexcept { - return &static_cast(node_.Data())->object; - } + ObjHeader* GetObjHeader() noexcept { return reinterpret_cast(node_.Data())->object(); } bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; } @@ -550,37 +581,21 @@ public: ThreadQueue(ObjectFactory& owner, Allocator allocator) noexcept : producer_(owner.storage_, std::move(allocator)) {} - static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept { - RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - // Only used for already allocated objects. Cannot overflow size_t - auto allocSize = ObjectAllocatedDataSize(typeInfo); - return Storage::Node::GetSizeForDataSize(allocSize); - } - ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - size_t allocSize = ObjectAllocatedDataSize(typeInfo); - auto& node = producer_.Insert(allocSize); - auto* heapObject = new (node.Data()) HeapObjHeader(); - auto* object = &heapObject->object; + auto descriptor = HeapObject::make_descriptor(typeInfo); + auto& heapObject = *descriptor.construct(producer_.Insert(descriptor.size()).Data()); + ObjHeader* object = heapObject.header(descriptor).object(); object->typeInfoOrMeta_ = const_cast(typeInfo); // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. return object; } - static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept { - RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - // Only used for already allocated arrays. Cannot overflow size_t - auto allocSize = ArrayAllocatedDataSize(typeInfo, count); - return Storage::Node::GetSizeForDataSize(allocSize); - } - ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - auto allocSize = ArrayAllocatedDataSize(typeInfo, count); - auto& node = producer_.Insert(allocSize); - auto* heapArray = new (node.Data()) HeapArrayHeader(); - auto* array = &heapArray->array; + auto descriptor = HeapArray::make_descriptor(typeInfo, count); + auto& heapArray = *descriptor.construct(producer_.Insert(descriptor.size()).Data()); + ArrayHeader* array = heapArray.header(descriptor).array(); array->typeInfoOrMeta_ = const_cast(typeInfo); array->count_ = count; // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. @@ -713,13 +728,8 @@ public: void ClearForTests() { storage_.ClearForTests(); } static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept { - RuntimeAssert(object->heap(), "Object must be a heap object"); - const auto* typeInfo = object->type_info(); - if (typeInfo->IsArray()) { - return ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_); - } else { - return ThreadQueue::ObjectAllocatedSize(typeInfo); - } + auto dataSize = GetDataSizeForAllocated(object); + return Storage::Node::GetSizeForDataSize(dataSize); } private: diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 3bc0ec6ccb9..d5519e9ca1e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -62,7 +62,7 @@ template std_support::vector Collect(ObjectFactoryStorage& storage) { std_support::vector result; for (auto& node : storage.LockForIter()) { - result.push_back(*static_cast(node.Data())); + result.push_back(*reinterpret_cast(node.Data())); } return result; } @@ -71,7 +71,7 @@ template std_support::vector Collect(Consumer>& consumer) { std_support::vector result; for (auto& node : consumer) { - result.push_back(*static_cast(node.Data())); + result.push_back(*reinterpret_cast(node.Data())); } return result; } @@ -710,7 +710,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) { } for (auto& node : iter) { - int element = *static_cast(node.Data()); + int element = *reinterpret_cast(node.Data()); actualBefore.push_back(element); } } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 8655566dd3a..77cee8eb21e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -11,7 +11,6 @@ #include "GlobalData.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" -#include "ObjectFactory.hpp" #include "ShadowStack.hpp" #include "SpecialRefRegistry.hpp" #include "ThreadLocalStorage.hpp" diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp index 03cf716b7f7..3e8a25f7bc2 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp @@ -7,6 +7,7 @@ #include "gtest/gtest.h" +#include "Porting.h" #include "ScopedThread.hpp" #include "ThreadData.hpp" diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index f1a79d3d99a..21be13de060 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -12,6 +12,7 @@ #include "CallsChecker.hpp" #include "Logging.hpp" +#include "Porting.h" #include "SafePoint.hpp" #include "StackTrace.hpp"