diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index c1c9c69edb2..e6bd80c8003 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -232,15 +232,11 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration val shouldCoverSources = configuration.getBoolean(KonanConfigKeys.COVERAGE) private val shouldCoverLibraries = !configuration.getList(KonanConfigKeys.LIBRARIES_TO_COVER).isNullOrEmpty() - private val defaultAllocationMode get() = when { - gc == GC.PARALLEL_MARK_CONCURRENT_SWEEP && sanitizer == null -> { + private val defaultAllocationMode get() = + if (sanitizer == null) AllocationMode.CUSTOM - } - target.supportsMimallocAllocator() && sanitizer == null -> { - AllocationMode.MIMALLOC - } - else -> AllocationMode.STD - } + else + AllocationMode.STD val allocationMode by lazy { when (configuration.get(KonanConfigKeys.ALLOCATION_MODE)) { @@ -262,13 +258,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration if (sanitizer != null) { configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Sanitizers are useful only with the std allocator") } - if (gc == GC.PARALLEL_MARK_CONCURRENT_SWEEP) { - AllocationMode.CUSTOM - } else { - configuration.report(CompilerMessageSeverity.STRONG_WARNING, - "Custom allocator is currently only integrated with concurrent mark and sweep gc. Using default mode.") - defaultAllocationMode - } + AllocationMode.CUSTOM } } } @@ -293,19 +283,17 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } if (allocationMode == AllocationMode.CUSTOM) { add("experimental_memory_manager_custom.bc") - add("concurrent_ms_gc_custom.bc") + when (gc) { + GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc") + GC.NOOP -> add("noop_gc_custom.bc") + GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc") + } } else { add("experimental_memory_manager.bc") when (gc) { - GC.STOP_THE_WORLD_MARK_AND_SWEEP -> { - add("same_thread_ms_gc.bc") - } - GC.NOOP -> { - add("noop_gc.bc") - } - GC.PARALLEL_MARK_CONCURRENT_SWEEP -> { - add("concurrent_ms_gc.bc") - } + GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc") + GC.NOOP -> add("noop_gc.bc") + GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc.bc") } } if (shouldCoverLibraries || shouldCoverSources) add("profileRuntime.bc") diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 6675e72f410..970fc52b356 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -158,7 +158,7 @@ bitcode { } module("custom_alloc") { - headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/cms/cpp")) + headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp")) sourceSets { main {} test {} @@ -195,6 +195,7 @@ bitcode { onlyIf { target.supportsCoreSymbolication() } } + module("source_info_libbacktrace") { srcRoot.set(layout.projectDirectory.dir("src/source_info/libbacktrace")) headersDirs.from(files("src/main/cpp", "src/libbacktrace/c/include")) @@ -299,6 +300,19 @@ bitcode { onlyIf { target.supportsThreads() } } + module("noop_gc_custom") { + srcRoot.set(layout.projectDirectory.dir("src/gc/noop")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + sourceSets { + main {} + testFixtures {} + } + + compilerArgs.add("-DCUSTOM_ALLOCATOR") + + onlyIf { target.supportsThreads() } + } + module("same_thread_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/stms")) headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) @@ -311,6 +325,20 @@ bitcode { onlyIf { target.supportsThreads() } } + module("same_thread_ms_gc_custom") { + srcRoot.set(layout.projectDirectory.dir("src/gc/stms")) + headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp")) + sourceSets { + main {} + testFixtures {} + test {} + } + + compilerArgs.add("-DCUSTOM_ALLOCATOR") + + onlyIf { target.supportsThreads() } + } + module("concurrent_ms_gc") { srcRoot.set(layout.projectDirectory.dir("src/gc/cms")) headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp")) @@ -394,6 +422,11 @@ bitcode { testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc") } + testsGroup("experimentalMM_custom_alloc_runtime_tests") { + testedModules.addAll("experimental_memory_manager_custom", "same_thread_ms_gc_custom") + testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + } + testsGroup("experimentalMM_cms_mimalloc_runtime_tests") { testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc") } @@ -402,6 +435,11 @@ bitcode { testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc") } + testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") { + testedModules.addAll("experimental_memory_manager_custom", "concurrent_ms_gc_custom") + testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + } + testsGroup("experimentalMM_noop_mimalloc_runtime_tests") { testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc") } @@ -410,6 +448,11 @@ bitcode { testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc") } + testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") { + testedModules.addAll("experimental_memory_manager_custom", "noop_gc_custom") + testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc") + } + testsGroup("aggressive_gcScheduler_runtime_tests") { testedModules.addAll("aggressive_gcScheduler") testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc") diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp index 81c02ff064c..ac107f85971 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp @@ -10,6 +10,7 @@ #include "KAssert.h" #include "Utils.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -82,6 +83,17 @@ public: RuntimeAssert(isEmpty(), "AtomicStack must be empty on destruction"); } + // Test method + std_support::vector GetElements() { + std_support::vector elements; + T* elm = stack_.load(); + while (elm) { + elements.push_back(elm); + elm = elm->next_; + } + return elements; + } + private: std::atomic stack_{nullptr}; }; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index 91d31a270d4..1262e83bbd4 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -12,18 +12,17 @@ #include #include -#include "ConcurrentMarkAndSweep.hpp" #include "CustomAllocConstants.hpp" #include "CustomLogging.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" +#include "GC.hpp" #include "GCScheduler.hpp" #include "KAssert.h" #include "SingleObjectPage.hpp" #include "NextFitPage.hpp" #include "Memory.h" #include "FixedBlockPage.hpp" -#include "GCImpl.hpp" #include "GCApi.hpp" #include "TypeInfo.h" @@ -31,7 +30,7 @@ namespace kotlin::alloc { size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); - return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); + return AlignUp(heapObjectHeaderSize + membersSize, kObjectAlignment); } uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { @@ -39,7 +38,7 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce // at about half of uint64_t max. uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; // Note: array body is aligned, but for size computation it is enough to align the sum. - return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + return AlignUp(heapArrayHeaderSize + membersSize, kObjectAlignment); } CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept : @@ -51,8 +50,8 @@ CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); size_t allocSize = ObjectAllocatedDataSize(typeInfo); - auto* heapObject = new (Allocate(allocSize)) HeapObjHeader(); - auto* object = &heapObject->object; + uint8_t* heapObject = Allocate(allocSize); + auto* object = reinterpret_cast(heapObject + gcDataSize); if (typeInfo->flags_ & TF_HAS_FINALIZER) { auto* extraObject = CreateExtraObject(); object->typeInfoOrMeta_ = reinterpret_cast(new (extraObject) mm::ExtraObjectData(object, typeInfo)); @@ -65,10 +64,11 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { } ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { + CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count); RuntimeAssert(typeInfo->IsArray(), "Must be an array"); auto allocSize = ArrayAllocatedDataSize(typeInfo, count); - auto* heapArray = new (Allocate(allocSize)) HeapArrayHeader(); - auto* array = &heapArray->array; + uint8_t* heapArray = Allocate(allocSize); + auto* array = reinterpret_cast(heapArray + gcDataSize); array->typeInfoOrMeta_ = const_cast(typeInfo); array->count_ = count; return array; @@ -96,10 +96,9 @@ mm::ExtraObjectData* CustomAllocator::CreateExtraObject() noexcept { return nullptr; } -// static mm::ExtraObjectData& CustomAllocator::CreateExtraObjectDataForObject( - mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept { - mm::ExtraObjectData* extraObject = threadData->gc().impl().alloc().CreateExtraObject(); + ObjHeader* baseObject, const TypeInfo* info) noexcept { + mm::ExtraObjectData* extraObject = CreateExtraObject(); return *new (extraObject) mm::ExtraObjectData(baseObject, info); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index a6cbcb33832..ce2a93ec411 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -29,8 +29,8 @@ public: mm::ExtraObjectData* CreateExtraObject() noexcept; - static mm::ExtraObjectData& CreateExtraObjectDataForObject( - mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept; + mm::ExtraObjectData& CreateExtraObjectDataForObject( + ObjHeader* baseObject, const TypeInfo* info) noexcept; void PrepareForGC() noexcept; @@ -38,6 +38,10 @@ public: static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; + Heap& heap() noexcept { + return heap_; + } + private: uint8_t* Allocate(uint64_t cellCount) noexcept; uint8_t* AllocateInSingleObjectPage(uint64_t cellCount) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp index d8356e10304..30fedd700bf 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -86,4 +86,22 @@ bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQ return nextFree_.first > 0 || nextFree_.last < end_; } +std_support::vector FixedBlockPage::GetAllocatedBlocks() noexcept { + std_support::vector allocated; + CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this); + FixedCellRange nextFree = nextFree_; // Accessing the previous free list structure. + for (uint32_t cell = 0 ; cell < end_ ; cell += blockSize_) { + for (; cell < nextFree.first ; cell += blockSize_) { + allocated.push_back(cells_[cell].data); + } + if (nextFree.last >= end_) { + break; + } + cell = nextFree.last; + nextFree = cells_[cell].nextFree; + } + return allocated; +} + + } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp index 80df4bce42e..ca760bfa920 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp @@ -12,6 +12,7 @@ #include "AtomicStack.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -43,6 +44,9 @@ public: bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; + // Testing method + std_support::vector GetAllocatedBlocks() noexcept; + private: explicit FixedBlockPage(uint32_t blockSize) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp index ef483f00a67..0f2338869e4 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp @@ -150,6 +150,15 @@ TEST(CustomAllocTest, FixedBlockPageRandomExercise) { } } EXPECT_EQ(page->Sweep(gcScope, finalizerQueue), !live.empty()); + uint8_t* prev = nullptr; + uint32_t allocCount = 0; + for (auto* obj : page->GetAllocatedBlocks()) { + EXPECT_LT(prev, obj); + prev = obj; + ++allocCount; + EXPECT_NE(live.find(obj), live.end()); + } + EXPECT_EQ(allocCount, live.size()); } while ((ptr = alloc(page, size))) live.insert(ptr); EXPECT_EQ(live.size(), BLOCK_COUNT); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 767c7f3bd62..62122039e8f 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -7,20 +7,22 @@ #include #include +#include #include #ifndef KONAN_WINDOWS #include #endif -#include "ConcurrentMarkAndSweep.hpp" #include "CompilerConstants.hpp" #include "CustomLogging.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" #include "FinalizerHooks.hpp" +#include "GC.hpp" #include "GCStatistics.hpp" #include "KAssert.h" +#include "Memory.h" #include "ObjectFactory.hpp" namespace { @@ -31,18 +33,17 @@ std::atomic allocatedBytesCounter; namespace kotlin::alloc { -bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { - HeapObjHeader* objHeader = reinterpret_cast(object); - if (objHeader->gcData.tryResetMark()) { - CustomAllocDebug("SweepObject(%p): still alive", object); +bool SweepObject(uint8_t* heapObjHeader, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { + if (gc::GC::SweepObject(heapObjHeader)) { + CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader); gcHandle.addKeptObject(); return true; } - auto* baseObject = &objHeader->object; - auto* extraObject = mm::ExtraObjectData::Get(baseObject); + auto* objHeader = reinterpret_cast(heapObjHeader + gcDataSize); + auto* extraObject = mm::ExtraObjectData::Get(objHeader); if (extraObject) { if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { - CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", object, extraObject); + CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject); extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); extraObject->ClearRegularWeakReferenceImpl(); CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject)); @@ -51,14 +52,14 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle:: return true; } if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { - CustomAllocDebug("SweepObject(%p): already waiting to be finalized", object); + CustomAllocDebug("SweepObject(%p): already waiting to be finalized", heapObjHeader); gcHandle.addMarkedObject(); return true; } extraObject->UnlinkFromBaseObject(); extraObject->setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); } - CustomAllocDebug("SweepObject(%p): can be reclaimed", object); + CustomAllocDebug("SweepObject(%p): can be reclaimed", heapObjHeader); gcHandle.addSweptObject(); return false; } @@ -98,10 +99,12 @@ void* SafeAlloc(uint64_t size) noexcept { konan::abort(); } allocatedBytesCounter.fetch_add(static_cast(size), std::memory_order_relaxed); + CustomAllocDebug("SafeAlloc(%zu) = %p", static_cast(size), memory); return memory; } void Free(void* ptr, size_t size) noexcept { + CustomAllocDebug("Free(%p, %zu)", ptr, size); if (compiler::disableMmap()) { free(ptr); } else { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index 36a4274d313..70fafee62f2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -11,27 +11,19 @@ #include #include +#include "Alignment.hpp" #include "AtomicStack.hpp" -#include "ConcurrentMarkAndSweep.hpp" #include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "Memory.h" +#include "GC.hpp" namespace kotlin::alloc { -// copied over from ObjectFactory - -using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData; - -struct HeapObjHeader { - ObjectData gcData; - alignas(kObjectAlignment) ObjHeader object; -}; - -struct HeapArrayHeader { - ObjectData gcData; - alignas(kObjectAlignment) ArrayHeader array; -}; +const size_t gcDataSize = AlignUp(gc::GC::objectDataSize, kObjectAlignment); +const size_t heapObjectHeaderSize = AlignUp(gcDataSize + sizeof(ObjHeader), kObjectAlignment); +const size_t heapArrayHeaderSize = AlignUp(gcDataSize + sizeof(ArrayHeader), kObjectAlignment); // Returns `true` if the `object` must be kept alive still. bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index 7d25a73519f..15095f991a2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -14,18 +14,17 @@ #include "CustomAllocConstants.hpp" #include "AtomicStack.hpp" #include "CustomLogging.hpp" +#include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" +#include "GCApi.hpp" +#include "Memory.h" #include "ThreadRegistry.hpp" -#include "GCImpl.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { void Heap::PrepareForGC() noexcept { CustomAllocDebug("Heap::PrepareForGC()"); - for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().alloc().PrepareForGC(); - } - nextFitPages_.PrepareForGC(); singleObjectPages_.PrepareForGC(); for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { @@ -36,6 +35,7 @@ void Heap::PrepareForGC() noexcept { FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept { FinalizerQueue finalizerQueue; + CustomAllocDebug("Heap: before sweep FinalizerQueue size == %zu", finalizerQueue.size()); CustomAllocDebug("Heap::Sweep()"); { auto sweepHandle = gcHandle.sweep(); @@ -45,13 +45,11 @@ FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept { nextFitPages_.Sweep(sweepHandle, finalizerQueue); singleObjectPages_.SweepAndFree(sweepHandle, finalizerQueue); } + CustomAllocDebug("Heap: before extra sweep FinalizerQueue size == %zu", finalizerQueue.size()); { auto sweepHandle = gcHandle.sweepExtraObjects(); extraObjectPages_.Sweep(sweepHandle, finalizerQueue); } - for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); - } CustomAllocDebug("Heap::Sweep done"); return finalizerQueue; } @@ -76,4 +74,41 @@ ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexce return extraObjectPages_.GetPage(0, finalizerQueue); } +std_support::vector Heap::GetAllocatedObjects() noexcept { + std_support::vector allocated; + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + for (auto* page : fixedBlockPages_[blockSize].GetPages()) { + for (auto* block : page->GetAllocatedBlocks()) { + allocated.push_back(reinterpret_cast(block + gcDataSize)); + } + } + } + for (auto* page : nextFitPages_.GetPages()) { + for (auto* block : page->GetAllocatedBlocks()) { + allocated.push_back(reinterpret_cast(block + gcDataSize)); + } + } + for (auto* page : singleObjectPages_.GetPages()) { + for (auto* block : page->GetAllocatedBlocks()) { + allocated.push_back(reinterpret_cast(block + gcDataSize)); + } + } + std_support::vector unfinalized; + for (auto* block: allocated) { + if (!block->has_meta_object() || !mm::ExtraObjectData::Get(block)->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { + unfinalized.push_back(block); + } + } + return unfinalized; +} + +void Heap::ClearForTests() noexcept { + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + fixedBlockPages_[blockSize].ClearForTests(); + } + nextFitPages_.ClearForTests(); + singleObjectPages_.ClearForTests(); + extraObjectPages_.ClearForTests(); +} + } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp index a3c206c0cda..d2fdf4f6db8 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp @@ -13,6 +13,7 @@ #include "CustomAllocConstants.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "Memory.h" #include "SingleObjectPage.hpp" #include "NextFitPage.hpp" #include "PageStore.hpp" @@ -35,6 +36,10 @@ public: SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept; ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept; + // Test method + std_support::vector GetAllocatedObjects() noexcept; + void ClearForTests() noexcept; + private: PageStore fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; PageStore nextFitPages_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index fb08351a9b5..89bed47057b 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -11,6 +11,7 @@ #include "CustomLogging.hpp" #include "CustomAllocConstants.hpp" #include "GCApi.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -104,4 +105,15 @@ bool NextFitPage::CheckInvariants() noexcept { } } +std_support::vector NextFitPage::GetAllocatedBlocks() noexcept { + std_support::vector allocated; + Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT; + for (Cell* block = cells_ + 1; block != end; block = block->Next()) { + if (block->isAllocated_) { + allocated.push_back(block->data_); + } + } + return allocated; +} + } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp index 085ac5b6f0e..e8bd30fbd82 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp @@ -13,6 +13,7 @@ #include "Cell.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -34,6 +35,9 @@ public: // Testing method bool CheckInvariants() noexcept; + // Testing method + std_support::vector GetAllocatedBlocks() noexcept; + private: explicit NextFitPage(uint32_t cellCount) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp index 88c3dfe0652..1190a10b9a7 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp @@ -109,9 +109,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); int unmarked = 0; - while (true) { - uint8_t* ptr = alloc(page, MIN_BLOCK_SIZE); - if (ptr == nullptr) break; + while (uint8_t* ptr = alloc(page, MIN_BLOCK_SIZE)) { if (r() & 1) { mark(ptr); } else { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp index 831b7dfdfa8..bf989d4ce29 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp @@ -7,10 +7,12 @@ #define CUSTOM_ALLOC_CPP_PAGESTORE_HPP_ #include +#include #include "AtomicStack.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -78,6 +80,8 @@ public: } private: + friend class Heap; + T* SweepSingle(GCSweepScope& sweepHandle, T* page, AtomicStack& from, AtomicStack& to, FinalizerQueue& finalizerQueue) noexcept { if (!page) { return nullptr; @@ -92,6 +96,22 @@ private: return nullptr; } + // Testing method + std_support::vector GetPages() noexcept { + std_support::vector pages; + for (T* page : ready_.GetElements()) pages.push_back(page); + for (T* page : used_.GetElements()) pages.push_back(page); + for (T* page : unswept_.GetElements()) pages.push_back(page); + return pages; + } + + void ClearForTests() noexcept { + while (T* page = empty_.Pop()) page->Destroy(); + while (T* page = ready_.Pop()) page->Destroy(); + while (T* page = used_.Pop()) page->Destroy(); + while (T* page = unswept_.Pop()) page->Destroy(); + } + AtomicStack empty_; AtomicStack ready_; AtomicStack used_; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp index e38d2d42a33..69998c294a0 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -11,6 +11,7 @@ #include "CustomLogging.hpp" #include "CustomAllocConstants.hpp" #include "GCApi.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -46,4 +47,12 @@ bool SingleObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalize return false; } +std_support::vector SingleObjectPage::GetAllocatedBlocks() noexcept { + std_support::vector allocated; + if (isAllocated_) { + allocated.push_back(data_); + } + return allocated; +} + } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index 51646a70c26..41dade484c8 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -12,6 +12,7 @@ #include "AtomicStack.hpp" #include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" +#include "std_support/Vector.hpp" namespace kotlin::alloc { @@ -33,9 +34,13 @@ public: private: friend class AtomicStack; + friend class Heap; explicit SingleObjectPage(size_t size) noexcept; + // Testing method + std_support::vector GetAllocatedBlocks() noexcept; + SingleObjectPage* next_; bool isAllocated_ = false; size_t size_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 1b10c25555a..2bf5b743100 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -167,6 +167,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gcHandle.threadsAreSuspended(); #ifdef CUSTOM_ALLOCATOR + // This should really be done by each individual thread while waiting + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + thread.gc().Allocator().PrepareForGC(); + } heap_.PrepareForGC(); #endif @@ -218,6 +222,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { #else // also sweeps extraObjects auto finalizerQueue = heap_.Sweep(gcHandle); + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + finalizerQueue.TransferAllFrom(thread.gc().Allocator().ExtractFinalizerQueue()); + } #endif state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 5dfc4629a99..9f6c8cdc01a 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -179,6 +179,9 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa } std_support::vector Alive(mm::ThreadData& threadData) { +#ifdef CUSTOM_ALLOCATOR + return threadData.gc().impl().alloc().heap().GetAllocatedObjects(); +#else std_support::vector objects; for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { objects.push_back(node.GetObjHeader()); @@ -187,6 +190,7 @@ std_support::vector Alive(mm::ThreadData& threadData) { objects.push_back(node.GetObjHeader()); } return objects; +#endif } bool IsMarked(ObjHeader* objHeader) { @@ -215,7 +219,9 @@ public: ~ConcurrentMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); +#ifndef CUSTOM_ALLOCATOR mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); +#endif mm::GlobalData::Instance().gc().ClearForTests(); } @@ -1016,6 +1022,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { } } + TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { std_support::vector mutators(kDefaultThreadCount); std_support::vector globals(2 * kDefaultThreadCount); @@ -1085,6 +1092,7 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { future.wait(); } +#ifndef CUSTOM_ALLOCATOR // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything). std_support::vector expectedAlive; @@ -1107,9 +1115,24 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]); EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread)); } +#else + // Custom allocator does not have a notion of objects alive only for some thread + std_support::vector expectedAlive; + for (int i = 0; i < kDefaultThreadCount; ++i) { + expectedAlive.push_back(globals[i]); + expectedAlive.push_back(locals[i]); + expectedAlive.push_back(reachables[i]); + expectedAlive.push_back(globals[kDefaultThreadCount + i]); + expectedAlive.push_back(locals[kDefaultThreadCount + i]); + expectedAlive.push_back(reachables[kDefaultThreadCount + i]); + // Unreachables for new threads were not collected. + expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); + } + // All threads see the same alive objects with the custom alloctor, enough to check a single mutator. + EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); +#endif // CUSTOM_ALLOCATOR } - TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { std_support::vector mutators(2); std::atomic*> object1 = nullptr; @@ -1138,7 +1161,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); done = true; }); - + auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { while (object1.load() == nullptr) {} ObjHolder holder; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index f4c0d6f90a0..14cf6bf6f58 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -5,6 +5,7 @@ #include "GCImpl.hpp" +#include "ConcurrentMarkAndSweep.hpp" #include "GC.hpp" #include "GCStatistics.hpp" #include "MarkAndSweepUtils.hpp" @@ -40,6 +41,8 @@ void gc::GC::ThreadData::Publish() noexcept { void gc::GC::ThreadData::ClearForTests() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->objectFactoryThreadQueue().ClearForTests(); +#else + impl_->alloc().PrepareForGC(); #endif } @@ -59,6 +62,12 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI #endif } +#ifdef CUSTOM_ALLOCATOR +alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { + return impl_->alloc(); +} +#endif + void gc::GC::ThreadData::OnSuspendForGC() noexcept { impl_->gc().OnSuspendForGC(); } @@ -88,6 +97,8 @@ void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); #ifndef CUSTOM_ALLOCATOR impl_->objectFactory().ClearForTests(); +#else + impl_->gc().heap().ClearForTests(); #endif GCHandle::ClearForTests(); } @@ -135,3 +146,11 @@ bool gc::isMarked(ObjHeader* object) noexcept { ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_RESULT_OF(gc::WeakRefRead, object); } + +// static +const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData); + +// static +ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { + return reinterpret_cast(objectData)->tryResetMark(); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp index 3c84c32877e..131801a41e1 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp @@ -26,6 +26,10 @@ auto collectCopy(T& iterable) { } // namespace void gc::AssertClear(GC& gc) noexcept { +#ifdef CUSTOM_ALLOCATOR + auto objects = gc.impl().gc().heap().GetAllocatedObjects(); +#else auto objects = gc.impl().objectFactory().LockForIter(); +#endif EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 9bd16aadbe6..b55223b2fbb 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -9,10 +9,13 @@ #include "GCScheduler.hpp" #include "Memory.h" -#include "Types.h" #include "Utils.hpp" #include "std_support/Memory.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "CustomAllocator.hpp" +#endif + namespace kotlin { namespace mm { @@ -44,6 +47,10 @@ public: ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; +#ifdef CUSTOM_ALLOCATOR + alloc::CustomAllocator& Allocator() noexcept; +#endif + void OnSuspendForGC() noexcept; void safePoint() noexcept; @@ -76,6 +83,9 @@ public: void WaitFinalizers(int64_t epoch) noexcept; void ScheduleAndWaitFullGCWithFinalizers() noexcept { WaitFinalizers(Schedule()); } + static const size_t objectDataSize; + static bool SweepObject(void* objectData) noexcept; + private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index cc4dd384844..4321d72ee1c 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -5,16 +5,21 @@ #include "GCImpl.hpp" +#include "Common.h" #include "GC.hpp" +#include "NoOpGC.hpp" #include "std_support/Memory.hpp" -#include "GlobalData.hpp" #include "GCStatistics.hpp" #include "ObjectOps.hpp" using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData&, mm::ThreadData& threadData) noexcept : +gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : +#ifdef CUSTOM_ALLOCATOR + impl_(std_support::make_unique(gc, gcScheduler, threadData)) {} +#else impl_(std_support::make_unique(gc, threadData)) {} +#endif gc::GC::ThreadData::~ThreadData() = default; @@ -31,21 +36,41 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { } void gc::GC::ThreadData::Publish() noexcept { +#ifndef CUSTOM_ALLOCATOR impl_->objectFactoryThreadQueue().Publish(); +#endif } void gc::GC::ThreadData::ClearForTests() noexcept { +#ifndef CUSTOM_ALLOCATOR impl_->objectFactoryThreadQueue().ClearForTests(); +#else + impl_->alloc().PrepareForGC(); +#endif } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { +#ifndef CUSTOM_ALLOCATOR return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +#else + return impl_->alloc().CreateObject(typeInfo); +#endif } ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { +#ifndef CUSTOM_ALLOCATOR return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +#else + return impl_->alloc().CreateArray(typeInfo, elements); +#endif } +#ifdef CUSTOM_ALLOCATOR +alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { + return impl_->alloc(); +} +#endif + void gc::GC::ThreadData::OnSuspendForGC() noexcept { } void gc::GC::ThreadData::safePoint() noexcept {} @@ -56,7 +81,11 @@ gc::GC::~GC() = default; // static size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { +#ifndef CUSTOM_ALLOCATOR return mm::ObjectFactory::GetAllocatedHeapSize(object); +#else + return alloc::CustomAllocator::GetAllocatedHeapSize(object); +#endif } size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { @@ -64,7 +93,11 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { } void gc::GC::ClearForTests() noexcept { +#ifndef CUSTOM_ALLOCATOR impl_->objectFactory().ClearForTests(); +#else + impl_->gc().heap().ClearForTests(); +#endif GCHandle::ClearForTests(); } @@ -98,3 +131,11 @@ bool gc::isMarked(ObjHeader* object) noexcept { ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_OBJ(object.load(std::memory_order_relaxed)); } + +// static +const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no_unique_address]] + +// static +ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { + return true; +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index 83c3d2494ce..d988b973b28 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -10,6 +10,11 @@ #include "NoOpGC.hpp" #include "ObjectFactory.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "CustomAllocator.hpp" +#include "GCScheduler.hpp" +#endif + namespace kotlin { namespace gc { @@ -19,25 +24,43 @@ class GC::Impl : private Pinned { public: Impl() noexcept = default; +#ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } +#endif GCImpl& gc() noexcept { return gc_; } private: +#ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; +#endif GCImpl gc_; }; class GC::ThreadData::Impl : private Pinned { public: +#ifdef CUSTOM_ALLOCATOR + Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : + alloc_(gc.impl_->gc().heap(), gcScheduler) {} +#else Impl(GC& gc, mm::ThreadData& threadData) noexcept : objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} +#endif GCImpl::ThreadData& gc() noexcept { return gc_; } +#ifdef CUSTOM_ALLOCATOR + alloc::CustomAllocator& alloc() noexcept { return alloc_; } +#else mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } +#endif private: GCImpl::ThreadData gc_; +#ifdef CUSTOM_ALLOCATOR + alloc::CustomAllocator alloc_; + /* gcScheduler::GCSchedulerThreadData; */ +#else mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; +#endif }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp index 3c84c32877e..131801a41e1 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp @@ -26,6 +26,10 @@ auto collectCopy(T& iterable) { } // namespace void gc::AssertClear(GC& gc) noexcept { +#ifdef CUSTOM_ALLOCATOR + auto objects = gc.impl().gc().heap().GetAllocatedObjects(); +#else auto objects = gc.impl().objectFactory().LockForIter(); +#endif EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); } diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 932a2362346..13cb7ab35ee 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -12,6 +12,10 @@ #include "Logging.hpp" #include "Utils.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "Heap.hpp" +#endif + namespace kotlin { namespace mm { @@ -51,7 +55,14 @@ public: NoOpGC() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); } ~NoOpGC() = default; +#ifdef CUSTOM_ALLOCATOR + alloc::Heap& heap() noexcept { return heap_; } +#endif + private: +#ifdef CUSTOM_ALLOCATOR + alloc::Heap heap_; +#endif }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 3d0b76cdd7a..76608237883 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -7,6 +7,7 @@ #include "GC.hpp" #include "MarkAndSweepUtils.hpp" +#include "SameThreadMarkAndSweep.hpp" #include "std_support/Memory.hpp" #include "GlobalData.hpp" #include "GCStatistics.hpp" @@ -32,21 +33,41 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { } void gc::GC::ThreadData::Publish() noexcept { +#ifndef CUSTOM_ALLOCATOR impl_->objectFactoryThreadQueue().Publish(); +#endif } void gc::GC::ThreadData::ClearForTests() noexcept { +#ifndef CUSTOM_ALLOCATOR impl_->objectFactoryThreadQueue().ClearForTests(); +#else + impl_->alloc().PrepareForGC(); +#endif } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { +#ifndef CUSTOM_ALLOCATOR return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +#else + return impl_->alloc().CreateObject(typeInfo); +#endif } ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { +#ifndef CUSTOM_ALLOCATOR return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +#else + return impl_->alloc().CreateArray(typeInfo, elements); +#endif } +#ifdef CUSTOM_ALLOCATOR +alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept { + return impl_->alloc(); +} +#endif + void gc::GC::ThreadData::OnSuspendForGC() noexcept { } void gc::GC::ThreadData::safePoint() noexcept {} @@ -57,7 +78,11 @@ gc::GC::~GC() = default; // static size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { +#ifdef CUSTOM_ALLOCATOR + return alloc::CustomAllocator::GetAllocatedHeapSize(object); +#else return mm::ObjectFactory::GetAllocatedHeapSize(object); +#endif } size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { @@ -66,7 +91,11 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); +#ifndef CUSTOM_ALLOCATOR impl_->objectFactory().ClearForTests(); +#else + impl_->gc().heap().ClearForTests(); +#endif GCHandle::ClearForTests(); } @@ -113,3 +142,11 @@ bool gc::isMarked(ObjHeader* object) noexcept { ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic& object) noexcept { RETURN_OBJ(object.load(std::memory_order_relaxed)); } + +// static +const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData); + +// static +ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept { + return reinterpret_cast(objectData)->tryResetMark(); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index 8027028891a..b8148ffef6a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -16,27 +16,46 @@ using GCImpl = SameThreadMarkAndSweep; class GC::Impl : private Pinned { public: +#ifdef CUSTOM_ALLOCATOR + explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {} +#else explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {} mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } +#endif GCImpl& gc() noexcept { return gc_; } private: +#ifndef CUSTOM_ALLOCATOR mm::ObjectFactory objectFactory_; +#endif GCImpl gc_; }; class GC::ThreadData::Impl : private Pinned { public: Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept : - gc_(gc.impl_->gc(), threadData, gcScheduler), objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} + gc_(gc.impl_->gc(), threadData, gcScheduler), +#ifdef CUSTOM_ALLOCATOR + alloc_(gc.impl_->gc().heap(), gcScheduler) {} +#else + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {} +#endif GCImpl::ThreadData& gc() noexcept { return gc_; } +#ifndef CUSTOM_ALLOCATOR mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } +#else + alloc::CustomAllocator& alloc() noexcept { return alloc_; } +#endif private: GCImpl::ThreadData gc_; +#ifdef CUSTOM_ALLOCATOR + alloc::CustomAllocator alloc_; +#else mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; +#endif }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp index 3c84c32877e..131801a41e1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp @@ -26,6 +26,10 @@ auto collectCopy(T& iterable) { } // namespace void gc::AssertClear(GC& gc) noexcept { +#ifdef CUSTOM_ALLOCATOR + auto objects = gc.impl().gc().heap().GetAllocatedObjects(); +#else auto objects = gc.impl().objectFactory().LockForIter(); +#endif EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 7debc925ae5..3b1702960d0 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -19,6 +19,10 @@ #include "ThreadRegistry.hpp" #include "ThreadSuspension.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "CustomFinalizerProcessor.hpp" +#endif + using namespace kotlin; namespace { @@ -82,9 +86,17 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { ScheduleAndWaitFullGC(); } +#ifdef CUSTOM_ALLOCATOR gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( - mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept : - objectFactory_(objectFactory), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { + gcScheduler::GCScheduler& gcScheduler) noexcept : +#else +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( + mm::ObjectFactory& objectFactory, + gcScheduler::GCScheduler& gcScheduler) noexcept : + + objectFactory_(objectFactory), +#endif + gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { GCHandle::getByEpoch(epoch).finalizersDone(); state_.finalized(epoch); }) { @@ -135,6 +147,14 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { state_.start(epoch); +#ifdef CUSTOM_ALLOCATOR + // This should really be done by each individual thread while waiting + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + thread.gc().Allocator().PrepareForGC(); + } + heap_.PrepareForGC(); +#endif + gc::collectRootSet(gcHandle, markQueue_, [](mm::ThreadData&) { return true; }); gc::Mark(gcHandle, markQueue_); @@ -143,6 +163,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); +#ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); @@ -153,6 +174,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto finalizerQueue = gc::Sweep(gcHandle, *objectFactoryIterable); objectFactoryIterable = std::nullopt; kotlin::compactObjectPoolInMainThread(); +#else + auto finalizerQueue = heap_.Sweep(gcHandle); +#endif mm::ResumeThreads(); gcHandle.threadsAreResumed(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 634397c096d..45ede2edcc7 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -17,6 +17,14 @@ #include "Types.h" #include "Utils.hpp" +#ifdef CUSTOM_ALLOCATOR +#include "CustomAllocator.hpp" +#include "CustomFinalizerProcessor.hpp" +#include "ExtraObjectPage.hpp" +#include "GCApi.hpp" +#include "Heap.hpp" +#endif + namespace kotlin { namespace mm { @@ -92,10 +100,18 @@ public: using Allocator = ThreadData::Allocator; +#ifdef CUSTOM_ALLOCATOR + using FinalizerQueue = alloc::FinalizerQueue; + using FinalizerQueueTraits = alloc::FinalizerQueueTraits; + + SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept; +#else using FinalizerQueue = mm::ObjectFactory::FinalizerQueue; using FinalizerQueueTraits = mm::ObjectFactory::FinalizerQueueTraits; SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept; +#endif + ~SameThreadMarkAndSweep(); void StartFinalizerThreadIfNeeded() noexcept; @@ -105,10 +121,18 @@ public: int64_t Schedule() noexcept { return state_.schedule(); } void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); } +#ifdef CUSTOM_ALLOCATOR + alloc::Heap& heap() noexcept { return heap_; } +#endif + private: void PerformFullGC(int64_t epoch) noexcept; +#ifndef CUSTOM_ALLOCATOR mm::ObjectFactory& objectFactory_; +#else + alloc::Heap heap_; +#endif gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 118be7c53be..3c3da0d6bb7 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -180,6 +180,9 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa } std_support::vector Alive(mm::ThreadData& threadData) { +#ifdef CUSTOM_ALLOCATOR + return threadData.gc().impl().alloc().heap().GetAllocatedObjects(); +#else std_support::vector objects; for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { objects.push_back(node.GetObjHeader()); @@ -188,6 +191,7 @@ std_support::vector Alive(mm::ThreadData& threadData) { objects.push_back(node.GetObjHeader()); } return objects; +#endif } bool IsMarked(ObjHeader* objHeader) { @@ -211,7 +215,12 @@ public: ~SameThreadMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); +#ifndef CUSTOM_ALLOCATOR mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); + mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests(); +#else + mm::GlobalData::Instance().gc().impl().gc().heap().ClearForTests(); +#endif mm::GlobalData::Instance().gc().ClearForTests(); } @@ -1081,6 +1090,7 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { future.wait(); } +#ifndef CUSTOM_ALLOCATOR // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything). std_support::vector expectedAlive; @@ -1103,6 +1113,22 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]); EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread)); } +#else + // Custom allocator does not have a notion of objects alive only for some thread + std_support::vector expectedAlive; + for (int i = 0; i < kDefaultThreadCount; ++i) { + expectedAlive.push_back(globals[i]); + expectedAlive.push_back(locals[i]); + expectedAlive.push_back(reachables[i]); + expectedAlive.push_back(globals[kDefaultThreadCount + i]); + expectedAlive.push_back(locals[kDefaultThreadCount + i]); + expectedAlive.push_back(reachables[kDefaultThreadCount + i]); + // Unreachables for new threads were not collected. + expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); + } + // All threads see the same alive objects with the custom alloctor, enough to check a single mutator. + EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); +#endif // CUSTOM_ALLOCATOR } @@ -1134,7 +1160,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header())); done = true; }); - + auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { while (object1.load() == nullptr) {} ObjHolder holder; diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index 5e4d2332978..89b829d9166 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -34,7 +34,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); #ifdef CUSTOM_ALLOCATOR - auto& data = alloc::CustomAllocator::CreateExtraObjectDataForObject(threadData, object, typeInfo); + auto& data = threadData->gc().Allocator().CreateExtraObjectDataForObject(object, typeInfo); if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { // Somebody else created `mm::ExtraObjectData` for this object. diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp index 41a3e245c4f..dd7d1462ff1 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp @@ -10,6 +10,7 @@ using namespace kotlin; +#ifndef CUSTOM_ALLOCATOR // static mm::ExtraObjectDataFactory& mm::ExtraObjectDataFactory::Instance() noexcept { return GlobalData::Instance().extraObjectDataFactory(); @@ -41,3 +42,4 @@ void mm::ExtraObjectDataFactory::ProcessThread(mm::ThreadData* threadData) noexc mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default; mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default; +#endif diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index 806cd4cb2fd..00532077eb3 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -29,7 +29,9 @@ public: ~ExtraObjectDataTest() { mm::GlobalsRegistry::Instance().ClearForTests(); +#ifndef CUSTOM_ALLOCATOR mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); +#endif mm::GlobalData::Instance().gc().ClearForTests(); } }; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index ad5a64ca6d6..f96e2fc7b41 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -27,7 +27,9 @@ public: ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; } GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; } +#ifndef CUSTOM_ALLOCATOR ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } +#endif gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } gc::GC& gc() noexcept { return gc_; } AppStateTracking& appStateTracking() noexcept { return appStateTracking_; } @@ -43,9 +45,14 @@ private: AppStateTracking appStateTracking_; GlobalsRegistry globalsRegistry_; SpecialRefRegistry specialRefRegistry_; - ExtraObjectDataFactory extraObjectDataFactory_; gcScheduler::GCScheduler gcScheduler_; gc::GC gc_{gcScheduler_}; +#ifndef CUSTOM_ALLOCATOR + // by being last, ommiting it will not affect the offsets of the other + // members, and we avoid having to have _custom versions of the gcScheduler + // modules. + ExtraObjectDataFactory extraObjectDataFactory_; +#endif }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 7cdb417bae1..bc1d6f5364b 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -442,6 +442,7 @@ class ObjectFactory : private Pinned { using Allocator = typename Traits::Allocator; struct HeapObjHeader { + [[no_unique_address]] // to account for GCs with empty ObjectData ObjectData gcData; alignas(kObjectAlignment) ObjHeader object; }; @@ -449,6 +450,7 @@ class ObjectFactory : private Pinned { // Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible // with `ObjHeader`: the former can always be casted to the other. struct HeapArrayHeader { + [[no_unique_address]] ObjectData gcData; alignas(kObjectAlignment) ArrayHeader array; }; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 36ec7b6c95f..a795beba1d5 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -21,6 +21,8 @@ #include "std_support/CStdlib.hpp" #include "std_support/Vector.hpp" +// ObjectFactory is not used by custom allocator +#ifndef CUSTOM_ALLOCATOR using namespace kotlin; using testing::_; @@ -1130,3 +1132,4 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount); } +#endif diff --git a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp index 6a041a3d55c..2c8ebbfbf90 100644 --- a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp +++ b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp @@ -45,12 +45,15 @@ auto collectPointers(T& iterable) { extern "C" void Kotlin_TestSupport_AssertClearGlobalState() { // Validate that global registries are empty. auto globals = mm::GlobalsRegistry::Instance().LockForIter(); - auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); auto specialRefs = mm::SpecialRefRegistry::instance().lockForIter(); auto threads = mm::ThreadRegistry::Instance().LockForIter(); - EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre()); +#ifndef CUSTOM_ALLOCATOR + auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); +#endif + + EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre()); gc::AssertClear(mm::GlobalData::Instance().gc()); diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index b58af879f51..792a5b98d07 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -34,7 +34,9 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), +#ifndef CUSTOM_ALLOCATOR extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()), +#endif gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), gc_(GlobalData::Instance().gc(), gcScheduler_, *this), suspensionData_(ThreadState::kNative, *this) {} @@ -49,7 +51,9 @@ public: SpecialRefRegistry::ThreadQueue& specialRefRegistry() noexcept { return specialRefRegistry_; } +#ifndef CUSTOM_ALLOCATOR ExtraObjectDataFactory::ThreadQueue& extraObjectDataThreadQueue() noexcept { return extraObjectDataThreadQueue_; } +#endif ThreadState state() noexcept { return suspensionData_.state(); } @@ -69,14 +73,18 @@ public: // TODO: These use separate locks, which is inefficient. globalsThreadQueue_.Publish(); specialRefRegistry_.publish(); +#ifndef CUSTOM_ALLOCATOR extraObjectDataThreadQueue_.Publish(); +#endif gc_.Publish(); } void ClearForTests() noexcept { globalsThreadQueue_.ClearForTests(); specialRefRegistry_.clearForTests(); +#ifndef CUSTOM_ALLOCATOR extraObjectDataThreadQueue_.ClearForTests(); +#endif gc_.ClearForTests(); } @@ -85,7 +93,9 @@ private: GlobalsRegistry::ThreadQueue globalsThreadQueue_; ThreadLocalStorage tls_; SpecialRefRegistry::ThreadQueue specialRefRegistry_; +#ifndef CUSTOM_ALLOCATOR ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_; +#endif ShadowStack shadowStack_; gcScheduler::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_;