From f47a09e55ec997003efa122f0826f6b778e059d9 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Fri, 1 Sep 2023 19:05:24 +0200 Subject: [PATCH] [K/N] Move allocator-specific data into allocators. ^KT-60928 --- kotlin-native/runtime/build.gradle.kts | 8 +- .../src/alloc/common/cpp/Allocator.hpp | 32 ++++++++ .../alloc/common/cpp/AllocatorTestSupport.hpp | 19 +++++ .../src/alloc/custom/cpp/AllocatorImpl.cpp | 46 +++++++++++ .../custom/cpp/AllocatorImplTestSupport.cpp | 35 ++++++++ .../runtime/src/alloc/custom/cpp/Heap.cpp | 6 +- .../runtime/src/alloc/custom/cpp/Heap.hpp | 2 +- .../src/alloc/legacy/cpp/AllocatorImpl.cpp | 50 ++++++++++++ .../legacy/cpp/AllocatorImplTestSupport.cpp} | 28 ++++--- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 24 +++--- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 7 +- .../gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp | 34 +++----- .../runtime/src/gc/cms/cpp/GCImpl.cpp | 76 +----------------- .../runtime/src/gc/cms/cpp/GCImpl.hpp | 11 +-- .../runtime/src/gc/cms/cpp/ObjectData.hpp | 2 +- .../runtime/src/gc/common/cpp/GC.hpp | 18 ++--- .../src/gc/common/cpp/GCTestSupport.hpp | 16 ---- .../common/cpp/MarkAndSweepUtilsMarkTest.cpp | 5 +- .../runtime/src/gc/noop/cpp/GCImpl.cpp | 80 ++----------------- .../runtime/src/gc/noop/cpp/GCImpl.hpp | 31 +------ .../src/gc/noop/cpp/GCImplTestSupport.cpp | 47 ----------- .../runtime/src/gc/stms/cpp/GCImpl.cpp | 70 +--------------- .../runtime/src/gc/stms/cpp/GCImpl.hpp | 9 +-- .../src/gc/stms/cpp/GCImplTestSupport.cpp | 47 ----------- .../runtime/src/gc/stms/cpp/ObjectData.hpp | 2 +- .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 26 +++--- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 4 +- .../stms/cpp/SameThreadMarkAndSweepTest.cpp | 33 +++----- .../src/main/cpp/MultiSourceQueueTest.cpp | 2 +- .../src/main/cpp/SingleLockListTest.cpp | 2 +- ...upport.hpp => StdAllocatorTestSupport.hpp} | 0 ...st.cpp => StdAllocatorTestSupportTest.cpp} | 10 +-- .../src/main/cpp/std_support/MemoryTest.cpp | 2 +- .../runtime/src/mm/cpp/ExtraObjectData.cpp | 6 +- .../src/mm/cpp/ExtraObjectDataTest.cpp | 4 +- .../runtime/src/mm/cpp/GlobalData.hpp | 5 +- kotlin-native/runtime/src/mm/cpp/Memory.cpp | 2 +- .../runtime/src/mm/cpp/ObjectOps.cpp | 9 ++- .../runtime/src/mm/cpp/TestSupport.cpp | 4 +- .../runtime/src/mm/cpp/ThreadData.hpp | 6 +- 40 files changed, 317 insertions(+), 503 deletions(-) create mode 100644 kotlin-native/runtime/src/alloc/common/cpp/AllocatorTestSupport.hpp create mode 100644 kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImplTestSupport.cpp rename kotlin-native/runtime/src/{gc/cms/cpp/GCImplTestSupport.cpp => alloc/legacy/cpp/AllocatorImplTestSupport.cpp} (51%) delete mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp delete mode 100644 kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp delete mode 100644 kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp rename kotlin-native/runtime/src/main/cpp/{AllocatorTestSupport.hpp => StdAllocatorTestSupport.hpp} (100%) rename kotlin-native/runtime/src/main/cpp/{AllocatorTestSupportTest.cpp => StdAllocatorTestSupportTest.cpp} (92%) diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 75fae6dda76..b2cf5de5a4d 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -172,6 +172,7 @@ bitcode { sourceSets { main {} test {} + testFixtures {} } compilerArgs.add("-DCUSTOM_ALLOCATOR") @@ -193,6 +194,7 @@ bitcode { sourceSets { main {} test {} + testFixtures {} } } @@ -270,7 +272,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} - testFixtures {} } } @@ -279,7 +280,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} - testFixtures {} } compilerArgs.add("-DCUSTOM_ALLOCATOR") @@ -290,7 +290,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} - testFixtures {} test {} } } @@ -300,7 +299,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} - testFixtures {} test {} } @@ -312,7 +310,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/legacy/cpp")) sourceSets { main {} - testFixtures {} test {} } } @@ -322,7 +319,6 @@ bitcode { headersDirs.from(files("src/alloc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/alloc/custom/cpp")) sourceSets { main {} - testFixtures {} test {} } diff --git a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp index ad11be79944..914966494f6 100644 --- a/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp @@ -5,6 +5,9 @@ #pragma once +#include +#include + #include "GC.hpp" #include "Utils.hpp" @@ -19,10 +22,37 @@ public: public: class Impl; + explicit ThreadData(Allocator& allocator) noexcept; + ~ThreadData(); + + Impl& impl() noexcept { return *impl_; } + + ObjHeader* allocateObject(const TypeInfo* typeInfo) noexcept; + ArrayHeader* allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; + mm::ExtraObjectData& allocateExtraObjectData(ObjHeader* object, const TypeInfo* typeInfo) noexcept; + void destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; + + void prepareForGC() noexcept; + + // TODO: Move into AllocatorTestSupport.hpp + void clearForTests() noexcept; + private: + std::unique_ptr impl_; }; + Allocator() noexcept; + ~Allocator(); + + Impl& impl() noexcept { return *impl_; } + + void prepareForGC() noexcept; + + // TODO: Move into AllocatorTestSupport.hpp + void clearForTests() noexcept; + private: + std::unique_ptr impl_; }; void initObjectPool() noexcept; @@ -36,4 +66,6 @@ ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept; size_t allocatedHeapSize(ObjHeader* object) noexcept; size_t allocatedBytes() noexcept; + +void destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; } diff --git a/kotlin-native/runtime/src/alloc/common/cpp/AllocatorTestSupport.hpp b/kotlin-native/runtime/src/alloc/common/cpp/AllocatorTestSupport.hpp new file mode 100644 index 00000000000..038bbd232af --- /dev/null +++ b/kotlin-native/runtime/src/alloc/common/cpp/AllocatorTestSupport.hpp @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Allocator.hpp" + +#include + +#include "ThreadData.hpp" + +namespace kotlin::alloc::test_support { + +void assertClear(Allocator& allocator) noexcept; +std::vector allocatedObjects(mm::ThreadData& threadData) noexcept; + +} // namespace kotlin::alloc::test_support diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp index a63590c5d51..b3cebf66eb4 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp @@ -9,6 +9,47 @@ using namespace kotlin; +alloc::Allocator::ThreadData::ThreadData(Allocator& allocator) noexcept : impl_(std::make_unique(allocator.impl())) {} + +alloc::Allocator::ThreadData::~ThreadData() = default; + +ALWAYS_INLINE ObjHeader* alloc::Allocator::ThreadData::allocateObject(const TypeInfo* typeInfo) noexcept { + return impl_->alloc().CreateObject(typeInfo); +} + +ALWAYS_INLINE ArrayHeader* alloc::Allocator::ThreadData::allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + return impl_->alloc().CreateArray(typeInfo, elements); +} + +ALWAYS_INLINE mm::ExtraObjectData& alloc::Allocator::ThreadData::allocateExtraObjectData( + ObjHeader* object, const TypeInfo* typeInfo) noexcept { + return impl_->alloc().CreateExtraObjectDataForObject(object, typeInfo); +} + +ALWAYS_INLINE void alloc::Allocator::ThreadData::destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { + extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); +} + +void alloc::Allocator::ThreadData::prepareForGC() noexcept { + impl_->alloc().PrepareForGC(); +} + +void alloc::Allocator::ThreadData::clearForTests() noexcept { + impl_->alloc().PrepareForGC(); +} + +alloc::Allocator::Allocator() noexcept : impl_(std::make_unique()) {} + +alloc::Allocator::~Allocator() = default; + +void alloc::Allocator::prepareForGC() noexcept { + impl_->heap().PrepareForGC(); +} + +void alloc::Allocator::clearForTests() noexcept { + impl_->heap().ClearForTests(); +} + void alloc::initObjectPool() noexcept {} void alloc::compactObjectPoolInCurrentThread() noexcept {} @@ -28,3 +69,8 @@ size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept { size_t alloc::allocatedBytes() noexcept { return GetAllocatedBytes(); } + +void alloc::destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { + extraObject.ReleaseAssociatedObject(); + extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); +} diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImplTestSupport.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImplTestSupport.cpp new file mode 100644 index 00000000000..ac3b76da346 --- /dev/null +++ b/kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImplTestSupport.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "AllocatorTestSupport.hpp" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "AllocatorImpl.hpp" + +using namespace kotlin; + +namespace { + +template +auto collectCopy(T& iterable) { + std::vector> result; + for (const auto& element : iterable) { + result.push_back(element); + } + return result; +} + +} // namespace + +void alloc::test_support::assertClear(Allocator& allocator) noexcept { + auto objects = allocator.impl().heap().GetAllocatedObjects(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); +} + +std::vector alloc::test_support::allocatedObjects(mm::ThreadData& threadData) noexcept { + return threadData.allocator().impl().alloc().heap().GetAllocatedObjects(); +} diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp index e49116dd288..aeafe108c20 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp @@ -88,8 +88,8 @@ FinalizerQueue Heap::ExtractFinalizerQueue() noexcept { return std::move(pendingFinalizerQueue_); } -std_support::vector Heap::GetAllocatedObjects() noexcept { - std_support::vector allocated; +std::vector Heap::GetAllocatedObjects() noexcept { + std::vector allocated; for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { for (auto* page : fixedBlockPages_[blockSize].GetPages()) { for (auto* block : page->GetAllocatedBlocks()) { @@ -107,7 +107,7 @@ std_support::vector Heap::GetAllocatedObjects() noexcept { allocated.push_back(reinterpret_cast(block)->object()); } } - std_support::vector unfinalized; + std::vector unfinalized; for (auto* block: allocated) { if (!block->has_meta_object() || !mm::ExtraObjectData::Get(block)->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { unfinalized.push_back(block); diff --git a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp index f3ce5fe5172..02087d74118 100644 --- a/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp @@ -41,7 +41,7 @@ public: FinalizerQueue ExtractFinalizerQueue() noexcept; // Test method - std_support::vector GetAllocatedObjects() noexcept; + std::vector GetAllocatedObjects() noexcept; void ClearForTests() noexcept; private: diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp index ebe15e0dc65..7e478e8efdc 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp @@ -5,8 +5,52 @@ #include "AllocatorImpl.hpp" +#include "ThreadData.hpp" + using namespace kotlin; +alloc::Allocator::ThreadData::ThreadData(Allocator& allocator) noexcept : impl_(std::make_unique(allocator.impl())) {} + +alloc::Allocator::ThreadData::~ThreadData() = default; + +ALWAYS_INLINE ObjHeader* alloc::Allocator::ThreadData::allocateObject(const TypeInfo* typeInfo) noexcept { + return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +} + +ALWAYS_INLINE ArrayHeader* alloc::Allocator::ThreadData::allocateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +} + +ALWAYS_INLINE mm::ExtraObjectData& alloc::Allocator::ThreadData::allocateExtraObjectData( + ObjHeader* object, const TypeInfo* typeInfo) noexcept { + return impl_->extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); +} + +ALWAYS_INLINE void alloc::Allocator::ThreadData::destroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { + impl_->extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +} + +void alloc::Allocator::ThreadData::prepareForGC() noexcept { + impl_->extraObjectDataFactoryThreadQueue().Publish(); + impl_->objectFactoryThreadQueue().Publish(); +} + +void alloc::Allocator::ThreadData::clearForTests() noexcept { + impl_->extraObjectDataFactoryThreadQueue().ClearForTests(); + impl_->objectFactoryThreadQueue().ClearForTests(); +} + +alloc::Allocator::Allocator() noexcept : impl_(std::make_unique()) {} + +alloc::Allocator::~Allocator() = default; + +void alloc::Allocator::prepareForGC() noexcept {} + +void alloc::Allocator::clearForTests() noexcept { + impl_->extraObjectDataFactory().ClearForTests(); + impl_->objectFactory().ClearForTests(); +} + gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept { return ObjectFactoryImpl::NodeRef::From(object).ObjectData(); } @@ -18,3 +62,9 @@ ObjHeader* alloc::objectForObjectData(gc::GC::ObjectData& objectData) noexcept { size_t alloc::allocatedHeapSize(ObjHeader* object) noexcept { return ObjectFactoryImpl::GetAllocatedHeapSize(object); } + +void alloc::destroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { + extraObject.Uninstall(); + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->allocator().impl().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImplTestSupport.cpp similarity index 51% rename from kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp rename to kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImplTestSupport.cpp index f75fd949b74..104f30cce83 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImplTestSupport.cpp @@ -1,14 +1,14 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ -#include "GCTestSupport.hpp" +#include "AllocatorTestSupport.hpp" #include "gtest/gtest.h" #include "gmock/gmock.h" -#include "GCImpl.hpp" +#include "AllocatorImpl.hpp" using namespace kotlin; @@ -34,14 +34,20 @@ auto collectPointers(T& iterable) { } // namespace -void gc::AssertClear(GC& gc) noexcept { -#ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); -#else - auto objects = gc.impl().allocator().objectFactory().LockForIter(); - auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); +void alloc::test_support::assertClear(Allocator& allocator) noexcept { + auto objects = allocator.impl().objectFactory().LockForIter(); + auto extraObjects = allocator.impl().extraObjectDataFactory().LockForIter(); EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); -#endif +} + +std::vector alloc::test_support::allocatedObjects(mm::ThreadData& threadData) noexcept { + std::vector objects; + for (auto node : threadData.allocator().impl().objectFactoryThreadQueue()) { + objects.push_back(node.GetObjHeader()); + } + for (auto node : mm::GlobalData::Instance().allocator().impl().objectFactory().LockForIter()) { + objects.push_back(node.GetObjHeader()); + } + return objects; } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index e5b5278d4be..27c1bef79a5 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -8,6 +8,7 @@ #include #include +#include "AllocatorImpl.hpp" #include "CallsChecker.hpp" #include "CompilerConstants.hpp" #include "GlobalData.hpp" @@ -90,8 +91,7 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const } gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( - alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept - : + alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept : allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) { @@ -178,21 +178,17 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // TODO outline as mark_.isolateMarkedHeapAndFinishMark() // By this point all the alive heap must be marked. // All the mutations (incl. allocations) after this method will be subject for the next GC. -#ifdef CUSTOM_ALLOCATOR // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().allocator().alloc().PrepareForGC(); - } - mm::GlobalData::Instance().gc().impl().allocator().heap().PrepareForGC(); -#else - for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().PublishObjectFactory(); + thread.allocator().prepareForGC(); } + allocator_.prepareForGC(); +#ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter(); - std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter(); + std::optional objectFactoryIterable = allocator_.impl().objectFactory().LockForIter(); + std::optional extraObjectFactoryIterable = allocator_.impl().extraObjectDataFactory().LockForIter(); checkMarkCorrectness(*objectFactoryIterable); #endif @@ -207,11 +203,11 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects - auto finalizerQueue = allocator_.heap().Sweep(gcHandle); + auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue()); } - finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif scheduler.onGCFinish(epoch, alloc::allocatedBytes()); state_.finish(epoch); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index ada4557bd6f..36e17824374 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -62,10 +62,7 @@ public: }; ConcurrentMarkAndSweep( - alloc::Allocator::Impl& allocator, - gcScheduler::GCScheduler& scheduler, - bool mutatorsCooperate, - std::size_t auxGCThreads) noexcept; + alloc::Allocator& allocator, gcScheduler::GCScheduler& scheduler, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; ~ConcurrentMarkAndSweep(); void StartFinalizerThreadIfNeeded() noexcept; @@ -81,7 +78,7 @@ private: void auxiliaryGCThreadBody(); void PerformFullGC(int64_t epoch) noexcept; - alloc::Allocator::Impl& allocator_; + alloc::Allocator& allocator_; gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index c0e3eedb62c..b6da1fa0044 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -13,6 +13,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "AllocatorTestSupport.hpp" #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" #include "GCImpl.hpp" @@ -179,19 +180,8 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa return test_support::Object::FromObjHeader(holder.obj()); } -std_support::vector Alive(mm::ThreadData& threadData) { -#ifdef CUSTOM_ALLOCATOR - return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects(); -#else - std_support::vector objects; - for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) { - objects.push_back(node.GetObjHeader()); - } - for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) { - objects.push_back(node.GetObjHeader()); - } - return objects; -#endif +std::vector Alive(mm::ThreadData& threadData) { + return alloc::test_support::allocatedObjects(threadData); } test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { @@ -232,6 +222,7 @@ public: mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); mm::GlobalData::Instance().gc().ClearForTests(); + mm::GlobalData::Instance().allocator().clearForTests(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } @@ -705,7 +696,7 @@ public: return holderRef; } - std_support::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } + std::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } private: struct Context { @@ -1065,7 +1056,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeakNewObj) { auto& extraObj = *mm::ExtraObjectData::Get(object.header()); extraObj.ClearRegularWeakReferenceImpl(); extraObj.Uninstall(); - mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj); + alloc::destroyExtraObjectData(extraObj); while (!gcDone.load(std::memory_order_relaxed)) { mm::safePoint(threadData); @@ -1162,23 +1153,20 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); } -#ifndef CUSTOM_ALLOCATOR // Force mutators to publish their internal heaps + // Really only needed for legacy allocators. std_support::vector> publishFutures; for (auto& mutator: mutators) { - publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().PublishObjectFactory(); - })); + publishFutures.emplace_back( + mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } for (auto& mutator: newMutators) { - publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().PublishObjectFactory(); - })); + publishFutures.emplace_back( + mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } for (auto& future : publishFutures) { future.wait(); } -#endif // All threads see the same alive objects, enough to check a single mutator. EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index d1db1d948be..a5d12725264 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -18,59 +18,8 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::PublishObjectFactory() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); - impl_->allocator().objectFactoryThreadQueue().Publish(); -#endif -} - -void gc::GC::ThreadData::ClearForTests() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->allocator().objectFactoryThreadQueue().ClearForTests(); -#else - impl_->allocator().alloc().PrepareForGC(); -#endif -} - -ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { - ObjHeader* obj; -#ifndef CUSTOM_ALLOCATOR - obj = impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); -#else - obj = impl_->allocator().alloc().CreateObject(typeInfo); -#endif - impl().gc().barriers().onAllocation(obj); - return obj; -} - -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { - ArrayHeader* arr; -#ifndef CUSTOM_ALLOCATOR - arr = impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); -#else - arr = impl_->allocator().alloc().CreateArray(typeInfo, elements); -#endif - impl().gc().barriers().onAllocation(arr->obj()); - return arr; -} - -ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( - ObjHeader* object, const TypeInfo* typeInfo) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); -#else - return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); -#endif -} - -ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); -#endif +ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept { + impl().gc().barriers().onAllocation(object); } void gc::GC::ThreadData::OnSuspendForGC() noexcept { @@ -85,18 +34,13 @@ void gc::GC::ThreadData::onThreadRegistration() noexcept { impl_->gc().onThreadRegistration(); } -gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} +gc::GC::GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : + impl_(std_support::make_unique(allocator, gcScheduler)) {} gc::GC::~GC() = default; void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactory().ClearForTests(); - impl_->allocator().objectFactory().ClearForTests(); -#else - impl_->allocator().heap().ClearForTests(); -#endif GCHandle::ClearForTests(); } @@ -151,18 +95,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { return objectData.tryResetMark(); } -// static -ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - extraObject.Uninstall(); - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.ReleaseAssociatedObject(); - extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); -#endif -} - // static ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { return sizeof(gc::GC::ObjectData); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp index 976a5d69d49..509a9ff7752 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -7,7 +7,6 @@ #include "GC.hpp" -#include "AllocatorImpl.hpp" #include "ConcurrentMarkAndSweep.hpp" namespace kotlin { @@ -15,27 +14,23 @@ namespace gc { class GC::Impl : private Pinned { public: - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : - gc_(allocator_, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {} + Impl(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : + gc_(allocator, gcScheduler, compiler::gcMutatorsCooperate(), compiler::auxGCThreads()) {} - alloc::Allocator::Impl& allocator() noexcept { return allocator_; } ConcurrentMarkAndSweep& gc() noexcept { return gc_; } private: - alloc::Allocator::Impl allocator_; ConcurrentMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {} + Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData) {} ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; } - alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } private: ConcurrentMarkAndSweep::ThreadData gc_; - alloc::Allocator::ThreadData::Impl allocator_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp index 346106ec0c4..72a9e871ebc 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp @@ -8,7 +8,7 @@ #include #include -#include "AllocatorImpl.hpp" +#include "Allocator.hpp" #include "GC.hpp" #include "IntrusiveList.hpp" #include "KAssert.h" diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 5f5c29920ff..5cf0dd79c5f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -16,6 +16,10 @@ namespace kotlin { +namespace alloc { +class Allocator; +} + namespace mm { class ThreadData; } @@ -35,20 +39,14 @@ public: Impl& impl() noexcept { return *impl_; } - void PublishObjectFactory() noexcept; - void ClearForTests() noexcept; - - ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; - ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; - mm::ExtraObjectData& CreateExtraObjectDataForObject(ObjHeader* object, const TypeInfo* typeInfo) noexcept; - void DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; - void OnSuspendForGC() noexcept; void safePoint() noexcept; void onThreadRegistration() noexcept; + void onAllocation(ObjHeader* object) noexcept; + private: std_support::unique_ptr impl_; }; @@ -61,7 +59,7 @@ public: // the destructor is a trivial one. class ObjectData; - explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept; + GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept; ~GC(); Impl& impl() noexcept { return *impl_; } @@ -81,8 +79,6 @@ public: void WaitFinished(int64_t epoch) noexcept; void WaitFinalizers(int64_t epoch) noexcept; - static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; - private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp deleted file mode 100644 index 8542acf9a40..00000000000 --- a/kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#include "GC.hpp" - -namespace kotlin { -namespace gc { - -void AssertClear(GC& gc) noexcept; - -} -} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index b597da85d64..6d5e1ed89d3 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -149,7 +149,10 @@ public: return handle.getMarked(); } - ~MarkAndSweepUtilsMarkTest() { mm::GlobalData::Instance().gc().ClearForTests(); } + ~MarkAndSweepUtilsMarkTest() { + mm::GlobalData::Instance().gc().ClearForTests(); + mm::GlobalData::Instance().allocator().clearForTests(); + } private: uint64_t epoch_ = 0; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 22e466266df..e5ba75a88c3 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -5,67 +5,17 @@ #include "GCImpl.hpp" -#include "Common.h" -#include "GC.hpp" #include "GCStatistics.hpp" -#include "ObjectOps.hpp" -#include "ThreadData.hpp" -#include "std_support/Memory.hpp" +#include "KAssert.h" +#include "Logging.hpp" using namespace kotlin; -gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(std_support::make_unique(gc, threadData)) {} +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept {} gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::PublishObjectFactory() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); - impl_->allocator().objectFactoryThreadQueue().Publish(); -#endif -} - -void gc::GC::ThreadData::ClearForTests() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->allocator().objectFactoryThreadQueue().ClearForTests(); -#else - impl_->allocator().alloc().PrepareForGC(); -#endif -} - -ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); -#else - return impl_->allocator().alloc().CreateObject(typeInfo); -#endif -} - -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); -#else - return impl_->allocator().alloc().CreateArray(typeInfo, elements); -#endif -} - -ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( - ObjHeader* object, const TypeInfo* typeInfo) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); -#else - return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); -#endif -} - -ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); -#endif -} +ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept {} void gc::GC::ThreadData::OnSuspendForGC() noexcept { } @@ -73,17 +23,13 @@ void gc::GC::ThreadData::safePoint() noexcept {} void gc::GC::ThreadData::onThreadRegistration() noexcept {} -gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique()) {} +gc::GC::GC(alloc::Allocator&, gcScheduler::GCScheduler&) noexcept { + RuntimeLogInfo({kTagGC}, "No-op GC initialized"); +} gc::GC::~GC() = default; void gc::GC::ClearForTests() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactory().ClearForTests(); - impl_->allocator().objectFactory().ClearForTests(); -#else - impl_->allocator().heap().ClearForTests(); -#endif GCHandle::ClearForTests(); } @@ -126,18 +72,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { return true; } -// static -ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - extraObject.Uninstall(); - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.ReleaseAssociatedObject(); - extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); -#endif -} - // static ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { return 0; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp index d748ca0ee41..b2efcfe0538 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -7,32 +7,9 @@ #include "GC.hpp" -#include "AllocatorImpl.hpp" -#include "Logging.hpp" -#include "Utils.hpp" +namespace kotlin::gc { -namespace kotlin { -namespace gc { +class GC::Impl {}; +class GC::ThreadData::Impl {}; -class GC::Impl : private Pinned { -public: - Impl() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); } - - alloc::Allocator::Impl& allocator() noexcept { return allocator_; } - -private: - alloc::Allocator::Impl allocator_; -}; - -class GC::ThreadData::Impl : private Pinned { -public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : allocator_(gc.impl_->allocator()) {} - - alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } - -private: - alloc::Allocator::ThreadData::Impl allocator_; -}; - -} // namespace gc -} // namespace kotlin +} // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp deleted file mode 100644 index f75fd949b74..00000000000 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCTestSupport.hpp" - -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "GCImpl.hpp" - -using namespace kotlin; - -namespace { - -template -auto collectCopy(T& iterable) { - std::vector> result; - for (const auto& element : iterable) { - result.push_back(element); - } - return result; -} - -template -auto collectPointers(T& iterable) { - std::vector*> result; - for (const auto& element : iterable) { - result.push_back(&element); - } - return result; -} - -} // namespace - -void gc::AssertClear(GC& gc) noexcept { -#ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); -#else - auto objects = gc.impl().allocator().objectFactory().LockForIter(); - auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); - EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); -#endif -} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 4ddca9e1948..1d6796500f9 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -19,54 +19,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::PublishObjectFactory() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().Publish(); - impl_->allocator().objectFactoryThreadQueue().Publish(); -#endif -} - -void gc::GC::ThreadData::ClearForTests() noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().ClearForTests(); - impl_->allocator().objectFactoryThreadQueue().ClearForTests(); -#else - impl_->allocator().alloc().PrepareForGC(); -#endif -} - -ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().objectFactoryThreadQueue().CreateObject(typeInfo); -#else - return impl_->allocator().alloc().CreateObject(typeInfo); -#endif -} - -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().objectFactoryThreadQueue().CreateArray(typeInfo, elements); -#else - return impl_->allocator().alloc().CreateArray(typeInfo, elements); -#endif -} - -ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( - ObjHeader* object, const TypeInfo* typeInfo) noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->allocator().extraObjectDataFactoryThreadQueue().CreateExtraObjectDataForObject(object, typeInfo); -#else - return impl_->allocator().alloc().CreateExtraObjectDataForObject(object, typeInfo); -#endif -} - -ALWAYS_INLINE void gc::GC::ThreadData::DestroyUnattachedExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); -#endif -} +ALWAYS_INLINE void gc::GC::ThreadData::onAllocation(ObjHeader* object) noexcept {} void gc::GC::ThreadData::OnSuspendForGC() noexcept { } @@ -74,18 +27,13 @@ void gc::GC::ThreadData::safePoint() noexcept {} void gc::GC::ThreadData::onThreadRegistration() noexcept {} -gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} +gc::GC::GC(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : + impl_(std_support::make_unique(allocator, gcScheduler)) {} gc::GC::~GC() = default; void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); -#ifndef CUSTOM_ALLOCATOR - impl_->allocator().extraObjectDataFactory().ClearForTests(); - impl_->allocator().objectFactory().ClearForTests(); -#else - impl_->allocator().heap().ClearForTests(); -#endif GCHandle::ClearForTests(); } @@ -140,18 +88,6 @@ ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept { return objectData.tryResetMark(); } -// static -ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept { -#ifndef CUSTOM_ALLOCATOR - extraObject.Uninstall(); - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - threadData->gc().impl().allocator().extraObjectDataFactoryThreadQueue().DestroyExtraObjectData(extraObject); -#else - extraObject.ReleaseAssociatedObject(); - extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); -#endif -} - // static ALWAYS_INLINE uint64_t type_layout::descriptor::type::size() noexcept { return sizeof(gc::GC::ObjectData); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp index be53c942a1c..930237e58e8 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -7,7 +7,6 @@ #include "GC.hpp" -#include "AllocatorImpl.hpp" #include "SameThreadMarkAndSweep.hpp" namespace kotlin { @@ -15,26 +14,22 @@ namespace gc { class GC::Impl : private Pinned { public: - explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator_, gcScheduler) {} + explicit Impl(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(allocator, gcScheduler) {} - alloc::Allocator::Impl& allocator() noexcept { return allocator_; } SameThreadMarkAndSweep& gc() noexcept { return gc_; } private: - alloc::Allocator::Impl allocator_; SameThreadMarkAndSweep gc_; }; class GC::ThreadData::Impl : private Pinned { public: - Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData), allocator_(gc.impl_->allocator()) {} + Impl(GC& gc, mm::ThreadData& threadData) noexcept : gc_(gc.impl_->gc(), threadData) {} SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; } - alloc::Allocator::ThreadData::Impl& allocator() noexcept { return allocator_; } private: SameThreadMarkAndSweep::ThreadData gc_; - alloc::Allocator::ThreadData::Impl allocator_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp deleted file mode 100644 index f75fd949b74..00000000000 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "GCTestSupport.hpp" - -#include "gtest/gtest.h" -#include "gmock/gmock.h" - -#include "GCImpl.hpp" - -using namespace kotlin; - -namespace { - -template -auto collectCopy(T& iterable) { - std::vector> result; - for (const auto& element : iterable) { - result.push_back(element); - } - return result; -} - -template -auto collectPointers(T& iterable) { - std::vector*> result; - for (const auto& element : iterable) { - result.push_back(&element); - } - return result; -} - -} // namespace - -void gc::AssertClear(GC& gc) noexcept { -#ifdef CUSTOM_ALLOCATOR - auto objects = gc.impl().allocator().heap().GetAllocatedObjects(); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); -#else - auto objects = gc.impl().allocator().objectFactory().LockForIter(); - auto extraObjects = gc.impl().allocator().extraObjectDataFactory().LockForIter(); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); - EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); -#endif -} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp b/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp index a372497969f..c825abdb336 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp @@ -7,7 +7,7 @@ #include -#include "AllocatorImpl.hpp" +#include "Allocator.hpp" #include "GC.hpp" #include "IntrusiveList.hpp" #include "KAssert.h" diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index b70e85b4c5e..012d0ebaa03 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -22,7 +22,7 @@ using namespace kotlin; -gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept : allocator_(allocator), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept { GCHandle::getByEpoch(epoch).finalizersDone(); @@ -70,29 +70,23 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { state_.start(epoch); -#ifdef CUSTOM_ALLOCATOR - // This should really be done by each individual thread while waiting - for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().allocator().alloc().PrepareForGC(); - } - allocator_.heap().PrepareForGC(); -#endif - gc::collectRootSet(gcHandle, markQueue_, [](mm::ThreadData&) { return true; }); gc::Mark(gcHandle, markQueue_); gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); -#ifndef CUSTOM_ALLOCATOR + // This should really be done by each individual thread while waiting for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().PublishObjectFactory(); + thread.allocator().prepareForGC(); } + allocator_.prepareForGC(); +#ifndef CUSTOM_ALLOCATOR // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. - std::optional extraObjectFactoryIterable = allocator_.extraObjectDataFactory().LockForIter(); - std::optional objectFactoryIterable = allocator_.objectFactory().LockForIter(); + std::optional extraObjectFactoryIterable = allocator_.impl().extraObjectDataFactory().LockForIter(); + std::optional objectFactoryIterable = allocator_.impl().objectFactory().LockForIter(); alloc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; @@ -101,11 +95,11 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { alloc::compactObjectPoolInMainThread(); #else // also sweeps extraObjects - auto finalizerQueue = allocator_.heap().Sweep(gcHandle); + auto finalizerQueue = allocator_.impl().heap().Sweep(gcHandle); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - finalizerQueue.TransferAllFrom(thread.gc().impl().allocator().alloc().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(thread.allocator().impl().alloc().ExtractFinalizerQueue()); } - finalizerQueue.TransferAllFrom(allocator_.heap().ExtractFinalizerQueue()); + finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue()); #endif scheduler.onGCFinish(epoch, alloc::allocatedBytes()); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 2b4fb425a23..cd2f86eb7d6 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -40,7 +40,7 @@ public: private: }; - SameThreadMarkAndSweep(alloc::Allocator::Impl& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept; + SameThreadMarkAndSweep(alloc::Allocator& allocator, gcScheduler::GCScheduler& gcScheduler) noexcept; ~SameThreadMarkAndSweep(); @@ -53,7 +53,7 @@ public: private: void PerformFullGC(int64_t epoch) noexcept; - alloc::Allocator::Impl& allocator_; + alloc::Allocator& allocator_; gcScheduler::GCScheduler& gcScheduler_; GCStateHolder state_; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 20510635df1..e84e88b5e4d 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -13,6 +13,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "AllocatorTestSupport.hpp" #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" #include "GCImpl.hpp" @@ -179,19 +180,8 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa return test_support::Object::FromObjHeader(holder.obj()); } -std_support::vector Alive(mm::ThreadData& threadData) { -#ifdef CUSTOM_ALLOCATOR - return threadData.gc().impl().allocator().alloc().heap().GetAllocatedObjects(); -#else - std_support::vector objects; - for (auto node : threadData.gc().impl().allocator().objectFactoryThreadQueue()) { - objects.push_back(node.GetObjHeader()); - } - for (auto node : mm::GlobalData::Instance().gc().impl().allocator().objectFactory().LockForIter()) { - objects.push_back(node.GetObjHeader()); - } - return objects; -#endif +std::vector Alive(mm::ThreadData& threadData) { + return alloc::test_support::allocatedObjects(threadData); } test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { @@ -211,6 +201,7 @@ public: mm::GlobalsRegistry::Instance().ClearForTests(); mm::SpecialRefRegistry::instance().clearForTests(); mm::GlobalData::Instance().gc().ClearForTests(); + mm::GlobalData::Instance().allocator().clearForTests(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } @@ -680,7 +671,7 @@ public: return holderRef; } - std_support::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } + std::vector Alive() { return ::Alive(*executor_.context().memory_->memoryState()->GetThreadData()); } private: struct Context { @@ -1040,7 +1031,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeakNewObj) { auto& extraObj = *mm::ExtraObjectData::Get(object.header()); extraObj.ClearRegularWeakReferenceImpl(); extraObj.Uninstall(); - mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj); + alloc::destroyExtraObjectData(extraObj); while (!gcDone.load(std::memory_order_relaxed)) { mm::safePoint(threadData); @@ -1137,23 +1128,19 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); } -#ifndef CUSTOM_ALLOCATOR // Force mutators to publish their internal heaps std_support::vector> publishFutures; for (auto& mutator: mutators) { - publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().PublishObjectFactory(); - })); + publishFutures.emplace_back( + mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } for (auto& mutator: newMutators) { - publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().PublishObjectFactory(); - })); + publishFutures.emplace_back( + mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.allocator().prepareForGC(); })); } for (auto& future : publishFutures) { future.wait(); } -#endif // All threads see the same alive objects, enough to check a single mutator. EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); diff --git a/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp b/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp index ca3b47b4b4a..0edce0e3e1a 100644 --- a/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp @@ -10,7 +10,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "AllocatorTestSupport.hpp" +#include "StdAllocatorTestSupport.hpp" #include "ScopedThread.hpp" #include "TestSupport.hpp" #include "std_support/Vector.hpp" diff --git a/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp b/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp index fa51e79c1ca..772c90d1197 100644 --- a/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp @@ -11,7 +11,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "AllocatorTestSupport.hpp" +#include "StdAllocatorTestSupport.hpp" #include "ScopedThread.hpp" #include "TestSupport.hpp" #include "std_support/Deque.hpp" diff --git a/kotlin-native/runtime/src/main/cpp/AllocatorTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupport.hpp similarity index 100% rename from kotlin-native/runtime/src/main/cpp/AllocatorTestSupport.hpp rename to kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupport.hpp diff --git a/kotlin-native/runtime/src/main/cpp/AllocatorTestSupportTest.cpp b/kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupportTest.cpp similarity index 92% rename from kotlin-native/runtime/src/main/cpp/AllocatorTestSupportTest.cpp rename to kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupportTest.cpp index ceb04f88b23..28cd6e4b6a7 100644 --- a/kotlin-native/runtime/src/main/cpp/AllocatorTestSupportTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/StdAllocatorTestSupportTest.cpp @@ -3,7 +3,7 @@ * that can be found in the LICENSE file. */ -#include "AllocatorTestSupport.hpp" +#include "StdAllocatorTestSupport.hpp" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -21,7 +21,7 @@ static_assert(sizeof(Class) > sizeof(EmptyClass)); } // namespace -TEST(AllocatorTestSupportTest, MockAllocate) { +TEST(StdAllocatorTestSupportTest, MockAllocate) { testing::StrictMock allocatorCore; auto allocator = test_support::MakeAllocator(allocatorCore); @@ -31,7 +31,7 @@ TEST(AllocatorTestSupportTest, MockAllocate) { EXPECT_THAT(ptr, expectedPtr); } -TEST(AllocatorTestSupportTest, MockDeallocate) { +TEST(StdAllocatorTestSupportTest, MockDeallocate) { testing::StrictMock allocatorCore; auto allocator = test_support::MakeAllocator(allocatorCore); @@ -40,7 +40,7 @@ TEST(AllocatorTestSupportTest, MockDeallocate) { std::allocator_traits::deallocate(allocator, ptr, 2); } -TEST(AllocatorTestSupportTest, MockAdjustType) { +TEST(StdAllocatorTestSupportTest, MockAdjustType) { testing::StrictMock allocatorCore; auto initial = test_support::MakeAllocator(allocatorCore); @@ -58,7 +58,7 @@ TEST(AllocatorTestSupportTest, MockAdjustType) { Traits::deallocate(allocator, ptr, 2); } -TEST(AllocatorTestSupportTest, Spy) { +TEST(StdAllocatorTestSupportTest, Spy) { test_support::SpyAllocatorCore allocatorCore; auto allocator = test_support::MakeAllocator(allocatorCore); diff --git a/kotlin-native/runtime/src/main/cpp/std_support/MemoryTest.cpp b/kotlin-native/runtime/src/main/cpp/std_support/MemoryTest.cpp index 8581cd41595..e4e4d94a8c2 100644 --- a/kotlin-native/runtime/src/main/cpp/std_support/MemoryTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/std_support/MemoryTest.cpp @@ -8,7 +8,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "AllocatorTestSupport.hpp" +#include "StdAllocatorTestSupport.hpp" #include "KAssert.h" #include "Utils.hpp" diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index 3facaa97852..2153e48a1ab 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -28,11 +28,11 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged"); - auto& gc = mm::ThreadRegistry::Instance().CurrentThreadData()->gc(); - auto& data = gc.CreateExtraObjectDataForObject(object, typeInfo); + auto& allocator = mm::ThreadRegistry::Instance().CurrentThreadData()->allocator(); + auto& data = allocator.allocateExtraObjectData(object, typeInfo); if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { // Somebody else created `mm::ExtraObjectData` for this object. - gc.DestroyUnattachedExtraObjectData(data); + allocator.destroyUnattachedExtraObjectData(data); return *reinterpret_cast(typeInfo); } diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index 9f4ac4c5822..ce9aac794d6 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -30,6 +30,7 @@ public: ~ExtraObjectDataTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().gc().ClearForTests(); + mm::GlobalData::Instance().allocator().clearForTests(); } }; @@ -78,7 +79,8 @@ TEST_F(ExtraObjectDataTest, ConcurrentInstall) { } auto& extraData = mm::ExtraObjectData::Install(object.header()); actual[i] = &extraData; - mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->gc().PublishObjectFactory(); + // Really only needed for legacy allocators. + mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->allocator().prepareForGC(); }); } diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index 660665d85c5..93772e020f4 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -6,6 +6,7 @@ #ifndef RUNTIME_MM_GLOBAL_DATA_H #define RUNTIME_MM_GLOBAL_DATA_H +#include "Allocator.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" #include "GCScheduler.hpp" @@ -26,6 +27,7 @@ public: GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; } gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } + alloc::Allocator& allocator() noexcept { return allocator_; } gc::GC& gc() noexcept { return gc_; } AppStateTracking& appStateTracking() noexcept { return appStateTracking_; } @@ -41,7 +43,8 @@ private: GlobalsRegistry globalsRegistry_; SpecialRefRegistry specialRefRegistry_; gcScheduler::GCScheduler gcScheduler_; - gc::GC gc_{gcScheduler_}; + alloc::Allocator allocator_; + gc::GC gc_{allocator_, gcScheduler_}; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 08edc382f93..cc5be1f1d07 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -80,7 +80,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { void ObjHeader::destroyMetaObject(ObjHeader* object) { RuntimeAssert(object->has_meta_object(), "Object must have a meta object set"); auto &extraObject = *mm::ExtraObjectData::Get(object); - gc::GC::DestroyExtraObjectData(extraObject); + alloc::destroyExtraObjectData(extraObject); } ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index 2f9b25c0ccc..702b71e6ca4 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -74,14 +74,15 @@ ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader* OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. - auto* object = threadData->gc().CreateObject(typeInfo); + auto* object = threadData->allocator().allocateObject(typeInfo); + threadData->gc().onAllocation(object); RETURN_OBJ(object); } OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. - auto* array = threadData->gc().CreateArray(typeInfo, static_cast(elements)); - // `ArrayHeader` and `ObjHeader` are expected to be compatible. - RETURN_OBJ(reinterpret_cast(array)); + auto* array = threadData->allocator().allocateArray(typeInfo, static_cast(elements)); + threadData->gc().onAllocation(array->obj()); + RETURN_OBJ(array->obj()); } diff --git a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp index 1fcdb59c5bb..f06d0eb1b6a 100644 --- a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp +++ b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp @@ -6,8 +6,8 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" +#include "AllocatorTestSupport.hpp" #include "GC.hpp" -#include "GCTestSupport.hpp" #include "GlobalData.hpp" #include "GlobalsRegistry.hpp" #include "TestSupport.hpp" @@ -51,7 +51,7 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() { EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre()); - gc::AssertClear(mm::GlobalData::Instance().gc()); + alloc::test_support::assertClear(mm::GlobalData::Instance().allocator()); } void kotlin::DeinitMemoryForTests(MemoryState* memoryState) { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 62ed319537d..503423a1bb0 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -32,6 +32,7 @@ public: globalsThreadQueue_(GlobalsRegistry::Instance()), specialRefRegistry_(SpecialRefRegistry::instance()), gcScheduler_(GlobalData::Instance().gcScheduler(), *this), + allocator_(GlobalData::Instance().allocator()), gc_(GlobalData::Instance().gc(), *this), suspensionData_(ThreadState::kNative, *this) {} @@ -55,6 +56,8 @@ public: gcScheduler::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; } + alloc::Allocator::ThreadData& allocator() noexcept { return allocator_; } + gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -68,7 +71,7 @@ public: void ClearForTests() noexcept { globalsThreadQueue_.ClearForTests(); specialRefRegistry_.clearForTests(); - gc_.ClearForTests(); + allocator_.clearForTests(); } private: @@ -78,6 +81,7 @@ private: SpecialRefRegistry::ThreadQueue specialRefRegistry_; ShadowStack shadowStack_; gcScheduler::GCScheduler::ThreadData gcScheduler_; + alloc::Allocator::ThreadData allocator_; gc::GC::ThreadData gc_; std_support::vector> initializingSingletons_; ThreadSuspensionData suspensionData_;