diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index de5d7545c66..3abcb40f9c3 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -234,20 +234,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration add("legacy_memory_manager.bc") } MemoryModel.EXPERIMENTAL -> { + add("common_gc.bc") + add("experimental_memory_manager.bc") when (gc) { GC.SAME_THREAD_MARK_AND_SWEEP -> { - add("common_gc_stms.bc") - add("experimental_memory_manager_stms.bc") add("same_thread_ms_gc.bc") } GC.NOOP -> { - add("common_gc_noop.bc") - add("experimental_memory_manager_noop.bc") add("noop_gc.bc") } GC.CONCURRENT_MARK_AND_SWEEP -> { - add("common_gc_cms.bc") - add("experimental_memory_manager_cms.bc") add("concurrent_ms_gc.bc") } } diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index fe028fd0c41..650f5a3ff3a 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -58,12 +58,8 @@ bitcode { "${target}Objc", "${target}ExceptionsSupport", "${target}LegacyMemoryManager", - "${target}ExperimentalMemoryManagerNoop", - "${target}ExperimentalMemoryManagerStms", - "${target}ExperimentalMemoryManagerCms", - "${target}CommonGcNoop", - "${target}CommonGcStms", - "${target}CommonGcCms", + "${target}ExperimentalMemoryManager", + "${target}CommonGc", "${target}SameThreadMsGc", "${target}ConcurrentMsGc", "${target}NoopGc" @@ -175,40 +171,16 @@ bitcode { includeRuntime() } - create("experimental_memory_manager_noop", file("src/mm")) { - headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp") + create("experimental_memory_manager", file("src/mm")) { + headersDirs += files("src/gc/common/cpp") includeRuntime() } - create("experimental_memory_manager_stms", file("src/mm")) { - headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp") + create("common_gc", file("src/gc/common")) { + headersDirs += files("src/mm/cpp") includeRuntime() } - create("experimental_memory_manager_cms", file("src/mm")) { - headersDirs += files("src/gc/cms/cpp", "src/gc/common/cpp") - includeRuntime() - - onlyIf { targetSupportsThreads(target) } - } - - create("common_gc_noop", file("src/gc/common")) { - headersDirs += files("src/gc/noop/cpp", "src/mm/cpp") - includeRuntime() - } - - create("common_gc_stms", file("src/gc/common")) { - headersDirs += files("src/gc/stms/cpp", "src/mm/cpp") - includeRuntime() - } - - create("common_gc_cms", file("src/gc/common")) { - headersDirs += files("src/gc/cms/cpp", "src/mm/cpp") - includeRuntime() - - onlyIf { targetSupportsThreads(target) } - } - create("noop_gc", file("src/gc/noop")) { headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp") includeRuntime() @@ -267,8 +239,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMMimallocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerStms", - "${targetName}CommonGcStms", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}SameThreadMsGc", "${targetName}Mimalloc", "${targetName}OptAlloc", @@ -285,8 +257,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMStdAllocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerStms", - "${targetName}CommonGcStms", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}SameThreadMsGc", "${targetName}StdAlloc", "${targetName}Objc" @@ -302,8 +274,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMCmsMimallocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerCms", - "${targetName}CommonGcCms", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}ConcurrentMsGc", "${targetName}Mimalloc", "${targetName}OptAlloc", @@ -320,8 +292,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMCmsStdAllocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerCms", - "${targetName}CommonGcCms", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}ConcurrentMsGc", "${targetName}StdAlloc", "${targetName}Objc" @@ -337,8 +309,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMNoOpMimallocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerNoop", - "${targetName}CommonGcNoop", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}NoopGc", "${targetName}Mimalloc", "${targetName}OptAlloc", @@ -355,8 +327,8 @@ targetList.forEach { targetName -> "${targetName}ExperimentalMMNoOpStdAllocRuntimeTests", listOf( "${targetName}Runtime", - "${targetName}ExperimentalMemoryManagerNoop", - "${targetName}CommonGcNoop", + "${targetName}ExperimentalMemoryManager", + "${targetName}CommonGc", "${targetName}NoopGc", "${targetName}StdAlloc", "${targetName}Objc" @@ -548,4 +520,4 @@ targetList.forEach { targetName -> } } -// endregion \ No newline at end of file +// endregion diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 7ff728205a0..366f3725b2f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -59,16 +59,8 @@ struct SweepTraits { } // namespace -ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept { - SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight); -} - -ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight); -} - void gc::ConcurrentMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - threadData_.gcScheduler().OnSafePointAllocation(size); + gcScheduler_.OnSafePointAllocation(size); mm::SuspendIfRequested(); } void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept { @@ -92,14 +84,12 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { ScheduleAndWaitFullGC(); } -ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { - threadData_.gcScheduler().OnSafePointRegular(weight); - mm::SuspendIfRequested(); -} - -gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep() noexcept : - finalizerProcessor_(make_unique([this](int64_t epoch) { state_.finalized(epoch);})) { - mm::GlobalData::Instance().gcScheduler().SetScheduleGC([this]() NO_EXTERNAL_CALLS_CHECK NO_INLINE { +gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( + mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : + objectFactory_(objectFactory), + gcScheduler_(gcScheduler), + finalizerProcessor_(make_unique([this](int64_t epoch) { state_.finalized(epoch); })) { + gcScheduler_.SetScheduleGC([this]() NO_EXTERNAL_CALLS_CHECK NO_INLINE { RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); state_.schedule(); }); @@ -115,7 +105,6 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep() noexcept : }); } - gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() { state_.shutdown(); gcThread_.join(); @@ -134,7 +123,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto timeSuspendUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); - auto& scheduler = mm::GlobalData::Instance().gcScheduler(); + auto& scheduler = gcScheduler_; scheduler.gcData().OnPerformFullGC(); state_.start(epoch); @@ -144,7 +133,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto timeRootSetUs = konan::getTimeMicros(); // Can be unsafe, because we've stopped the world. - auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); RuntimeLogInfo( {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(), timeRootSetUs - timeSuspendUs); @@ -156,7 +145,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto timeSweepExtraObjectsUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); - auto objectFactoryIterable = mm::GlobalData::Instance().objectFactory().LockForIter(); + auto objectFactoryIterable = objectFactory_.LockForIter(); mm::ResumeThreads(); auto timeResumeUs = konan::getTimeMicros(); @@ -170,7 +159,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs); // Can be unsafe, because we have a lock in objectFactoryIterable - auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto objectsCountAfter = objectFactory_.GetSizeUnsafe(); auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); auto finalizersCount = finalizerQueue.size(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 32738d76694..4a289adbab6 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -49,12 +49,10 @@ public: using ObjectData = ConcurrentMarkAndSweep::ObjectData; using Allocator = AllocatorWithGC; - explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} + explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : + gc_(gc), gcScheduler_(gcScheduler) {} ~ThreadData() = default; - void SafePointFunctionPrologue() noexcept; - void SafePointLoopBody() noexcept; - void SafePointExceptionUnwind() noexcept; void SafePointAllocation(size_t size) noexcept; void ScheduleAndWaitFullGC() noexcept; @@ -66,21 +64,22 @@ public: Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } private: - void SafePointRegular(size_t weight) noexcept; - ConcurrentMarkAndSweep& gc_; - mm::ThreadData& threadData_; + GCSchedulerThreadData& gcScheduler_; }; using Allocator = ThreadData::Allocator; - ConcurrentMarkAndSweep() noexcept; + ConcurrentMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& scheduler) noexcept; ~ConcurrentMarkAndSweep(); private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). bool PerformFullGC(int64_t epoch) noexcept; + mm::ObjectFactory& objectFactory_; + GCScheduler& gcScheduler_; + uint64_t lastGCTimestampUs_ = 0; GCStateHolder state_; std::thread gcThread_; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 9d72b9dc85a..fae62ac218a 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -15,6 +15,7 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" +#include "GCImpl.hpp" #include "GlobalData.hpp" #include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" @@ -187,10 +188,10 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa KStdVector Alive(mm::ThreadData& threadData) { KStdVector objects; - for (auto node : threadData.objectFactoryThreadQueue()) { + for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); } - for (auto node : mm::GlobalData::Instance().objectFactory().LockForIter()) { + for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) { objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); } return objects; @@ -219,7 +220,7 @@ public: ~ConcurrentMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); - mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gc().ClearForTests(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } @@ -338,7 +339,7 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().StopFinalizerThreadForTests(); + threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -358,7 +359,7 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().StopFinalizerThreadForTests(); + threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre()); }); @@ -509,7 +510,7 @@ TEST_F(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); - threadData.gc().StopFinalizerThreadForTests(); + threadData.gc().impl().gc().StopFinalizerThreadForTests(); EXPECT_THAT( Alive(threadData), diff --git a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessorTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessorTest.cpp index ba0d74dd640..50ed0788561 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessorTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessorTest.cpp @@ -15,6 +15,7 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" +#include "GCImpl.hpp" #include "GlobalData.hpp" #include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" @@ -55,7 +56,7 @@ public: ~FinalizerProcessorTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); - mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gc().ClearForTests(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } @@ -94,7 +95,7 @@ TEST_F(FinalizerProcessorTest, RemoveObject) { mm::ObjectFactory::FinalizerQueue queue; auto &object = AllocateObjectWithFinalizer(*mm::ThreadRegistry::Instance().CurrentThreadData()); mm::ThreadRegistry::Instance().CurrentThreadData()->Publish(); - auto &factory = mm::GlobalData::Instance().objectFactory(); + auto& factory = mm::GlobalData::Instance().gc().impl().objectFactory(); auto iter = factory.LockForIter(); auto iterator = iter.begin(); iter.MoveAndAdvance(queue, iterator); @@ -121,7 +122,7 @@ TEST_F(FinalizerProcessorTest, ScheduleTasksWhileFinalizing) { auto& object = AllocateObjectWithFinalizer(*mm::ThreadRegistry::Instance().CurrentThreadData()); headers.push_back(object.header()); } - auto& factory = mm::GlobalData::Instance().objectFactory(); + auto& factory = mm::GlobalData::Instance().gc().impl().objectFactory(); mm::ThreadRegistry::Instance().CurrentThreadData()->Publish(); auto iter = factory.LockForIter(); mm::ObjectFactory::FinalizerQueue queue; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp new file mode 100644 index 00000000000..3cb45f18429 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCImpl.hpp" + +#include "GC.hpp" +#include "ThreadSuspension.hpp" + +using namespace kotlin; + +namespace { + +ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept { + threadData.impl().gcScheduler().OnSafePointRegular(weight); + mm::SuspendIfRequested(); +} + +} // namespace + +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique(gc, threadData)) {} + +gc::GC::ThreadData::~ThreadData() = default; + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept { + SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight); +} + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept { + SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { + impl_->gc().ScheduleAndWaitFullGC(); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { + impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); +} + +void gc::GC::ThreadData::Publish() noexcept { + impl_->objectFactoryThreadQueue().Publish(); +} + +void gc::GC::ThreadData::ClearForTests() noexcept { + impl_->objectFactoryThreadQueue().ClearForTests(); +} + +ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { + return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +} + +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +} + +void gc::GC::ThreadData::OnStoppedForGC() noexcept { + impl_->gcScheduler().OnStoppedForGC(); +} + +gc::GC::GC() noexcept : impl_(make_unique()) {} + +gc::GC::~GC() = default; + +// static +size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { + return mm::ObjectFactory::GetAllocatedHeapSize(object); +} + +gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { + return impl_->gcScheduler().config(); +} + +void gc::GC::ClearForTests() noexcept { + impl_->objectFactory().ClearForTests(); +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp new file mode 100644 index 00000000000..61ffee7f1d0 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp @@ -0,0 +1,49 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GC.hpp" + +#include "ConcurrentMarkAndSweep.hpp" + +namespace kotlin { +namespace gc { + +using GCImpl = ConcurrentMarkAndSweep; + +class GC::Impl : private Pinned { +public: + Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + + mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + GCScheduler& gcScheduler() noexcept { return gcScheduler_; } + GCImpl& gc() noexcept { return gc_; } + +private: + mm::ObjectFactory objectFactory_; + GCScheduler gcScheduler_; + GCImpl gc_; +}; + +class GC::ThreadData::Impl : private Pinned { +public: + Impl(GC& gc, mm::ThreadData& threadData) noexcept : + gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), + gc_(gc.impl_->gc(), threadData, gcScheduler_), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {} + + GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } + GCImpl::ThreadData& gc() noexcept { return gc_; } + mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + +private: + GCSchedulerThreadData gcScheduler_; + GCImpl::ThreadData gc_; + mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; +}; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp new file mode 100644 index 00000000000..3c84c32877e --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCTestSupport.hpp" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "GCImpl.hpp" + +using namespace kotlin; + +namespace { + +template +auto collectCopy(T& iterable) { + std::vector> result; + for (const auto& element : iterable) { + result.push_back(element); + } + return result; +} + +} // namespace + +void gc::AssertClear(GC& gc) noexcept { + auto objects = gc.impl().objectFactory().LockForIter(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp new file mode 100644 index 00000000000..35a89dcdb81 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -0,0 +1,70 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GCScheduler.hpp" +#include "Memory.h" +#include "Types.h" +#include "Utils.hpp" + +namespace kotlin { + +namespace mm { +class ThreadData; +} + +namespace gc { + +class GC : private Pinned { +public: + class Impl; + + class ThreadData : private Pinned { + public: + class Impl; + + ThreadData(GC& gc, mm::ThreadData& threadData) noexcept; + ~ThreadData(); + + Impl& impl() noexcept { return *impl_; } + + void SafePointFunctionPrologue() noexcept; + void SafePointLoopBody() noexcept; + + void ScheduleAndWaitFullGC() noexcept; + void ScheduleAndWaitFullGCWithFinalizers() noexcept; + + void Publish() noexcept; + void ClearForTests() noexcept; + + ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; + + void OnStoppedForGC() noexcept; + + private: + KStdUniquePtr impl_; + }; + + GC() noexcept; + ~GC(); + + Impl& impl() noexcept { return *impl_; } + + static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; + + gc::GCSchedulerConfig& gcSchedulerConfig() noexcept; + + void ClearForTests() noexcept; + +private: + KStdUniquePtr impl_; +}; + +inline constexpr bool kSupportsMultipleMutators = true; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp similarity index 58% rename from kotlin-native/runtime/src/gc/cms/cpp/GC.hpp rename to kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp index 2e309a267c3..8542acf9a40 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp @@ -5,15 +5,12 @@ #pragma once -#include "ConcurrentMarkAndSweep.hpp" +#include "GC.hpp" namespace kotlin { namespace gc { -using GC = kotlin::gc::ConcurrentMarkAndSweep; +void AssertClear(GC& gc) noexcept; -inline constexpr bool kSupportsMultipleMutators = true; - -} // namespace gc +} } // namespace kotlin - diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp index 09867b9c56e..ddde4ed8ab7 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp @@ -18,7 +18,7 @@ KStdVector kotlin::gc::collectRootSet() { for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { // TODO: Maybe it's more efficient to do by the suspending thread? thread.Publish(); - thread.gcScheduler().OnStoppedForGC(); + thread.gc().OnStoppedForGC(); size_t stack = 0; size_t tls = 0; for (auto value : mm::ThreadRootSet(thread)) { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index 2361ab10566..05c5ffbe6ba 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -9,6 +9,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "Allocator.hpp" #include "FinalizerHooksTestSupport.hpp" #include "ObjectFactory.hpp" #include "ObjectTestSupport.hpp" diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GC.hpp deleted file mode 100644 index 43b333eade7..00000000000 --- a/kotlin-native/runtime/src/gc/noop/cpp/GC.hpp +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#ifndef RUNTIME_GC_NOOP_GC_H -#define RUNTIME_GC_NOOP_GC_H - -#include "NoOpGC.hpp" - -namespace kotlin { -namespace gc { - -using GC = kotlin::gc::NoOpGC; - -inline constexpr bool kSupportsMultipleMutators = true; - -} // namespace gc -} // namespace kotlin - -#endif // RUNTIME_GC_NOOP_GC_H diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp new file mode 100644 index 00000000000..62e64c651bf --- /dev/null +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -0,0 +1,67 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCImpl.hpp" + +#include "GC.hpp" + +using namespace kotlin; + +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique(gc, threadData)) {} + +gc::GC::ThreadData::~ThreadData() = default; + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept { + impl_->gc().SafePointFunctionPrologue(); +} + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept { + impl_->gc().SafePointLoopBody(); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { + impl_->gc().ScheduleAndWaitFullGC(); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { + impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); +} + +void gc::GC::ThreadData::Publish() noexcept { + impl_->objectFactoryThreadQueue().Publish(); +} + +void gc::GC::ThreadData::ClearForTests() noexcept { + impl_->objectFactoryThreadQueue().ClearForTests(); +} + +ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { + return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +} + +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +} + +void gc::GC::ThreadData::OnStoppedForGC() noexcept { + impl_->gcScheduler().OnStoppedForGC(); +} + +gc::GC::GC() noexcept : impl_(make_unique()) {} + +gc::GC::~GC() = default; + +// static +size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { + return mm::ObjectFactory::GetAllocatedHeapSize(object); +} + +gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { + return impl_->gcScheduler().config(); +} + +void gc::GC::ClearForTests() noexcept { + impl_->objectFactory().ClearForTests(); +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp new file mode 100644 index 00000000000..61df62fe855 --- /dev/null +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp @@ -0,0 +1,49 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GC.hpp" + +#include "NoOpGC.hpp" + +namespace kotlin { +namespace gc { + +using GCImpl = NoOpGC; + +class GC::Impl : private Pinned { +public: + Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + + mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + GCScheduler& gcScheduler() noexcept { return gcScheduler_; } + GCImpl& gc() noexcept { return gc_; } + +private: + mm::ObjectFactory objectFactory_; + GCScheduler gcScheduler_; + GCImpl gc_; +}; + +class GC::ThreadData::Impl : private Pinned { +public: + Impl(GC& gc, mm::ThreadData& threadData) noexcept : + gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), + gc_(gc.impl_->gc(), threadData, gcScheduler_), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {} + + GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } + GCImpl::ThreadData& gc() noexcept { return gc_; } + mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + +private: + GCSchedulerThreadData gcScheduler_; + GCImpl::ThreadData gc_; + mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; +}; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp new file mode 100644 index 00000000000..3c84c32877e --- /dev/null +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCTestSupport.hpp" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "GCImpl.hpp" + +using namespace kotlin; + +namespace { + +template +auto collectCopy(T& iterable) { + std::vector> result; + for (const auto& element : iterable) { + result.push_back(element); + } + return result; +} + +} // namespace + +void gc::AssertClear(GC& gc) noexcept { + auto objects = gc.impl().objectFactory().LockForIter(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); +} diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 4d5b75501ac..ae1d57bf1eb 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -10,6 +10,7 @@ #include "Allocator.hpp" #include "GCScheduler.hpp" +#include "ObjectFactory.hpp" #include "Utils.hpp" #include "Types.h" @@ -33,7 +34,7 @@ public: public: using ObjectData = NoOpGC::ObjectData; - explicit ThreadData(NoOpGC& gc, mm::ThreadData& threadData) noexcept {} + ThreadData(NoOpGC& gc, mm::ThreadData& threadData, GCSchedulerThreadData&) noexcept {} ~ThreadData() = default; void SafePointFunctionPrologue() noexcept {} @@ -50,7 +51,7 @@ public: private: }; - NoOpGC() noexcept {} + NoOpGC(mm::ObjectFactory&, GCScheduler&) noexcept {} ~NoOpGC() = default; GCScheduler& scheduler() noexcept { return scheduler_; } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp deleted file mode 100644 index c08722760c4..00000000000 --- a/kotlin-native/runtime/src/gc/stms/cpp/GC.hpp +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#ifndef RUNTIME_GC_STMS_GC_H -#define RUNTIME_GC_STMS_GC_H - -#include "SameThreadMarkAndSweep.hpp" - -namespace kotlin { -namespace gc { - -using GC = kotlin::gc::SameThreadMarkAndSweep; - -inline constexpr bool kSupportsMultipleMutators = true; - -} // namespace gc -} // namespace kotlin - -#endif // RUNTIME_GC_STMS_GC_H diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp new file mode 100644 index 00000000000..e3d9bcdeb0e --- /dev/null +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -0,0 +1,79 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCImpl.hpp" + +#include "GC.hpp" + +using namespace kotlin; + +namespace { + +ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept { + threadData.impl().gcScheduler().OnSafePointRegular(weight); + auto flag = gc::internal::loadSafepointFlag(); + if (flag != gc::SameThreadMarkAndSweep::SafepointFlag::kNone) { + threadData.impl().gc().SafePointSlowPath(flag); + } +} + +} // namespace + +gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique(gc, threadData)) {} + +gc::GC::ThreadData::~ThreadData() = default; + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept { + SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight); +} + +ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept { + SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept { + impl_->gc().ScheduleAndWaitFullGC(); +} + +void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept { + impl_->gc().ScheduleAndWaitFullGCWithFinalizers(); +} + +void gc::GC::ThreadData::Publish() noexcept { + impl_->objectFactoryThreadQueue().Publish(); +} + +void gc::GC::ThreadData::ClearForTests() noexcept { + impl_->objectFactoryThreadQueue().ClearForTests(); +} + +ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { + return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); +} + +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); +} + +void gc::GC::ThreadData::OnStoppedForGC() noexcept { + impl_->gcScheduler().OnStoppedForGC(); +} + +gc::GC::GC() noexcept : impl_(make_unique()) {} + +gc::GC::~GC() = default; + +// static +size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { + return mm::ObjectFactory::GetAllocatedHeapSize(object); +} + +gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { + return impl_->gcScheduler().config(); +} + +void gc::GC::ClearForTests() noexcept { + impl_->objectFactory().ClearForTests(); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp new file mode 100644 index 00000000000..69c151e214c --- /dev/null +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp @@ -0,0 +1,49 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "GC.hpp" + +#include "SameThreadMarkAndSweep.hpp" + +namespace kotlin { +namespace gc { + +using GCImpl = SameThreadMarkAndSweep; + +class GC::Impl : private Pinned { +public: + Impl() noexcept : gc_(objectFactory_, gcScheduler_) {} + + mm::ObjectFactory& objectFactory() noexcept { return objectFactory_; } + GCScheduler& gcScheduler() noexcept { return gcScheduler_; } + GCImpl& gc() noexcept { return gc_; } + +private: + mm::ObjectFactory objectFactory_; + GCScheduler gcScheduler_; + GCImpl gc_; +}; + +class GC::ThreadData::Impl : private Pinned { +public: + Impl(GC& gc, mm::ThreadData& threadData) noexcept : + gcScheduler_(gc.impl_->gcScheduler().NewThreadData()), + gc_(gc.impl_->gc(), threadData, gcScheduler_), + objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {} + + GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } + GCImpl::ThreadData& gc() noexcept { return gc_; } + mm::ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + +private: + GCSchedulerThreadData gcScheduler_; + GCImpl::ThreadData gc_; + mm::ObjectFactory::ThreadQueue objectFactoryThreadQueue_; +}; + +} // namespace gc +} // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp new file mode 100644 index 00000000000..3c84c32877e --- /dev/null +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCTestSupport.hpp" + +#include "gtest/gtest.h" +#include "gmock/gmock.h" + +#include "GCImpl.hpp" + +using namespace kotlin; + +namespace { + +template +auto collectCopy(T& iterable) { + std::vector> result; + for (const auto& element : iterable) { + result.push_back(element); + } + return result; +} + +} // namespace + +void gc::AssertClear(GC& gc) noexcept { + auto objects = gc.impl().objectFactory().LockForIter(); + EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index fc8d8a80bbc..ba59bd40c75 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -64,16 +64,8 @@ std::atomic gSafepointFlag = gc::Same } // namespace -ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept { - SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight); -} - -ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight); -} - void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - threadData_.gcScheduler().OnSafePointAllocation(size); + gcScheduler_.OnSafePointAllocation(size); SafepointFlag flag = gSafepointFlag.load(); if (flag != SafepointFlag::kNone) { SafePointSlowPath(flag); @@ -85,7 +77,7 @@ void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept { if (!didGC) { // If we failed to suspend threads, someone else might be asking to suspend them. - threadData_.suspensionData().suspendIfRequested(); + mm::SuspendIfRequested(); } } @@ -94,21 +86,13 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { ScheduleAndWaitFullGC(); } -ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { - threadData_.gcScheduler().OnSafePointRegular(weight); - SafepointFlag flag = gSafepointFlag.load(); - if (flag != SafepointFlag::kNone) { - SafePointSlowPath(flag); - } -} - NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept { switch (flag) { case SafepointFlag::kNone: RuntimeAssert(false, "Must've been handled by the caller"); return; case SafepointFlag::kNeedsSuspend: - threadData_.suspensionData().suspendIfRequested(); + mm::SuspendIfRequested(); return; case SafepointFlag::kNeedsGC: RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint"); @@ -117,8 +101,10 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi } } -gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept { - mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() { +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( + mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : + objectFactory_(objectFactory), gcScheduler_(gcScheduler) { + gcScheduler_.SetScheduleGC([]() { RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); gSafepointFlag = SafepointFlag::kNeedsGC; }); @@ -146,7 +132,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { auto timeSuspendUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); - auto& scheduler = mm::GlobalData::Instance().gcScheduler(); + auto& scheduler = gcScheduler_; scheduler.gcData().OnPerformFullGC(); RuntimeLogInfo( @@ -154,7 +140,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { auto graySet = collectRootSet(); auto timeRootSetUs = konan::getTimeMicros(); // Can be unsafe, because we've stopped the world. - auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); RuntimeLogInfo( {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(), @@ -166,12 +152,12 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); auto timeSweepExtraObjectsUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); - finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); + finalizerQueue = gc::Sweep(objectFactory_); auto timeSweepUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs); // Can be unsafe, because we've stopped the world. - auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto objectsCountAfter = objectFactory_.GetSizeUnsafe(); auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); gSafepointFlag = SafepointFlag::kNone; @@ -206,3 +192,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { return true; } + +gc::SameThreadMarkAndSweep::SafepointFlag gc::internal::loadSafepointFlag() noexcept { + return gSafepointFlag.load(); +} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index fa94d183a0f..288f83a1fdb 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -51,12 +51,11 @@ public: using ObjectData = SameThreadMarkAndSweep::ObjectData; using Allocator = AllocatorWithGC; - explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} + ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : + gc_(gc), gcScheduler_(gcScheduler) {} ~ThreadData() = default; - void SafePointFunctionPrologue() noexcept; - void SafePointLoopBody() noexcept; - void SafePointExceptionUnwind() noexcept; + void SafePointSlowPath(SafepointFlag flag) noexcept; void SafePointAllocation(size_t size) noexcept; void ScheduleAndWaitFullGC() noexcept; @@ -67,16 +66,14 @@ public: Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } private: - void SafePointRegular(size_t weight) noexcept; - void SafePointSlowPath(SafepointFlag flag) noexcept; SameThreadMarkAndSweep& gc_; - mm::ThreadData& threadData_; + GCSchedulerThreadData& gcScheduler_; }; using Allocator = ThreadData::Allocator; - SameThreadMarkAndSweep() noexcept; + SameThreadMarkAndSweep(mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept; ~SameThreadMarkAndSweep() = default; void StopFinalizerThreadForTests() noexcept {} @@ -86,8 +83,17 @@ private: size_t epoch_ = 0; uint64_t lastGCTimestampUs_ = 0; + + mm::ObjectFactory& objectFactory_; + GCScheduler& gcScheduler_; }; +namespace internal { + +SameThreadMarkAndSweep::SafepointFlag loadSafepointFlag() noexcept; + +} // namespace internal + } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 94a9c1000be..6b6e280dbd1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -15,6 +15,7 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooksTestSupport.hpp" +#include "GCImpl.hpp" #include "GlobalData.hpp" #include "ObjectOps.hpp" #include "ObjectTestSupport.hpp" @@ -187,10 +188,10 @@ test_support::Object& AllocateObjectWithFinalizer(mm::ThreadData& threa KStdVector Alive(mm::ThreadData& threadData) { KStdVector objects; - for (auto node : threadData.objectFactoryThreadQueue()) { + for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) { objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); } - for (auto node : mm::GlobalData::Instance().objectFactory().LockForIter()) { + for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) { objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader()); } return objects; @@ -215,11 +216,10 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader class SameThreadMarkAndSweepTest : public testing::Test { public: - ~SameThreadMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); - mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests(); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index f946dc46060..6b9af6b8049 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -30,7 +30,7 @@ public: ~ExtraObjectDataTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests(); - mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gc().ClearForTests(); } }; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index a0d652e11b0..b1fa0049b45 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -27,8 +27,6 @@ public: GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; } ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } - ObjectFactory& objectFactory() noexcept { return objectFactory_; } - gc::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } gc::GC& gc() noexcept { return gc_; } private: @@ -42,8 +40,6 @@ private: GlobalsRegistry globalsRegistry_; StableRefRegistry stableRefRegistry_; ExtraObjectDataFactory extraObjectDataFactory_; - ObjectFactory objectFactory_; - gc::GCScheduler gcScheduler_; gc::GC gc_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp b/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp index 5e0ab7c3d1e..5878c3800e4 100644 --- a/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp @@ -35,7 +35,7 @@ public: ~InitSingletonTest() { globalConstructor_ = nullptr; - mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gc().ClearForTests(); mm::GlobalData::Instance().globalsRegistry().ClearForTests(); } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index d302576206e..0cd5fa13283 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -324,11 +324,11 @@ extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, int32_t value if (value < 0) { ThrowIllegalArgumentException(); } - mm::GlobalData::Instance().gcScheduler().config().threshold = static_cast(value); + mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = static_cast(value); } extern "C" int32_t Kotlin_native_internal_GC_getThreshold(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gcScheduler().config().threshold.load(); + auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load(); auto maxValue = std::numeric_limits::max(); if (threshold > static_cast(maxValue)) { return maxValue; @@ -350,11 +350,11 @@ extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, in if (value < 0) { ThrowIllegalArgumentException(); } - mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = static_cast(value); + mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = static_cast(value); } extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load(); + auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load(); auto maxValue = std::numeric_limits::max(); if (threshold > static_cast(maxValue)) { return maxValue; @@ -363,11 +363,11 @@ extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) } extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) { - mm::GlobalData::Instance().gcScheduler().config().autoTune = value; + mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune = value; } extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) { - return mm::GlobalData::Instance().gcScheduler().config().autoTune.load(); + return mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune.load(); } extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 68c56932fd1..567964217ea 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -691,6 +691,16 @@ public: void ClearForTests() { storage_.ClearForTests(); } + static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept { + RuntimeAssert(object->heap(), "Object must be a heap object"); + const auto* typeInfo = object->type_info(); + if (typeInfo->IsArray()) { + return ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_); + } else { + return ThreadQueue::ObjectAllocatedSize(typeInfo); + } + } + private: Storage storage_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index 3e286a5bc53..b72df3f89fa 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -58,24 +58,18 @@ ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHea OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. - auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo); + auto* object = threadData->gc().CreateObject(typeInfo); RETURN_OBJ(object); } OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. - auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast(elements)); + auto* array = threadData->gc().CreateArray(typeInfo, static_cast(elements)); // `ArrayHeader` and `ObjHeader` are expected to be compatible. RETURN_OBJ(reinterpret_cast(array)); } size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept { - RuntimeAssert(object->heap(), "Object must be a heap object"); - const auto* typeInfo = object->type_info(); - if (typeInfo->IsArray()) { - return mm::ObjectFactory::ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_); - } else { - return mm::ObjectFactory::ThreadQueue::ObjectAllocatedSize(typeInfo); - } + return gc::GC::GetAllocatedHeapSize(object); } diff --git a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp index d1b84d46af1..69b76f59396 100644 --- a/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp +++ b/kotlin-native/runtime/src/mm/cpp/TestSupport.cpp @@ -7,6 +7,7 @@ #include "gmock/gmock.h" #include "GC.hpp" +#include "GCTestSupport.hpp" #include "GlobalData.hpp" #include "GlobalsRegistry.hpp" #include "TestSupport.hpp" @@ -45,15 +46,14 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() { // Validate that global registries are empty. auto globals = mm::GlobalsRegistry::Instance().LockForIter(); auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter(); - auto objects = mm::GlobalData::Instance().objectFactory().LockForIter(); auto stableRefs = mm::StableRefRegistry::Instance().LockForIter(); auto threads = mm::ThreadRegistry::Instance().LockForIter(); EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre()); - EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre()); EXPECT_THAT(collectCopy(stableRefs), testing::UnorderedElementsAre()); EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre()); + gc::AssertClear(mm::GlobalData::Instance().gc()); } void kotlin::DeinitMemoryForTests(MemoryState* memoryState) { @@ -63,4 +63,4 @@ void kotlin::DeinitMemoryForTests(MemoryState* memoryState) { std::ostream& kotlin::operator<<(std::ostream& stream, ThreadState state) { return stream << ThreadStateName(state); -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index f6b20598dcb..3ae4edbc6a7 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -35,9 +35,7 @@ public: globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()), extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()), - gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), gc_(GlobalData::Instance().gc(), *this), - objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_), suspensionData_(ThreadState::kNative) {} ~ThreadData() = default; @@ -56,14 +54,10 @@ public: ThreadState setState(ThreadState state) noexcept { return suspensionData_.setState(state); } - ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } - ShadowStack& shadowStack() noexcept { return shadowStack_; } KStdVector>& initializingSingletons() noexcept { return initializingSingletons_; } - gc::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } - gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -72,15 +66,15 @@ public: // TODO: These use separate locks, which is inefficient. globalsThreadQueue_.Publish(); stableRefThreadQueue_.Publish(); - objectFactoryThreadQueue_.Publish(); extraObjectDataThreadQueue_.Publish(); + gc_.Publish(); } void ClearForTests() noexcept { globalsThreadQueue_.ClearForTests(); stableRefThreadQueue_.ClearForTests(); - objectFactoryThreadQueue_.ClearForTests(); extraObjectDataThreadQueue_.ClearForTests(); + gc_.ClearForTests(); } private: @@ -90,9 +84,7 @@ private: StableRefRegistry::ThreadQueue stableRefThreadQueue_; ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_; ShadowStack shadowStack_; - gc::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_; KStdVector> initializingSingletons_; ThreadSuspensionData suspensionData_; };