[K/N] Separate MM from GC implementations

^KT-50291

Merge-request: KT-MR-5191
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -234,20 +234,16 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
                 add("legacy_memory_manager.bc")
             }
             MemoryModel.EXPERIMENTAL -> {
+                add("common_gc.bc")
+                add("experimental_memory_manager.bc")
                 when (gc) {
                     GC.SAME_THREAD_MARK_AND_SWEEP -> {
-                        add("common_gc_stms.bc")
-                        add("experimental_memory_manager_stms.bc")
                         add("same_thread_ms_gc.bc")
                     }
                     GC.NOOP -> {
-                        add("common_gc_noop.bc")
-                        add("experimental_memory_manager_noop.bc")
                         add("noop_gc.bc")
                     }
                     GC.CONCURRENT_MARK_AND_SWEEP -> {
-                        add("common_gc_cms.bc")
-                        add("experimental_memory_manager_cms.bc")
                         add("concurrent_ms_gc.bc")
                     }
                 }

kotlin-native/runtime/build.gradle.kts

@@ -58,12 +58,8 @@ bitcode {
             "${target}Objc",
             "${target}ExceptionsSupport",
             "${target}LegacyMemoryManager",
-            "${target}ExperimentalMemoryManagerNoop",
-            "${target}ExperimentalMemoryManagerStms",
-            "${target}ExperimentalMemoryManagerCms",
-            "${target}CommonGcNoop",
-            "${target}CommonGcStms",
-            "${target}CommonGcCms",
+            "${target}ExperimentalMemoryManager",
+            "${target}CommonGc",
             "${target}SameThreadMsGc",
             "${target}ConcurrentMsGc",
             "${target}NoopGc"
@@ -175,40 +171,16 @@ bitcode {
         includeRuntime()
     }
 
-    create("experimental_memory_manager_noop", file("src/mm")) {
-        headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp")
+    create("experimental_memory_manager", file("src/mm")) {
+        headersDirs += files("src/gc/common/cpp")
         includeRuntime()
     }
 
-    create("experimental_memory_manager_stms", file("src/mm")) {
-        headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp")
+    create("common_gc", file("src/gc/common")) {
+        headersDirs += files("src/mm/cpp")
         includeRuntime()
     }
 
-    create("experimental_memory_manager_cms", file("src/mm")) {
-        headersDirs += files("src/gc/cms/cpp", "src/gc/common/cpp")
-        includeRuntime()
-
-        onlyIf { targetSupportsThreads(target) }
-    }
-
-    create("common_gc_noop", file("src/gc/common")) {
-        headersDirs += files("src/gc/noop/cpp", "src/mm/cpp")
-        includeRuntime()
-    }
-
-    create("common_gc_stms", file("src/gc/common")) {
-        headersDirs += files("src/gc/stms/cpp", "src/mm/cpp")
-        includeRuntime()
-    }
-
-    create("common_gc_cms", file("src/gc/common")) {
-        headersDirs += files("src/gc/cms/cpp", "src/mm/cpp")
-        includeRuntime()
-
-        onlyIf { targetSupportsThreads(target) }
-    }
-
     create("noop_gc", file("src/gc/noop")) {
         headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp")
         includeRuntime()
@@ -267,8 +239,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMMimallocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManagerStms",
-                "${targetName}CommonGcStms",
+                "${targetName}ExperimentalMemoryManager",
+                "${targetName}CommonGc",
                 "${targetName}SameThreadMsGc",
                 "${targetName}Mimalloc",
                 "${targetName}OptAlloc",
@@ -285,8 +257,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMStdAllocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManagerStms",
-                "${targetName}CommonGcStms",
+                "${targetName}ExperimentalMemoryManager",
+                "${targetName}CommonGc",
                 "${targetName}SameThreadMsGc",
                 "${targetName}StdAlloc",
                 "${targetName}Objc"
@@ -302,8 +274,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMCmsMimallocRuntimeTests",
             listOf(
                     "${targetName}Runtime",
-                    "${targetName}ExperimentalMemoryManagerCms",
-                    "${targetName}CommonGcCms",
+                    "${targetName}ExperimentalMemoryManager",
+                    "${targetName}CommonGc",
                     "${targetName}ConcurrentMsGc",
                     "${targetName}Mimalloc",
                     "${targetName}OptAlloc",
@@ -320,8 +292,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMCmsStdAllocRuntimeTests",
             listOf(
                     "${targetName}Runtime",
-                    "${targetName}ExperimentalMemoryManagerCms",
-                    "${targetName}CommonGcCms",
+                    "${targetName}ExperimentalMemoryManager",
+                    "${targetName}CommonGc",
                     "${targetName}ConcurrentMsGc",
                     "${targetName}StdAlloc",
                     "${targetName}Objc"
@@ -337,8 +309,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMNoOpMimallocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManagerNoop",
-                "${targetName}CommonGcNoop",
+                "${targetName}ExperimentalMemoryManager",
+                "${targetName}CommonGc",
                 "${targetName}NoopGc",
                 "${targetName}Mimalloc",
                 "${targetName}OptAlloc",
@@ -355,8 +327,8 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMNoOpStdAllocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManagerNoop",
-                "${targetName}CommonGcNoop",
+                "${targetName}ExperimentalMemoryManager",
+                "${targetName}CommonGc",
                 "${targetName}NoopGc",
                 "${targetName}StdAlloc",
                 "${targetName}Objc"
@@ -548,4 +520,4 @@ targetList.forEach { targetName ->
     }
 }
 
-// endregion
+// endregion

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -59,16 +59,8 @@ struct SweepTraits {
 
 } // namespace
 
-ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept {
-    SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight);
-}
-
-ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
-    SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight);
-}
-
 void gc::ConcurrentMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
-    threadData_.gcScheduler().OnSafePointAllocation(size);
+    gcScheduler_.OnSafePointAllocation(size);
     mm::SuspendIfRequested();
 }
 void gc::ConcurrentMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept {
@@ -92,14 +84,12 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
     ScheduleAndWaitFullGC();
 }
 
-ALWAYS_INLINE void gc::ConcurrentMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
-    threadData_.gcScheduler().OnSafePointRegular(weight);
-    mm::SuspendIfRequested();
-}
-
-gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep() noexcept :
-    finalizerProcessor_(make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch);})) {
-    mm::GlobalData::Instance().gcScheduler().SetScheduleGC([this]() NO_EXTERNAL_CALLS_CHECK NO_INLINE {
+gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
+        mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
+    objectFactory_(objectFactory),
+    gcScheduler_(gcScheduler),
+    finalizerProcessor_(make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch); })) {
+    gcScheduler_.SetScheduleGC([this]() NO_EXTERNAL_CALLS_CHECK NO_INLINE {
         RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
         state_.schedule();
     });
@@ -115,7 +105,6 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep() noexcept :
     });
 }
 
-
 gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() {
     state_.shutdown();
     gcThread_.join();
@@ -134,7 +123,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     auto timeSuspendUs = konan::getTimeMicros();
     RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
 
-    auto& scheduler = mm::GlobalData::Instance().gcScheduler();
+    auto& scheduler = gcScheduler_;
     scheduler.gcData().OnPerformFullGC();
 
     state_.start(epoch);
@@ -144,7 +133,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     auto timeRootSetUs = konan::getTimeMicros();
     // Can be unsafe, because we've stopped the world.
 
-    auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+    auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
     RuntimeLogInfo(
             {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(),
             timeRootSetUs - timeSuspendUs);
@@ -156,7 +145,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     auto timeSweepExtraObjectsUs = konan::getTimeMicros();
     RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
 
-    auto objectFactoryIterable = mm::GlobalData::Instance().objectFactory().LockForIter();
+    auto objectFactoryIterable = objectFactory_.LockForIter();
 
     mm::ResumeThreads();
     auto timeResumeUs = konan::getTimeMicros();
@@ -170,7 +159,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs);
 
     // Can be unsafe, because we have a lock in objectFactoryIterable
-    auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+    auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
     auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
 
     auto finalizersCount = finalizerQueue.size();

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -49,12 +49,10 @@ public:
         using ObjectData = ConcurrentMarkAndSweep::ObjectData;
         using Allocator = AllocatorWithGC<AlignedAllocator, ThreadData>;
 
-        explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
+        explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept :
+            gc_(gc), gcScheduler_(gcScheduler) {}
         ~ThreadData() = default;
 
-        void SafePointFunctionPrologue() noexcept;
-        void SafePointLoopBody() noexcept;
-        void SafePointExceptionUnwind() noexcept;
         void SafePointAllocation(size_t size) noexcept;
 
         void ScheduleAndWaitFullGC() noexcept;
@@ -66,21 +64,22 @@ public:
         Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); }
 
     private:
-        void SafePointRegular(size_t weight) noexcept;
-
         ConcurrentMarkAndSweep& gc_;
-        mm::ThreadData& threadData_;
+        GCSchedulerThreadData& gcScheduler_;
     };
 
     using Allocator = ThreadData::Allocator;
 
-    ConcurrentMarkAndSweep() noexcept;
+    ConcurrentMarkAndSweep(mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, GCScheduler& scheduler) noexcept;
     ~ConcurrentMarkAndSweep();
 
 private:
     // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
     bool PerformFullGC(int64_t epoch) noexcept;
 
+    mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
+    GCScheduler& gcScheduler_;
+
     uint64_t lastGCTimestampUs_ = 0;
     GCStateHolder state_;
     std::thread gcThread_;

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -15,6 +15,7 @@
 
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "GCImpl.hpp"
 #include "GlobalData.hpp"
 #include "ObjectOps.hpp"
 #include "ObjectTestSupport.hpp"
@@ -187,10 +188,10 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
 
 KStdVector<ObjHeader*> Alive(mm::ThreadData& threadData) {
     KStdVector<ObjHeader*> objects;
-    for (auto node : threadData.objectFactoryThreadQueue()) {
+    for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
         objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader());
     }
-    for (auto node : mm::GlobalData::Instance().objectFactory().LockForIter()) {
+    for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) {
         objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader());
     }
     return objects;
@@ -219,7 +220,7 @@ public:
     ~ConcurrentMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-        mm::GlobalData::Instance().objectFactory().ClearForTests();
+        mm::GlobalData::Instance().gc().ClearForTests();
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }
@@ -338,7 +339,7 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
         EXPECT_CALL(finalizerHook(), Call(object1.header()));
         EXPECT_CALL(finalizerHook(), Call(object2.header()));
         threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
-        threadData.gc().StopFinalizerThreadForTests();
+        threadData.gc().impl().gc().StopFinalizerThreadForTests();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
     });
@@ -358,7 +359,7 @@ TEST_F(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
         ASSERT_THAT(weak1->referred, object1.header());
 
         threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
-        threadData.gc().StopFinalizerThreadForTests();
+        threadData.gc().impl().gc().StopFinalizerThreadForTests();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre());
     });
@@ -509,7 +510,7 @@ TEST_F(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
         EXPECT_CALL(finalizerHook(), Call(object5.header()));
         EXPECT_CALL(finalizerHook(), Call(object6.header()));
         threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
-        threadData.gc().StopFinalizerThreadForTests();
+        threadData.gc().impl().gc().StopFinalizerThreadForTests();
 
         EXPECT_THAT(
                 Alive(threadData),

kotlin-native/runtime/src/gc/cms/cpp/FinalizerProcessorTest.cpp

@@ -15,6 +15,7 @@
 
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "GCImpl.hpp"
 #include "GlobalData.hpp"
 #include "ObjectOps.hpp"
 #include "ObjectTestSupport.hpp"
@@ -55,7 +56,7 @@ public:
    ~FinalizerProcessorTest() {
        mm::GlobalsRegistry::Instance().ClearForTests();
        mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-       mm::GlobalData::Instance().objectFactory().ClearForTests();
+       mm::GlobalData::Instance().gc().ClearForTests();
    }
 
    testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }
@@ -94,7 +95,7 @@ TEST_F(FinalizerProcessorTest, RemoveObject) {
         mm::ObjectFactory<kotlin::gc::ConcurrentMarkAndSweep>::FinalizerQueue queue;
         auto &object = AllocateObjectWithFinalizer(*mm::ThreadRegistry::Instance().CurrentThreadData());
         mm::ThreadRegistry::Instance().CurrentThreadData()->Publish();
-        auto &factory = mm::GlobalData::Instance().objectFactory();
+        auto& factory = mm::GlobalData::Instance().gc().impl().objectFactory();
         auto iter = factory.LockForIter();
         auto iterator = iter.begin();
         iter.MoveAndAdvance(queue, iterator);
@@ -121,7 +122,7 @@ TEST_F(FinalizerProcessorTest, ScheduleTasksWhileFinalizing) {
                 auto& object = AllocateObjectWithFinalizer(*mm::ThreadRegistry::Instance().CurrentThreadData());
                 headers.push_back(object.header());
             }
-            auto& factory = mm::GlobalData::Instance().objectFactory();
+            auto& factory = mm::GlobalData::Instance().gc().impl().objectFactory();
             mm::ThreadRegistry::Instance().CurrentThreadData()->Publish();
             auto iter = factory.LockForIter();
             mm::ObjectFactory<kotlin::gc::ConcurrentMarkAndSweep>::FinalizerQueue queue;

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCImpl.hpp"
+
+#include "GC.hpp"
+#include "ThreadSuspension.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept {
+    threadData.impl().gcScheduler().OnSafePointRegular(weight);
+    mm::SuspendIfRequested();
+}
+
+} // namespace
+
+gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique<Impl>(gc, threadData)) {}
+
+gc::GC::ThreadData::~ThreadData() = default;
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept {
+    SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight);
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept {
+    SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight);
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
+    impl_->gc().ScheduleAndWaitFullGC();
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
+    impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
+}
+
+void gc::GC::ThreadData::Publish() noexcept {
+    impl_->objectFactoryThreadQueue().Publish();
+}
+
+void gc::GC::ThreadData::ClearForTests() noexcept {
+    impl_->objectFactoryThreadQueue().ClearForTests();
+}
+
+ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+}
+
+ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+}
+
+void gc::GC::ThreadData::OnStoppedForGC() noexcept {
+    impl_->gcScheduler().OnStoppedForGC();
+}
+
+gc::GC::GC() noexcept : impl_(make_unique<Impl>()) {}
+
+gc::GC::~GC() = default;
+
+// static
+size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
+    return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
+}
+
+gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
+    return impl_->gcScheduler().config();
+}
+
+void gc::GC::ClearForTests() noexcept {
+    impl_->objectFactory().ClearForTests();
+}

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "GC.hpp"
+
+#include "ConcurrentMarkAndSweep.hpp"
+
+namespace kotlin {
+namespace gc {
+
+using GCImpl = ConcurrentMarkAndSweep;
+
+class GC::Impl : private Pinned {
+public:
+    Impl() noexcept : gc_(objectFactory_, gcScheduler_) {}
+
+    mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl& gc() noexcept { return gc_; }
+
+private:
+    mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    GCScheduler gcScheduler_;
+    GCImpl gc_;
+};
+
+class GC::ThreadData::Impl : private Pinned {
+public:
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
+        gcScheduler_(gc.impl_->gcScheduler().NewThreadData()),
+        gc_(gc.impl_->gc(), threadData, gcScheduler_),
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {}
+
+    GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl::ThreadData& gc() noexcept { return gc_; }
+    mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+
+private:
+    GCSchedulerThreadData gcScheduler_;
+    GCImpl::ThreadData gc_;
+    mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+};
+
+} // namespace gc
+} // namespace kotlin

kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCTestSupport.hpp"
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+
+#include "GCImpl.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+template <typename T>
+auto collectCopy(T& iterable) {
+    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
+    for (const auto& element : iterable) {
+        result.push_back(element);
+    }
+    return result;
+}
+
+} // namespace
+
+void gc::AssertClear(GC& gc) noexcept {
+    auto objects = gc.impl().objectFactory().LockForIter();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+}

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "GCScheduler.hpp"
+#include "Memory.h"
+#include "Types.h"
+#include "Utils.hpp"
+
+namespace kotlin {
+
+namespace mm {
+class ThreadData;
+}
+
+namespace gc {
+
+class GC : private Pinned {
+public:
+    class Impl;
+
+    class ThreadData : private Pinned {
+    public:
+        class Impl;
+
+        ThreadData(GC& gc, mm::ThreadData& threadData) noexcept;
+        ~ThreadData();
+
+        Impl& impl() noexcept { return *impl_; }
+
+        void SafePointFunctionPrologue() noexcept;
+        void SafePointLoopBody() noexcept;
+
+        void ScheduleAndWaitFullGC() noexcept;
+        void ScheduleAndWaitFullGCWithFinalizers() noexcept;
+
+        void Publish() noexcept;
+        void ClearForTests() noexcept;
+
+        ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
+        ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept;
+
+        void OnStoppedForGC() noexcept;
+
+    private:
+        KStdUniquePtr<Impl> impl_;
+    };
+
+    GC() noexcept;
+    ~GC();
+
+    Impl& impl() noexcept { return *impl_; }
+
+    static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
+
+    gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
+
+    void ClearForTests() noexcept;
+
+private:
+    KStdUniquePtr<Impl> impl_;
+};
+
+inline constexpr bool kSupportsMultipleMutators = true;
+
+} // namespace gc
+} // namespace kotlin

kotlin-native/runtime/src/gc/common/cpp/GCTestSupport.hpp

@@ -5,15 +5,12 @@
 
 #pragma once
 
-#include "ConcurrentMarkAndSweep.hpp"
+#include "GC.hpp"
 
 namespace kotlin {
 namespace gc {
 
-using GC = kotlin::gc::ConcurrentMarkAndSweep;
+void AssertClear(GC& gc) noexcept;
 
-inline constexpr bool kSupportsMultipleMutators = true;
-
-} // namespace gc
+}
 } // namespace kotlin
-

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp

@@ -18,7 +18,7 @@ KStdVector<ObjHeader*> kotlin::gc::collectRootSet() {
     for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
         // TODO: Maybe it's more efficient to do by the suspending thread?
         thread.Publish();
-        thread.gcScheduler().OnStoppedForGC();
+        thread.gc().OnStoppedForGC();
         size_t stack = 0;
         size_t tls = 0;
         for (auto value : mm::ThreadRootSet(thread)) {

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -9,6 +9,7 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "Allocator.hpp"
 #include "FinalizerHooksTestSupport.hpp"
 #include "ObjectFactory.hpp"
 #include "ObjectTestSupport.hpp"

kotlin-native/runtime/src/gc/noop/cpp/GC.hpp

@@ -1,21 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#ifndef RUNTIME_GC_NOOP_GC_H
-#define RUNTIME_GC_NOOP_GC_H
-
-#include "NoOpGC.hpp"
-
-namespace kotlin {
-namespace gc {
-
-using GC = kotlin::gc::NoOpGC;
-
-inline constexpr bool kSupportsMultipleMutators = true;
-
-} // namespace gc
-} // namespace kotlin
-
-#endif // RUNTIME_GC_NOOP_GC_H

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCImpl.hpp"
+
+#include "GC.hpp"
+
+using namespace kotlin;
+
+gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique<Impl>(gc, threadData)) {}
+
+gc::GC::ThreadData::~ThreadData() = default;
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept {
+    impl_->gc().SafePointFunctionPrologue();
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept {
+    impl_->gc().SafePointLoopBody();
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
+    impl_->gc().ScheduleAndWaitFullGC();
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
+    impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
+}
+
+void gc::GC::ThreadData::Publish() noexcept {
+    impl_->objectFactoryThreadQueue().Publish();
+}
+
+void gc::GC::ThreadData::ClearForTests() noexcept {
+    impl_->objectFactoryThreadQueue().ClearForTests();
+}
+
+ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+}
+
+ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+}
+
+void gc::GC::ThreadData::OnStoppedForGC() noexcept {
+    impl_->gcScheduler().OnStoppedForGC();
+}
+
+gc::GC::GC() noexcept : impl_(make_unique<Impl>()) {}
+
+gc::GC::~GC() = default;
+
+// static
+size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
+    return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
+}
+
+gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
+    return impl_->gcScheduler().config();
+}
+
+void gc::GC::ClearForTests() noexcept {
+    impl_->objectFactory().ClearForTests();
+}

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "GC.hpp"
+
+#include "NoOpGC.hpp"
+
+namespace kotlin {
+namespace gc {
+
+using GCImpl = NoOpGC;
+
+class GC::Impl : private Pinned {
+public:
+    Impl() noexcept : gc_(objectFactory_, gcScheduler_) {}
+
+    mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl& gc() noexcept { return gc_; }
+
+private:
+    mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    GCScheduler gcScheduler_;
+    GCImpl gc_;
+};
+
+class GC::ThreadData::Impl : private Pinned {
+public:
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
+        gcScheduler_(gc.impl_->gcScheduler().NewThreadData()),
+        gc_(gc.impl_->gc(), threadData, gcScheduler_),
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {}
+
+    GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl::ThreadData& gc() noexcept { return gc_; }
+    mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+
+private:
+    GCSchedulerThreadData gcScheduler_;
+    GCImpl::ThreadData gc_;
+    mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+};
+
+} // namespace gc
+} // namespace kotlin

kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCTestSupport.hpp"
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+
+#include "GCImpl.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+template <typename T>
+auto collectCopy(T& iterable) {
+    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
+    for (const auto& element : iterable) {
+        result.push_back(element);
+    }
+    return result;
+}
+
+} // namespace
+
+void gc::AssertClear(GC& gc) noexcept {
+    auto objects = gc.impl().objectFactory().LockForIter();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+}

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp

@@ -10,6 +10,7 @@
 
 #include "Allocator.hpp"
 #include "GCScheduler.hpp"
+#include "ObjectFactory.hpp"
 #include "Utils.hpp"
 #include "Types.h"
 
@@ -33,7 +34,7 @@ public:
     public:
         using ObjectData = NoOpGC::ObjectData;
 
-        explicit ThreadData(NoOpGC& gc, mm::ThreadData& threadData) noexcept {}
+        ThreadData(NoOpGC& gc, mm::ThreadData& threadData, GCSchedulerThreadData&) noexcept {}
         ~ThreadData() = default;
 
         void SafePointFunctionPrologue() noexcept {}
@@ -50,7 +51,7 @@ public:
     private:
     };
 
-    NoOpGC() noexcept {}
+    NoOpGC(mm::ObjectFactory<NoOpGC>&, GCScheduler&) noexcept {}
     ~NoOpGC() = default;
 
     GCScheduler& scheduler() noexcept { return scheduler_; }

kotlin-native/runtime/src/gc/stms/cpp/GC.hpp

@@ -1,21 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#ifndef RUNTIME_GC_STMS_GC_H
-#define RUNTIME_GC_STMS_GC_H
-
-#include "SameThreadMarkAndSweep.hpp"
-
-namespace kotlin {
-namespace gc {
-
-using GC = kotlin::gc::SameThreadMarkAndSweep;
-
-inline constexpr bool kSupportsMultipleMutators = true;
-
-} // namespace gc
-} // namespace kotlin
-
-#endif // RUNTIME_GC_STMS_GC_H

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCImpl.hpp"
+
+#include "GC.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+ALWAYS_INLINE void SafePointRegular(gc::GC::ThreadData& threadData, size_t weight) noexcept {
+    threadData.impl().gcScheduler().OnSafePointRegular(weight);
+    auto flag = gc::internal::loadSafepointFlag();
+    if (flag != gc::SameThreadMarkAndSweep::SafepointFlag::kNone) {
+        threadData.impl().gc().SafePointSlowPath(flag);
+    }
+}
+
+} // namespace
+
+gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : impl_(make_unique<Impl>(gc, threadData)) {}
+
+gc::GC::ThreadData::~ThreadData() = default;
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointFunctionPrologue() noexcept {
+    SafePointRegular(*this, GCSchedulerThreadData::kFunctionPrologueWeight);
+}
+
+ALWAYS_INLINE void gc::GC::ThreadData::SafePointLoopBody() noexcept {
+    SafePointRegular(*this, GCSchedulerThreadData::kLoopBodyWeight);
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGC() noexcept {
+    impl_->gc().ScheduleAndWaitFullGC();
+}
+
+void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
+    impl_->gc().ScheduleAndWaitFullGCWithFinalizers();
+}
+
+void gc::GC::ThreadData::Publish() noexcept {
+    impl_->objectFactoryThreadQueue().Publish();
+}
+
+void gc::GC::ThreadData::ClearForTests() noexcept {
+    impl_->objectFactoryThreadQueue().ClearForTests();
+}
+
+ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+}
+
+ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+}
+
+void gc::GC::ThreadData::OnStoppedForGC() noexcept {
+    impl_->gcScheduler().OnStoppedForGC();
+}
+
+gc::GC::GC() noexcept : impl_(make_unique<Impl>()) {}
+
+gc::GC::~GC() = default;
+
+// static
+size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
+    return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
+}
+
+gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
+    return impl_->gcScheduler().config();
+}
+
+void gc::GC::ClearForTests() noexcept {
+    impl_->objectFactory().ClearForTests();
+}

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "GC.hpp"
+
+#include "SameThreadMarkAndSweep.hpp"
+
+namespace kotlin {
+namespace gc {
+
+using GCImpl = SameThreadMarkAndSweep;
+
+class GC::Impl : private Pinned {
+public:
+    Impl() noexcept : gc_(objectFactory_, gcScheduler_) {}
+
+    mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
+    GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl& gc() noexcept { return gc_; }
+
+private:
+    mm::ObjectFactory<gc::GCImpl> objectFactory_;
+    GCScheduler gcScheduler_;
+    GCImpl gc_;
+};
+
+class GC::ThreadData::Impl : private Pinned {
+public:
+    Impl(GC& gc, mm::ThreadData& threadData) noexcept :
+        gcScheduler_(gc.impl_->gcScheduler().NewThreadData()),
+        gc_(gc.impl_->gc(), threadData, gcScheduler_),
+        objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_) {}
+
+    GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
+    GCImpl::ThreadData& gc() noexcept { return gc_; }
+    mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+
+private:
+    GCSchedulerThreadData gcScheduler_;
+    GCImpl::ThreadData gc_;
+    mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
+};
+
+} // namespace gc
+} // namespace kotlin

kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GCTestSupport.hpp"
+
+#include "gtest/gtest.h"
+#include "gmock/gmock.h"
+
+#include "GCImpl.hpp"
+
+using namespace kotlin;
+
+namespace {
+
+template <typename T>
+auto collectCopy(T& iterable) {
+    std::vector<std::remove_reference_t<decltype(*iterable.begin())>> result;
+    for (const auto& element : iterable) {
+        result.push_back(element);
+    }
+    return result;
+}
+
+} // namespace
+
+void gc::AssertClear(GC& gc) noexcept {
+    auto objects = gc.impl().objectFactory().LockForIter();
+    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
+}

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -64,16 +64,8 @@ std::atomic<gc::SameThreadMarkAndSweep::SafepointFlag> gSafepointFlag = gc::Same
 
 } // namespace
 
-ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept {
-    SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight);
-}
-
-ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
-    SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight);
-}
-
 void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
-    threadData_.gcScheduler().OnSafePointAllocation(size);
+    gcScheduler_.OnSafePointAllocation(size);
     SafepointFlag flag = gSafepointFlag.load();
     if (flag != SafepointFlag::kNone) {
         SafePointSlowPath(flag);
@@ -85,7 +77,7 @@ void gc::SameThreadMarkAndSweep::ThreadData::ScheduleAndWaitFullGC() noexcept {
 
     if (!didGC) {
         // If we failed to suspend threads, someone else might be asking to suspend them.
-        threadData_.suspensionData().suspendIfRequested();
+        mm::SuspendIfRequested();
     }
 }
 
@@ -94,21 +86,13 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
     ScheduleAndWaitFullGC();
 }
 
-ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept {
-    threadData_.gcScheduler().OnSafePointRegular(weight);
-    SafepointFlag flag = gSafepointFlag.load();
-    if (flag != SafepointFlag::kNone) {
-        SafePointSlowPath(flag);
-    }
-}
-
 NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept {
     switch (flag) {
         case SafepointFlag::kNone:
             RuntimeAssert(false, "Must've been handled by the caller");
             return;
         case SafepointFlag::kNeedsSuspend:
-            threadData_.suspensionData().suspendIfRequested();
+            mm::SuspendIfRequested();
             return;
         case SafepointFlag::kNeedsGC:
             RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint");
@@ -117,8 +101,10 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi
     }
 }
 
-gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept {
-    mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() {
+gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
+        mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
+    objectFactory_(objectFactory), gcScheduler_(gcScheduler) {
+    gcScheduler_.SetScheduleGC([]() {
         RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
         gSafepointFlag = SafepointFlag::kNeedsGC;
     });
@@ -146,7 +132,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         auto timeSuspendUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
 
-        auto& scheduler = mm::GlobalData::Instance().gcScheduler();
+        auto& scheduler = gcScheduler_;
         scheduler.gcData().OnPerformFullGC();
 
         RuntimeLogInfo(
@@ -154,7 +140,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         auto graySet = collectRootSet();
         auto timeRootSetUs = konan::getTimeMicros();
         // Can be unsafe, because we've stopped the world.
-        auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+        auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
 
         RuntimeLogInfo(
                 {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet.size(),
@@ -166,12 +152,12 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
         auto timeSweepExtraObjectsUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
-        finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
+        finalizerQueue = gc::Sweep<SweepTraits>(objectFactory_);
         auto timeSweepUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs);
 
         // Can be unsafe, because we've stopped the world.
-        auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+        auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
         auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
 
         gSafepointFlag = SafepointFlag::kNone;
@@ -206,3 +192,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
     return true;
 }
+
+gc::SameThreadMarkAndSweep::SafepointFlag gc::internal::loadSafepointFlag() noexcept {
+    return gSafepointFlag.load();
+}

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -51,12 +51,11 @@ public:
         using ObjectData = SameThreadMarkAndSweep::ObjectData;
         using Allocator = AllocatorWithGC<AlignedAllocator, ThreadData>;
 
-        explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
+        ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept :
+            gc_(gc), gcScheduler_(gcScheduler) {}
         ~ThreadData() = default;
 
-        void SafePointFunctionPrologue() noexcept;
-        void SafePointLoopBody() noexcept;
-        void SafePointExceptionUnwind() noexcept;
+        void SafePointSlowPath(SafepointFlag flag) noexcept;
         void SafePointAllocation(size_t size) noexcept;
 
         void ScheduleAndWaitFullGC() noexcept;
@@ -67,16 +66,14 @@ public:
         Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); }
 
     private:
-        void SafePointRegular(size_t weight) noexcept;
-        void SafePointSlowPath(SafepointFlag flag) noexcept;
 
         SameThreadMarkAndSweep& gc_;
-        mm::ThreadData& threadData_;
+        GCSchedulerThreadData& gcScheduler_;
     };
 
     using Allocator = ThreadData::Allocator;
 
-    SameThreadMarkAndSweep() noexcept;
+    SameThreadMarkAndSweep(mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept;
     ~SameThreadMarkAndSweep() = default;
     void StopFinalizerThreadForTests() noexcept {}
 
@@ -86,8 +83,17 @@ private:
 
     size_t epoch_ = 0;
     uint64_t lastGCTimestampUs_ = 0;
+
+    mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
+    GCScheduler& gcScheduler_;
 };
 
+namespace internal {
+
+SameThreadMarkAndSweep::SafepointFlag loadSafepointFlag() noexcept;
+
+} // namespace internal
+
 } // namespace gc
 } // namespace kotlin
 

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -15,6 +15,7 @@
 
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "GCImpl.hpp"
 #include "GlobalData.hpp"
 #include "ObjectOps.hpp"
 #include "ObjectTestSupport.hpp"
@@ -187,10 +188,10 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
 
 KStdVector<ObjHeader*> Alive(mm::ThreadData& threadData) {
     KStdVector<ObjHeader*> objects;
-    for (auto node : threadData.objectFactoryThreadQueue()) {
+    for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
         objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader());
     }
-    for (auto node : mm::GlobalData::Instance().objectFactory().LockForIter()) {
+    for (auto node : mm::GlobalData::Instance().gc().impl().objectFactory().LockForIter()) {
         objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader());
     }
     return objects;
@@ -215,11 +216,10 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader
 
 class SameThreadMarkAndSweepTest : public testing::Test {
 public:
-
     ~SameThreadMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-        mm::GlobalData::Instance().objectFactory().ClearForTests();
+        mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests();
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -30,7 +30,7 @@ public:
     ~ExtraObjectDataTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
-        mm::GlobalData::Instance().objectFactory().ClearForTests();
+        mm::GlobalData::Instance().gc().ClearForTests();
     }
 };
 

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -27,8 +27,6 @@ public:
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
     StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; }
     ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
-    ObjectFactory<gc::GC>& objectFactory() noexcept { return objectFactory_; }
-    gc::GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
     gc::GC& gc() noexcept { return gc_; }
 
 private:
@@ -42,8 +40,6 @@ private:
     GlobalsRegistry globalsRegistry_;
     StableRefRegistry stableRefRegistry_;
     ExtraObjectDataFactory extraObjectDataFactory_;
-    ObjectFactory<gc::GC> objectFactory_;
-    gc::GCScheduler gcScheduler_;
     gc::GC gc_;
 };
 

kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp

@@ -35,7 +35,7 @@ public:
 
     ~InitSingletonTest() {
         globalConstructor_ = nullptr;
-        mm::GlobalData::Instance().objectFactory().ClearForTests();
+        mm::GlobalData::Instance().gc().ClearForTests();
         mm::GlobalData::Instance().globalsRegistry().ClearForTests();
     }
 

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -324,11 +324,11 @@ extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, int32_t value
     if (value < 0) {
         ThrowIllegalArgumentException();
     }
-    mm::GlobalData::Instance().gcScheduler().config().threshold = static_cast<size_t>(value);
+    mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold = static_cast<size_t>(value);
 }
 
 extern "C" int32_t Kotlin_native_internal_GC_getThreshold(ObjHeader*) {
-    auto threshold = mm::GlobalData::Instance().gcScheduler().config().threshold.load();
+    auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().threshold.load();
     auto maxValue = std::numeric_limits<int32_t>::max();
     if (threshold > static_cast<size_t>(maxValue)) {
         return maxValue;
@@ -350,11 +350,11 @@ extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, in
     if (value < 0) {
         ThrowIllegalArgumentException();
     }
-    mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = static_cast<size_t>(value);
+    mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes = static_cast<size_t>(value);
 }
 
 extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) {
-    auto threshold = mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load();
+    auto threshold = mm::GlobalData::Instance().gc().gcSchedulerConfig().allocationThresholdBytes.load();
     auto maxValue = std::numeric_limits<int64_t>::max();
     if (threshold > static_cast<size_t>(maxValue)) {
         return maxValue;
@@ -363,11 +363,11 @@ extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*)
 }
 
 extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) {
-    mm::GlobalData::Instance().gcScheduler().config().autoTune = value;
+    mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune = value;
 }
 
 extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) {
-    return mm::GlobalData::Instance().gcScheduler().config().autoTune.load();
+    return mm::GlobalData::Instance().gc().gcSchedulerConfig().autoTune.load();
 }
 
 extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) {

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -691,6 +691,16 @@ public:
 
     void ClearForTests() { storage_.ClearForTests(); }
 
+    static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept {
+        RuntimeAssert(object->heap(), "Object must be a heap object");
+        const auto* typeInfo = object->type_info();
+        if (typeInfo->IsArray()) {
+            return ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_);
+        } else {
+            return ThreadQueue::ObjectAllocatedSize(typeInfo);
+        }
+    }
+
 private:
     Storage storage_;
 };

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -58,24 +58,18 @@ ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHea
 OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept {
     AssertThreadState(threadData, ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
-    auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo);
+    auto* object = threadData->gc().CreateObject(typeInfo);
     RETURN_OBJ(object);
 }
 
 OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept {
     AssertThreadState(threadData, ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.
-    auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast<uint32_t>(elements));
+    auto* array = threadData->gc().CreateArray(typeInfo, static_cast<uint32_t>(elements));
     // `ArrayHeader` and `ObjHeader` are expected to be compatible.
     RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
 }
 
 size_t mm::GetAllocatedHeapSize(ObjHeader* object) noexcept {
-    RuntimeAssert(object->heap(), "Object must be a heap object");
-    const auto* typeInfo = object->type_info();
-    if (typeInfo->IsArray()) {
-        return mm::ObjectFactory<gc::GC>::ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_);
-    } else {
-        return mm::ObjectFactory<gc::GC>::ThreadQueue::ObjectAllocatedSize(typeInfo);
-    }
+    return gc::GC::GetAllocatedHeapSize(object);
 }

kotlin-native/runtime/src/mm/cpp/TestSupport.cpp

@@ -7,6 +7,7 @@
 #include "gmock/gmock.h"
 
 #include "GC.hpp"
+#include "GCTestSupport.hpp"
 #include "GlobalData.hpp"
 #include "GlobalsRegistry.hpp"
 #include "TestSupport.hpp"
@@ -45,15 +46,14 @@ extern "C" void Kotlin_TestSupport_AssertClearGlobalState() {
     // Validate that global registries are empty.
     auto globals = mm::GlobalsRegistry::Instance().LockForIter();
     auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
-    auto objects = mm::GlobalData::Instance().objectFactory().LockForIter();
     auto stableRefs = mm::StableRefRegistry::Instance().LockForIter();
     auto threads = mm::ThreadRegistry::Instance().LockForIter();
 
     EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
-    EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
     EXPECT_THAT(collectCopy(stableRefs), testing::UnorderedElementsAre());
     EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre());
+    gc::AssertClear(mm::GlobalData::Instance().gc());
 }
 
 void kotlin::DeinitMemoryForTests(MemoryState* memoryState) {
@@ -63,4 +63,4 @@ void kotlin::DeinitMemoryForTests(MemoryState* memoryState) {
 
 std::ostream& kotlin::operator<<(std::ostream& stream, ThreadState state) {
     return stream << ThreadStateName(state);
-}
+}

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -35,9 +35,7 @@ public:
         globalsThreadQueue_(GlobalsRegistry::Instance()),
         stableRefThreadQueue_(StableRefRegistry::Instance()),
         extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()),
-        gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()),
         gc_(GlobalData::Instance().gc(), *this),
-        objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_),
         suspensionData_(ThreadState::kNative) {}
 
     ~ThreadData() = default;
@@ -56,14 +54,10 @@ public:
 
     ThreadState setState(ThreadState state) noexcept { return suspensionData_.setState(state); }
 
-    ObjectFactory<gc::GC>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
-
     ShadowStack& shadowStack() noexcept { return shadowStack_; }
 
     KStdVector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
 
-    gc::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; }
-
     gc::GC::ThreadData& gc() noexcept { return gc_; }
 
     ThreadSuspensionData& suspensionData() { return suspensionData_; }
@@ -72,15 +66,15 @@ public:
         // TODO: These use separate locks, which is inefficient.
         globalsThreadQueue_.Publish();
         stableRefThreadQueue_.Publish();
-        objectFactoryThreadQueue_.Publish();
         extraObjectDataThreadQueue_.Publish();
+        gc_.Publish();
     }
 
     void ClearForTests() noexcept {
         globalsThreadQueue_.ClearForTests();
         stableRefThreadQueue_.ClearForTests();
-        objectFactoryThreadQueue_.ClearForTests();
         extraObjectDataThreadQueue_.ClearForTests();
+        gc_.ClearForTests();
     }
 
 private:
@@ -90,9 +84,7 @@ private:
     StableRefRegistry::ThreadQueue stableRefThreadQueue_;
     ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_;
     ShadowStack shadowStack_;
-    gc::GCSchedulerThreadData gcScheduler_;
     gc::GC::ThreadData gc_;
-    ObjectFactory<gc::GC>::ThreadQueue objectFactoryThreadQueue_;
     KStdVector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
     ThreadSuspensionData suspensionData_;
 };