Put GC implementations into separate modules.

The mm module now compiles separately for each GC implementation.

kotlin-native/runtime/build.gradle.kts

@@ -41,10 +41,13 @@ bitcode {
             "${target}Objc",
             "${target}ExceptionsSupport",
             "${target}LegacyMemoryManager",
-            "${target}ExperimentalMemoryManager"
+            "${target}ExperimentalMemoryManagerNoop",
+            "${target}ExperimentalMemoryManagerStms",
+            "${target}CommonGc",
+            "${target}SingleThreadMsGc",
+            "${target}NoopGc"
         )
         includeRuntime()
-        // TODO: Should depend on the sanitizer.
     }
 
     create("mimalloc") {
@@ -103,7 +106,28 @@ bitcode {
         includeRuntime()
     }
 
-    create("experimental_memory_manager", file("src/mm")) {
+    create("experimental_memory_manager_noop", file("src/mm")) {
+        headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp")
+        includeRuntime()
+    }
+
+    create("experimental_memory_manager_stms", file("src/mm")) {
+        headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp")
+        includeRuntime()
+    }
+
+    create("common_gc", file("src/gc/common")) {
+        headersDirs += files("src/mm/cpp")
+        includeRuntime()
+    }
+
+    create("noop_gc", file("src/gc/noop")) {
+        headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp")
+        includeRuntime()
+    }
+
+    create("single_thread_ms_gc", file("src/gc/stms")) {
+        headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp", "src/mm/cpp")
         includeRuntime()
     }
 }
@@ -148,12 +172,15 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMMimallocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManager",
+                "${targetName}ExperimentalMemoryManagerStms",
+                "${targetName}CommonGc",
+                "${targetName}SingleThreadMsGc",
                 "${targetName}Release",
                 "${targetName}Mimalloc",
                 "${targetName}OptAlloc"
             )
     ) {
+        headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp", "src/mm/cpp")
         includeRuntime()
     })
 
@@ -163,11 +190,49 @@ targetList.forEach { targetName ->
             "${targetName}ExperimentalMMStdAllocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
-                "${targetName}ExperimentalMemoryManager",
+                "${targetName}ExperimentalMemoryManagerStms",
+                "${targetName}CommonGc",
+                "${targetName}SingleThreadMsGc",
                 "${targetName}Release",
                 "${targetName}StdAlloc"
             )
     ) {
+        headersDirs += files("src/gc/stms/cpp", "src/gc/common/cpp", "src/mm/cpp")
+        includeRuntime()
+    })
+
+    allTests.addAll(createTestTasks(
+            project,
+            targetName,
+            "${targetName}ExperimentalMMNoOpMimallocRuntimeTests",
+            listOf(
+                "${targetName}Runtime",
+                "${targetName}ExperimentalMemoryManagerNoop",
+                "${targetName}CommonGc",
+                "${targetName}NoopGc",
+                "${targetName}Release",
+                "${targetName}Mimalloc",
+                "${targetName}OptAlloc"
+            )
+    ) {
+        headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp")
+        includeRuntime()
+    })
+
+    allTests.addAll(createTestTasks(
+            project,
+            targetName,
+            "${targetName}ExperimentalMMNoOpStdAllocRuntimeTests",
+            listOf(
+                "${targetName}Runtime",
+                "${targetName}ExperimentalMemoryManagerNoop",
+                "${targetName}CommonGc",
+                "${targetName}NoopGc",
+                "${targetName}Release",
+                "${targetName}StdAlloc"
+            )
+    ) {
+        headersDirs += files("src/gc/noop/cpp", "src/gc/common/cpp", "src/mm/cpp")
         includeRuntime()
     })
 

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp

@@ -0,0 +1,6 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "MarkAndSweepUtils.hpp"

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -3,10 +3,10 @@
  * that can be found in the LICENSE file.
  */
 
-#ifndef RUNTIME_MM_MARK_AND_SWEEP_UTILS_H
-#define RUNTIME_MM_MARK_AND_SWEEP_UTILS_H
+#ifndef RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
+#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
 
-#include "../ExtraObjectData.hpp"
+#include "ExtraObjectData.hpp"
 #include "FinalizerHooks.hpp"
 #include "Memory.h"
 #include "ObjectTraversal.hpp"
@@ -14,7 +14,7 @@
 #include "Types.h"
 
 namespace kotlin {
-namespace mm {
+namespace gc {
 
 // TODO: Because of `graySet` this implementation may allocate heap memory during GC.
 template <typename Traits>
@@ -73,7 +73,7 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact
     return finalizerQueue;
 }
 
-} // namespace mm
+} // namespace gc
 } // namespace kotlin
 
-#endif // RUNTIME_MM_MARK_AND_SWEEP_UTILS_H
+#endif // RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -138,7 +138,7 @@ public:
     void Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
         KStdVector<ObjHeader*> objects;
         for (auto& object : graySet) objects.push_back(object.get().GetObjHeader());
-        mm::Mark<ScopedMarkTraits>(std::move(objects));
+        gc::Mark<ScopedMarkTraits>(std::move(objects));
     }
 
 private:

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -8,8 +8,8 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
-#include "../ObjectFactory.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "ObjectFactory.hpp"
 #include "ObjectTestSupport.hpp"
 
 using namespace kotlin;
@@ -184,7 +184,7 @@ public:
     }
 
     KStdVector<ObjHeader*> Sweep() {
-        auto finalizers = mm::Sweep<SweepTraits>(objectFactory_);
+        auto finalizers = gc::Sweep<SweepTraits>(objectFactory_);
         KStdVector<ObjHeader*> objects;
         for (auto node : finalizers.IterForTests()) {
             objects.push_back(node.IsArray() ? node.GetArrayHeader()->obj() : node.GetObjHeader());

kotlin-native/runtime/src/gc/noop/cpp/GC.hpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_GC_NOOP_GC_H
+#define RUNTIME_GC_NOOP_GC_H
+
+#include "NoOpGC.hpp"
+
+namespace kotlin {
+namespace gc {
+
+using GC = kotlin::gc::NoOpGC;
+
+} // namespace gc
+} // namespace kotlin
+
+#endif // RUNTIME_GC_NOOP_GC_H

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.cpp

@@ -0,0 +1,6 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "NoOpGC.hpp"

kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp

@@ -3,15 +3,15 @@
  * that can be found in the LICENSE file.
  */
 
-#ifndef RUNTIME_MM_NOOP_GC_H
-#define RUNTIME_MM_NOOP_GC_H
+#ifndef RUNTIME_GC_NOOP_NOOP_GC_H
+#define RUNTIME_GC_NOOP_NOOP_GC_H
 
 #include <cstddef>
 
 #include "Utils.hpp"
 
 namespace kotlin {
-namespace mm {
+namespace gc {
 
 // No-op GC is a GC that does not free memory.
 // TODO: It can be made more efficient.
@@ -52,7 +52,7 @@ private:
     size_t allocationThresholdBytes_ = 0;
 };
 
-} // namespace mm
+} // namespace gc
 } // namespace kotlin
 
-#endif // RUNTIME_MM_NOOP_GC_H
+#endif // RUNTIME_GC_NOOP_NOOP_GC_H

kotlin-native/runtime/src/gc/stms/cpp/GC.hpp

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_GC_STMS_GC_H
+#define RUNTIME_GC_STMS_GC_H
+
+#include "SingleThreadMarkAndSweep.hpp"
+
+namespace kotlin {
+namespace gc {
+
+using GC = kotlin::gc::SingleThreadMarkAndSweep;
+
+} // namespace gc
+} // namespace kotlin
+
+#endif // RUNTIME_GC_STMS_GC_H

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.cpp

@@ -5,13 +5,13 @@
 
 #include "SingleThreadMarkAndSweep.hpp"
 
-#include "../GlobalData.hpp"
-#include "../RootSet.hpp"
-#include "../ThreadData.hpp"
-#include "../ThreadRegistry.hpp"
+#include "GlobalData.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "Memory.h"
+#include "RootSet.hpp"
 #include "Runtime.h"
+#include "ThreadData.hpp"
+#include "ThreadRegistry.hpp"
 
 using namespace kotlin;
 
@@ -19,57 +19,57 @@ namespace {
 
 struct MarkTraits {
     static bool IsMarked(ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<mm::SingleThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        return objectData.color() == mm::SingleThreadMarkAndSweep::ObjectData::Color::kBlack;
+        auto& objectData = mm::ObjectFactory<gc::SingleThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
+        return objectData.color() == gc::SingleThreadMarkAndSweep::ObjectData::Color::kBlack;
     }
 
     static bool TryMark(ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<mm::SingleThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        if (objectData.color() == mm::SingleThreadMarkAndSweep::ObjectData::Color::kBlack) return false;
-        objectData.setColor(mm::SingleThreadMarkAndSweep::ObjectData::Color::kBlack);
+        auto& objectData = mm::ObjectFactory<gc::SingleThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
+        if (objectData.color() == gc::SingleThreadMarkAndSweep::ObjectData::Color::kBlack) return false;
+        objectData.setColor(gc::SingleThreadMarkAndSweep::ObjectData::Color::kBlack);
         return true;
     };
 };
 
 struct SweepTraits {
-    using ObjectFactory = mm::ObjectFactory<mm::SingleThreadMarkAndSweep>;
+    using ObjectFactory = mm::ObjectFactory<gc::SingleThreadMarkAndSweep>;
 
     static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
         auto& objectData = node.GCObjectData();
-        if (objectData.color() == mm::SingleThreadMarkAndSweep::ObjectData::Color::kWhite) return false;
-        objectData.setColor(mm::SingleThreadMarkAndSweep::ObjectData::Color::kWhite);
+        if (objectData.color() == gc::SingleThreadMarkAndSweep::ObjectData::Color::kWhite) return false;
+        objectData.setColor(gc::SingleThreadMarkAndSweep::ObjectData::Color::kWhite);
         return true;
     }
 };
 
 struct FinalizeTraits {
-    using ObjectFactory = mm::ObjectFactory<mm::SingleThreadMarkAndSweep>;
+    using ObjectFactory = mm::ObjectFactory<gc::SingleThreadMarkAndSweep>;
 };
 
 } // namespace
 
-void mm::SingleThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept {
     if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
         PerformFullGC();
     }
     ++safePointsCounter_;
 }
 
-void mm::SingleThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept {
     if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
         PerformFullGC();
     }
     ++safePointsCounter_;
 }
 
-void mm::SingleThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept {
     if (gc_.GetThreshold() == 0 || safePointsCounter_ % gc_.GetThreshold() == 0) {
         PerformFullGC();
     }
     ++safePointsCounter_;
 }
 
-void mm::SingleThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept {
     size_t allocationOverhead =
             gc_.GetAllocationThresholdBytes() == 0 ? allocatedBytes_ : allocatedBytes_ % gc_.GetAllocationThresholdBytes();
     if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) {
@@ -78,15 +78,15 @@ void mm::SingleThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size)
     allocatedBytes_ += size;
 }
 
-void mm::SingleThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept {
     gc_.PerformFullGC();
 }
 
-void mm::SingleThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
+void gc::SingleThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
     PerformFullGC();
 }
 
-void mm::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
+void gc::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
     RuntimeAssert(running_ == false, "Cannot have been called during another collection");
     running_ = true;
 
@@ -106,8 +106,8 @@ void mm::SingleThreadMarkAndSweep::PerformFullGC() noexcept {
         }
     }
 
-    mm::Mark<MarkTraits>(std::move(graySet));
-    auto finalizerQueue = mm::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
+    gc::Mark<MarkTraits>(std::move(graySet));
+    auto finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
 
     running_ = false;
 

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweep.hpp

@@ -3,8 +3,8 @@
  * that can be found in the LICENSE file.
  */
 
-#ifndef RUNTIME_MM_GC_SINGLE_THREAD_MARK_AND_SWEEP_H
-#define RUNTIME_MM_GC_SINGLE_THREAD_MARK_AND_SWEEP_H
+#ifndef RUNTIME_GC_STMS_SINGLE_THREAD_MARK_AND_SWEEP_H
+#define RUNTIME_GC_STMS_SINGLE_THREAD_MARK_AND_SWEEP_H
 
 #include <cstddef>
 
@@ -12,7 +12,7 @@
 #include "Utils.hpp"
 
 namespace kotlin {
-namespace mm {
+namespace gc {
 
 // Stop-the-world Mark-and-Sweep for a single mutator
 class SingleThreadMarkAndSweep : private Pinned {
@@ -72,7 +72,7 @@ private:
     size_t allocationThresholdBytes_ = 10000;
 };
 
-} // namespace mm
+} // namespace gc
 } // namespace kotlin
 
-#endif // RUNTIME_MM_GC_SINGLE_THREAD_MARK_AND_SWEEP_H
+#endif // RUNTIME_GC_STMS_SINGLE_THREAD_MARK_AND_SWEEP_H

kotlin-native/runtime/src/gc/stms/cpp/SingleThreadMarkAndSweepTest.cpp

@@ -8,12 +8,13 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
-#include "../ExtraObjectData.hpp"
-#include "../GlobalData.hpp"
-#include "../ObjectOps.hpp"
-#include "../TestSupport.hpp"
+#include "ExtraObjectData.hpp"
 #include "FinalizerHooksTestSupport.hpp"
+#include "GlobalData.hpp"
+#include "ObjectOps.hpp"
 #include "ObjectTestSupport.hpp"
+#include "TestSupport.hpp"
+#include "ThreadData.hpp"
 
 using namespace kotlin;
 
@@ -186,10 +187,10 @@ KStdVector<ObjHeader*> Alive(mm::ThreadData& threadData) {
     return objects;
 }
 
-using Color = mm::SingleThreadMarkAndSweep::ObjectData::Color;
+using Color = gc::SingleThreadMarkAndSweep::ObjectData::Color;
 
 Color GetColor(ObjHeader* objHeader) {
-    auto nodeRef = mm::ObjectFactory<mm::SingleThreadMarkAndSweep>::NodeRef::From(objHeader);
+    auto nodeRef = mm::ObjectFactory<gc::SingleThreadMarkAndSweep>::NodeRef::From(objHeader);
     return nodeRef.GCObjectData().color();
 }
 
@@ -591,7 +592,8 @@ TEST_F(SingleThreadMarkAndSweepTest, PermanentObjects) {
         GlobalPermanentObjectHolder global1{threadData};
         GlobalObjectHolder global2{threadData};
         test_support::Object<Payload> permanentObject{typeHolder.typeInfo()};
-        permanentObject.header()->typeInfoOrMeta_ = setPointerBits(permanentObject.header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
+        permanentObject.header()->typeInfoOrMeta_ =
+                setPointerBits(permanentObject.header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
         RuntimeAssert(permanentObject.header()->permanent(), "Must be permanent");
 
         global1->field1 = permanentObject.header();

kotlin-native/runtime/src/mm/cpp/GC.hpp

@@ -1,23 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#ifndef RUNTIME_MM_GC_H
-#define RUNTIME_MM_GC_H
-
-#include "gc/SingleThreadMarkAndSweep.hpp"
-#include "gc/NoOpGC.hpp"
-
-namespace kotlin {
-namespace mm {
-
-// TODO: GC should be extracted into a separate module, so that we can do different GCs without
-//       the need to redo the entire MM. For now changing GCs can be done by modifying `using` below.
-
-using GC = SingleThreadMarkAndSweep;
-
-} // namespace mm
-} // namespace kotlin
-
-#endif // RUNTIME_MM_GC_H

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -24,8 +24,8 @@ public:
     ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; }
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
     StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; }
-    ObjectFactory<GC>& objectFactory() noexcept { return objectFactory_; }
-    GC& gc() noexcept { return gc_; }
+    ObjectFactory<gc::GC>& objectFactory() noexcept { return objectFactory_; }
+    gc::GC& gc() noexcept { return gc_; }
 
 private:
     GlobalData();
@@ -37,8 +37,8 @@ private:
     ThreadRegistry threadRegistry_;
     GlobalsRegistry globalsRegistry_;
     StableRefRegistry stableRefRegistry_;
-    ObjectFactory<GC> objectFactory_;
-    GC gc_;
+    ObjectFactory<gc::GC> objectFactory_;
+    gc::GC gc_;
 };
 
 } // namespace mm

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -13,7 +13,6 @@
 #include "gtest/gtest.h"
 
 #include "FinalizerHooksTestSupport.hpp"
-#include "GC.hpp"
 #include "ObjectTestSupport.hpp"
 #include "TestSupport.hpp"
 #include "Types.h"

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -51,13 +51,13 @@ public:
 
     ThreadState setState(ThreadState state) noexcept { return state_.exchange(state); }
 
-    ObjectFactory<GC>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    ObjectFactory<gc::GC>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
 
     ShadowStack& shadowStack() noexcept { return shadowStack_; }
 
     KStdVector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
 
-    GC::ThreadData& gc() noexcept { return gc_; }
+    gc::GC::ThreadData& gc() noexcept { return gc_; }
 
     void Publish() noexcept {
         // TODO: These use separate locks, which is inefficient.
@@ -79,8 +79,8 @@ private:
     StableRefRegistry::ThreadQueue stableRefThreadQueue_;
     std::atomic<ThreadState> state_;
     ShadowStack shadowStack_;
-    GC::ThreadData gc_;
-    ObjectFactory<GC>::ThreadQueue objectFactoryThreadQueue_;
+    gc::GC::ThreadData gc_;
+    ObjectFactory<gc::GC>::ThreadQueue objectFactoryThreadQueue_;
     KStdVector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
 };