Sketch GC interface (#4682)

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -85,6 +85,8 @@ struct ObjHeader {
     return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
   }
 
+  inline bool heap() const { return getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK) == 0; }
+
   static MetaObjHeader* createMetaObject(ObjHeader* object);
   static void destroyMetaObject(ObjHeader* object);
 };

kotlin-native/runtime/src/mm/cpp/GC.hpp

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_GC_H
+#define RUNTIME_MM_GC_H
+
+#include "gc/NoOpGC.hpp"
+
+namespace kotlin {
+namespace mm {
+
+// TODO: GC should be extracted into a separate module, so that we can do different GCs without
+//       the need to redo the entire MM. For now changing GCs can be done by modifying `using` below.
+
+using GC = NoOpGC;
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_GC_H

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -8,6 +8,7 @@
 
 #include "ObjectFactory.hpp"
 #include "GlobalsRegistry.hpp"
+#include "GC.hpp"
 #include "StableRefRegistry.hpp"
 #include "ThreadRegistry.hpp"
 #include "Utils.hpp"
@@ -23,7 +24,8 @@ public:
     ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; }
     GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
     StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; }
-    ObjectFactory& objectFactory() noexcept { return objectFactory_; }
+    ObjectFactory<GC>& objectFactory() noexcept { return objectFactory_; }
+    GC& gc() noexcept { return gc_; }
 
 private:
     GlobalData();
@@ -34,7 +36,8 @@ private:
     ThreadRegistry threadRegistry_;
     GlobalsRegistry globalsRegistry_;
     StableRefRegistry stableRefRegistry_;
-    ObjectFactory objectFactory_;
+    ObjectFactory<GC> objectFactory_;
+    GC gc_;
 };
 
 } // namespace mm

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -253,6 +253,11 @@ extern "C" RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) {
     // Nothing to do
 }
 
+extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) {
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().PerformFullGC();
+}
+
 extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) {
     // TODO: Remove when legacy MM is gone.
     ThrowIllegalArgumentException();
@@ -289,6 +294,10 @@ extern "C" void Kotlin_Any_share(ObjHeader* thiz) {
     // Nothing to do
 }
 
+extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) {
+    GetThreadData(memory)->gc().PerformFullGC();
+}
+
 extern "C" RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
     // TODO: Remove when legacy MM is gone.
     return true;
@@ -366,7 +375,22 @@ extern "C" void AdoptReferenceFromSharedVariable(ObjHeader* object) {
     // Nothing to do.
 }
 
-void CheckGlobalsAccessible() {
+extern "C" void CheckGlobalsAccessible() {
     // TODO: Remove when legacy MM is gone.
     // Always accessible
 }
+
+extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() {
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().SafePointFunctionEpilogue();
+}
+
+extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() {
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().SafePointLoopBody();
+}
+
+extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() {
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+    threadData->gc().SafePointExceptionUnwind();
+}

kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp

@@ -1,61 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-#include "ObjectFactory.hpp"
-
-#include "Alignment.hpp"
-#include "Alloc.h"
-#include "GlobalData.hpp"
-#include "Types.h"
-
-using namespace kotlin;
-
-ObjHeader* mm::ObjectFactory::ThreadQueue::CreateObject(const TypeInfo* typeInfo) noexcept {
-    RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
-    size_t allocSize = typeInfo->instanceSize_;
-    auto& node = producer_.Insert(allocSize);
-    auto* object = static_cast<ObjHeader*>(node.Data());
-    object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
-    return object;
-}
-
-ArrayHeader* mm::ObjectFactory::ThreadQueue::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
-    RuntimeAssert(typeInfo->IsArray(), "Must be an array");
-    uint32_t arraySize = static_cast<uint32_t>(-typeInfo->instanceSize_) * count;
-    // Note: array body is aligned, but for size computation it is enough to align the sum.
-    size_t allocSize = AlignUp(sizeof(ArrayHeader) + arraySize, kObjectAlignment);
-    auto& node = producer_.Insert(allocSize);
-    auto* array = static_cast<ArrayHeader*>(node.Data());
-    array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
-    array->count_ = count;
-    return array;
-}
-
-bool mm::ObjectFactory::Iterator::IsArray() noexcept {
-    // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can
-    // be always casted to the other.
-    auto* object = static_cast<ObjHeader*>((*iterator_).Data());
-    return object->type_info()->IsArray();
-}
-
-ObjHeader* mm::ObjectFactory::Iterator::GetObjHeader() noexcept {
-    auto* object = static_cast<ObjHeader*>((*iterator_).Data());
-    RuntimeAssert(!object->type_info()->IsArray(), "Must not be an array");
-    return object;
-}
-
-ArrayHeader* mm::ObjectFactory::Iterator::GetArrayHeader() noexcept {
-    auto* array = static_cast<ArrayHeader*>((*iterator_).Data());
-    RuntimeAssert(array->type_info()->IsArray(), "Must be an array");
-    return array;
-}
-
-mm::ObjectFactory::ObjectFactory() noexcept = default;
-mm::ObjectFactory::~ObjectFactory() = default;
-
-// static
-mm::ObjectFactory& mm::ObjectFactory::Instance() noexcept {
-    return GlobalData::Instance().objectFactory();
-}

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -25,12 +25,24 @@ namespace internal {
 
 // A queue that is constructed by collecting subqueues from several `Producer`s.
 // This is essentially a heterogeneous `MultiSourceQueue` on top of a singly linked list that
-// uses `konanAllocMemory` and `konanFreeMemory`
+// uses `Allocator` to allocate and free memory.
 // TODO: Consider merging with `MultiSourceQueue` somehow.
-template <size_t DataAlignment>
+template <size_t DataAlignment, typename Allocator>
 class ObjectFactoryStorage : private Pinned {
     static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment");
 
+    template <typename T>
+    class Deleter {
+    public:
+        void operator()(T* instance) noexcept {
+            instance->~T();
+            Allocator::Free(instance);
+        }
+    };
+
+    template <typename T>
+    using unique_ptr = std::unique_ptr<T, Deleter<T>>;
+
 public:
     // This class does not know its size at compile-time. Does not inherit from `KonanAllocatorAware` because
     // in `KonanAllocatorAware::operator new(size_t size, KonanAllocTag)` `size` would be incorrect.
@@ -40,6 +52,13 @@ public:
     public:
         ~Node() = default;
 
+        static Node& FromData(void* data) noexcept {
+            constexpr size_t kDataOffset = DataOffset();
+            Node* node = reinterpret_cast<Node*>(reinterpret_cast<uintptr_t>(data) - kDataOffset);
+            RuntimeAssert(node->Data() == data, "Node layout has broken");
+            return *node;
+        }
+
         // Note: This can only be trivially destructible data, as nobody can invoke its destructor.
         void* Data() noexcept {
             constexpr size_t kDataOffset = DataOffset();
@@ -59,37 +78,36 @@ public:
 
         Node() noexcept = default;
 
-        static KStdUniquePtr<Node> Create(size_t dataSize) noexcept {
+        static unique_ptr<Node> Create(Allocator& allocator, size_t dataSize) noexcept {
             size_t dataSizeAligned = AlignUp(dataSize, DataAlignment);
             size_t totalAlignment = std::max(alignof(Node), DataAlignment);
             size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment);
             RuntimeAssert(
                     DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize,
                     DataOffset());
-            void* ptr = konanAllocAlignedMemory(totalSize, totalAlignment);
+            void* ptr = allocator.Alloc(totalSize, totalAlignment);
             if (!ptr) {
-                // TODO: Try doing GC first.
+                konan::consoleErrorf("Out of memory trying to allocate %zu bytes. Aborting.\n", totalSize);
-                konan::consoleErrorf("Out of memory trying to allocate %zu. Aborting.\n", totalSize);
                 konan::abort();
             }
             RuntimeAssert(IsAligned(ptr, totalAlignment), "Allocator returned unaligned to %zu pointer %p", totalAlignment, ptr);
-            return KStdUniquePtr<Node>(new (ptr) Node());
+            return unique_ptr<Node>(new (ptr) Node());
         }
 
-        KStdUniquePtr<Node> next_;
+        unique_ptr<Node> next_;
         // There's some more data of an unknown (at compile-time) size here, but it cannot be represented
         // with C++ members.
     };
 
     class Producer : private MoveOnly {
     public:
-        explicit Producer(ObjectFactoryStorage& owner) noexcept : owner_(owner) {}
+        Producer(ObjectFactoryStorage& owner, Allocator allocator) noexcept : owner_(owner), allocator_(std::move(allocator)) {}
 
         ~Producer() { Publish(); }
 
         Node& Insert(size_t dataSize) noexcept {
             AssertCorrect();
-            auto node = Node::Create(dataSize);
+            auto node = Node::Create(allocator_, dataSize);
             auto* nodePtr = node.get();
             if (!root_) {
                 root_ = std::move(node);
@@ -159,7 +177,8 @@ public:
         }
 
         ObjectFactoryStorage& owner_; // weak
-        KStdUniquePtr<Node> root_;
+        Allocator allocator_;
+        unique_ptr<Node> root_;
         Node* last_ = nullptr;
     };
 
@@ -245,35 +264,160 @@ private:
         }
     }
 
-    KStdUniquePtr<Node> root_;
+    unique_ptr<Node> root_;
     Node* last_ = nullptr;
     SpinLock mutex_;
 };
 
+class SimpleAllocator {
+public:
+    void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); }
+
+    static void Free(void* instance) noexcept { konanFreeMemory(instance); }
+};
+
+template <typename BaseAllocator, typename GC>
+class AllocatorWithGC {
+public:
+    AllocatorWithGC(BaseAllocator base, GC& gc) noexcept : base_(std::move(base)), gc_(gc) {}
+
+    void* Alloc(size_t size, size_t alignment) noexcept {
+        gc_.SafePointAllocation(size);
+        if (void* ptr = base_.Alloc(size, alignment)) {
+            return ptr;
+        }
+        // Tell GC that we failed to allocate, and try one more time.
+        gc_.OnOOM(size);
+        return base_.Alloc(size, alignment);
+    }
+
+    static void Free(void* instance) noexcept { BaseAllocator::Free(instance); }
+
+private:
+    BaseAllocator base_;
+    GC& gc_;
+};
+
 } // namespace internal
 
+template <typename GC>
 class ObjectFactory : private Pinned {
+    using GCObjectData = typename GC::ObjectData;
+    using GCThreadData = typename GC::ThreadData;
+
+    using Allocator = internal::AllocatorWithGC<internal::SimpleAllocator, GCThreadData>;
+
+    struct HeapObjHeader {
+        GCObjectData gcData;
+        alignas(kObjectAlignment) ObjHeader object;
+    };
+
+    // Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
+    // with `ObjHeader`: the former can always be casted to the other.
+    struct HeapArrayHeader {
+        GCObjectData gcData;
+        alignas(kObjectAlignment) ArrayHeader array;
+    };
+
 public:
-    using Storage = internal::ObjectFactoryStorage<kObjectAlignment>;
+    using Storage = internal::ObjectFactoryStorage<kObjectAlignment, Allocator>;
+
+    class NodeRef {
+    public:
+        explicit NodeRef(typename Storage::Node& node) noexcept : node_(node) {}
+
+        static NodeRef From(ObjHeader* object) noexcept {
+            RuntimeAssert(object->heap(), "Must be a heap object");
+            auto* heapObject = reinterpret_cast<HeapObjHeader*>(reinterpret_cast<uintptr_t>(object) - offsetof(HeapObjHeader, object));
+            RuntimeAssert(&heapObject->object == object, "HeapObjHeader layout has broken");
+            return NodeRef(Storage::Node::FromData(heapObject));
+        }
+
+        static NodeRef From(ArrayHeader* array) noexcept {
+            // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can
+            // be always casted to the other.
+            RuntimeAssert(reinterpret_cast<ObjHeader*>(array)->heap(), "Must be a heap object");
+            auto* heapArray = reinterpret_cast<HeapArrayHeader*>(reinterpret_cast<uintptr_t>(array) - offsetof(HeapArrayHeader, array));
+            RuntimeAssert(&heapArray->array == array, "HeapArrayHeader layout has broken");
+            return NodeRef(Storage::Node::FromData(heapArray));
+        }
+
+        NodeRef* operator->() noexcept { return this; }
+
+        GCObjectData& GCObjectData() noexcept {
+            // `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can
+            // be always casted to the other.
+            return static_cast<HeapObjHeader*>(node_.Data())->gcData;
+        }
+
+        bool IsArray() const noexcept {
+            // `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can
+            // be always casted to the other.
+            auto* object = &static_cast<HeapObjHeader*>(node_.Data())->object;
+            return object->type_info()->IsArray();
+        }
+
+        ObjHeader* GetObjHeader() noexcept {
+            auto* object = &static_cast<HeapObjHeader*>(node_.Data())->object;
+            RuntimeAssert(!object->type_info()->IsArray(), "Must not be an array");
+            return object;
+        }
+
+        ArrayHeader* GetArrayHeader() noexcept {
+            auto* array = &static_cast<HeapArrayHeader*>(node_.Data())->array;
+            RuntimeAssert(array->type_info()->IsArray(), "Must be an array");
+            return array;
+        }
+
+        bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; }
+
+        bool operator!=(const NodeRef& rhs) const noexcept { return !(*this == rhs); }
+
+    private:
+        typename Storage::Node& node_;
+    };
 
     class ThreadQueue : private MoveOnly {
     public:
-        explicit ThreadQueue(ObjectFactory& owner) noexcept : producer_(owner.storage_) {}
+        ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept :
+            producer_(owner.storage_, internal::AllocatorWithGC(internal::SimpleAllocator(), gc)) {}
 
-        ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
+        ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept {
-        ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept;
+            RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
+            size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
+            size_t allocSize = AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
+            auto& node = producer_.Insert(allocSize);
+            auto* heapObject = new (node.Data()) HeapObjHeader();
+            auto* object = &heapObject->object;
+            object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
+            return object;
+        }
+
+        ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
+            RuntimeAssert(typeInfo->IsArray(), "Must be an array");
+            uint32_t membersSize = static_cast<uint32_t>(-typeInfo->instanceSize_) * count;
+            // Note: array body is aligned, but for size computation it is enough to align the sum.
+            size_t allocSize = AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
+            auto& node = producer_.Insert(allocSize);
+            auto* heapArray = new (node.Data()) HeapArrayHeader();
+            auto* array = &heapArray->array;
+            array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
+            array->count_ = count;
+            return array;
+        }
 
         void Publish() noexcept { producer_.Publish(); }
 
         void ClearForTests() noexcept { producer_.ClearForTests(); }
 
     private:
-        Storage::Producer producer_;
+        typename Storage::Producer producer_;
     };
 
     class Iterator {
     public:
-        Storage::Node& operator*() noexcept { return *iterator_; }
+        NodeRef operator*() noexcept { return NodeRef(*iterator_); }
+        NodeRef operator->() noexcept { return NodeRef(*iterator_); }
 
         Iterator& operator++() noexcept {
             ++iterator_;
@@ -284,17 +428,12 @@ public:
 
         bool operator!=(const Iterator& rhs) const noexcept { return iterator_ != rhs.iterator_; }
 
-        bool IsArray() noexcept;
-
-        ObjHeader* GetObjHeader() noexcept;
-        ArrayHeader* GetArrayHeader() noexcept;
-
     private:
         friend class ObjectFactory;
 
-        explicit Iterator(Storage::Iterator iterator) noexcept : iterator_(std::move(iterator)) {}
+        explicit Iterator(typename Storage::Iterator iterator) noexcept : iterator_(std::move(iterator)) {}
 
-        Storage::Iterator iterator_;
+        typename Storage::Iterator iterator_;
     };
 
     class Iterable {
@@ -307,13 +446,11 @@ public:
         void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); }
 
     private:
-        Storage::Iterable iter_;
+        typename Storage::Iterable iter_;
     };
 
-    ObjectFactory() noexcept;
+    ObjectFactory() noexcept = default;
-    ~ObjectFactory();
+    ~ObjectFactory() = default;
-
-    static ObjectFactory& Instance() noexcept;
 
     Iterable Iter() noexcept { return Iterable(*this); }
 

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -12,17 +12,25 @@
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
+#include "GC.hpp"
 #include "TestSupport.hpp"
 #include "Types.h"
 
 using namespace kotlin;
 
+using testing::_;
+
+namespace {
+
+using SimpleAllocator = mm::internal::SimpleAllocator;
+
 template <size_t DataAlignment>
-using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment>;
+using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator>;
 
 using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
 
-namespace {
+template <typename Storage>
+using Producer = typename Storage::Producer;
 
 template <size_t DataAlignment>
 KStdVector<void*> Collect(ObjectFactoryStorage<DataAlignment>& storage) {
@@ -76,7 +84,7 @@ TEST(ObjectFactoryStorageTest, Empty) {
 
 TEST(ObjectFactoryStorageTest, DoNotPublish) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<int>(2);
@@ -88,8 +96,8 @@ TEST(ObjectFactoryStorageTest, DoNotPublish) {
 
 TEST(ObjectFactoryStorageTest, Publish) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer1(storage);
+    Producer<ObjectFactoryStorageRegular> producer1(storage, SimpleAllocator());
-    ObjectFactoryStorageRegular::Producer producer2(storage);
+    Producer<ObjectFactoryStorageRegular> producer2(storage, SimpleAllocator());
 
     producer1.Insert<int>(1);
     producer1.Insert<int>(2);
@@ -106,7 +114,7 @@ TEST(ObjectFactoryStorageTest, Publish) {
 
 TEST(ObjectFactoryStorageTest, PublishDifferentTypes) {
     ObjectFactoryStorage<alignof(MaxAlignedData)> storage;
-    ObjectFactoryStorage<alignof(MaxAlignedData)>::Producer producer(storage);
+    Producer<ObjectFactoryStorage<alignof(MaxAlignedData)>> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<size_t>(2);
@@ -139,7 +147,7 @@ TEST(ObjectFactoryStorageTest, PublishDifferentTypes) {
 
 TEST(ObjectFactoryStorageTest, PublishSeveralTimes) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     // Add 2 elements and publish.
     producer.Insert<int>(1);
@@ -167,7 +175,7 @@ TEST(ObjectFactoryStorageTest, PublishInDestructor) {
     ObjectFactoryStorageRegular storage;
 
     {
-        ObjectFactoryStorageRegular::Producer producer(storage);
+        Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
         producer.Insert<int>(1);
         producer.Insert<int>(2);
     }
@@ -177,9 +185,22 @@ TEST(ObjectFactoryStorageTest, PublishInDestructor) {
     EXPECT_THAT(actual, testing::ElementsAre(1, 2));
 }
 
+TEST(ObjectFactoryStorageTest, FindNode) {
+    ObjectFactoryStorageRegular storage;
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
+
+    auto& node1 = producer.Insert<int>(1);
+    auto& node2 = producer.Insert<int>(2);
+
+    producer.Publish();
+
+    EXPECT_THAT(&ObjectFactoryStorageRegular::Node::FromData(node1.Data()), &node1);
+    EXPECT_THAT(&ObjectFactoryStorageRegular::Node::FromData(node2.Data()), &node2);
+}
+
 TEST(ObjectFactoryStorageTest, EraseFirst) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<int>(2);
@@ -205,7 +226,7 @@ TEST(ObjectFactoryStorageTest, EraseFirst) {
 
 TEST(ObjectFactoryStorageTest, EraseMiddle) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<int>(2);
@@ -231,7 +252,7 @@ TEST(ObjectFactoryStorageTest, EraseMiddle) {
 
 TEST(ObjectFactoryStorageTest, EraseLast) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<int>(2);
@@ -257,7 +278,7 @@ TEST(ObjectFactoryStorageTest, EraseLast) {
 
 TEST(ObjectFactoryStorageTest, EraseAll) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
     producer.Insert<int>(2);
@@ -279,7 +300,7 @@ TEST(ObjectFactoryStorageTest, EraseAll) {
 
 TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) {
     ObjectFactoryStorageRegular storage;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
 
     producer.Insert<int>(1);
 
@@ -307,7 +328,7 @@ TEST(ObjectFactoryStorageTest, ConcurrentPublish) {
     for (int i = 0; i < kThreadCount; ++i) {
         expected.push_back(i);
         threads.emplace_back([i, &storage, &canStart, &readyCount]() {
-            ObjectFactoryStorageRegular::Producer producer(storage);
+            Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
             producer.Insert<int>(i);
             ++readyCount;
             while (!canStart) {
@@ -335,7 +356,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) {
 
     KStdVector<int> expectedBefore;
     KStdVector<int> expectedAfter;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
     for (int i = 0; i < kStartCount; ++i) {
         expectedBefore.push_back(i);
         expectedAfter.push_back(i);
@@ -351,7 +372,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) {
         int j = i + kStartCount;
         expectedAfter.push_back(j);
         threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() {
-            ObjectFactoryStorageRegular::Producer producer(storage);
+            Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
             producer.Insert<int>(j);
             ++readyCount;
             while (!canStart) {
@@ -393,7 +414,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
     constexpr int kThreadCount = kDefaultThreadCount;
 
     KStdVector<int> expectedAfter;
-    ObjectFactoryStorageRegular::Producer producer(storage);
+    Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
     for (int i = 0; i < kStartCount; ++i) {
         if (i % 2 == 0) {
             expectedAfter.push_back(i);
@@ -410,7 +431,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
         int j = i + kStartCount;
         expectedAfter.push_back(j);
         threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() {
-            ObjectFactoryStorageRegular::Producer producer(storage);
+            Producer<ObjectFactoryStorageRegular> producer(storage, SimpleAllocator());
             producer.Insert<int>(j);
             ++readyCount;
             while (!canStart) {
@@ -446,10 +467,103 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
     EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expectedAfter));
 }
 
-using mm::ObjectFactory;
+using mm::internal::AllocatorWithGC;
 
 namespace {
 
+class MockAllocator {
+public:
+    MOCK_METHOD(void*, Alloc, (size_t, size_t));
+};
+
+class MockAllocatorWrapper {
+public:
+    MockAllocator& operator*() { return *mock_; }
+
+    void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); }
+
+private:
+    KStdUniquePtr<testing::StrictMock<MockAllocator>> mock_ = make_unique<testing::StrictMock<MockAllocator>>();
+};
+
+class MockGC {
+public:
+    MOCK_METHOD(void, SafePointAllocation, (size_t));
+    MOCK_METHOD(void, OnOOM, (size_t));
+};
+
+} // namespace
+
+TEST(AllocatorWithGCTest, AllocateWithoutOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    void* nonNull = reinterpret_cast<void*>(1);
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
+        EXPECT_CALL(gc, OnOOM(_)).Times(0);
+    }
+    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nonNull);
+}
+
+TEST(AllocatorWithGCTest, AllocateWithFixableOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    void* nonNull = reinterpret_cast<void*>(1);
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+        EXPECT_CALL(gc, OnOOM(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull));
+    }
+    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nonNull);
+}
+
+TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) {
+    constexpr size_t size = 256;
+    constexpr size_t alignment = 8;
+    MockAllocatorWrapper baseAllocator;
+    testing::StrictMock<MockGC> gc;
+    {
+        testing::InSequence seq;
+        EXPECT_CALL(gc, SafePointAllocation(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+        EXPECT_CALL(gc, OnOOM(size));
+        EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr));
+    }
+    AllocatorWithGC<MockAllocatorWrapper, MockGC> allocator(std::move(baseAllocator), gc);
+    void* ptr = allocator.Alloc(size, alignment);
+    EXPECT_THAT(ptr, nullptr);
+}
+
+namespace {
+
+class GC {
+public:
+    struct ObjectData {
+        uint32_t flags = 42;
+    };
+
+    class ThreadData {
+    public:
+        void SafePointAllocation(size_t size) noexcept {}
+
+        void OnOOM(size_t size) noexcept {}
+    };
+};
+
+using ObjectFactory = mm::ObjectFactory<GC>;
+
 KStdUniquePtr<TypeInfo> MakeObjectTypeInfo(int32_t size) {
     auto typeInfo = make_unique<TypeInfo>();
     typeInfo->typeInfo_ = typeInfo.get();
@@ -468,32 +582,42 @@ KStdUniquePtr<TypeInfo> MakeArrayTypeInfo(int32_t elementSize) {
 
 TEST(ObjectFactoryTest, CreateObject) {
     auto typeInfo = MakeObjectTypeInfo(24);
+    GC::ThreadData gc;
     ObjectFactory objectFactory;
-    ObjectFactory::ThreadQueue threadQueue(objectFactory);
+    ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
     auto* object = threadQueue.CreateObject(typeInfo.get());
     threadQueue.Publish();
 
+    auto node = ObjectFactory::NodeRef::From(object);
+    EXPECT_FALSE(node.IsArray());
+    EXPECT_THAT(node.GetObjHeader(), object);
+    EXPECT_THAT(node.GCObjectData().flags, 42);
+
     auto iter = objectFactory.Iter();
     auto it = iter.begin();
-    EXPECT_FALSE(it.IsArray());
+    EXPECT_THAT(*it, node);
-    EXPECT_THAT(it.GetObjHeader(), object);
     ++it;
     EXPECT_THAT(it, iter.end());
 }
 
 TEST(ObjectFactoryTest, CreateArray) {
     auto typeInfo = MakeArrayTypeInfo(24);
+    GC::ThreadData gc;
     ObjectFactory objectFactory;
-    ObjectFactory::ThreadQueue threadQueue(objectFactory);
+    ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
     auto* array = threadQueue.CreateArray(typeInfo.get(), 3);
     threadQueue.Publish();
 
+    auto node = ObjectFactory::NodeRef::From(array);
+    EXPECT_TRUE(node.IsArray());
+    EXPECT_THAT(node.GetArrayHeader(), array);
+    EXPECT_THAT(node.GCObjectData().flags, 42);
+
     auto iter = objectFactory.Iter();
     auto it = iter.begin();
-    EXPECT_TRUE(it.IsArray());
+    EXPECT_THAT(*it, node);
-    EXPECT_THAT(it.GetArrayHeader(), array);
     ++it;
     EXPECT_THAT(it, iter.end());
 }
@@ -501,8 +625,9 @@ TEST(ObjectFactoryTest, CreateArray) {
 TEST(ObjectFactoryTest, Erase) {
     auto objectTypeInfo = MakeObjectTypeInfo(24);
     auto arrayTypeInfo = MakeArrayTypeInfo(24);
+    GC::ThreadData gc;
     ObjectFactory objectFactory;
-    ObjectFactory::ThreadQueue threadQueue(objectFactory);
+    ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
 
     for (int i = 0; i < 10; ++i) {
         threadQueue.CreateObject(objectTypeInfo.get());
@@ -514,7 +639,7 @@ TEST(ObjectFactoryTest, Erase) {
     {
         auto iter = objectFactory.Iter();
         for (auto it = iter.begin(); it != iter.end();) {
-            if (it.IsArray()) {
+            if (it->IsArray()) {
                 iter.EraseAndAdvance(it);
             } else {
                 ++it;
@@ -526,7 +651,7 @@ TEST(ObjectFactoryTest, Erase) {
         auto iter = objectFactory.Iter();
         int count = 0;
         for (auto it = iter.begin(); it != iter.end(); ++it, ++count) {
-            EXPECT_FALSE(it.IsArray());
+            EXPECT_FALSE(it->IsArray());
         }
         EXPECT_THAT(count, 10);
     }
@@ -544,7 +669,8 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
 
     for (int i = 0; i < kThreadCount; ++i) {
         threads.emplace_back([&typeInfo, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() {
-            ObjectFactory::ThreadQueue threadQueue(objectFactory);
+            GC::ThreadData gc;
+            ObjectFactory::ThreadQueue threadQueue(objectFactory, gc);
             auto* object = threadQueue.CreateObject(typeInfo.get());
             {
                 std::lock_guard<std::mutex> guard(expectedMutex);
@@ -567,7 +693,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
     auto iter = objectFactory.Iter();
     KStdVector<ObjHeader*> actual;
     for (auto it = iter.begin(); it != iter.end(); ++it) {
-        actual.push_back(it.GetObjHeader());
+        actual.push_back(it->GetObjHeader());
     }
 
     EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));

kotlin-native/runtime/src/mm/cpp/Stubs.cpp

@@ -26,10 +26,6 @@ void EnsureNeverFrozen(ObjHeader* obj) {
     TODO();
 }
 
-void Kotlin_native_internal_GC_collect(ObjHeader*) {
-    TODO();
-}
-
 void Kotlin_native_internal_GC_suspend(ObjHeader*) {
     TODO();
 }
@@ -78,10 +74,6 @@ bool Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) {
     TODO();
 }
 
-RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) {
-    TODO();
-}
-
 bool TryAddHeapRef(const ObjHeader* object) {
     TODO();
 }
@@ -98,16 +90,4 @@ ForeignRefContext InitLocalForeignRef(ObjHeader* object) {
     TODO();
 }
 
-RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() {
-    // TODO: Unimplemented
-}
-
-RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() {
-    // TODO: Unimplemented
-}
-
-RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() {
-    // TODO: Unimplemented
-}
-
 } // extern "C"

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -9,7 +9,9 @@
 #include <atomic>
 #include <pthread.h>
 
+#include "GlobalData.hpp"
 #include "GlobalsRegistry.hpp"
+#include "GC.hpp"
 #include "ObjectFactory.hpp"
 #include "ShadowStack.hpp"
 #include "StableRefRegistry.hpp"
@@ -32,7 +34,8 @@ public:
         globalsThreadQueue_(GlobalsRegistry::Instance()),
         stableRefThreadQueue_(StableRefRegistry::Instance()),
         state_(ThreadState::kRunnable),
-        objectFactoryThreadQueue_(ObjectFactory::Instance()) {}
+        gc_(GlobalData::Instance().gc()),
+        objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_) {}
 
     ~ThreadData() = default;
 
@@ -48,20 +51,23 @@ public:
 
     ThreadState setState(ThreadState state) noexcept { return state_.exchange(state); }
 
-    ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
+    ObjectFactory<GC>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
 
     ShadowStack& shadowStack() noexcept { return shadowStack_; }
 
     KStdVector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
 
+    GC::ThreadData& gc() noexcept { return gc_; }
+
 private:
     const pthread_t threadId_;
     GlobalsRegistry::ThreadQueue globalsThreadQueue_;
     ThreadLocalStorage tls_;
     StableRefRegistry::ThreadQueue stableRefThreadQueue_;
     std::atomic<ThreadState> state_;
-    ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
     ShadowStack shadowStack_;
+    GC::ThreadData gc_;
+    ObjectFactory<GC>::ThreadQueue objectFactoryThreadQueue_;
     KStdVector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
 };
 

kotlin-native/runtime/src/mm/cpp/gc/NoOpGC.hpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_NOOP_GC_H
+#define RUNTIME_MM_NOOP_GC_H
+
+#include <cstddef>
+
+#include "Utils.hpp"
+
+namespace kotlin {
+namespace mm {
+
+// No-op GC is a GC that does not free memory.
+// TODO: It can be made more efficient.
+class NoOpGC : private Pinned {
+public:
+    class ObjectData {};
+
+    class ThreadData : private Pinned {
+    public:
+        using ObjectData = NoOpGC::ObjectData;
+
+        explicit ThreadData(NoOpGC& gc) noexcept {}
+        ~ThreadData() = default;
+
+        void SafePointFunctionEpilogue() noexcept {}
+        void SafePointLoopBody() noexcept {}
+        void SafePointExceptionUnwind() noexcept {}
+        void SafePointAllocation(size_t size) noexcept {}
+
+        void PerformFullGC() noexcept {}
+
+        void OnOOM(size_t size) noexcept {}
+
+    private:
+    };
+
+    NoOpGC() noexcept = default;
+    ~NoOpGC() = default;
+
+private:
+};
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_NOOP_GC_H