[K/N] Use intrusive list for mark queue ^KT-51436

Merge-request: KT-MR-5912
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -36,16 +36,17 @@ struct MarkTraits {
     }
 
     static ObjHeader* dequeue(MarkQueue& queue) noexcept {
-        auto top = queue.back();
-        queue.pop_back();
-        return top;
+        auto& top = queue.front();
+        queue.pop_front();
+        auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(top);
+        return node->GetObjHeader();
     }
 
     static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
         if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return;
         objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack);
-        queue.push_back(object);
+        queue.push_front(objectData);
     }
 };
 
@@ -96,7 +97,6 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
     objectFactory_(objectFactory),
     gcScheduler_(gcScheduler),
     finalizerProcessor_(make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch); })) {
-    markQueue_.reserve(1000);
     gcScheduler_.SetScheduleGC([this]() NO_INLINE {
         RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
         // This call acquires a lock, so we need to ensure that we're in the safe state.

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -9,6 +9,7 @@
 
 #include "Allocator.hpp"
 #include "GCScheduler.hpp"
+#include "IntrusiveList.hpp"
 #include "ObjectFactory.hpp"
 #include "ScopedThread.hpp"
 #include "Types.h"
@@ -29,24 +30,39 @@ class FinalizerProcessor;
 // TODO: Also make mark concurrent.
 class ConcurrentMarkAndSweep : private Pinned {
 public:
-    // This implementation of mark queue allocates memory during collection.
-    using MarkQueue = KStdVector<ObjHeader*>;
-
     class ObjectData {
+        static inline constexpr unsigned colorMask = (1 << 1) - 1;
+
     public:
-        enum class Color {
+        enum class Color : unsigned {
             kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
                         // All new objects are allocated with this color.
             kBlack, // Objects encountered during mark phase.
         };
 
-        Color color() const noexcept { return color_; }
-        void setColor(Color color) noexcept { color_ = color; }
+        Color color() const noexcept { return static_cast<Color>(getPointerBits(next_, colorMask)); }
+        void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast<unsigned>(color)); }
+
+        ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); }
+        void setNext(ObjectData* next) noexcept {
+            RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
+            auto bits = getPointerBits(next_, colorMask);
+            next_ = setPointerBits(next, bits);
+        }
 
     private:
-        Color color_ = Color::kWhite;
+        // Color is encoded in low bits.
+        ObjectData* next_ = nullptr;
     };
 
+    struct MarkQueueTraits {
+        static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
+
+        static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
+    };
+
+    using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
+
     class ThreadData : private Pinned {
     public:
         using ObjectData = ConcurrentMarkAndSweep::ObjectData;

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -34,16 +34,17 @@ struct MarkTraits {
     }
 
     static ObjHeader* dequeue(MarkQueue& queue) noexcept {
-        auto top = queue.back();
-        queue.pop_back();
-        return top;
+        auto& top = queue.front();
+        queue.pop_front();
+        auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(top);
+        return node->GetObjHeader();
     }
 
     static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
         if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
         objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
-        queue.push_back(object);
+        queue.push_front(objectData);
     }
 };
 
@@ -115,7 +116,6 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi
 gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
         mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
     objectFactory_(objectFactory), gcScheduler_(gcScheduler) {
-    markQueue_.reserve(1000);
     gcScheduler_.SetScheduleGC([]() {
         // TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running.
         //       Investigate, if it's possible to move this logic into the scheduler.

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -10,6 +10,7 @@
 
 #include "Allocator.hpp"
 #include "GCScheduler.hpp"
+#include "IntrusiveList.hpp"
 #include "ObjectFactory.hpp"
 #include "Types.h"
 #include "Utils.hpp"
@@ -25,9 +26,6 @@ namespace gc {
 // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads.
 class SameThreadMarkAndSweep : private Pinned {
 public:
-    // This implementation of mark queue allocates memory during collection.
-    using MarkQueue = KStdVector<ObjHeader*>;
-
     enum class SafepointFlag {
         kNone,
         kNeedsSuspend,
@@ -35,20 +33,38 @@ public:
     };
 
     class ObjectData {
+        static inline constexpr unsigned colorMask = (1 << 1) - 1;
+
     public:
-        enum class Color {
+        enum class Color : unsigned {
             kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
                         // All new objects are allocated with this color.
             kBlack, // Objects encountered during mark phase.
         };
 
-        Color color() const noexcept { return color_; }
-        void setColor(Color color) noexcept { color_ = color; }
+        Color color() const noexcept { return static_cast<Color>(getPointerBits(next_, colorMask)); }
+        void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast<unsigned>(color)); }
+
+        ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); }
+        void setNext(ObjectData* next) noexcept {
+            RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
+            auto bits = getPointerBits(next_, colorMask);
+            next_ = setPointerBits(next, bits);
+        }
 
     private:
-        Color color_ = Color::kWhite;
+        // Color is encoded in low bits.
+        ObjectData* next_ = nullptr;
     };
 
+    struct MarkQueueTraits {
+        static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
+
+        static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
+    };
+
+    using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
+
     class ThreadData : private Pinned {
     public:
         using ObjectData = SameThreadMarkAndSweep::ObjectData;

kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp

@@ -12,6 +12,7 @@
 
 #include "Mutex.hpp"
 #include "Types.h"
+#include "Utils.hpp"
 
 namespace kotlin {
 
@@ -34,7 +35,7 @@ public:
 
         static Node& fromValue(T& t) noexcept {
             static_assert(std::is_base_of_v<Pinned, T>, "fromValue function only makes sense for non-movable object");
-            return *reinterpret_cast<Node*>(reinterpret_cast<uintptr_t>(&t) - offsetof(Node, value_));
+            return ownerOf(Node, value_, t);
         }
 
     private:

kotlin-native/runtime/src/main/cpp/Utils.hpp

@@ -80,6 +80,8 @@ private:
 
 size_t CombineHash(size_t seed, size_t value);
 
+#define ownerOf(type, field, ref) *reinterpret_cast<type*>(reinterpret_cast<char*>(&ref) - offsetof(type, field))
+
 } // namespace kotlin
 
 #endif // RUNTIME_UTILS_H

kotlin-native/runtime/src/main/cpp/UtilsTest.cpp

@@ -7,6 +7,7 @@
 
 #include <type_traits>
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 using namespace kotlin;
@@ -48,3 +49,27 @@ TEST(UtilsTest, PinnedImpl) {
     static_assert(!std::is_move_assignable_v<PinnedImpl>, "Must not be move assignable");
     static_assert(sizeof(PinnedImpl) == sizeof(A), "Must not increase size");
 }
+
+namespace {
+
+class Container {
+public:
+    static Container& fromX(int32_t& x) { return ownerOf(Container, x_, x); }
+
+    static Container& fromY(void*& y) { return ownerOf(Container, y_, y); }
+
+    int32_t& x() { return x_; }
+    void*& y() { return y_; }
+
+private:
+    int32_t x_ = 0;
+    void* y_ = nullptr;
+};
+
+} // namespace
+
+TEST(UtilsTest, OwnerOf) {
+    Container c;
+    EXPECT_THAT(&c, &Container::fromX(c.x()));
+    EXPECT_THAT(&c, &Container::fromY(c.y()));
+}

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -453,18 +453,19 @@ public:
 
         static NodeRef From(ObjHeader* object) noexcept {
             RuntimeAssert(object->heap(), "Must be a heap object");
-            auto* heapObject = reinterpret_cast<HeapObjHeader*>(reinterpret_cast<uintptr_t>(object) - offsetof(HeapObjHeader, object));
-            RuntimeAssert(&heapObject->object == object, "HeapObjHeader layout has broken");
-            return NodeRef(Storage::Node::FromData(heapObject));
+            auto& heapObject = ownerOf(HeapObjHeader, object, *object);
+            return NodeRef(Storage::Node::FromData(&heapObject));
         }
 
         static NodeRef From(ArrayHeader* array) noexcept {
-            // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can
-            // be always casted to the other.
-            RuntimeAssert(reinterpret_cast<ObjHeader*>(array)->heap(), "Must be a heap object");
-            auto* heapArray = reinterpret_cast<HeapArrayHeader*>(reinterpret_cast<uintptr_t>(array) - offsetof(HeapArrayHeader, array));
-            RuntimeAssert(&heapArray->array == array, "HeapArrayHeader layout has broken");
-            return NodeRef(Storage::Node::FromData(heapArray));
+            RuntimeAssert(array->obj()->heap(), "Must be a heap object");
+            auto& heapArray = ownerOf(HeapArrayHeader, array, *array);
+            return NodeRef(Storage::Node::FromData(&heapArray));
+        }
+
+        static NodeRef From(ObjectData& objectData) noexcept {
+            auto& heapObject = ownerOf(HeapObjHeader, gcData, objectData);
+            return NodeRef(Storage::Node::FromData(&heapObject));
         }
 
         NodeRef* operator->() noexcept { return this; }

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -869,7 +869,7 @@ TEST(ObjectFactoryTest, CreateObject) {
     testing::Mock::VerifyAndClearExpectations(&allocator);
     EXPECT_THAT(allocSize, testing::Gt<size_t>(type.typeInfo()->instanceSize_));
     EXPECT_THAT(allocAddress, testing::Ne(nullptr));
-    EXPECT_THAT(mm::GetAllocatedHeapSize(object), allocSize);
+    EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(object), allocSize);
     EXPECT_THAT(object->type_info(), type.typeInfo());
 
     threadQueue.Publish();
@@ -904,7 +904,7 @@ TEST(ObjectFactoryTest, CreateObjectArray) {
     testing::Mock::VerifyAndClearExpectations(&allocator);
     EXPECT_THAT(allocSize, testing::Gt<size_t>(-theArrayTypeInfo->instanceSize_ * 3));
     EXPECT_THAT(allocAddress, testing::Ne(nullptr));
-    EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize);
+    EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(array->obj()), allocSize);
     EXPECT_THAT(array->type_info(), theArrayTypeInfo);
 
     threadQueue.Publish();
@@ -939,7 +939,7 @@ TEST(ObjectFactoryTest, CreateCharArray) {
     testing::Mock::VerifyAndClearExpectations(&allocator);
     EXPECT_THAT(allocSize, testing::Gt<size_t>(-theCharArrayTypeInfo->instanceSize_ * 3));
     EXPECT_THAT(allocAddress, testing::Ne(nullptr));
-    EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize);
+    EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(array->obj()), allocSize);
     EXPECT_THAT(array->type_info(), theCharArrayTypeInfo);
 
     threadQueue.Publish();