[K/N] Generalize Mark queue handling ^KT-51436

Merge-request: KT-MR-5904
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -25,17 +25,28 @@ using namespace kotlin;
 namespace {
 
 struct MarkTraits {
-    static bool IsMarked(ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        return objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack;
+    using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue;
+
+    static bool isEmpty(const MarkQueue& queue) noexcept {
+        return queue.empty();
     }
 
-    static bool TryMark(ObjHeader* object) noexcept {
+    static void clear(MarkQueue& queue) noexcept {
+        queue.clear();
+    }
+
+    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+        auto top = queue.back();
+        queue.pop_back();
+        return top;
+    }
+
+    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return false;
+        if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return;
         objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack);
-        return true;
-    };
+        queue.push_back(object);
+    }
 };
 
 struct SweepTraits {
@@ -85,7 +96,7 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
     objectFactory_(objectFactory),
     gcScheduler_(gcScheduler),
     finalizerProcessor_(make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch); })) {
-    graySet_.reserve(1000);
+    markQueue_.reserve(1000);
     gcScheduler_.SetScheduleGC([this]() NO_INLINE {
         RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
         // This call acquires a lock, so we need to ensure that we're in the safe state.
@@ -143,17 +154,17 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     state_.start(epoch);
     RuntimeLogInfo(
             {kTagGC}, "Started GC epoch %" PRId64 ". Time since last GC %" PRIu64 " microseconds", epoch, timeStartUs - lastGCTimestampUs_);
-    collectRootSet(graySet_);
+    gc::collectRootSet<MarkTraits>(markQueue_);
     auto timeRootSetUs = konan::getTimeMicros();
     // Can be unsafe, because we've stopped the world.
 
     auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
     RuntimeLogInfo(
-            {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet_.size(),
+            {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(),
             timeRootSetUs - timeSuspendUs);
-    auto markStats = gc::Mark<MarkTraits>(graySet_);
+    auto markStats = gc::Mark<MarkTraits>(markQueue_);
     auto timeMarkUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries);
+    RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs);
     scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
     gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
     auto timeSweepExtraObjectsUs = konan::getTimeMicros();

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -29,6 +29,8 @@ class FinalizerProcessor;
 // TODO: Also make mark concurrent.
 class ConcurrentMarkAndSweep : private Pinned {
 public:
+    // This implementation of mark queue allocates memory during collection.
+    using MarkQueue = KStdVector<ObjHeader*>;
 
     class ObjectData {
     public:
@@ -89,7 +91,7 @@ private:
     ScopedThread gcThread_;
     KStdUniquePtr<FinalizerProcessor> finalizerProcessor_;
 
-    KStdVector<ObjHeader*> graySet_;
+    MarkQueue markQueue_;
 };
 
 } // namespace gc

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp

@@ -4,53 +4,3 @@
  */
 
 #include "MarkAndSweepUtils.hpp"
-#include "CompilerConstants.hpp"
-#include "GlobalData.hpp"
-#include "Logging.hpp"
-#include "Memory.h"
-#include "RootSet.hpp"
-#include "ThreadData.hpp"
-
-using namespace kotlin;
-
-void kotlin::gc::collectRootSet(KStdVector<ObjHeader*>& graySet) {
-    graySet.clear();
-    for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
-        // TODO: Maybe it's more efficient to do by the suspending thread?
-        thread.Publish();
-        thread.gc().OnStoppedForGC();
-        size_t stack = 0;
-        size_t tls = 0;
-        for (auto value : mm::ThreadRootSet(thread)) {
-            if (!isNullOrMarker(value.object)) {
-                graySet.push_back(value.object);
-                switch (value.source) {
-                    case mm::ThreadRootSet::Source::kStack:
-                        ++stack;
-                        break;
-                        case mm::ThreadRootSet::Source::kTLS:
-                            ++tls;
-                            break;
-                }
-            }
-        }
-        RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
-    }
-    mm::StableRefRegistry::Instance().ProcessDeletions();
-    size_t global = 0;
-    size_t stableRef = 0;
-    for (auto value : mm::GlobalRootSet()) {
-        if (!isNullOrMarker(value.object)) {
-            graySet.push_back(value.object);
-            switch (value.source) {
-                case mm::GlobalRootSet::Source::kGlobal:
-                    ++global;
-                    break;
-                    case mm::GlobalRootSet::Source::kStableRef:
-                        ++stableRef;
-                        break;
-            }
-        }
-    }
-    RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
-}

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -8,10 +8,15 @@
 
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooks.hpp"
+#include "GlobalData.hpp"
+#include "Logging.hpp"
 #include "Memory.h"
 #include "ObjectOps.hpp"
 #include "ObjectTraversal.hpp"
+#include "RootSet.hpp"
 #include "Runtime.h"
+#include "StableRefRegistry.hpp"
+#include "ThreadData.hpp"
 #include "Types.h"
 
 namespace kotlin {
@@ -22,41 +27,33 @@ struct MarkStats {
     size_t aliveHeapSet = 0;
     // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself.
     size_t aliveHeapSetBytes = 0;
-    // How many times a marked object was found in the mark queue.
-    size_t duplicateEntries = 0;
 };
 
-// TODO: Because of `graySet` this implementation may allocate heap memory during GC.
 template <typename Traits>
-MarkStats Mark(KStdVector<ObjHeader*>& graySet) noexcept {
+MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
     MarkStats stats;
-    while (!graySet.empty()) {
-        ObjHeader* top = graySet.back();
-        graySet.pop_back();
+    while (!Traits::isEmpty(markQueue)) {
+        ObjHeader* top = Traits::dequeue(markQueue);
 
-        RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in gray set", top);
+        RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top);
+        RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local());
 
-        if (top->heap()) {
-            if (!Traits::TryMark(top)) {
-                ++stats.duplicateEntries;
-                continue;
+        stats.aliveHeapSet++;
+        stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
+
+        traverseReferredObjects(top, [&](ObjHeader* field) noexcept {
+            if (!isNullOrMarker(field) && field->heap()) {
+                Traits::enqueue(markQueue, field);
             }
-            stats.aliveHeapSet++;
-            stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
-        }
-
-        if (top->heap() || top->local()) {
-            traverseReferredObjects(top, [&graySet](ObjHeader* field) noexcept {
-                if (!isNullOrMarker(field) && field->heap() && !Traits::IsMarked(field)) {
-                    graySet.push_back(field);
-                }
-            });
-        }
+        });
 
         if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) {
             auto weakCounter = extraObjectData->GetWeakReferenceCounter();
             if (!isNullOrMarker(weakCounter)) {
-                graySet.push_back(weakCounter);
+                RuntimeAssert(
+                        weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", top,
+                        weakCounter, weakCounter->permanent(), weakCounter->local());
+                Traits::enqueue(markQueue, weakCounter);
             }
         }
     }
@@ -111,7 +108,71 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact
     return Sweep<Traits>(iter);
 }
 
-void collectRootSet(KStdVector<ObjHeader*>& graySet);
+// TODO: This needs some tests now.
+template <typename Traits>
+void collectRootSet(typename Traits::MarkQueue& markQueue) noexcept {
+    Traits::clear(markQueue);
+    for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
+        // TODO: Maybe it's more efficient to do by the suspending thread?
+        thread.Publish();
+        thread.gc().OnStoppedForGC();
+        size_t stack = 0;
+        size_t tls = 0;
+        for (auto value : mm::ThreadRootSet(thread)) {
+            auto* object = value.object;
+            if (!isNullOrMarker(object)) {
+                if (object->heap()) {
+                    Traits::enqueue(markQueue, object);
+                } else {
+                    traverseReferredObjects(object, [&](ObjHeader* field) noexcept {
+                        // Each permanent and stack object has own entry in the root set.
+                        if (field->heap() && !isNullOrMarker(field)) {
+                            Traits::enqueue(markQueue, field);
+                        }
+                    });
+                    RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object);
+                }
+                switch (value.source) {
+                    case mm::ThreadRootSet::Source::kStack:
+                        ++stack;
+                        break;
+                    case mm::ThreadRootSet::Source::kTLS:
+                        ++tls;
+                        break;
+                }
+            }
+        }
+        RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
+    }
+    mm::StableRefRegistry::Instance().ProcessDeletions();
+    size_t global = 0;
+    size_t stableRef = 0;
+    for (auto value : mm::GlobalRootSet()) {
+        auto* object = value.object;
+        if (!isNullOrMarker(object)) {
+            if (object->heap()) {
+                Traits::enqueue(markQueue, object);
+            } else {
+                traverseReferredObjects(object, [&](ObjHeader* field) noexcept {
+                    // Each permanent and stack object has own entry in the root set.
+                    if (field->heap() && !isNullOrMarker(field)) {
+                        Traits::enqueue(markQueue, field);
+                    }
+                });
+                RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object);
+            }
+            switch (value.source) {
+                case mm::GlobalRootSet::Source::kGlobal:
+                    ++global;
+                    break;
+                case mm::GlobalRootSet::Source::kStableRef:
+                    ++stableRef;
+                    break;
+            }
+        }
+    }
+    RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
+}
 
 } // namespace gc
 } // namespace kotlin

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -108,6 +108,8 @@ public:
 
 class ScopedMarkTraits : private Pinned {
 public:
+    using MarkQueue = KStdVector<ObjHeader*>;
+
     ScopedMarkTraits() {
         RuntimeAssert(instance_ == nullptr, "Only one ScopedMarkTraits is allowed");
         instance_ = this;
@@ -120,8 +122,26 @@ public:
 
     const KStdUnorderedSet<ObjHeader*>& marked() const { return marked_; }
 
-    static bool TryMark(ObjHeader* object) noexcept { return instance_->marked_.insert(object).second; }
-    static bool IsMarked(ObjHeader* object) noexcept { return instance_->marked_.find(object) != instance_->marked_.end(); }
+    static bool isEmpty(const MarkQueue& queue) noexcept {
+        return queue.empty();
+    }
+
+    static void clear(MarkQueue& queue) noexcept {
+        queue.clear();
+    }
+
+    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+        auto top = queue.back();
+        queue.pop_back();
+        return top;
+    }
+
+    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+        auto result = instance_->marked_.insert(object);
+        if (result.second) {
+            queue.push_back(object);
+        }
+    }
 
 private:
     static ScopedMarkTraits* instance_;
@@ -146,7 +166,7 @@ public:
 
     gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
         KStdVector<ObjHeader*> objects;
-        for (auto& object : graySet) objects.push_back(object.get().GetObjHeader());
+        for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader());
         return gc::Mark<ScopedMarkTraits>(objects);
     }
 
@@ -177,7 +197,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkNothing) {
     auto stats = Mark({});
 
     EXPECT_MARKED(stats);
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) {
@@ -186,7 +205,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) {
     auto stats = Mark({object});
 
     EXPECT_MARKED(stats, object);
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) {
@@ -195,8 +213,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) {
@@ -205,71 +221,8 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObject) {
-    Object object{BaseObject::Kind::kPermanent};
-
-    auto stats = Mark({object});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObjectArray) {
-    ObjectArray array{BaseObject::Kind::kPermanent};
-
-    auto stats = Mark({array});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentCharArray) {
-    CharArray array{BaseObject::Kind::kPermanent};
-
-    auto stats = Mark({array});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObject) {
-    Object object{BaseObject::Kind::kStackLocal};
-
-    auto stats = Mark({object});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObjectArray) {
-    ObjectArray array{BaseObject::Kind::kStackLocal};
-
-    auto stats = Mark({array});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackCharArray) {
-    CharArray array{BaseObject::Kind::kStackLocal};
-
-    auto stats = Mark({array});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) {
     Object object;
     object->field1 = kInitializingSingleton;
@@ -277,8 +230,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) {
     auto stats = Mark({object});
 
     EXPECT_MARKED(stats, object);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) {
@@ -288,8 +239,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) {
@@ -301,8 +250,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) {
@@ -312,8 +259,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) {
     auto stats = Mark({object});
 
     EXPECT_MARKED(stats, object);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) {
@@ -323,8 +268,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) {
@@ -334,8 +277,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) {
@@ -347,8 +288,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) {
     auto stats = Mark({object});
 
     EXPECT_MARKED(stats, object, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) {
@@ -360,8 +299,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) {
@@ -373,8 +310,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) {
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) {
@@ -387,8 +322,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCount
     auto stats = Mark({object});
 
     EXPECT_MARKED(stats, object, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) {
@@ -401,8 +334,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeak
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) {
@@ -417,8 +348,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter
     auto stats = Mark({array});
 
     EXPECT_MARKED(stats, array, weakCounter);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) {
@@ -443,105 +372,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) {
     EXPECT_MARKED(
             stats, root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2,
             root_field3_element3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) {
-    Object root{BaseObject::Kind::kPermanent};
-    Object root_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field2{BaseObject::Kind::kPermanent};
-    ObjectArray root_field3{BaseObject::Kind::kPermanent};
-    Object root_field3_element1{BaseObject::Kind::kPermanent};
-    ObjectArray root_field3_element2{BaseObject::Kind::kPermanent};
-    CharArray root_field3_element3{BaseObject::Kind::kPermanent};
-    root->field1 = root_field1.header();
-    root_field1->field1 = root_field1_field1.header();
-    root_field1->field2 = root_field1_field2.header();
-    root->field3 = root_field3.header();
-    root_field3.elements()[0] = root_field3_element1.header();
-    root_field3.elements()[1] = root_field3_element2.header();
-    root_field3.elements()[2] = root_field3_element3.header();
-
-    auto stats = Mark({root});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) {
-    Object root;
-    Object root_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field2{BaseObject::Kind::kPermanent};
-    ObjectArray root_field3;
-    Object root_field3_element1;
-    ObjectArray root_field3_element2;
-    CharArray root_field3_element3;
-    root->field1 = root_field1.header();
-    root_field1->field1 = root_field1_field1.header();
-    root_field1->field2 = root_field1_field2.header();
-    root->field3 = root_field3.header();
-    root_field3.elements()[0] = root_field3_element1.header();
-    root_field3.elements()[1] = root_field3_element2.header();
-    root_field3.elements()[2] = root_field3_element3.header();
-
-    auto stats = Mark({root});
-
-    EXPECT_MARKED(stats, root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) {
-    Object root;
-    Object root_field1;
-    Object root_field1_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field2;
-    ObjectArray root_field3;
-    Object root_field3_element1;
-    ObjectArray root_field3_element2;
-    CharArray root_field3_element3;
-    root->field1 = root_field1.header();
-    root_field1->field1 = root_field1_field1.header();
-    root_field1->field2 = root_field1_field2.header();
-    root->field3 = root_field3.header();
-    root_field3.elements()[0] = root_field3_element1.header();
-    root_field3.elements()[1] = root_field3_element2.header();
-    root_field3.elements()[2] = root_field3_element3.header();
-
-    auto stats = Mark({root});
-
-    EXPECT_MARKED(
-            stats, root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) {
-    Object root{BaseObject::Kind::kStackLocal};
-    Object root_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field1{BaseObject::Kind::kPermanent};
-    Object root_field1_field2{BaseObject::Kind::kPermanent};
-    ObjectArray root_field3{BaseObject::Kind::kStackLocal};
-    Object root_field3_element1{BaseObject::Kind::kHeapLike};
-    ObjectArray root_field3_element2{BaseObject::Kind::kHeapLike};
-    CharArray root_field3_element3{BaseObject::Kind::kPermanent};
-    root->field1 = root_field1.header();
-    root_field1->field1 = root_field1_field1.header();
-    root_field1->field2 = root_field1_field2.header();
-    root->field3 = root_field3.header();
-    root_field3.elements()[0] = root_field3_element1.header();
-    root_field3.elements()[1] = root_field3_element2.header();
-    root_field3.elements()[2] = root_field3_element3.header();
-
-    auto stats = Mark({root, root_field3});
-
-    EXPECT_MARKED(stats, root_field3_element1, root_field3_element2);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) {
@@ -555,46 +385,8 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) {
     auto stats = Mark({root});
 
     EXPECT_MARKED(stats, root, inner1, inner2);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
-TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) {
-    Object root{BaseObject::Kind::kPermanent};
-    Object inner1{BaseObject::Kind::kPermanent};
-    ObjectArray inner2{BaseObject::Kind::kPermanent};
-    root->field1 = inner1.header();
-    inner1->field1 = inner2.header();
-    inner2.elements()[0] = root.header();
-
-    auto stats = Mark({root});
-
-    EXPECT_MARKED(stats);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) {
-    Object root{BaseObject::Kind::kStackLocal};
-    Object inner1{BaseObject::Kind::kStackLocal};
-    ObjectArray inner2{BaseObject::Kind::kStackLocal};
-    Object inner3{BaseObject::Kind::kHeapLike};
-    Object inner2_element1{BaseObject::Kind::kHeapLike};
-    root->field1 = inner1.header();
-    inner1->field1 = inner2.header();
-    inner2.elements()[0] = root.header();
-    inner2.elements()[1] = inner2_element1.header();
-    root->field2 = inner3.header();
-    inner3->field1 = inner2.header();
-
-    auto stats = Mark({root, inner1, inner2});
-
-    EXPECT_MARKED(stats, inner3, inner2_element1);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) {
     Object root1;
     ObjectArray root2;
@@ -603,44 +395,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) {
     auto stats = Mark({root1, root2, root3});
 
     EXPECT_MARKED(stats, root1, root2, root3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) {
-    Object root1{BaseObject::Kind::kPermanent};
-    ObjectArray root2;
-    Object root3;
-
-    auto stats = Mark({root1, root2, root3});
-
-    EXPECT_MARKED(stats, root2, root3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) {
-    Object root1;
-    ObjectArray root2{BaseObject::Kind::kPermanent};
-    Object root3;
-
-    auto stats = Mark({root1, root2, root3});
-
-    EXPECT_MARKED(stats, root1, root3);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
-}
-
-TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) {
-    Object root1;
-    ObjectArray root2;
-    Object root3{BaseObject::Kind::kPermanent};
-
-    auto stats = Mark({root1, root2, root3});
-
-    EXPECT_MARKED(stats, root1, root2);
-
-    EXPECT_THAT(stats.duplicateEntries, 0);
 }
 
 TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) {
@@ -655,6 +409,4 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) {
     auto stats = Mark({root1, root2, root3});
 
     EXPECT_MARKED(stats, root1, root2, root3);
-
-    EXPECT_THAT(stats.duplicateEntries, 2);
 }

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -23,17 +23,28 @@ using namespace kotlin;
 namespace {
 
 struct MarkTraits {
-    static bool IsMarked(ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
+    using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue;
+
+    static bool isEmpty(const MarkQueue& queue) noexcept {
+        return queue.empty();
     }
 
-    static bool TryMark(ObjHeader* object) noexcept {
+    static void clear(MarkQueue& queue) noexcept {
+        queue.clear();
+    }
+
+    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+        auto top = queue.back();
+        queue.pop_back();
+        return top;
+    }
+
+    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).GCObjectData();
-        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return false;
+        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
         objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
-        return true;
-    };
+        queue.push_back(object);
+    }
 };
 
 struct SweepTraits {
@@ -104,7 +115,7 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi
 gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
         mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
     objectFactory_(objectFactory), gcScheduler_(gcScheduler) {
-    graySet_.reserve(1000);
+    markQueue_.reserve(1000);
     gcScheduler_.SetScheduleGC([]() {
         // TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running.
         //       Investigate, if it's possible to move this logic into the scheduler.
@@ -143,17 +154,17 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
         RuntimeLogInfo(
                 {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
-        collectRootSet(graySet_);
+        gc::collectRootSet<MarkTraits>(markQueue_);
         auto timeRootSetUs = konan::getTimeMicros();
         // Can be unsafe, because we've stopped the world.
         auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
 
         RuntimeLogInfo(
-                {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet_.size(),
+                {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(),
                 timeRootSetUs - timeSuspendUs);
-        auto markStats = gc::Mark<MarkTraits>(graySet_);
+        auto markStats = gc::Mark<MarkTraits>(markQueue_);
         auto timeMarkUs = konan::getTimeMicros();
-        RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries);
+        RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs);
         scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
         gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
         auto timeSweepExtraObjectsUs = konan::getTimeMicros();

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -25,6 +25,9 @@ namespace gc {
 // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads.
 class SameThreadMarkAndSweep : private Pinned {
 public:
+    // This implementation of mark queue allocates memory during collection.
+    using MarkQueue = KStdVector<ObjHeader*>;
+
     enum class SafepointFlag {
         kNone,
         kNeedsSuspend,
@@ -86,7 +89,7 @@ private:
     mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
     GCScheduler& gcScheduler_;
 
-    KStdVector<ObjHeader*> graySet_;
+    MarkQueue markQueue_;
 };
 
 namespace internal {