[K/N] Optimize intrusive_forward_list usage in GCs ^KT-54163

* Add fallible push to intrusive_forward_list.
* Add fallible pop to intrusive_forward_list.
* Get rid of size_ member in intrusive_forward_list.
* Force all intrusive_forward_list items to have non-null next().
* Get rid of explicit colors in STMS and CMS.
* Do not queue weak reference for later processing in Mark.

Merge-request: KT-MR-7263
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -38,14 +38,12 @@ struct SweepTraits {
         auto *baseObject = object.GetBaseObject();
         if (!baseObject->heap()) return true;
         auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
-        return objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack;
+        return objectData.marked();
     }
 
     static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
         auto& objectData = node.ObjectData();
-        if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite) return false;
+        return objectData.tryResetMark();
-        objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite);
-        return true;
     }
 };
 

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -35,48 +35,39 @@ class FinalizerProcessor;
 class ConcurrentMarkAndSweep : private Pinned {
 public:
     class ObjectData {
-        static inline constexpr unsigned colorMask = (1 << 1) - 1;
-
     public:
-        enum class Color : unsigned {
+        bool tryMark() noexcept {
-            kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
+            return trySetNext(reinterpret_cast<ObjectData*>(1));
-                        // All new objects are allocated with this color.
-            kBlack, // Objects encountered during mark phase.
-        };
-
-        Color color() const noexcept { return static_cast<Color>(getPointerBits(next_.load(std::memory_order_relaxed), colorMask)); }
-        void setColor(Color color) noexcept { next_.store(setPointerBits(clearPointerBits(next_.load(std::memory_order_relaxed), colorMask), static_cast<unsigned>(color)), std::memory_order_relaxed); }
-        bool atomicSetToBlack() noexcept {
-            ObjectData* before = next_.load(std::memory_order_relaxed);
-            if (getPointerBits(before, colorMask) != static_cast<unsigned>(Color::kWhite))
-                return false;
-            ObjectData* black = setPointerBits(before, static_cast<unsigned>(Color::kBlack));
-            bool success = next_.compare_exchange_strong(before, black, std::memory_order_relaxed);
-            RuntimeAssert(success || hasPointerBits(before, colorMask), "next_ must have been marked black");
-            return success;
         }
 
-        ObjectData* next() const noexcept { return clearPointerBits(next_.load(std::memory_order_relaxed), colorMask); }
+        bool marked() const noexcept { return next() != nullptr; }
-        void setNext(ObjectData* next) noexcept {
+
-            RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
+        bool tryResetMark() noexcept {
-            auto bits = getPointerBits(next_.load(std::memory_order_relaxed), colorMask);
+            if (next() == nullptr) return false;
-            next_.store(setPointerBits(next, bits), std::memory_order_relaxed);
+            next_.store(nullptr, std::memory_order_relaxed);
+            return true;
         }
 
     private:
-        // Color is encoded in low bits.
+        friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
+
+        ObjectData* next() const noexcept { return next_.load(std::memory_order_relaxed); }
+        void setNext(ObjectData* next) noexcept {
+            RuntimeAssert(next, "next cannot be nullptr");
+            next_.store(next, std::memory_order_relaxed);
+        }
+        bool trySetNext(ObjectData* next) noexcept {
+            RuntimeAssert(next, "next cannot be nullptr");
+            ObjectData* expected = nullptr;
+            return next_.compare_exchange_strong(expected, next, std::memory_order_relaxed);
+        }
+
         std::atomic<ObjectData*> next_ = nullptr;
     };
 
-    struct MarkQueueTraits {
-        static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
-
-        static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
-    };
-
     enum MarkingBehavior { kMarkOwnStack, kDoNotMark };
 
-    using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
+    using MarkQueue = intrusive_forward_list<ObjectData>;
 
     class ThreadData : private Pinned {
     public:
@@ -138,21 +129,24 @@ namespace internal {
 struct MarkTraits {
     using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue;
 
-    static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
-
     static void clear(MarkQueue& queue) noexcept { queue.clear(); }
 
-    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+    static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
-        auto& top = queue.front();
+        if (auto* top = queue.try_pop_front()) {
-        queue.pop_front();
+            auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(*top);
-        auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(top);
+            return node->GetObjHeader();
-        return node->GetObjHeader();
+        }
+        return nullptr;
     }
 
-    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+    static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
-        if (!objectData.atomicSetToBlack()) return;
+        return queue.try_push_front(objectData);
-        queue.push_front(objectData);
+    }
+
+    static bool tryMark(ObjHeader* object) noexcept {
+        auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
+        return objectData.tryMark();
     }
 
     static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -199,11 +199,9 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
     return objects;
 }
 
-using Color = gc::ConcurrentMarkAndSweep::ObjectData::Color;
+bool IsMarked(ObjHeader* objHeader) {
-
-Color GetColor(ObjHeader* objHeader) {
     auto nodeRef = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(objHeader);
-    return nodeRef.ObjectData().color();
+    return nodeRef.ObjectData().marked();
 }
 
 WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -250,12 +248,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global2.header()), false);
-        ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global3.header()), false);
-        ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack1.header()), false);
-        ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack2.header()), false);
-        ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack3.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -263,12 +261,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global1.header()), false);
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
-        EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global3.header()), false);
-        EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack1.header()), false);
-        EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack2.header()), false);
-        EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack3.header()), false);
     });
 }
 
@@ -296,12 +294,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global2.header()), false);
-        ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global3.header()), false);
-        ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack1.header()), false);
-        ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack2.header()), false);
-        ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack3.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -309,12 +307,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global1.header()), false);
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
-        EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global3.header()), false);
-        EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack1.header()), false);
-        EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack2.header()), false);
-        EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack3.header()), false);
     });
 }
 
@@ -324,8 +322,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) {
         auto& object2 = AllocateObject(threadData);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -339,8 +337,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
         auto& object2 = AllocateObjectWithFinalizer(threadData);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         EXPECT_CALL(finalizerHook(), Call(object1.header()));
         EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -359,8 +357,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
         })();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak1.header()), false);
         ASSERT_THAT(weak1->referred, object1.header());
 
         threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
@@ -376,14 +374,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
         auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak1.header()), false);
         ASSERT_THAT(weak1->referred, object1.header());
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
-        EXPECT_THAT(GetColor(weak1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(weak1.header()), false);
         EXPECT_THAT(weak1->referred, nullptr);
     });
 }
@@ -406,12 +404,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -419,12 +417,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -453,14 +451,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -468,12 +466,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -502,14 +500,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         EXPECT_CALL(finalizerHook(), Call(object5.header()));
         EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -519,12 +517,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -541,18 +539,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
         object2->field1 = stack.header();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
     });
 }
 
@@ -581,14 +579,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
         threadData.gc().ScheduleAndWaitFullGC();
@@ -597,12 +595,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -619,12 +617,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) {
         global2->field1 = global1.header();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
     });
 }
 
@@ -638,14 +636,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) {
 
         ASSERT_THAT(global.header(), stack.header());
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(object.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(object.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object.header()), false);
     });
 }
 
@@ -1114,13 +1112,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
         auto& object1_local = AllocateObject(threadData);
         object1 = &object1_local;
         global1->field1 = object1_local.header();
-        while (weak.load() == nullptr);
+        while (weak.load() == nullptr)
+            ;
         threadData.gc().ScheduleAndWaitFullGC();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1_local.header()), false);
-        ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak.load()->header()), false);
         ASSERT_THAT((*weak.load())->referred, object1_local.header());
 
         global1->field1 = nullptr;
@@ -1187,8 +1186,8 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
     for (int i = 0; i < kDefaultThreadCount; ++i) {
         gcFutures[i].wait();
         // Verify that threads marked their own locals (but not their globals) according to the configured marking behavior:
-        ASSERT_THAT(GetColor(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack ? Color::kBlack : Color::kWhite);
+        ASSERT_THAT(IsMarked(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack);
-        ASSERT_THAT(GetColor(reachablesGlobals[i]), Color::kWhite);
+        ASSERT_THAT(IsMarked(reachablesGlobals[i]), false);
     }
 
     for (auto& future : gcFutures) {

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp

@@ -23,6 +23,7 @@ struct MemoryUsage {
 };
 
 class GCHandle {
+public:
     class GCStageScopeUsTimer {
     protected:
         uint64_t startTime_ = konan::getTimeMicros();
@@ -86,6 +87,7 @@ class GCHandle {
         }
     };
 
+private:
     uint64_t epoch_;
     explicit GCHandle(uint64_t epoch) : epoch_(epoch) {}
 

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -29,7 +29,7 @@ template <typename Traits>
 void processFieldInMark(void* state, ObjHeader* field) noexcept {
     auto& markQueue = *static_cast<typename Traits::MarkQueue*>(state);
     if (field->heap()) {
-        Traits::enqueue(markQueue, field);
+        Traits::tryEnqueue(markQueue, field);
     }
 }
 
@@ -62,7 +62,7 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc
         return false;
 
     if (object->heap()) {
-        Traits::enqueue(markQueue, object);
+        Traits::tryEnqueue(markQueue, object);
     } else {
         // Each permanent and stack object has own entry in the root set, so it's okay to only process objects in heap.
         Traits::processInMark(markQueue, object);
@@ -71,43 +71,38 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc
     return true;
 }
 
-} // namespace internal
+// TODO: Consider making it noinline to keep loop in `Mark` small.
-
+template <typename Traits>
-struct MarkStats {
+void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQueue, mm::ExtraObjectData& extraObjectData, ObjHeader* object) noexcept {
-    // How many objects are alive.
+    if (auto weakCounter = extraObjectData.GetWeakReferenceCounter()) {
-    size_t aliveHeapSet = 0;
+        RuntimeAssert(
-    // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself.
+                weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", object, weakCounter,
-    size_t aliveHeapSetBytes = 0;
+                weakCounter->permanent(), weakCounter->local());
-    // How many roots are were marked.
+        // Do not schedule WeakReferenceCounter but process it right away.
-    size_t rootSetSize = 0;
+        // This will skip markQueue interaction.
-
+        if (Traits::tryMark(weakCounter)) {
-    void merge(MarkStats other) {
+            markHandle.addObject(mm::GetAllocatedHeapSize(weakCounter));
-        aliveHeapSet += other.aliveHeapSet;
+            // WeakReferenceCounter is empty, but keeping this just in case.
-        aliveHeapSetBytes += other.aliveHeapSetBytes;
+            Traits::processInMark(markQueue, weakCounter);
-        rootSetSize += other.rootSetSize;
+        }
     }
-};
+}
+
+} // namespace internal
 
 template <typename Traits>
 void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept {
     auto markHandle = handle.mark();
-    while (!Traits::isEmpty(markQueue)) {
+    while (ObjHeader* top = Traits::tryDequeue(markQueue)) {
-        ObjHeader* top = Traits::dequeue(markQueue);
+        // TODO: Consider moving it to the sweep phase to make this loop more tight.
-
+        //       This, however, requires care with scheduler interoperation.
-        RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top);
-        RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local());
-
         markHandle.addObject(mm::GetAllocatedHeapSize(top));
 
         Traits::processInMark(markQueue, top);
 
+        // TODO: Consider moving it before processInMark to make the latter something of a tail call.
         if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) {
-            if (auto weakCounter = extraObjectData->GetWeakReferenceCounter()) {
+            internal::processExtraObjectData<Traits>(markHandle, markQueue, *extraObjectData, top);
-                RuntimeAssert(
-                        weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", top,
-                        weakCounter, weakCounter->permanent(), weakCounter->local());
-                Traits::enqueue(markQueue, weakCounter);
-            }
         }
     }
 }

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -124,25 +124,28 @@ public:
 
     const std_support::unordered_set<ObjHeader*>& marked() const { return marked_; }
 
-    static bool isEmpty(const MarkQueue& queue) noexcept {
-        return queue.empty();
-    }
-
     static void clear(MarkQueue& queue) noexcept {
         queue.clear();
     }
 
-    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+    static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
+        if (queue.empty()) return nullptr;
         auto top = queue.back();
         queue.pop_back();
         return top;
     }
 
-    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+    static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto result = instance_->marked_.insert(object);
         if (result.second) {
             queue.push_back(object);
         }
+        return result.second;
+    }
+
+    static bool tryMark(ObjHeader* object) noexcept {
+        auto result = instance_->marked_.insert(object);
+        return result.second;
     }
 
     static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
@@ -176,7 +179,7 @@ public:
 
     gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
         std_support::vector<ObjHeader*> objects;
-        for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader());
+        for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().GetObjHeader());
         auto handle = gc::GCHandle::create(epoch_++);
         gc::Mark<ScopedMarkTraits>(handle, objects);
         handle.finished();

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -31,14 +31,12 @@ struct SweepTraits {
         auto *baseObject = object.GetBaseObject();
         if (!baseObject->heap()) return true;
         auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
-        return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
+        return objectData.marked();
     }
 
     static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
         auto& objectData = node.ObjectData();
-        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite) return false;
+        return objectData.tryResetMark();
-        objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite);
-        return true;
     }
 };
 

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -33,37 +33,40 @@ public:
     };
 
     class ObjectData {
-        static inline constexpr unsigned colorMask = (1 << 1) - 1;
-
     public:
-        enum class Color : unsigned {
+        bool tryMark() noexcept {
-            kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected.
+            return trySetNext(reinterpret_cast<ObjectData*>(1));
-                        // All new objects are allocated with this color.
+        }
-            kBlack, // Objects encountered during mark phase.
-        };
 
-        Color color() const noexcept { return static_cast<Color>(getPointerBits(next_, colorMask)); }
+        bool marked() const noexcept { return next_ != nullptr; }
-        void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast<unsigned>(color)); }
 
-        ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); }
+        bool tryResetMark() noexcept {
-        void setNext(ObjectData* next) noexcept {
+            if (next_ == nullptr) return false;
-            RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next);
+            next_ = nullptr;
-            auto bits = getPointerBits(next_, colorMask);
+            return true;
-            next_ = setPointerBits(next, bits);
         }
 
     private:
-        // Color is encoded in low bits.
+        friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
+
+        ObjectData* next() const noexcept { return next_; }
+        void setNext(ObjectData* next) noexcept {
+            RuntimeAssert(next, "next cannot be nullptr");
+            next_ = next;
+        }
+        bool trySetNext(ObjectData* next) noexcept {
+            RuntimeAssert(next, "next cannot be nullptr");
+            if (next_ != nullptr) {
+                return false;
+            }
+            next_ = next;
+            return true;
+        }
+
         ObjectData* next_ = nullptr;
     };
 
-    struct MarkQueueTraits {
+    using MarkQueue = intrusive_forward_list<ObjectData>;
-        static ObjectData* next(const ObjectData& value) noexcept { return value.next(); }
-
-        static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); }
-    };
-
-    using MarkQueue = intrusive_forward_list<ObjectData, MarkQueueTraits>;
 
     class ThreadData : private Pinned {
     public:
@@ -112,22 +115,24 @@ namespace internal {
 struct MarkTraits {
     using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue;
 
-    static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
-
     static void clear(MarkQueue& queue) noexcept { queue.clear(); }
 
-    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+    static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
-        auto& top = queue.front();
+        if (auto* top = queue.try_pop_front()) {
-        queue.pop_front();
+            auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(*top);
-        auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(top);
+            return node->GetObjHeader();
-        return node->GetObjHeader();
+        }
+        return nullptr;
     }
 
-    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+    static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
         auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
-        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
+        return queue.try_push_front(objectData);
-        objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
+    }
-        queue.push_front(objectData);
+
+    static bool tryMark(ObjHeader* object) noexcept {
+        auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
+        return objectData.tryMark();
     }
 
     static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -200,11 +200,9 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
     return objects;
 }
 
-using Color = gc::SameThreadMarkAndSweep::ObjectData::Color;
+bool IsMarked(ObjHeader* objHeader) {
-
-Color GetColor(ObjHeader* objHeader) {
     auto nodeRef = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(objHeader);
-    return nodeRef.ObjectData().color();
+    return nodeRef.ObjectData().marked();
 }
 
 WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
@@ -246,12 +244,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global2.header()), false);
-        ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global3.header()), false);
-        ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack1.header()), false);
-        ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack2.header()), false);
-        ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack3.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -259,12 +257,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global1.header()), false);
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
-        EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global3.header()), false);
-        EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack1.header()), false);
-        EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack2.header()), false);
-        EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack3.header()), false);
     });
 }
 
@@ -292,12 +290,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(global2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global2.header()), false);
-        ASSERT_THAT(GetColor(global3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global3.header()), false);
-        ASSERT_THAT(GetColor(stack1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack1.header()), false);
-        ASSERT_THAT(GetColor(stack2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack2.header()), false);
-        ASSERT_THAT(GetColor(stack3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack3.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -305,12 +303,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
-        EXPECT_THAT(GetColor(global1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global1.header()), false);
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
-        EXPECT_THAT(GetColor(global3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global3.header()), false);
-        EXPECT_THAT(GetColor(stack1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack1.header()), false);
-        EXPECT_THAT(GetColor(stack2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack2.header()), false);
-        EXPECT_THAT(GetColor(stack3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack3.header()), false);
     });
 }
 
@@ -320,8 +318,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) {
         auto& object2 = AllocateObject(threadData);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -335,8 +333,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
         auto& object2 = AllocateObjectWithFinalizer(threadData);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         EXPECT_CALL(finalizerHook(), Call(object1.header()));
         EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -355,8 +353,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
         })();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak1.header()), false);
         ASSERT_THAT(weak1->referred, object1.header());
 
         threadData.gc().ScheduleAndWaitFullGC();
@@ -372,14 +370,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
         auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(weak1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak1.header()), false);
         ASSERT_THAT(weak1->referred, object1.header());
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
-        EXPECT_THAT(GetColor(weak1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(weak1.header()), false);
         EXPECT_THAT(weak1->referred, nullptr);
     });
 }
@@ -402,12 +400,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -415,12 +413,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -449,14 +447,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
@@ -464,12 +462,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -498,14 +496,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         EXPECT_CALL(finalizerHook(), Call(object5.header()));
         EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -515,12 +513,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -537,18 +535,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
         object2->field1 = stack.header();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
     });
 }
 
@@ -577,14 +575,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
                         object5.header(), object6.header()));
-        ASSERT_THAT(GetColor(global.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global.header()), false);
-        ASSERT_THAT(GetColor(stack.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(stack.header()), false);
-        ASSERT_THAT(GetColor(object1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1.header()), false);
-        ASSERT_THAT(GetColor(object2.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object2.header()), false);
-        ASSERT_THAT(GetColor(object3.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object3.header()), false);
-        ASSERT_THAT(GetColor(object4.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object4.header()), false);
-        ASSERT_THAT(GetColor(object5.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object5.header()), false);
-        ASSERT_THAT(GetColor(object6.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object6.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
         threadData.gc().ScheduleAndWaitFullGC();
@@ -593,12 +591,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
                 Alive(threadData),
                 testing::UnorderedElementsAre(
                         global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(stack.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(stack.header()), false);
-        EXPECT_THAT(GetColor(object1.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object1.header()), false);
-        EXPECT_THAT(GetColor(object2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object2.header()), false);
-        EXPECT_THAT(GetColor(object3.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object3.header()), false);
-        EXPECT_THAT(GetColor(object4.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object4.header()), false);
     });
 }
 
@@ -615,12 +613,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) {
         global2->field1 = global1.header();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
-        EXPECT_THAT(GetColor(global2.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global2.header()), false);
     });
 }
 
@@ -634,14 +632,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) {
 
         ASSERT_THAT(global.header(), stack.header());
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(object.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object.header()), false);
 
         threadData.gc().ScheduleAndWaitFullGC();
 
         EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
-        EXPECT_THAT(GetColor(global.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(global.header()), false);
-        EXPECT_THAT(GetColor(object.header()), Color::kWhite);
+        EXPECT_THAT(IsMarked(object.header()), false);
     });
 }
 
@@ -1098,13 +1096,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
         auto& object1_local = AllocateObject(threadData);
         object1 = &object1_local;
         global1->field1 = object1_local.header();
-        while (weak.load() == nullptr);
+        while (weak.load() == nullptr)
+            ;
         threadData.gc().ScheduleAndWaitFullGC();
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
-        ASSERT_THAT(GetColor(global1.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(global1.header()), false);
-        ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(object1_local.header()), false);
-        ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite);
+        ASSERT_THAT(IsMarked(weak.load()->header()), false);
         ASSERT_THAT((*weak.load())->referred, object1_local.header());
 
         global1->field1 = nullptr;

kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp

@@ -11,29 +11,41 @@
 
 #include "KAssert.h"
 #include "Utils.hpp"
+#include "std_support/Optional.hpp"
 
 namespace kotlin {
 
 template <typename T>
 struct DefaultIntrusiveForwardListTraits {
-    static T* next(const T& value) noexcept { return value.next_; }
+    static T* next(const T& value) noexcept { return value.next(); }
 
-    static void setNext(T& value, T* next) noexcept { value.next_ = next; }
+    static void setNext(T& value, T* next) noexcept { value.setNext(next); }
+
+    static bool trySetNext(T& value, T* next) noexcept { return value.trySetNext(next); }
 };
 
-// Intrusive variant of `std::forward_list`. Notable differences:
+// Intrusive variant of `std::forward_list`.
-// * The container does not own nodes. Care must be taken not to allow a node
-//   to be in two containers at once, or twice into the same container.
-// * The container is move-only, and moving invalidates `before_begin` iterator.
-// * insert_after, erase_after take `iterator` instead of `const_iterator`, because
-//   they do in fact require mutability via `Traits::setNext`.
-// * When the node leaves the container, nothing clears `next` pointer inside it.
 //
-// `Traits` must have 2 methods:
+// The container does not own nodes. The list structure is maintained by `T` itself via
-// static T* next(const T& value);
+// `Traits`. `Traits` must provide 3 operations:
-// static void setNext(T& value, T* next);
+//
-// NOTE: `setNext` and `next` must be callable even on uninitialized `T` (i.e. they
+// static T* next(const T& value); // obtain the next pointer
-//       should only access storage inside `T`).
+// static void setNext(T& value, T* next); // set the next pointer
+// static bool trySetNext(T& value, T* next); // try to set the next pointer or return `false` if it's not possible. Used by `try_push_front`.
+// The default `Traits` implementation expects `T` to provide all operations as member functions.
+//
+// Notable differences from regular containers:
+// * To put `T` into different intrusive lists simultaneously it should
+//   provide custom `Traits` implementation to regulate which next pointer is used
+//   by which list.
+// * It's not possible to keep the same `T` twice in the same list.
+// * The container is move-only and moving invalidates `before_begin` iterator.
+// * `insert_after`, `erase_after` take `iterator` instead of `const_iterator`,
+//   because they require mutability.
+// * When the node is inserted into the container its next pointer is set
+//   to something non-null, but when its removed from the container nothing
+//   nulls the next pointer.
+// * `Traits::trySetNext`, `Traits::setNext` and `Traits::next` must be callable even on uninitialized `T` (i.e. they should only access storage inside `T`).
 template <typename T, typename Traits = DefaultIntrusiveForwardListTraits<T>>
 class intrusive_forward_list : private MoveOnly {
 public:
@@ -123,159 +135,237 @@ public:
         pointer node_ = nullptr;
     };
 
-    intrusive_forward_list() noexcept {
+    // Complexity: O(1)
-        setNext(head(), nullptr);
+    intrusive_forward_list() noexcept { clear(); }
-    }
 
-    intrusive_forward_list(intrusive_forward_list&& rhs) noexcept : size_(rhs.size_) {
+    // Complexity: O(1)
+    intrusive_forward_list(intrusive_forward_list&& rhs) noexcept {
+        // Since tail() is shared, there's no need to update the last node's next_.
         setNext(head(), next(rhs.head()));
-        setNext(rhs.head(), nullptr);
+        rhs.clear();
-        rhs.size_ = 0;
     }
 
+
+    // `InputIt` should dereference into `T&`.
+    // Complexity: O(last - first)
     template <typename InputIt>
-    intrusive_forward_list(InputIt first, InputIt last) noexcept {
+    intrusive_forward_list(InputIt first, InputIt last) noexcept : intrusive_forward_list() {
-        setNext(head(), nullptr);
         assign(std::move(first), std::move(last));
     }
 
+    // Complexity: O(1)
     ~intrusive_forward_list() = default;
 
+    // Complexity: O(1)
     intrusive_forward_list& operator=(intrusive_forward_list&& rhs) noexcept {
         intrusive_forward_list tmp(std::move(rhs));
         swap(tmp);
         return *this;
     }
 
+    // Complexity: O(1)
     void swap(intrusive_forward_list& rhs) noexcept {
+        // Since tail() is shared, there's no need to swap the last nodes' next_.
         using std::swap;
         auto thisNext = next(head());
         auto rhsNext = next(rhs.head());
         swap(thisNext, rhsNext);
         setNext(head(), thisNext);
         setNext(rhs.head(), rhsNext);
-        swap(size_, rhs.size_);
     }
 
+    // Rewrite the contents of `this` with nodes from range `[first, last)`.
+    // `InputIt` should dereference into `T&`.
+    // Complexity: O(last - first)
     template <typename InputIt>
     void assign(InputIt first, InputIt last) noexcept {
         clear();
         insert_after(before_begin(), std::move(first), std::move(last));
     }
 
+    // Complexity: O(1)
     reference front() noexcept { return *next(head()); }
+    // Complexity: O(1)
     const_reference front() const noexcept { return *next(head()); }
 
+    // Iterator before the first node. Cannot be dereferenced.
+    // Complexity: O(1)
     iterator before_begin() noexcept { return iterator(head()); }
+    // Iterator before the first node. Cannot be dereferenced.
+    // Complexity: O(1)
     const_iterator before_begin() const noexcept { return const_iterator(head()); }
+    // Iterator before the first node. Cannot be dereferenced.
+    // Complexity: O(1)
     const_iterator cbefore_begin() const noexcept { return const_iterator(head()); }
 
+    // Complexity: O(1)
     iterator begin() noexcept { return iterator(next(head())); }
+    // Complexity: O(1)
     const_iterator begin() const noexcept { return const_iterator(next(head())); }
+    // Complexity: O(1)
     const_iterator cbegin() const noexcept { return const_iterator(next(head())); }
 
-    iterator end() noexcept { return iterator(); }
+    // Complexity: O(1)
-    const_iterator end() const noexcept { return const_iterator(); }
+    iterator end() noexcept { return iterator(tail()); }
-    const_iterator cend() const noexcept { return const_iterator(); }
+    // Complexity: O(1)
+    const_iterator end() const noexcept { return const_iterator(tail()); }
+    // Complexity: O(1)
+    const_iterator cend() const noexcept { return const_iterator(tail()); }
 
-    bool empty() const noexcept { return size_ == 0; }
+    // Complexity: O(1)
+    bool empty() const noexcept { return next(head()) == tail(); }
 
+    // Complexity: O(1)
     size_type max_size() const noexcept { return std::numeric_limits<size_type>::max(); }
 
-    void clear() noexcept { setNext(head(), nullptr); size_ = 0; }
+    // Complexity: O(1)
+    void clear() noexcept { setNext(head(), tail()); }
 
+    // Insert `value` after `pos`. `pos` can be in range `[before_begin(), end())`.
+    // Returns iterator to the newly inserted element
+    // Complexity: O(1)
     iterator insert_after(iterator pos, reference value) noexcept {
-        pointer nextNode = next(pos.node_);
+        RuntimeAssert(pos != end(), "Attempted to insert_after end()");
+        RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator");
+        setNext(&value, next(pos.node_));
         setNext(pos.node_, &value);
-        setNext(&value, nextNode);
-        ++size_;
         return iterator(&value);
     }
 
+    // Insert `[first, last)` after `pos`. `pos` can be in range `[before_begin(), end())`.
+    // `InputIt` should dereference into `T&`.
+    // Returns iterator to the last inserted element.
+    // Complexity: O(last - first)
     template <typename InputIt>
     iterator insert_after(iterator pos, InputIt first, InputIt last) noexcept {
+        RuntimeAssert(pos != end(), "Attempted to insert_after end()");
+        RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator");
         pointer nextNode = next(pos.node_);
         pointer prevNode = pos.node_;
-        size_t newSize = size_;
         for (auto it = first; it != last; ++it) {
-            setNext(prevNode, &*it);
+            pointer newNode = &*it;
-            prevNode = &*it;
+            setNext(prevNode, newNode);
-            ++newSize;
+            prevNode = newNode;
         }
         setNext(prevNode, nextNode);
-        size_ = newSize;
         return iterator(prevNode);
     }
 
+    // Erase a node after `pos`. `pos` can be in range `[begin(), end() - 1)`.
+    // This does not destroy the erased element, and it does not change its next pointer.
+    // Returns iterator to the next node of the erased one.
+    // Complexity: O(1)
     iterator erase_after(iterator pos) noexcept {
-        pointer prevNode = pos.node_;
+        RuntimeAssert(pos != end(), "Attempted to erase_after end()");
-        pointer nodeToErase = next(pos.node_);
+        RuntimeAssert(pos != iterator(), "Attempted to erase_after empty iterator");
-        if (!nodeToErase) {
+        RuntimeAssert(next(pos.node_) != tail(), "Attempted to erase_after the last node");
-            return end();
+        pointer nextNode = next(next(pos.node_));
-        }
+        setNext(pos.node_, nextNode);
-        pointer nextNode = next(nodeToErase);
-        setNext(prevNode, nextNode);
-        setNext(nodeToErase, nullptr);
-        --size_;
         return iterator(nextNode);
     }
 
+    // Erase all nodes in range `(first, last)`.
+    // `first` can be in range `[before_begin(), last)`.
+    // `last` can be in range `(first, end()]`.
+    // This does not destroy erased elements, and it does not change their next pointers.
+    // Returns iterator to the next node of the last erased (i.e. returns `last`).
+    // Complexity: O(1)
     iterator erase_after(iterator first, iterator last) noexcept {
-        size_ -= std::distance(first, last) - 1;
+        RuntimeAssert(first != end(), "Attempted to erase_after starting at end()");
+        RuntimeAssert(first != iterator(), "Attempted to erase_after starting at empty iterator");
+        RuntimeAssert(next(first.node_) != tail(), "Attempted to erase_after starting at the last node");
+        RuntimeAssert(last != iterator(), "Attempted to erase_after ending at empty iterator");
         setNext(first.node_, last.node_);
         return last;
     }
 
+    // Insert a new node to the front.
+    // Equivalent to `insert_after(before_begin(), value)`.
+    // Complexity: O(1)
     void push_front(reference value) noexcept { insert_after(before_begin(), value); }
 
+    // Try to insert a new node to the front.
+    // When setting the next node of `value` uses `Traits::trySetNext`.
+    // If `Traits::trySetNext` returns `true`, this operates like `push_front` and returns `true`.
+    // If `Traits::trySetNext` returns `false`, this doesn't change anything else and returns `false`.
+    // Complexity: O(1)
+    bool try_push_front(reference value) noexcept { return try_insert_after(before_begin(), value) != std::nullopt; }
+
+    // Erase a node at the front.
+    // This does not destroy the erased node and does not change its next pointer.
+    // Complexity: O(1)
     void pop_front() noexcept { erase_after(before_begin()); }
 
-    void remove(reference value) noexcept {
+    // Try to erase node at the front.
-        // TODO: no need to move on after finding the first match.
+    // If this list is empty, returns `nullptr`.
-        return remove_if([&value](const_reference x) { return &x == &value; });
+    // Otherwise returns pointer to the node at the front and erases it.
+    // This does not destroy the erased node and does not change its next pointer.
+    // Complexity: O(1)
+    pointer try_pop_front() noexcept {
+        pointer top = next(head());
+        if (top == tail()) {
+            return nullptr;
+        }
+        setNext(head(), next(top));
+        return top;
     }
 
+    // Erase node `value`.
+    // If the `value` is not in the list, does nothing.
+    // This does not destroy the erased node and does not change its next pointer.
+    // Complexity: O(n)
+    void remove(reference value) noexcept {
+        // TODO: no need to move on after finding the first match.
+        return remove_if([&value](const_reference x) noexcept { return &x == &value; });
+    }
+
+    // Erase all nodes satisfying predicate `P`.
+    // This does not destroy erased nodes and does not change their next pointer.
+    // Complexity: O(n)
     template <typename P>
-    void remove_if(P p) noexcept {
+    void remove_if(P p) noexcept(noexcept(p(std::declval<const_reference>()))) {
-        size_t newSize = size_;
         pointer prev = head();
         pointer node = next(prev);
-        while (node) {
+        while (node != tail()) {
             if (p(*node)) {
                 // The node is being removed.
                 node = next(node);
                 setNext(prev, node);
-                --newSize;
             } else {
                 // The node is staying.
                 prev = node;
                 node = next(node);
             }
         }
-        size_ = newSize;
     }
 
     // TODO: Implement splice_after.
 
-    size_type size() const noexcept {
-        return size_;
-    }
-
 private:
     static pointer next(const_pointer node) noexcept { return Traits::next(*node); }
 
     static void setNext(pointer node, pointer next) noexcept { return Traits::setNext(*node, next); }
 
-    pointer head() noexcept {
+    static bool trySetNext(pointer node, pointer next) noexcept { return Traits::trySetNext(*node, next); }
-        return reinterpret_cast<pointer>(headStorage_);
+
+    pointer head() noexcept { return reinterpret_cast<pointer>(headStorage_); }
+    const_pointer head() const noexcept { return reinterpret_cast<const_pointer>(headStorage_); }
+
+    static pointer tail() noexcept { return reinterpret_cast<pointer>(tailStorage_); }
+
+    // TODO: Consider making public.
+    std::optional<iterator> try_insert_after(iterator pos, reference value) noexcept {
+        RuntimeAssert(pos != end(), "Attempted to try_insert_after end()");
+        RuntimeAssert(pos != iterator(), "Attempted to try_insert_after empty iterator");
+        if (!trySetNext(&value, next(pos.node_))) {
+            return std::nullopt;
+        }
+        setNext(pos.node_, &value);
+        return iterator(&value);
     }
 
-    const_pointer head() const noexcept {
+    alignas(value_type) char headStorage_[sizeof(value_type)] = {0};
-        return reinterpret_cast<const_pointer>(headStorage_);
+    alignas(value_type) static inline char tailStorage_[sizeof(value_type)] = {0};
-    }
-
-    alignas(value_type) char headStorage_[sizeof(value_type)] = { 0 };
-    size_t size_ = 0;
 };
 
 template <typename InputIt>

kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp

@@ -45,9 +45,23 @@ public:
     int& operator*() { return value_; }
     const int& operator*() const { return value_; }
 
+    void clearNext() noexcept { next_ = nullptr; }
+
 private:
     friend struct DefaultIntrusiveForwardListTraits<Node>;
 
+    Node* next() const noexcept { return next_; }
+    void setNext(Node* next) noexcept {
+        RuntimeAssert(next, "next cannot be nullptr");
+        next_ = next;
+    }
+    bool trySetNext(Node* next) noexcept {
+        RuntimeAssert(next, "next cannot be nullptr");
+        if (next_) return false;
+        next_ = next;
+        return true;
+    }
+
     int value_;
     // Use non-null marker to make sure inserting into the list properly updates this value.
     Node* next_ = reinterpret_cast<Node*>(0x1);
@@ -68,16 +82,8 @@ MATCHER_P(isEmpty, expected, (expected == !negation) ? "is empty" : "is not empt
     return expected == actual;
 }
 
-size_t getSize(const std::forward_list<Element>& list) {
-    return std::distance(list.begin(), list.end());
-}
-
-size_t getSize(const intrusive_forward_list<Node>& list) {
-    return list.size();
-}
-
 MATCHER_P(hasSize, expected, "") {
-    size_t actual = getSize(arg);
+    size_t actual = std::distance(arg.begin(), arg.end());
     *result_listener << "of size " << actual;
     return expected == actual;
 }
@@ -622,3 +628,72 @@ TYPED_TEST(ForwardListTest, EraseAfterEmptyRangeFront) {
     EXPECT_THAT(result, list.begin());
     EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4);
 }
+
+TEST(InstrusiveForwardListTest, TryPushFrontSuccess) {
+    using List = intrusive_forward_list<Node>;
+    auto values = create<List>({1, 2, 3, 4});
+    List list(values.begin(), values.end());
+    typename List::value_type value(5);
+    value.clearNext();
+    auto result = list.try_push_front(value);
+    EXPECT_TRUE(result);
+    EXPECT_ELEMENTS_ARE(list, 5, 1, 2, 3, 4);
+}
+
+TEST(InstrusiveForwardListTest, TryPushFrontFailure) {
+    using List = intrusive_forward_list<Node>;
+    auto values = create<List>({1, 2, 3, 4});
+    List list(values.begin(), values.end());
+    typename List::value_type value(5);
+    auto result = list.try_push_front(value);
+    EXPECT_FALSE(result);
+    EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4);
+}
+
+TEST(InstrusiveForwardListTest, TryPushFrontEmptySuccess) {
+    using List = intrusive_forward_list<Node>;
+    List list;
+    typename List::value_type value(5);
+    value.clearNext();
+    auto result = list.try_push_front(value);
+    EXPECT_TRUE(result);
+    EXPECT_ELEMENTS_ARE(list, 5);
+}
+
+TEST(InstrusiveForwardListTest, TryPushFrontEmptyFailure) {
+    using List = intrusive_forward_list<Node>;
+    List list;
+    typename List::value_type value(5);
+    auto result = list.try_push_front(value);
+    EXPECT_FALSE(result);
+    EXPECT_ELEMENTS_ARE(list);
+}
+
+TEST(IntrusiveForwardListTest, TryPopFront) {
+    using List = intrusive_forward_list<Node>;
+    auto values = create<List>({1, 2, 3, 4});
+    List list(values.begin(), values.end());
+    auto& front = list.front();
+    auto result = list.try_pop_front();
+    EXPECT_THAT(result, &front);
+    EXPECT_ELEMENTS_ARE(list, 2, 3, 4);
+}
+
+TEST(IntrusiveForwardListTest, TryPopFrontIntoEmpty) {
+    using List = intrusive_forward_list<Node>;
+    auto values = create<List>({1});
+    List list(values.begin(), values.end());
+    auto& front = list.front();
+    auto result = list.try_pop_front();
+    EXPECT_THAT(result, &front);
+    EXPECT_ELEMENTS_ARE(list);
+}
+
+TEST(IntrusiveForwardListTest, TryPopFrontFromEmpty) {
+    using List = intrusive_forward_list<Node>;
+    auto values = create<List>({});
+    List list(values.begin(), values.end());
+    auto result = list.try_pop_front();
+    EXPECT_THAT(result, nullptr);
+    EXPECT_ELEMENTS_ARE(list);
+}