diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index a5fc9b896be..c3264c62063 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -38,14 +38,12 @@ struct SweepTraits { auto *baseObject = object.GetBaseObject(); if (!baseObject->heap()) return true; auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).ObjectData(); - return objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack; + return objectData.marked(); } static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { auto& objectData = node.ObjectData(); - if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite) return false; - objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kWhite); - return true; + return objectData.tryResetMark(); } }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 252876c6d1b..ecdef1a9ea4 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -35,48 +35,39 @@ class FinalizerProcessor; class ConcurrentMarkAndSweep : private Pinned { public: class ObjectData { - static inline constexpr unsigned colorMask = (1 << 1) - 1; - public: - enum class Color : unsigned { - kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected. - // All new objects are allocated with this color. - kBlack, // Objects encountered during mark phase. - }; - - Color color() const noexcept { return static_cast(getPointerBits(next_.load(std::memory_order_relaxed), colorMask)); } - void setColor(Color color) noexcept { next_.store(setPointerBits(clearPointerBits(next_.load(std::memory_order_relaxed), colorMask), static_cast(color)), std::memory_order_relaxed); } - bool atomicSetToBlack() noexcept { - ObjectData* before = next_.load(std::memory_order_relaxed); - if (getPointerBits(before, colorMask) != static_cast(Color::kWhite)) - return false; - ObjectData* black = setPointerBits(before, static_cast(Color::kBlack)); - bool success = next_.compare_exchange_strong(before, black, std::memory_order_relaxed); - RuntimeAssert(success || hasPointerBits(before, colorMask), "next_ must have been marked black"); - return success; + bool tryMark() noexcept { + return trySetNext(reinterpret_cast(1)); } - ObjectData* next() const noexcept { return clearPointerBits(next_.load(std::memory_order_relaxed), colorMask); } - void setNext(ObjectData* next) noexcept { - RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next); - auto bits = getPointerBits(next_.load(std::memory_order_relaxed), colorMask); - next_.store(setPointerBits(next, bits), std::memory_order_relaxed); + bool marked() const noexcept { return next() != nullptr; } + + bool tryResetMark() noexcept { + if (next() == nullptr) return false; + next_.store(nullptr, std::memory_order_relaxed); + return true; } private: - // Color is encoded in low bits. + friend struct DefaultIntrusiveForwardListTraits; + + ObjectData* next() const noexcept { return next_.load(std::memory_order_relaxed); } + void setNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + next_.store(next, std::memory_order_relaxed); + } + bool trySetNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + ObjectData* expected = nullptr; + return next_.compare_exchange_strong(expected, next, std::memory_order_relaxed); + } + std::atomic next_ = nullptr; }; - struct MarkQueueTraits { - static ObjectData* next(const ObjectData& value) noexcept { return value.next(); } - - static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); } - }; - enum MarkingBehavior { kMarkOwnStack, kDoNotMark }; - using MarkQueue = intrusive_forward_list; + using MarkQueue = intrusive_forward_list; class ThreadData : private Pinned { public: @@ -138,21 +129,24 @@ namespace internal { struct MarkTraits { using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue; - static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); } - static void clear(MarkQueue& queue) noexcept { queue.clear(); } - static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto& top = queue.front(); - queue.pop_front(); - auto node = mm::ObjectFactory::NodeRef::From(top); - return node->GetObjHeader(); + static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { + if (auto* top = queue.try_pop_front()) { + auto node = mm::ObjectFactory::NodeRef::From(*top); + return node->GetObjHeader(); + } + return nullptr; } - static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - if (!objectData.atomicSetToBlack()) return; - queue.push_front(objectData); + return queue.try_push_front(objectData); + } + + static bool tryMark(ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); + return objectData.tryMark(); } static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 7c0140a1621..1786c624691 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -199,11 +199,9 @@ std_support::vector Alive(mm::ThreadData& threadData) { return objects; } -using Color = gc::ConcurrentMarkAndSweep::ObjectData::Color; - -Color GetColor(ObjHeader* objHeader) { +bool IsMarked(ObjHeader* objHeader) { auto nodeRef = mm::ObjectFactory::NodeRef::From(objHeader); - return nodeRef.ObjectData().color(); + return nodeRef.ObjectData().marked(); } WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { @@ -250,12 +248,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(global2.header()), Color::kWhite); - ASSERT_THAT(GetColor(global3.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(global2.header()), false); + ASSERT_THAT(IsMarked(global3.header()), false); + ASSERT_THAT(IsMarked(stack1.header()), false); + ASSERT_THAT(IsMarked(stack2.header()), false); + ASSERT_THAT(IsMarked(stack3.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -263,12 +261,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(GetColor(global1.header()), Color::kWhite); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); - EXPECT_THAT(GetColor(global3.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global1.header()), false); + EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(IsMarked(global3.header()), false); + EXPECT_THAT(IsMarked(stack1.header()), false); + EXPECT_THAT(IsMarked(stack2.header()), false); + EXPECT_THAT(IsMarked(stack3.header()), false); }); } @@ -296,12 +294,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(global2.header()), Color::kWhite); - ASSERT_THAT(GetColor(global3.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(global2.header()), false); + ASSERT_THAT(IsMarked(global3.header()), false); + ASSERT_THAT(IsMarked(stack1.header()), false); + ASSERT_THAT(IsMarked(stack2.header()), false); + ASSERT_THAT(IsMarked(stack3.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -309,12 +307,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(GetColor(global1.header()), Color::kWhite); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); - EXPECT_THAT(GetColor(global3.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global1.header()), false); + EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(IsMarked(global3.header()), false); + EXPECT_THAT(IsMarked(stack1.header()), false); + EXPECT_THAT(IsMarked(stack2.header()), false); + EXPECT_THAT(IsMarked(stack3.header()), false); }); } @@ -324,8 +322,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) { auto& object2 = AllocateObject(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -339,8 +337,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { auto& object2 = AllocateObjectWithFinalizer(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); @@ -359,8 +357,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); @@ -376,14 +374,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) { auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(GetColor(weak1.header()), Color::kWhite); + EXPECT_THAT(IsMarked(weak1.header()), false); EXPECT_THAT(weak1->referred, nullptr); }); } @@ -406,12 +404,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -419,12 +417,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -453,14 +451,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -468,12 +466,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -502,14 +500,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); @@ -519,12 +517,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -541,18 +539,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { object2->field1 = stack.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); }); } @@ -581,14 +579,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); threadData.gc().ScheduleAndWaitFullGC(); threadData.gc().ScheduleAndWaitFullGC(); @@ -597,12 +595,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -619,12 +617,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) { global2->field1 = global1.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global2.header()), false); }); } @@ -638,14 +636,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) { ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(object.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(object.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(object.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(object.header()), false); }); } @@ -1114,13 +1112,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { auto& object1_local = AllocateObject(threadData); object1 = &object1_local; global1->field1 = object1_local.header(); - while (weak.load() == nullptr); + while (weak.load() == nullptr) + ; threadData.gc().ScheduleAndWaitFullGC(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(object1_local.header()), false); + ASSERT_THAT(IsMarked(weak.load()->header()), false); ASSERT_THAT((*weak.load())->referred, object1_local.header()); global1->field1 = nullptr; @@ -1187,8 +1186,8 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) { for (int i = 0; i < kDefaultThreadCount; ++i) { gcFutures[i].wait(); // Verify that threads marked their own locals (but not their globals) according to the configured marking behavior: - ASSERT_THAT(GetColor(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack ? Color::kBlack : Color::kWhite); - ASSERT_THAT(GetColor(reachablesGlobals[i]), Color::kWhite); + ASSERT_THAT(IsMarked(reachablesLocals[i]), GetParam() == kotlin::gc::ConcurrentMarkAndSweep::kMarkOwnStack); + ASSERT_THAT(IsMarked(reachablesGlobals[i]), false); } for (auto& future : gcFutures) { diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 2cf6dd8d9c2..9b07f6b96a2 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -23,6 +23,7 @@ struct MemoryUsage { }; class GCHandle { +public: class GCStageScopeUsTimer { protected: uint64_t startTime_ = konan::getTimeMicros(); @@ -86,6 +87,7 @@ class GCHandle { } }; +private: uint64_t epoch_; explicit GCHandle(uint64_t epoch) : epoch_(epoch) {} @@ -118,4 +120,4 @@ public: MemoryUsage getMarked(); }; -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index d39ab318865..7aa968ed07e 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -29,7 +29,7 @@ template void processFieldInMark(void* state, ObjHeader* field) noexcept { auto& markQueue = *static_cast(state); if (field->heap()) { - Traits::enqueue(markQueue, field); + Traits::tryEnqueue(markQueue, field); } } @@ -62,7 +62,7 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc return false; if (object->heap()) { - Traits::enqueue(markQueue, object); + Traits::tryEnqueue(markQueue, object); } else { // Each permanent and stack object has own entry in the root set, so it's okay to only process objects in heap. Traits::processInMark(markQueue, object); @@ -71,43 +71,38 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc return true; } -} // namespace internal - -struct MarkStats { - // How many objects are alive. - size_t aliveHeapSet = 0; - // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself. - size_t aliveHeapSetBytes = 0; - // How many roots are were marked. - size_t rootSetSize = 0; - - void merge(MarkStats other) { - aliveHeapSet += other.aliveHeapSet; - aliveHeapSetBytes += other.aliveHeapSetBytes; - rootSetSize += other.rootSetSize; +// TODO: Consider making it noinline to keep loop in `Mark` small. +template +void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQueue, mm::ExtraObjectData& extraObjectData, ObjHeader* object) noexcept { + if (auto weakCounter = extraObjectData.GetWeakReferenceCounter()) { + RuntimeAssert( + weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", object, weakCounter, + weakCounter->permanent(), weakCounter->local()); + // Do not schedule WeakReferenceCounter but process it right away. + // This will skip markQueue interaction. + if (Traits::tryMark(weakCounter)) { + markHandle.addObject(mm::GetAllocatedHeapSize(weakCounter)); + // WeakReferenceCounter is empty, but keeping this just in case. + Traits::processInMark(markQueue, weakCounter); + } } -}; +} + +} // namespace internal template void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept { auto markHandle = handle.mark(); - while (!Traits::isEmpty(markQueue)) { - ObjHeader* top = Traits::dequeue(markQueue); - - RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top); - RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local()); - + while (ObjHeader* top = Traits::tryDequeue(markQueue)) { + // TODO: Consider moving it to the sweep phase to make this loop more tight. + // This, however, requires care with scheduler interoperation. markHandle.addObject(mm::GetAllocatedHeapSize(top)); Traits::processInMark(markQueue, top); + // TODO: Consider moving it before processInMark to make the latter something of a tail call. if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) { - if (auto weakCounter = extraObjectData->GetWeakReferenceCounter()) { - RuntimeAssert( - weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", top, - weakCounter, weakCounter->permanent(), weakCounter->local()); - Traits::enqueue(markQueue, weakCounter); - } + internal::processExtraObjectData(markHandle, markQueue, *extraObjectData, top); } } } diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index 5da15cb0ab5..689f823c696 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -124,25 +124,28 @@ public: const std_support::unordered_set& marked() const { return marked_; } - static bool isEmpty(const MarkQueue& queue) noexcept { - return queue.empty(); - } - static void clear(MarkQueue& queue) noexcept { queue.clear(); } - static ObjHeader* dequeue(MarkQueue& queue) noexcept { + static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { + if (queue.empty()) return nullptr; auto top = queue.back(); queue.pop_back(); return top; } - static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto result = instance_->marked_.insert(object); if (result.second) { queue.push_back(object); } + return result.second; + } + + static bool tryMark(ObjHeader* object) noexcept { + auto result = instance_->marked_.insert(object); + return result.second; } static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { @@ -176,7 +179,7 @@ public: gc::MemoryUsage Mark(std::initializer_list> graySet) { std_support::vector objects; - for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader()); + for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().GetObjHeader()); auto handle = gc::GCHandle::create(epoch_++); gc::Mark(handle, objects); handle.finished(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index ba229fed7d7..526621fa7c3 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -31,14 +31,12 @@ struct SweepTraits { auto *baseObject = object.GetBaseObject(); if (!baseObject->heap()) return true; auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).ObjectData(); - return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack; + return objectData.marked(); } static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { auto& objectData = node.ObjectData(); - if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite) return false; - objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kWhite); - return true; + return objectData.tryResetMark(); } }; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 95e216e4c06..1da48e442b2 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -33,37 +33,40 @@ public: }; class ObjectData { - static inline constexpr unsigned colorMask = (1 << 1) - 1; - public: - enum class Color : unsigned { - kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected. - // All new objects are allocated with this color. - kBlack, // Objects encountered during mark phase. - }; + bool tryMark() noexcept { + return trySetNext(reinterpret_cast(1)); + } - Color color() const noexcept { return static_cast(getPointerBits(next_, colorMask)); } - void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast(color)); } + bool marked() const noexcept { return next_ != nullptr; } - ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); } - void setNext(ObjectData* next) noexcept { - RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next); - auto bits = getPointerBits(next_, colorMask); - next_ = setPointerBits(next, bits); + bool tryResetMark() noexcept { + if (next_ == nullptr) return false; + next_ = nullptr; + return true; } private: - // Color is encoded in low bits. + friend struct DefaultIntrusiveForwardListTraits; + + ObjectData* next() const noexcept { return next_; } + void setNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + next_ = next; + } + bool trySetNext(ObjectData* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + if (next_ != nullptr) { + return false; + } + next_ = next; + return true; + } + ObjectData* next_ = nullptr; }; - struct MarkQueueTraits { - static ObjectData* next(const ObjectData& value) noexcept { return value.next(); } - - static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); } - }; - - using MarkQueue = intrusive_forward_list; + using MarkQueue = intrusive_forward_list; class ThreadData : private Pinned { public: @@ -112,22 +115,24 @@ namespace internal { struct MarkTraits { using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue; - static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); } - static void clear(MarkQueue& queue) noexcept { queue.clear(); } - static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto& top = queue.front(); - queue.pop_front(); - auto node = mm::ObjectFactory::NodeRef::From(top); - return node->GetObjHeader(); + static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { + if (auto* top = queue.try_pop_front()) { + auto node = mm::ObjectFactory::NodeRef::From(*top); + return node->GetObjHeader(); + } + return nullptr; } - static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return; - objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); - queue.push_front(objectData); + return queue.try_push_front(objectData); + } + + static bool tryMark(ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); + return objectData.tryMark(); } static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 7c7cf480c01..0b90eb431aa 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -200,11 +200,9 @@ std_support::vector Alive(mm::ThreadData& threadData) { return objects; } -using Color = gc::SameThreadMarkAndSweep::ObjectData::Color; - -Color GetColor(ObjHeader* objHeader) { +bool IsMarked(ObjHeader* objHeader) { auto nodeRef = mm::ObjectFactory::NodeRef::From(objHeader); - return nodeRef.ObjectData().color(); + return nodeRef.ObjectData().marked(); } WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { @@ -246,12 +244,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(global2.header()), Color::kWhite); - ASSERT_THAT(GetColor(global3.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(global2.header()), false); + ASSERT_THAT(IsMarked(global3.header()), false); + ASSERT_THAT(IsMarked(stack1.header()), false); + ASSERT_THAT(IsMarked(stack2.header()), false); + ASSERT_THAT(IsMarked(stack3.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -259,12 +257,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(GetColor(global1.header()), Color::kWhite); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); - EXPECT_THAT(GetColor(global3.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global1.header()), false); + EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(IsMarked(global3.header()), false); + EXPECT_THAT(IsMarked(stack1.header()), false); + EXPECT_THAT(IsMarked(stack2.header()), false); + EXPECT_THAT(IsMarked(stack3.header()), false); }); } @@ -292,12 +290,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(global2.header()), Color::kWhite); - ASSERT_THAT(GetColor(global3.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack1.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack2.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack3.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(global2.header()), false); + ASSERT_THAT(IsMarked(global3.header()), false); + ASSERT_THAT(IsMarked(stack1.header()), false); + ASSERT_THAT(IsMarked(stack2.header()), false); + ASSERT_THAT(IsMarked(stack3.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -305,12 +303,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) { Alive(threadData), testing::UnorderedElementsAre( global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); - EXPECT_THAT(GetColor(global1.header()), Color::kWhite); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); - EXPECT_THAT(GetColor(global3.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack1.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack2.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack3.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global1.header()), false); + EXPECT_THAT(IsMarked(global2.header()), false); + EXPECT_THAT(IsMarked(global3.header()), false); + EXPECT_THAT(IsMarked(stack1.header()), false); + EXPECT_THAT(IsMarked(stack2.header()), false); + EXPECT_THAT(IsMarked(stack3.header()), false); }); } @@ -320,8 +318,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) { auto& object2 = AllocateObject(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -335,8 +333,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { auto& object2 = AllocateObjectWithFinalizer(threadData); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object2.header())); @@ -355,8 +353,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGC(); @@ -372,14 +370,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak1.header()), Color::kWhite); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(weak1->referred, object1.header()); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); - EXPECT_THAT(GetColor(weak1.header()), Color::kWhite); + EXPECT_THAT(IsMarked(weak1.header()), false); EXPECT_THAT(weak1->referred, nullptr); }); } @@ -402,12 +400,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -415,12 +413,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -449,14 +447,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); threadData.gc().ScheduleAndWaitFullGC(); @@ -464,12 +462,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -498,14 +496,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object6.header())); @@ -515,12 +513,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -537,18 +535,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) { object2->field1 = stack.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); }); } @@ -577,14 +575,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), object5.header(), object6.header())); - ASSERT_THAT(GetColor(global.header()), Color::kWhite); - ASSERT_THAT(GetColor(stack.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object2.header()), Color::kWhite); - ASSERT_THAT(GetColor(object3.header()), Color::kWhite); - ASSERT_THAT(GetColor(object4.header()), Color::kWhite); - ASSERT_THAT(GetColor(object5.header()), Color::kWhite); - ASSERT_THAT(GetColor(object6.header()), Color::kWhite); + ASSERT_THAT(IsMarked(global.header()), false); + ASSERT_THAT(IsMarked(stack.header()), false); + ASSERT_THAT(IsMarked(object1.header()), false); + ASSERT_THAT(IsMarked(object2.header()), false); + ASSERT_THAT(IsMarked(object3.header()), false); + ASSERT_THAT(IsMarked(object4.header()), false); + ASSERT_THAT(IsMarked(object5.header()), false); + ASSERT_THAT(IsMarked(object6.header()), false); threadData.gc().ScheduleAndWaitFullGC(); threadData.gc().ScheduleAndWaitFullGC(); @@ -593,12 +591,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) { Alive(threadData), testing::UnorderedElementsAre( global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(stack.header()), Color::kWhite); - EXPECT_THAT(GetColor(object1.header()), Color::kWhite); - EXPECT_THAT(GetColor(object2.header()), Color::kWhite); - EXPECT_THAT(GetColor(object3.header()), Color::kWhite); - EXPECT_THAT(GetColor(object4.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(stack.header()), false); + EXPECT_THAT(IsMarked(object1.header()), false); + EXPECT_THAT(IsMarked(object2.header()), false); + EXPECT_THAT(IsMarked(object3.header()), false); + EXPECT_THAT(IsMarked(object4.header()), false); }); } @@ -615,12 +613,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) { global2->field1 = global1.header(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global2.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); - EXPECT_THAT(GetColor(global2.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global2.header()), false); }); } @@ -634,14 +632,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) { ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(object.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(object.header()), false); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); - EXPECT_THAT(GetColor(global.header()), Color::kWhite); - EXPECT_THAT(GetColor(object.header()), Color::kWhite); + EXPECT_THAT(IsMarked(global.header()), false); + EXPECT_THAT(IsMarked(object.header()), false); }); } @@ -1098,13 +1096,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { auto& object1_local = AllocateObject(threadData); object1 = &object1_local; global1->field1 = object1_local.header(); - while (weak.load() == nullptr); + while (weak.load() == nullptr) + ; threadData.gc().ScheduleAndWaitFullGC(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); - ASSERT_THAT(GetColor(global1.header()), Color::kWhite); - ASSERT_THAT(GetColor(object1_local.header()), Color::kWhite); - ASSERT_THAT(GetColor(weak.load()->header()), Color::kWhite); + ASSERT_THAT(IsMarked(global1.header()), false); + ASSERT_THAT(IsMarked(object1_local.header()), false); + ASSERT_THAT(IsMarked(weak.load()->header()), false); ASSERT_THAT((*weak.load())->referred, object1_local.header()); global1->field1 = nullptr; diff --git a/kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp b/kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp index a0c9f439916..c99a7b8b633 100644 --- a/kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp +++ b/kotlin-native/runtime/src/main/cpp/IntrusiveList.hpp @@ -11,29 +11,41 @@ #include "KAssert.h" #include "Utils.hpp" +#include "std_support/Optional.hpp" namespace kotlin { template struct DefaultIntrusiveForwardListTraits { - static T* next(const T& value) noexcept { return value.next_; } + static T* next(const T& value) noexcept { return value.next(); } - static void setNext(T& value, T* next) noexcept { value.next_ = next; } + static void setNext(T& value, T* next) noexcept { value.setNext(next); } + + static bool trySetNext(T& value, T* next) noexcept { return value.trySetNext(next); } }; -// Intrusive variant of `std::forward_list`. Notable differences: -// * The container does not own nodes. Care must be taken not to allow a node -// to be in two containers at once, or twice into the same container. -// * The container is move-only, and moving invalidates `before_begin` iterator. -// * insert_after, erase_after take `iterator` instead of `const_iterator`, because -// they do in fact require mutability via `Traits::setNext`. -// * When the node leaves the container, nothing clears `next` pointer inside it. +// Intrusive variant of `std::forward_list`. // -// `Traits` must have 2 methods: -// static T* next(const T& value); -// static void setNext(T& value, T* next); -// NOTE: `setNext` and `next` must be callable even on uninitialized `T` (i.e. they -// should only access storage inside `T`). +// The container does not own nodes. The list structure is maintained by `T` itself via +// `Traits`. `Traits` must provide 3 operations: +// +// static T* next(const T& value); // obtain the next pointer +// static void setNext(T& value, T* next); // set the next pointer +// static bool trySetNext(T& value, T* next); // try to set the next pointer or return `false` if it's not possible. Used by `try_push_front`. +// The default `Traits` implementation expects `T` to provide all operations as member functions. +// +// Notable differences from regular containers: +// * To put `T` into different intrusive lists simultaneously it should +// provide custom `Traits` implementation to regulate which next pointer is used +// by which list. +// * It's not possible to keep the same `T` twice in the same list. +// * The container is move-only and moving invalidates `before_begin` iterator. +// * `insert_after`, `erase_after` take `iterator` instead of `const_iterator`, +// because they require mutability. +// * When the node is inserted into the container its next pointer is set +// to something non-null, but when its removed from the container nothing +// nulls the next pointer. +// * `Traits::trySetNext`, `Traits::setNext` and `Traits::next` must be callable even on uninitialized `T` (i.e. they should only access storage inside `T`). template > class intrusive_forward_list : private MoveOnly { public: @@ -123,159 +135,237 @@ public: pointer node_ = nullptr; }; - intrusive_forward_list() noexcept { - setNext(head(), nullptr); - } + // Complexity: O(1) + intrusive_forward_list() noexcept { clear(); } - intrusive_forward_list(intrusive_forward_list&& rhs) noexcept : size_(rhs.size_) { + // Complexity: O(1) + intrusive_forward_list(intrusive_forward_list&& rhs) noexcept { + // Since tail() is shared, there's no need to update the last node's next_. setNext(head(), next(rhs.head())); - setNext(rhs.head(), nullptr); - rhs.size_ = 0; + rhs.clear(); } + + // `InputIt` should dereference into `T&`. + // Complexity: O(last - first) template - intrusive_forward_list(InputIt first, InputIt last) noexcept { - setNext(head(), nullptr); + intrusive_forward_list(InputIt first, InputIt last) noexcept : intrusive_forward_list() { assign(std::move(first), std::move(last)); } + // Complexity: O(1) ~intrusive_forward_list() = default; + // Complexity: O(1) intrusive_forward_list& operator=(intrusive_forward_list&& rhs) noexcept { intrusive_forward_list tmp(std::move(rhs)); swap(tmp); return *this; } + // Complexity: O(1) void swap(intrusive_forward_list& rhs) noexcept { + // Since tail() is shared, there's no need to swap the last nodes' next_. using std::swap; auto thisNext = next(head()); auto rhsNext = next(rhs.head()); swap(thisNext, rhsNext); setNext(head(), thisNext); setNext(rhs.head(), rhsNext); - swap(size_, rhs.size_); } + // Rewrite the contents of `this` with nodes from range `[first, last)`. + // `InputIt` should dereference into `T&`. + // Complexity: O(last - first) template void assign(InputIt first, InputIt last) noexcept { clear(); insert_after(before_begin(), std::move(first), std::move(last)); } + // Complexity: O(1) reference front() noexcept { return *next(head()); } + // Complexity: O(1) const_reference front() const noexcept { return *next(head()); } + // Iterator before the first node. Cannot be dereferenced. + // Complexity: O(1) iterator before_begin() noexcept { return iterator(head()); } + // Iterator before the first node. Cannot be dereferenced. + // Complexity: O(1) const_iterator before_begin() const noexcept { return const_iterator(head()); } + // Iterator before the first node. Cannot be dereferenced. + // Complexity: O(1) const_iterator cbefore_begin() const noexcept { return const_iterator(head()); } + // Complexity: O(1) iterator begin() noexcept { return iterator(next(head())); } + // Complexity: O(1) const_iterator begin() const noexcept { return const_iterator(next(head())); } + // Complexity: O(1) const_iterator cbegin() const noexcept { return const_iterator(next(head())); } - iterator end() noexcept { return iterator(); } - const_iterator end() const noexcept { return const_iterator(); } - const_iterator cend() const noexcept { return const_iterator(); } + // Complexity: O(1) + iterator end() noexcept { return iterator(tail()); } + // Complexity: O(1) + const_iterator end() const noexcept { return const_iterator(tail()); } + // Complexity: O(1) + const_iterator cend() const noexcept { return const_iterator(tail()); } - bool empty() const noexcept { return size_ == 0; } + // Complexity: O(1) + bool empty() const noexcept { return next(head()) == tail(); } + // Complexity: O(1) size_type max_size() const noexcept { return std::numeric_limits::max(); } - void clear() noexcept { setNext(head(), nullptr); size_ = 0; } + // Complexity: O(1) + void clear() noexcept { setNext(head(), tail()); } + // Insert `value` after `pos`. `pos` can be in range `[before_begin(), end())`. + // Returns iterator to the newly inserted element + // Complexity: O(1) iterator insert_after(iterator pos, reference value) noexcept { - pointer nextNode = next(pos.node_); + RuntimeAssert(pos != end(), "Attempted to insert_after end()"); + RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator"); + setNext(&value, next(pos.node_)); setNext(pos.node_, &value); - setNext(&value, nextNode); - ++size_; return iterator(&value); } + // Insert `[first, last)` after `pos`. `pos` can be in range `[before_begin(), end())`. + // `InputIt` should dereference into `T&`. + // Returns iterator to the last inserted element. + // Complexity: O(last - first) template iterator insert_after(iterator pos, InputIt first, InputIt last) noexcept { + RuntimeAssert(pos != end(), "Attempted to insert_after end()"); + RuntimeAssert(pos != iterator(), "Attempted to insert_after empty iterator"); pointer nextNode = next(pos.node_); pointer prevNode = pos.node_; - size_t newSize = size_; for (auto it = first; it != last; ++it) { - setNext(prevNode, &*it); - prevNode = &*it; - ++newSize; + pointer newNode = &*it; + setNext(prevNode, newNode); + prevNode = newNode; } setNext(prevNode, nextNode); - size_ = newSize; return iterator(prevNode); } + // Erase a node after `pos`. `pos` can be in range `[begin(), end() - 1)`. + // This does not destroy the erased element, and it does not change its next pointer. + // Returns iterator to the next node of the erased one. + // Complexity: O(1) iterator erase_after(iterator pos) noexcept { - pointer prevNode = pos.node_; - pointer nodeToErase = next(pos.node_); - if (!nodeToErase) { - return end(); - } - pointer nextNode = next(nodeToErase); - setNext(prevNode, nextNode); - setNext(nodeToErase, nullptr); - --size_; + RuntimeAssert(pos != end(), "Attempted to erase_after end()"); + RuntimeAssert(pos != iterator(), "Attempted to erase_after empty iterator"); + RuntimeAssert(next(pos.node_) != tail(), "Attempted to erase_after the last node"); + pointer nextNode = next(next(pos.node_)); + setNext(pos.node_, nextNode); return iterator(nextNode); } + // Erase all nodes in range `(first, last)`. + // `first` can be in range `[before_begin(), last)`. + // `last` can be in range `(first, end()]`. + // This does not destroy erased elements, and it does not change their next pointers. + // Returns iterator to the next node of the last erased (i.e. returns `last`). + // Complexity: O(1) iterator erase_after(iterator first, iterator last) noexcept { - size_ -= std::distance(first, last) - 1; + RuntimeAssert(first != end(), "Attempted to erase_after starting at end()"); + RuntimeAssert(first != iterator(), "Attempted to erase_after starting at empty iterator"); + RuntimeAssert(next(first.node_) != tail(), "Attempted to erase_after starting at the last node"); + RuntimeAssert(last != iterator(), "Attempted to erase_after ending at empty iterator"); setNext(first.node_, last.node_); return last; } + // Insert a new node to the front. + // Equivalent to `insert_after(before_begin(), value)`. + // Complexity: O(1) void push_front(reference value) noexcept { insert_after(before_begin(), value); } + // Try to insert a new node to the front. + // When setting the next node of `value` uses `Traits::trySetNext`. + // If `Traits::trySetNext` returns `true`, this operates like `push_front` and returns `true`. + // If `Traits::trySetNext` returns `false`, this doesn't change anything else and returns `false`. + // Complexity: O(1) + bool try_push_front(reference value) noexcept { return try_insert_after(before_begin(), value) != std::nullopt; } + + // Erase a node at the front. + // This does not destroy the erased node and does not change its next pointer. + // Complexity: O(1) void pop_front() noexcept { erase_after(before_begin()); } - void remove(reference value) noexcept { - // TODO: no need to move on after finding the first match. - return remove_if([&value](const_reference x) { return &x == &value; }); + // Try to erase node at the front. + // If this list is empty, returns `nullptr`. + // Otherwise returns pointer to the node at the front and erases it. + // This does not destroy the erased node and does not change its next pointer. + // Complexity: O(1) + pointer try_pop_front() noexcept { + pointer top = next(head()); + if (top == tail()) { + return nullptr; + } + setNext(head(), next(top)); + return top; } + // Erase node `value`. + // If the `value` is not in the list, does nothing. + // This does not destroy the erased node and does not change its next pointer. + // Complexity: O(n) + void remove(reference value) noexcept { + // TODO: no need to move on after finding the first match. + return remove_if([&value](const_reference x) noexcept { return &x == &value; }); + } + + // Erase all nodes satisfying predicate `P`. + // This does not destroy erased nodes and does not change their next pointer. + // Complexity: O(n) template - void remove_if(P p) noexcept { - size_t newSize = size_; + void remove_if(P p) noexcept(noexcept(p(std::declval()))) { pointer prev = head(); pointer node = next(prev); - while (node) { + while (node != tail()) { if (p(*node)) { // The node is being removed. node = next(node); setNext(prev, node); - --newSize; } else { // The node is staying. prev = node; node = next(node); } } - size_ = newSize; } // TODO: Implement splice_after. - size_type size() const noexcept { - return size_; - } - private: static pointer next(const_pointer node) noexcept { return Traits::next(*node); } static void setNext(pointer node, pointer next) noexcept { return Traits::setNext(*node, next); } - pointer head() noexcept { - return reinterpret_cast(headStorage_); + static bool trySetNext(pointer node, pointer next) noexcept { return Traits::trySetNext(*node, next); } + + pointer head() noexcept { return reinterpret_cast(headStorage_); } + const_pointer head() const noexcept { return reinterpret_cast(headStorage_); } + + static pointer tail() noexcept { return reinterpret_cast(tailStorage_); } + + // TODO: Consider making public. + std::optional try_insert_after(iterator pos, reference value) noexcept { + RuntimeAssert(pos != end(), "Attempted to try_insert_after end()"); + RuntimeAssert(pos != iterator(), "Attempted to try_insert_after empty iterator"); + if (!trySetNext(&value, next(pos.node_))) { + return std::nullopt; + } + setNext(pos.node_, &value); + return iterator(&value); } - const_pointer head() const noexcept { - return reinterpret_cast(headStorage_); - } - - alignas(value_type) char headStorage_[sizeof(value_type)] = { 0 }; - size_t size_ = 0; + alignas(value_type) char headStorage_[sizeof(value_type)] = {0}; + alignas(value_type) static inline char tailStorage_[sizeof(value_type)] = {0}; }; template diff --git a/kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp b/kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp index 8ef698873c6..5321d8257d9 100644 --- a/kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/IntrusiveListTest.cpp @@ -45,9 +45,23 @@ public: int& operator*() { return value_; } const int& operator*() const { return value_; } + void clearNext() noexcept { next_ = nullptr; } + private: friend struct DefaultIntrusiveForwardListTraits; + Node* next() const noexcept { return next_; } + void setNext(Node* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + next_ = next; + } + bool trySetNext(Node* next) noexcept { + RuntimeAssert(next, "next cannot be nullptr"); + if (next_) return false; + next_ = next; + return true; + } + int value_; // Use non-null marker to make sure inserting into the list properly updates this value. Node* next_ = reinterpret_cast(0x1); @@ -68,16 +82,8 @@ MATCHER_P(isEmpty, expected, (expected == !negation) ? "is empty" : "is not empt return expected == actual; } -size_t getSize(const std::forward_list& list) { - return std::distance(list.begin(), list.end()); -} - -size_t getSize(const intrusive_forward_list& list) { - return list.size(); -} - MATCHER_P(hasSize, expected, "") { - size_t actual = getSize(arg); + size_t actual = std::distance(arg.begin(), arg.end()); *result_listener << "of size " << actual; return expected == actual; } @@ -622,3 +628,72 @@ TYPED_TEST(ForwardListTest, EraseAfterEmptyRangeFront) { EXPECT_THAT(result, list.begin()); EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4); } + +TEST(InstrusiveForwardListTest, TryPushFrontSuccess) { + using List = intrusive_forward_list; + auto values = create({1, 2, 3, 4}); + List list(values.begin(), values.end()); + typename List::value_type value(5); + value.clearNext(); + auto result = list.try_push_front(value); + EXPECT_TRUE(result); + EXPECT_ELEMENTS_ARE(list, 5, 1, 2, 3, 4); +} + +TEST(InstrusiveForwardListTest, TryPushFrontFailure) { + using List = intrusive_forward_list; + auto values = create({1, 2, 3, 4}); + List list(values.begin(), values.end()); + typename List::value_type value(5); + auto result = list.try_push_front(value); + EXPECT_FALSE(result); + EXPECT_ELEMENTS_ARE(list, 1, 2, 3, 4); +} + +TEST(InstrusiveForwardListTest, TryPushFrontEmptySuccess) { + using List = intrusive_forward_list; + List list; + typename List::value_type value(5); + value.clearNext(); + auto result = list.try_push_front(value); + EXPECT_TRUE(result); + EXPECT_ELEMENTS_ARE(list, 5); +} + +TEST(InstrusiveForwardListTest, TryPushFrontEmptyFailure) { + using List = intrusive_forward_list; + List list; + typename List::value_type value(5); + auto result = list.try_push_front(value); + EXPECT_FALSE(result); + EXPECT_ELEMENTS_ARE(list); +} + +TEST(IntrusiveForwardListTest, TryPopFront) { + using List = intrusive_forward_list; + auto values = create({1, 2, 3, 4}); + List list(values.begin(), values.end()); + auto& front = list.front(); + auto result = list.try_pop_front(); + EXPECT_THAT(result, &front); + EXPECT_ELEMENTS_ARE(list, 2, 3, 4); +} + +TEST(IntrusiveForwardListTest, TryPopFrontIntoEmpty) { + using List = intrusive_forward_list; + auto values = create({1}); + List list(values.begin(), values.end()); + auto& front = list.front(); + auto result = list.try_pop_front(); + EXPECT_THAT(result, &front); + EXPECT_ELEMENTS_ARE(list); +} + +TEST(IntrusiveForwardListTest, TryPopFrontFromEmpty) { + using List = intrusive_forward_list; + auto values = create({}); + List list(values.begin(), values.end()); + auto result = list.try_pop_front(); + EXPECT_THAT(result, nullptr); + EXPECT_ELEMENTS_ARE(list); +}