diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index b76a448289b..c9b9e59756a 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -36,16 +36,17 @@ struct MarkTraits { } static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto top = queue.back(); - queue.pop_back(); - return top; + auto& top = queue.front(); + queue.pop_front(); + auto node = mm::ObjectFactory::NodeRef::From(top); + return node->GetObjHeader(); } static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return; objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack); - queue.push_back(object); + queue.push_front(objectData); } }; @@ -96,7 +97,6 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( objectFactory_(objectFactory), gcScheduler_(gcScheduler), finalizerProcessor_(make_unique([this](int64_t epoch) { state_.finalized(epoch); })) { - markQueue_.reserve(1000); gcScheduler_.SetScheduleGC([this]() NO_INLINE { RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); // This call acquires a lock, so we need to ensure that we're in the safe state. diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index a6d0ee1a525..ccb93ea58cb 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -9,6 +9,7 @@ #include "Allocator.hpp" #include "GCScheduler.hpp" +#include "IntrusiveList.hpp" #include "ObjectFactory.hpp" #include "ScopedThread.hpp" #include "Types.h" @@ -29,24 +30,39 @@ class FinalizerProcessor; // TODO: Also make mark concurrent. class ConcurrentMarkAndSweep : private Pinned { public: - // This implementation of mark queue allocates memory during collection. - using MarkQueue = KStdVector; - class ObjectData { + static inline constexpr unsigned colorMask = (1 << 1) - 1; + public: - enum class Color { + enum class Color : unsigned { kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected. // All new objects are allocated with this color. kBlack, // Objects encountered during mark phase. }; - Color color() const noexcept { return color_; } - void setColor(Color color) noexcept { color_ = color; } + Color color() const noexcept { return static_cast(getPointerBits(next_, colorMask)); } + void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast(color)); } + + ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); } + void setNext(ObjectData* next) noexcept { + RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next); + auto bits = getPointerBits(next_, colorMask); + next_ = setPointerBits(next, bits); + } private: - Color color_ = Color::kWhite; + // Color is encoded in low bits. + ObjectData* next_ = nullptr; }; + struct MarkQueueTraits { + static ObjectData* next(const ObjectData& value) noexcept { return value.next(); } + + static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); } + }; + + using MarkQueue = intrusive_forward_list; + class ThreadData : private Pinned { public: using ObjectData = ConcurrentMarkAndSweep::ObjectData; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 960ed326698..849d7164da4 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -34,16 +34,17 @@ struct MarkTraits { } static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto top = queue.back(); - queue.pop_back(); - return top; + auto& top = queue.front(); + queue.pop_front(); + auto node = mm::ObjectFactory::NodeRef::From(top); + return node->GetObjHeader(); } static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return; objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); - queue.push_back(object); + queue.push_front(objectData); } }; @@ -115,7 +116,6 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), gcScheduler_(gcScheduler) { - markQueue_.reserve(1000); gcScheduler_.SetScheduleGC([]() { // TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running. // Investigate, if it's possible to move this logic into the scheduler. diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 6f461f3b59c..a928e6674a9 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -10,6 +10,7 @@ #include "Allocator.hpp" #include "GCScheduler.hpp" +#include "IntrusiveList.hpp" #include "ObjectFactory.hpp" #include "Types.h" #include "Utils.hpp" @@ -25,9 +26,6 @@ namespace gc { // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads. class SameThreadMarkAndSweep : private Pinned { public: - // This implementation of mark queue allocates memory during collection. - using MarkQueue = KStdVector; - enum class SafepointFlag { kNone, kNeedsSuspend, @@ -35,20 +33,38 @@ public: }; class ObjectData { + static inline constexpr unsigned colorMask = (1 << 1) - 1; + public: - enum class Color { + enum class Color : unsigned { kWhite = 0, // Initial color at the start of collection cycles. Objects with this color at the end of GC cycle are collected. // All new objects are allocated with this color. kBlack, // Objects encountered during mark phase. }; - Color color() const noexcept { return color_; } - void setColor(Color color) noexcept { color_ = color; } + Color color() const noexcept { return static_cast(getPointerBits(next_, colorMask)); } + void setColor(Color color) noexcept { next_ = setPointerBits(clearPointerBits(next_, colorMask), static_cast(color)); } + + ObjectData* next() const noexcept { return clearPointerBits(next_, colorMask); } + void setNext(ObjectData* next) noexcept { + RuntimeAssert(!hasPointerBits(next, colorMask), "next must be untagged: %p", next); + auto bits = getPointerBits(next_, colorMask); + next_ = setPointerBits(next, bits); + } private: - Color color_ = Color::kWhite; + // Color is encoded in low bits. + ObjectData* next_ = nullptr; }; + struct MarkQueueTraits { + static ObjectData* next(const ObjectData& value) noexcept { return value.next(); } + + static void setNext(ObjectData& value, ObjectData* next) noexcept { value.setNext(next); } + }; + + using MarkQueue = intrusive_forward_list; + class ThreadData : private Pinned { public: using ObjectData = SameThreadMarkAndSweep::ObjectData; diff --git a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp index 6a4171a27ed..d8483efb33b 100644 --- a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp +++ b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp @@ -12,6 +12,7 @@ #include "Mutex.hpp" #include "Types.h" +#include "Utils.hpp" namespace kotlin { @@ -34,7 +35,7 @@ public: static Node& fromValue(T& t) noexcept { static_assert(std::is_base_of_v, "fromValue function only makes sense for non-movable object"); - return *reinterpret_cast(reinterpret_cast(&t) - offsetof(Node, value_)); + return ownerOf(Node, value_, t); } private: diff --git a/kotlin-native/runtime/src/main/cpp/Utils.hpp b/kotlin-native/runtime/src/main/cpp/Utils.hpp index 6258f1194ef..2574f3382b7 100644 --- a/kotlin-native/runtime/src/main/cpp/Utils.hpp +++ b/kotlin-native/runtime/src/main/cpp/Utils.hpp @@ -80,6 +80,8 @@ private: size_t CombineHash(size_t seed, size_t value); +#define ownerOf(type, field, ref) *reinterpret_cast(reinterpret_cast(&ref) - offsetof(type, field)) + } // namespace kotlin #endif // RUNTIME_UTILS_H diff --git a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp index 178da999c9a..279455862f2 100644 --- a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp @@ -7,6 +7,7 @@ #include +#include "gmock/gmock.h" #include "gtest/gtest.h" using namespace kotlin; @@ -48,3 +49,27 @@ TEST(UtilsTest, PinnedImpl) { static_assert(!std::is_move_assignable_v, "Must not be move assignable"); static_assert(sizeof(PinnedImpl) == sizeof(A), "Must not increase size"); } + +namespace { + +class Container { +public: + static Container& fromX(int32_t& x) { return ownerOf(Container, x_, x); } + + static Container& fromY(void*& y) { return ownerOf(Container, y_, y); } + + int32_t& x() { return x_; } + void*& y() { return y_; } + +private: + int32_t x_ = 0; + void* y_ = nullptr; +}; + +} // namespace + +TEST(UtilsTest, OwnerOf) { + Container c; + EXPECT_THAT(&c, &Container::fromX(c.x())); + EXPECT_THAT(&c, &Container::fromY(c.y())); +} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 397875dc951..cf17a1b795a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -453,18 +453,19 @@ public: static NodeRef From(ObjHeader* object) noexcept { RuntimeAssert(object->heap(), "Must be a heap object"); - auto* heapObject = reinterpret_cast(reinterpret_cast(object) - offsetof(HeapObjHeader, object)); - RuntimeAssert(&heapObject->object == object, "HeapObjHeader layout has broken"); - return NodeRef(Storage::Node::FromData(heapObject)); + auto& heapObject = ownerOf(HeapObjHeader, object, *object); + return NodeRef(Storage::Node::FromData(&heapObject)); } static NodeRef From(ArrayHeader* array) noexcept { - // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can - // be always casted to the other. - RuntimeAssert(reinterpret_cast(array)->heap(), "Must be a heap object"); - auto* heapArray = reinterpret_cast(reinterpret_cast(array) - offsetof(HeapArrayHeader, array)); - RuntimeAssert(&heapArray->array == array, "HeapArrayHeader layout has broken"); - return NodeRef(Storage::Node::FromData(heapArray)); + RuntimeAssert(array->obj()->heap(), "Must be a heap object"); + auto& heapArray = ownerOf(HeapArrayHeader, array, *array); + return NodeRef(Storage::Node::FromData(&heapArray)); + } + + static NodeRef From(ObjectData& objectData) noexcept { + auto& heapObject = ownerOf(HeapObjHeader, gcData, objectData); + return NodeRef(Storage::Node::FromData(&heapObject)); } NodeRef* operator->() noexcept { return this; } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 5c4dd31b76c..4f7565eabe5 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -869,7 +869,7 @@ TEST(ObjectFactoryTest, CreateObject) { testing::Mock::VerifyAndClearExpectations(&allocator); EXPECT_THAT(allocSize, testing::Gt(type.typeInfo()->instanceSize_)); EXPECT_THAT(allocAddress, testing::Ne(nullptr)); - EXPECT_THAT(mm::GetAllocatedHeapSize(object), allocSize); + EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(object), allocSize); EXPECT_THAT(object->type_info(), type.typeInfo()); threadQueue.Publish(); @@ -904,7 +904,7 @@ TEST(ObjectFactoryTest, CreateObjectArray) { testing::Mock::VerifyAndClearExpectations(&allocator); EXPECT_THAT(allocSize, testing::Gt(-theArrayTypeInfo->instanceSize_ * 3)); EXPECT_THAT(allocAddress, testing::Ne(nullptr)); - EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize); + EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(array->obj()), allocSize); EXPECT_THAT(array->type_info(), theArrayTypeInfo); threadQueue.Publish(); @@ -939,7 +939,7 @@ TEST(ObjectFactoryTest, CreateCharArray) { testing::Mock::VerifyAndClearExpectations(&allocator); EXPECT_THAT(allocSize, testing::Gt(-theCharArrayTypeInfo->instanceSize_ * 3)); EXPECT_THAT(allocAddress, testing::Ne(nullptr)); - EXPECT_THAT(mm::GetAllocatedHeapSize(array->obj()), allocSize); + EXPECT_THAT(ObjectFactory::GetAllocatedHeapSize(array->obj()), allocSize); EXPECT_THAT(array->type_info(), theCharArrayTypeInfo); threadQueue.Publish();