[K/N] Separate sweep pass for extra-objects

^KT-49325

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -48,6 +48,20 @@ void Mark(KStdVector<ObjHeader*> graySet) noexcept {
     }
 }
 
+template <typename Traits>
+void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noexcept {
+    objectFactory.ProcessDeletions();
+    auto iter = objectFactory.LockForIter();
+    for (auto it = iter.begin(); it != iter.end();) {
+        auto &extraObject = *it;
+        if (!Traits::IsMarkedByExtraObject(extraObject)) {
+            extraObject.ClearWeakReferenceCounter();
+            extraObject.DetachAssociatedObject();
+        }
+        ++it;
+    }
+}
+
 template <typename Traits>
 typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory& objectFactory) noexcept {
     typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
@@ -59,10 +73,6 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact
             continue;
         }
         auto* objHeader = it->IsArray() ? it->GetArrayHeader()->obj() : it->GetObjHeader();
-        if (auto* extraObject = mm::ExtraObjectData::Get(objHeader)) {
-            extraObject->ClearWeakReferenceCounter();
-            extraObject->DetachAssociatedObject();
-        }
         if (HasFinalizers(objHeader)) {
             iter.MoveAndAdvance(finalizerQueue, it);
         } else {

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -12,6 +12,7 @@
 #include "FinalizerHooksTestSupport.hpp"
 #include "ObjectFactory.hpp"
 #include "ObjectTestSupport.hpp"
+#include "ExtraObjectDataFactory.hpp"
 
 using namespace kotlin;
 
@@ -59,6 +60,7 @@ struct GC {
 };
 
 using ObjectFactory = mm::ObjectFactory<GC>;
+using ExtraObjectsDataFactory = mm::ExtraObjectDataFactory;
 
 class Object : public test_support::Object<Payload> {
 public:
@@ -68,7 +70,6 @@ public:
 
     static Object& FromObjHeader(ObjHeader* obj) { return static_cast<Object&>(test_support::Object<Payload>::FromObjHeader(obj)); }
 
-    void InstallExtraData() { mm::ExtraObjectData::Install(header()); }
 
     bool HasWeakCounter() {
         if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
@@ -95,8 +96,6 @@ public:
         return static_cast<ObjectArray&>(test_support::ObjectArray<3>::FromArrayHeader(array));
     }
 
-    void InstallExtraData() { mm::ExtraObjectData::Install(header()); }
-
     bool HasWeakCounter() {
         if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
             return extraObjectData->HasWeakReferenceCounter();
@@ -122,8 +121,6 @@ public:
         return static_cast<CharArray&>(test_support::CharArray<3>::FromArrayHeader(array));
     }
 
-    void InstallExtraData() { mm::ExtraObjectData::Install(header()); }
-
     bool HasWeakCounter() {
         if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
             return extraObjectData->HasWeakReferenceCounter();
@@ -151,6 +148,12 @@ GC::ObjectData::State GetWeakCounterState(WeakCounter& counter) {
 
 struct SweepTraits {
     using ObjectFactory = ObjectFactory;
+    using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
+
+    static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
+        auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).GCObjectData();
+        return objectData.state != GC::ObjectData::State::kUnmarked;
+    }
 
     static bool TryResetMark(ObjectFactory::NodeRef node) {
         GC::ObjectData& objectData = node.GCObjectData();
@@ -169,7 +172,20 @@ struct SweepTraits {
 class MarkAndSweepUtilsSweepTest : public ::testing::Test {
 public:
     ~MarkAndSweepUtilsSweepTest() override {
+        auto deallocExtraObject = [this](ObjHeader* obj) {
+            auto *extraObject = mm::ExtraObjectData::Get(obj);
+            EXPECT_NE(extraObject, nullptr);
+            obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(obj->type_info());
+            extraObjectFactory_.DestroyExtraObjectData(extraObjectFactoryThreadQueue_, *extraObject);
+            extraObjectFactoryThreadQueue_.Publish();
+        };
         for (auto& finalizerQueue : finalizers_) {
+            for (auto node : finalizerQueue.IterForTests()) {
+                auto *object = node->IsArray() ? node->GetArrayHeader()->obj() : node->GetObjHeader();
+                if (object->has_meta_object()) {
+                    deallocExtraObject(object);
+                }
+            }
             finalizerQueue.Finalize();
         }
         testing::Mock::VerifyAndClear(&finalizerHook());
@@ -179,12 +195,14 @@ public:
             auto* obj = node->IsArray() ? node->GetArrayHeader()->obj() : node->GetObjHeader();
             if (auto* extraObject = mm::ExtraObjectData::Get(obj)) {
                 extraObject->ClearWeakReferenceCounter();
+                deallocExtraObject(obj);
             }
             RunFinalizers(obj);
         }
     }
 
     KStdVector<ObjHeader*> Sweep() {
+        gc::SweepExtraObjects<SweepTraits>(extraObjectFactory_);
         auto finalizers = gc::Sweep<SweepTraits>(objectFactory_);
         KStdVector<ObjHeader*> objects;
         for (auto node : finalizers.IterForTests()) {
@@ -220,11 +238,18 @@ public:
         return CharArray::FromArrayHeader(array);
     }
 
-    WeakCounter& InstallWeakCounter(ObjHeader* objHeader) {
+    mm::ExtraObjectData& InstallExtraData(ObjHeader *objHeader) {
+        auto& extraObjectData = extraObjectFactory_.CreateExtraObjectDataForObject(extraObjectFactoryThreadQueue_, objHeader, objHeader->type_info());
+        extraObjectFactoryThreadQueue_.Publish();
+        objHeader->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(&extraObjectData);
+        return *mm::ExtraObjectData::Get(objHeader);
+    }
+
+    WeakCounter& InstallWeakCounter(ObjHeader *objHeader) {
         auto* weakCounterHeader = objectFactoryThreadQueue_.CreateObject(typeHolderWeakCounter.typeInfo());
         objectFactoryThreadQueue_.Publish();
         auto& weakCounter = WeakCounter::FromObjHeader(weakCounterHeader);
-        auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader);
+        auto& extraObjectData = InstallExtraData(objHeader);
         auto *setHeader = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header());
         EXPECT_EQ(setHeader,  weakCounter.header());
         weakCounter->referred = objHeader;
@@ -240,6 +265,8 @@ private:
     GC::ThreadData gcThreadData_;
     ObjectFactory objectFactory_;
     ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gcThreadData_};
+    ExtraObjectsDataFactory extraObjectFactory_;
+    ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_};
     KStdVector<ObjectFactory::FinalizerQueue> finalizers_;
 };
 
@@ -322,7 +349,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArray) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithExtraData) {
     auto& object = AllocateObject();
-    object.InstallExtraData();
+    InstallExtraData(object.header());
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header()));
 
     auto finalizers = Sweep();
@@ -334,7 +361,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithExtraData) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithExtraData) {
     auto& array = AllocateObjectArray();
-    array.InstallExtraData();
+    InstallExtraData(array.header());
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header()));
 
     auto finalizers = Sweep();
@@ -346,7 +373,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithExtraData) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithExtraData) {
     auto& array = AllocateCharArray();
-    array.InstallExtraData();
+    InstallExtraData(array.header());
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header()));
 
     auto finalizers = Sweep();
@@ -358,7 +385,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithExtraData) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithExtraData) {
     auto& object = AllocateObject();
-    object.InstallExtraData();
+    InstallExtraData(object.header());
     object.Mark();
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header()));
 
@@ -371,7 +398,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithExtraData) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithExtraData) {
     auto& array = AllocateObjectArray();
-    array.InstallExtraData();
+    InstallExtraData(array.header());
     array.Mark();
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header()));
 
@@ -384,7 +411,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithExtraData) {
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithExtraData) {
     auto& array = AllocateCharArray();
-    array.InstallExtraData();
+    InstallExtraData(array.header());
     array.Mark();
     ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header()));
 
@@ -473,6 +500,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakCounter) {
     EXPECT_THAT(object.state(), GC::ObjectData::State::kMarkReset);
     EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset);
     EXPECT_TRUE(object.HasWeakCounter());
+    EXPECT_NE(weakCounter->referred, nullptr);
 }
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter) {
@@ -489,6 +517,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter)
     EXPECT_THAT(array.state(), GC::ObjectData::State::kMarkReset);
     EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset);
     EXPECT_TRUE(array.HasWeakCounter());
+    EXPECT_NE(weakCounter->referred, nullptr);
 }
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) {
@@ -505,6 +534,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) {
     EXPECT_THAT(array.state(), GC::ObjectData::State::kMarkReset);
     EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset);
     EXPECT_TRUE(array.HasWeakCounter());
+    EXPECT_NE(weakCounter->referred, nullptr);
 }
 
 TEST_F(MarkAndSweepUtilsSweepTest, SweepObjects) {

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -38,6 +38,14 @@ struct MarkTraits {
 
 struct SweepTraits {
     using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
+    using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
+
+    static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
+        auto *baseObject = object.GetBaseObject();
+        if (!baseObject->heap()) return true;
+        auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).GCObjectData();
+        return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack;
+    }
 
     static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
         auto& objectData = node.GCObjectData();
@@ -192,12 +200,16 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         gc::Mark<MarkTraits>(std::move(graySet));
         auto timeMarkUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs);
+        gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
+        auto timeSweepExtraObjectsUs = konan::getTimeMicros();
+        RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
         finalizerQueue = gc::Sweep<SweepTraits>(mm::GlobalData::Instance().objectFactory());
         auto timeSweepUs = konan::getTimeMicros();
-        RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs);
+        RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs);
 
         // Can be unsafe, because we've stopped the world.
         auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe();
+        auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
 
         gSafepointFlag = SafepointFlag::kNone;
         mm::ResumeThreads();
@@ -210,9 +222,9 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
         RuntimeLogInfo(
                 {kTagGC},
-                "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64
+                "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
                 " microseconds",
-                epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs);
+                epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs);
         ++epoch_;
         lastGCTimestampUs_ = timeResumeUs;
     }

kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp

@@ -165,6 +165,10 @@ public:
         deletionQueue_.clear();
     }
 
+    size_t GetSizeUnsafe() noexcept {
+        return queue_.size();
+    }
+
 private:
     // Using `KStdList` as it allows to implement `Collect` without memory allocations,
     // which is important for GC mark phase.

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp

@@ -51,6 +51,8 @@ public:
 
     void ClearForTests() noexcept { extraObjects_.ClearForTests(); }
 
+    size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); }
+
 private:
     Queue extraObjects_;
 };