diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 1e48b0ab918..909892d22b9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -48,6 +48,20 @@ void Mark(KStdVector graySet) noexcept { } } +template +void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noexcept { + objectFactory.ProcessDeletions(); + auto iter = objectFactory.LockForIter(); + for (auto it = iter.begin(); it != iter.end();) { + auto &extraObject = *it; + if (!Traits::IsMarkedByExtraObject(extraObject)) { + extraObject.ClearWeakReferenceCounter(); + extraObject.DetachAssociatedObject(); + } + ++it; + } +} + template typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory& objectFactory) noexcept { typename Traits::ObjectFactory::FinalizerQueue finalizerQueue; @@ -59,10 +73,6 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact continue; } auto* objHeader = it->IsArray() ? it->GetArrayHeader()->obj() : it->GetObjHeader(); - if (auto* extraObject = mm::ExtraObjectData::Get(objHeader)) { - extraObject->ClearWeakReferenceCounter(); - extraObject->DetachAssociatedObject(); - } if (HasFinalizers(objHeader)) { iter.MoveAndAdvance(finalizerQueue, it); } else { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index 60d738b55b8..fc0df507893 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -12,6 +12,7 @@ #include "FinalizerHooksTestSupport.hpp" #include "ObjectFactory.hpp" #include "ObjectTestSupport.hpp" +#include "ExtraObjectDataFactory.hpp" using namespace kotlin; @@ -59,6 +60,7 @@ struct GC { }; using ObjectFactory = mm::ObjectFactory; +using ExtraObjectsDataFactory = mm::ExtraObjectDataFactory; class Object : public test_support::Object { public: @@ -68,7 +70,6 @@ public: static Object& FromObjHeader(ObjHeader* obj) { return static_cast(test_support::Object::FromObjHeader(obj)); } - void InstallExtraData() { mm::ExtraObjectData::Install(header()); } bool HasWeakCounter() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { @@ -95,8 +96,6 @@ public: return static_cast(test_support::ObjectArray<3>::FromArrayHeader(array)); } - void InstallExtraData() { mm::ExtraObjectData::Install(header()); } - bool HasWeakCounter() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { return extraObjectData->HasWeakReferenceCounter(); @@ -122,8 +121,6 @@ public: return static_cast(test_support::CharArray<3>::FromArrayHeader(array)); } - void InstallExtraData() { mm::ExtraObjectData::Install(header()); } - bool HasWeakCounter() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { return extraObjectData->HasWeakReferenceCounter(); @@ -151,6 +148,12 @@ GC::ObjectData::State GetWeakCounterState(WeakCounter& counter) { struct SweepTraits { using ObjectFactory = ObjectFactory; + using ExtraObjectsFactory = mm::ExtraObjectDataFactory; + + static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { + auto& objectData = ObjectFactory::NodeRef::From(object.GetBaseObject()).GCObjectData(); + return objectData.state != GC::ObjectData::State::kUnmarked; + } static bool TryResetMark(ObjectFactory::NodeRef node) { GC::ObjectData& objectData = node.GCObjectData(); @@ -169,7 +172,20 @@ struct SweepTraits { class MarkAndSweepUtilsSweepTest : public ::testing::Test { public: ~MarkAndSweepUtilsSweepTest() override { + auto deallocExtraObject = [this](ObjHeader* obj) { + auto *extraObject = mm::ExtraObjectData::Get(obj); + EXPECT_NE(extraObject, nullptr); + obj->typeInfoOrMeta_ = const_cast(obj->type_info()); + extraObjectFactory_.DestroyExtraObjectData(extraObjectFactoryThreadQueue_, *extraObject); + extraObjectFactoryThreadQueue_.Publish(); + }; for (auto& finalizerQueue : finalizers_) { + for (auto node : finalizerQueue.IterForTests()) { + auto *object = node->IsArray() ? node->GetArrayHeader()->obj() : node->GetObjHeader(); + if (object->has_meta_object()) { + deallocExtraObject(object); + } + } finalizerQueue.Finalize(); } testing::Mock::VerifyAndClear(&finalizerHook()); @@ -179,12 +195,14 @@ public: auto* obj = node->IsArray() ? node->GetArrayHeader()->obj() : node->GetObjHeader(); if (auto* extraObject = mm::ExtraObjectData::Get(obj)) { extraObject->ClearWeakReferenceCounter(); + deallocExtraObject(obj); } RunFinalizers(obj); } } KStdVector Sweep() { + gc::SweepExtraObjects(extraObjectFactory_); auto finalizers = gc::Sweep(objectFactory_); KStdVector objects; for (auto node : finalizers.IterForTests()) { @@ -220,11 +238,18 @@ public: return CharArray::FromArrayHeader(array); } - WeakCounter& InstallWeakCounter(ObjHeader* objHeader) { + mm::ExtraObjectData& InstallExtraData(ObjHeader *objHeader) { + auto& extraObjectData = extraObjectFactory_.CreateExtraObjectDataForObject(extraObjectFactoryThreadQueue_, objHeader, objHeader->type_info()); + extraObjectFactoryThreadQueue_.Publish(); + objHeader->typeInfoOrMeta_ = reinterpret_cast(&extraObjectData); + return *mm::ExtraObjectData::Get(objHeader); + } + + WeakCounter& InstallWeakCounter(ObjHeader *objHeader) { auto* weakCounterHeader = objectFactoryThreadQueue_.CreateObject(typeHolderWeakCounter.typeInfo()); objectFactoryThreadQueue_.Publish(); auto& weakCounter = WeakCounter::FromObjHeader(weakCounterHeader); - auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); + auto& extraObjectData = InstallExtraData(objHeader); auto *setHeader = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header()); EXPECT_EQ(setHeader, weakCounter.header()); weakCounter->referred = objHeader; @@ -240,6 +265,8 @@ private: GC::ThreadData gcThreadData_; ObjectFactory objectFactory_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gcThreadData_}; + ExtraObjectsDataFactory extraObjectFactory_; + ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_}; KStdVector finalizers_; }; @@ -322,7 +349,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArray) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithExtraData) { auto& object = AllocateObject(); - object.InstallExtraData(); + InstallExtraData(object.header()); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header())); auto finalizers = Sweep(); @@ -334,7 +361,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithExtraData) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithExtraData) { auto& array = AllocateObjectArray(); - array.InstallExtraData(); + InstallExtraData(array.header()); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header())); auto finalizers = Sweep(); @@ -346,7 +373,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithExtraData) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithExtraData) { auto& array = AllocateCharArray(); - array.InstallExtraData(); + InstallExtraData(array.header()); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header())); auto finalizers = Sweep(); @@ -358,7 +385,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithExtraData) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithExtraData) { auto& object = AllocateObject(); - object.InstallExtraData(); + InstallExtraData(object.header()); object.Mark(); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header())); @@ -371,7 +398,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithExtraData) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithExtraData) { auto& array = AllocateObjectArray(); - array.InstallExtraData(); + InstallExtraData(array.header()); array.Mark(); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header())); @@ -384,7 +411,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithExtraData) { TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithExtraData) { auto& array = AllocateCharArray(); - array.InstallExtraData(); + InstallExtraData(array.header()); array.Mark(); ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header())); @@ -473,6 +500,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakCounter) { EXPECT_THAT(object.state(), GC::ObjectData::State::kMarkReset); EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset); EXPECT_TRUE(object.HasWeakCounter()); + EXPECT_NE(weakCounter->referred, nullptr); } TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter) { @@ -489,6 +517,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter) EXPECT_THAT(array.state(), GC::ObjectData::State::kMarkReset); EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset); EXPECT_TRUE(array.HasWeakCounter()); + EXPECT_NE(weakCounter->referred, nullptr); } TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) { @@ -505,6 +534,7 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) { EXPECT_THAT(array.state(), GC::ObjectData::State::kMarkReset); EXPECT_THAT(GetWeakCounterState(weakCounter), GC::ObjectData::State::kMarkReset); EXPECT_TRUE(array.HasWeakCounter()); + EXPECT_NE(weakCounter->referred, nullptr); } TEST_F(MarkAndSweepUtilsSweepTest, SweepObjects) { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index e5bda47ad4f..8b9fb350698 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -38,6 +38,14 @@ struct MarkTraits { struct SweepTraits { using ObjectFactory = mm::ObjectFactory; + using ExtraObjectsFactory = mm::ExtraObjectDataFactory; + + static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept { + auto *baseObject = object.GetBaseObject(); + if (!baseObject->heap()) return true; + auto& objectData = mm::ObjectFactory::NodeRef::From(baseObject).GCObjectData(); + return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack; + } static bool TryResetMark(ObjectFactory::NodeRef node) noexcept { auto& objectData = node.GCObjectData(); @@ -192,12 +200,16 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { gc::Mark(std::move(graySet)); auto timeMarkUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs); + gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); + auto timeSweepExtraObjectsUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); auto timeSweepUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs); + RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs); // Can be unsafe, because we've stopped the world. auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); gSafepointFlag = SafepointFlag::kNone; mm::ResumeThreads(); @@ -210,9 +222,9 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogInfo( {kTagGC}, - "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64 + "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64 " microseconds", - epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs); + epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs); ++epoch_; lastGCTimestampUs_ = timeResumeUs; } diff --git a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp index af2d47d2a67..48ccae4575b 100644 --- a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp +++ b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp @@ -165,6 +165,10 @@ public: deletionQueue_.clear(); } + size_t GetSizeUnsafe() noexcept { + return queue_.size(); + } + private: // Using `KStdList` as it allows to implement `Collect` without memory allocations, // which is important for GC mark phase. diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp index e9be894e649..a04c29619d1 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp @@ -51,6 +51,8 @@ public: void ClearForTests() noexcept { extraObjects_.ClearForTests(); } + size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); } + private: Queue extraObjects_; };