diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index b77b4016f95..521f057cd96 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -25,17 +25,28 @@ using namespace kotlin; namespace { struct MarkTraits { - static bool IsMarked(ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); - return objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack; + using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue; + + static bool isEmpty(const MarkQueue& queue) noexcept { + return queue.empty(); } - static bool TryMark(ObjHeader* object) noexcept { + static void clear(MarkQueue& queue) noexcept { + queue.clear(); + } + + static ObjHeader* dequeue(MarkQueue& queue) noexcept { + auto top = queue.back(); + queue.pop_back(); + return top; + } + + static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); - if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return false; + if (objectData.color() == gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack) return; objectData.setColor(gc::ConcurrentMarkAndSweep::ObjectData::Color::kBlack); - return true; - }; + queue.push_back(object); + } }; struct SweepTraits { @@ -85,7 +96,7 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( objectFactory_(objectFactory), gcScheduler_(gcScheduler), finalizerProcessor_(make_unique([this](int64_t epoch) { state_.finalized(epoch); })) { - graySet_.reserve(1000); + markQueue_.reserve(1000); gcScheduler_.SetScheduleGC([this]() NO_INLINE { RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); // This call acquires a lock, so we need to ensure that we're in the safe state. @@ -143,17 +154,17 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { state_.start(epoch); RuntimeLogInfo( {kTagGC}, "Started GC epoch %" PRId64 ". Time since last GC %" PRIu64 " microseconds", epoch, timeStartUs - lastGCTimestampUs_); - collectRootSet(graySet_); + gc::collectRootSet(markQueue_); auto timeRootSetUs = konan::getTimeMicros(); // Can be unsafe, because we've stopped the world. auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); RuntimeLogInfo( - {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet_.size(), + {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(), timeRootSetUs - timeSuspendUs); - auto markStats = gc::Mark(graySet_); + auto markStats = gc::Mark(markQueue_); auto timeMarkUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries); + RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs); scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); auto timeSweepExtraObjectsUs = konan::getTimeMicros(); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index e4c0960fdc8..a6d0ee1a525 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -29,6 +29,8 @@ class FinalizerProcessor; // TODO: Also make mark concurrent. class ConcurrentMarkAndSweep : private Pinned { public: + // This implementation of mark queue allocates memory during collection. + using MarkQueue = KStdVector; class ObjectData { public: @@ -89,7 +91,7 @@ private: ScopedThread gcThread_; KStdUniquePtr finalizerProcessor_; - KStdVector graySet_; + MarkQueue markQueue_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp index 9d167e2bd98..69e6f9c51bf 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp @@ -4,53 +4,3 @@ */ #include "MarkAndSweepUtils.hpp" -#include "CompilerConstants.hpp" -#include "GlobalData.hpp" -#include "Logging.hpp" -#include "Memory.h" -#include "RootSet.hpp" -#include "ThreadData.hpp" - -using namespace kotlin; - -void kotlin::gc::collectRootSet(KStdVector& graySet) { - graySet.clear(); - for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { - // TODO: Maybe it's more efficient to do by the suspending thread? - thread.Publish(); - thread.gc().OnStoppedForGC(); - size_t stack = 0; - size_t tls = 0; - for (auto value : mm::ThreadRootSet(thread)) { - if (!isNullOrMarker(value.object)) { - graySet.push_back(value.object); - switch (value.source) { - case mm::ThreadRootSet::Source::kStack: - ++stack; - break; - case mm::ThreadRootSet::Source::kTLS: - ++tls; - break; - } - } - } - RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls); - } - mm::StableRefRegistry::Instance().ProcessDeletions(); - size_t global = 0; - size_t stableRef = 0; - for (auto value : mm::GlobalRootSet()) { - if (!isNullOrMarker(value.object)) { - graySet.push_back(value.object); - switch (value.source) { - case mm::GlobalRootSet::Source::kGlobal: - ++global; - break; - case mm::GlobalRootSet::Source::kStableRef: - ++stableRef; - break; - } - } - } - RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef); -} diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index d5a3de6e661..bb0dee15229 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -8,10 +8,15 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooks.hpp" +#include "GlobalData.hpp" +#include "Logging.hpp" #include "Memory.h" #include "ObjectOps.hpp" #include "ObjectTraversal.hpp" +#include "RootSet.hpp" #include "Runtime.h" +#include "StableRefRegistry.hpp" +#include "ThreadData.hpp" #include "Types.h" namespace kotlin { @@ -22,41 +27,33 @@ struct MarkStats { size_t aliveHeapSet = 0; // How many objects are alive in bytes. Note: this does not include overhead of malloc/mimalloc itself. size_t aliveHeapSetBytes = 0; - // How many times a marked object was found in the mark queue. - size_t duplicateEntries = 0; }; -// TODO: Because of `graySet` this implementation may allocate heap memory during GC. template -MarkStats Mark(KStdVector& graySet) noexcept { +MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept { MarkStats stats; - while (!graySet.empty()) { - ObjHeader* top = graySet.back(); - graySet.pop_back(); + while (!Traits::isEmpty(markQueue)) { + ObjHeader* top = Traits::dequeue(markQueue); - RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in gray set", top); + RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top); + RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local()); - if (top->heap()) { - if (!Traits::TryMark(top)) { - ++stats.duplicateEntries; - continue; + stats.aliveHeapSet++; + stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top); + + traverseReferredObjects(top, [&](ObjHeader* field) noexcept { + if (!isNullOrMarker(field) && field->heap()) { + Traits::enqueue(markQueue, field); } - stats.aliveHeapSet++; - stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top); - } - - if (top->heap() || top->local()) { - traverseReferredObjects(top, [&graySet](ObjHeader* field) noexcept { - if (!isNullOrMarker(field) && field->heap() && !Traits::IsMarked(field)) { - graySet.push_back(field); - } - }); - } + }); if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) { auto weakCounter = extraObjectData->GetWeakReferenceCounter(); if (!isNullOrMarker(weakCounter)) { - graySet.push_back(weakCounter); + RuntimeAssert( + weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", top, + weakCounter, weakCounter->permanent(), weakCounter->local()); + Traits::enqueue(markQueue, weakCounter); } } } @@ -111,7 +108,71 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact return Sweep(iter); } -void collectRootSet(KStdVector& graySet); +// TODO: This needs some tests now. +template +void collectRootSet(typename Traits::MarkQueue& markQueue) noexcept { + Traits::clear(markQueue); + for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { + // TODO: Maybe it's more efficient to do by the suspending thread? + thread.Publish(); + thread.gc().OnStoppedForGC(); + size_t stack = 0; + size_t tls = 0; + for (auto value : mm::ThreadRootSet(thread)) { + auto* object = value.object; + if (!isNullOrMarker(object)) { + if (object->heap()) { + Traits::enqueue(markQueue, object); + } else { + traverseReferredObjects(object, [&](ObjHeader* field) noexcept { + // Each permanent and stack object has own entry in the root set. + if (field->heap() && !isNullOrMarker(field)) { + Traits::enqueue(markQueue, field); + } + }); + RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object); + } + switch (value.source) { + case mm::ThreadRootSet::Source::kStack: + ++stack; + break; + case mm::ThreadRootSet::Source::kTLS: + ++tls; + break; + } + } + } + RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls); + } + mm::StableRefRegistry::Instance().ProcessDeletions(); + size_t global = 0; + size_t stableRef = 0; + for (auto value : mm::GlobalRootSet()) { + auto* object = value.object; + if (!isNullOrMarker(object)) { + if (object->heap()) { + Traits::enqueue(markQueue, object); + } else { + traverseReferredObjects(object, [&](ObjHeader* field) noexcept { + // Each permanent and stack object has own entry in the root set. + if (field->heap() && !isNullOrMarker(field)) { + Traits::enqueue(markQueue, field); + } + }); + RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object); + } + switch (value.source) { + case mm::GlobalRootSet::Source::kGlobal: + ++global; + break; + case mm::GlobalRootSet::Source::kStableRef: + ++stableRef; + break; + } + } + } + RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef); +} } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index e1b3f92fd7e..fa899cc4548 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -108,6 +108,8 @@ public: class ScopedMarkTraits : private Pinned { public: + using MarkQueue = KStdVector; + ScopedMarkTraits() { RuntimeAssert(instance_ == nullptr, "Only one ScopedMarkTraits is allowed"); instance_ = this; @@ -120,8 +122,26 @@ public: const KStdUnorderedSet& marked() const { return marked_; } - static bool TryMark(ObjHeader* object) noexcept { return instance_->marked_.insert(object).second; } - static bool IsMarked(ObjHeader* object) noexcept { return instance_->marked_.find(object) != instance_->marked_.end(); } + static bool isEmpty(const MarkQueue& queue) noexcept { + return queue.empty(); + } + + static void clear(MarkQueue& queue) noexcept { + queue.clear(); + } + + static ObjHeader* dequeue(MarkQueue& queue) noexcept { + auto top = queue.back(); + queue.pop_back(); + return top; + } + + static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + auto result = instance_->marked_.insert(object); + if (result.second) { + queue.push_back(object); + } + } private: static ScopedMarkTraits* instance_; @@ -146,7 +166,7 @@ public: gc::MarkStats Mark(std::initializer_list> graySet) { KStdVector objects; - for (auto& object : graySet) objects.push_back(object.get().GetObjHeader()); + for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader()); return gc::Mark(objects); } @@ -177,7 +197,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkNothing) { auto stats = Mark({}); EXPECT_MARKED(stats); - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) { @@ -186,7 +205,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObject) { auto stats = Mark({object}); EXPECT_MARKED(stats, object); - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) { @@ -195,8 +213,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArray) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) { @@ -205,71 +221,8 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArray) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObject) { - Object object{BaseObject::Kind::kPermanent}; - - auto stats = Mark({object}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentObjectArray) { - ObjectArray array{BaseObject::Kind::kPermanent}; - - auto stats = Mark({array}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkSinglePermanentCharArray) { - CharArray array{BaseObject::Kind::kPermanent}; - - auto stats = Mark({array}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObject) { - Object object{BaseObject::Kind::kStackLocal}; - - auto stats = Mark({object}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackObjectArray) { - ObjectArray array{BaseObject::Kind::kStackLocal}; - - auto stats = Mark({array}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleStackCharArray) { - CharArray array{BaseObject::Kind::kStackLocal}; - - auto stats = Mark({array}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - - TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) { Object object; object->field1 = kInitializingSingleton; @@ -277,8 +230,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFields) { auto stats = Mark({object}); EXPECT_MARKED(stats, object); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) { @@ -288,8 +239,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFields) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) { @@ -301,8 +250,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) { @@ -312,8 +259,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) { auto stats = Mark({object}); EXPECT_MARKED(stats, object); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) { @@ -323,8 +268,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) { @@ -334,8 +277,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) { auto stats = Mark({array}); EXPECT_MARKED(stats, array); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) { @@ -347,8 +288,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) { auto stats = Mark({object}); EXPECT_MARKED(stats, object, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) { @@ -360,8 +299,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) { auto stats = Mark({array}); EXPECT_MARKED(stats, array, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) { @@ -373,8 +310,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) { auto stats = Mark({array}); EXPECT_MARKED(stats, array, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) { @@ -387,8 +322,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCount auto stats = Mark({object}); EXPECT_MARKED(stats, object, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) { @@ -401,8 +334,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeak auto stats = Mark({array}); EXPECT_MARKED(stats, array, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) { @@ -417,8 +348,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter auto stats = Mark({array}); EXPECT_MARKED(stats, array, weakCounter); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) { @@ -443,105 +372,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) { EXPECT_MARKED( stats, root, root_field1, root_field1_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentRoot) { - Object root{BaseObject::Kind::kPermanent}; - Object root_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field2{BaseObject::Kind::kPermanent}; - ObjectArray root_field3{BaseObject::Kind::kPermanent}; - Object root_field3_element1{BaseObject::Kind::kPermanent}; - ObjectArray root_field3_element2{BaseObject::Kind::kPermanent}; - CharArray root_field3_element3{BaseObject::Kind::kPermanent}; - root->field1 = root_field1.header(); - root_field1->field1 = root_field1_field1.header(); - root_field1->field2 = root_field1_field2.header(); - root->field3 = root_field3.header(); - root_field3.elements()[0] = root_field3_element1.header(); - root_field3.elements()[1] = root_field3_element2.header(); - root_field3.elements()[2] = root_field3_element3.header(); - - auto stats = Mark({root}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentMiddle) { - Object root; - Object root_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field2{BaseObject::Kind::kPermanent}; - ObjectArray root_field3; - Object root_field3_element1; - ObjectArray root_field3_element2; - CharArray root_field3_element3; - root->field1 = root_field1.header(); - root_field1->field1 = root_field1_field1.header(); - root_field1->field2 = root_field1_field2.header(); - root->field3 = root_field3.header(); - root_field3.elements()[0] = root_field3_element1.header(); - root_field3.elements()[1] = root_field3_element2.header(); - root_field3.elements()[2] = root_field3_element3.header(); - - auto stats = Mark({root}); - - EXPECT_MARKED(stats, root, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithPermanentLeaf) { - Object root; - Object root_field1; - Object root_field1_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field2; - ObjectArray root_field3; - Object root_field3_element1; - ObjectArray root_field3_element2; - CharArray root_field3_element3; - root->field1 = root_field1.header(); - root_field1->field1 = root_field1_field1.header(); - root_field1->field2 = root_field1_field2.header(); - root->field3 = root_field3.header(); - root_field3.elements()[0] = root_field3_element1.header(); - root_field3.elements()[1] = root_field3_element2.header(); - root_field3.elements()[2] = root_field3_element3.header(); - - auto stats = Mark({root}); - - EXPECT_MARKED( - stats, root, root_field1, root_field1_field2, root_field3, root_field3_element1, root_field3_element2, root_field3_element3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkTreeWithStackRoot) { - Object root{BaseObject::Kind::kStackLocal}; - Object root_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field1{BaseObject::Kind::kPermanent}; - Object root_field1_field2{BaseObject::Kind::kPermanent}; - ObjectArray root_field3{BaseObject::Kind::kStackLocal}; - Object root_field3_element1{BaseObject::Kind::kHeapLike}; - ObjectArray root_field3_element2{BaseObject::Kind::kHeapLike}; - CharArray root_field3_element3{BaseObject::Kind::kPermanent}; - root->field1 = root_field1.header(); - root_field1->field1 = root_field1_field1.header(); - root_field1->field2 = root_field1_field2.header(); - root->field3 = root_field3.header(); - root_field3.elements()[0] = root_field3_element1.header(); - root_field3.elements()[1] = root_field3_element2.header(); - root_field3.elements()[2] = root_field3_element3.header(); - - auto stats = Mark({root, root_field3}); - - EXPECT_MARKED(stats, root_field3_element1, root_field3_element2); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) { @@ -555,46 +385,8 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTree) { auto stats = Mark({root}); EXPECT_MARKED(stats, root, inner1, inner2); - - EXPECT_THAT(stats.duplicateEntries, 0); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithPermanentRoot) { - Object root{BaseObject::Kind::kPermanent}; - Object inner1{BaseObject::Kind::kPermanent}; - ObjectArray inner2{BaseObject::Kind::kPermanent}; - root->field1 = inner1.header(); - inner1->field1 = inner2.header(); - inner2.elements()[0] = root.header(); - - auto stats = Mark({root}); - - EXPECT_MARKED(stats); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkRecursiveTreeWithStackRoot) { - Object root{BaseObject::Kind::kStackLocal}; - Object inner1{BaseObject::Kind::kStackLocal}; - ObjectArray inner2{BaseObject::Kind::kStackLocal}; - Object inner3{BaseObject::Kind::kHeapLike}; - Object inner2_element1{BaseObject::Kind::kHeapLike}; - root->field1 = inner1.header(); - inner1->field1 = inner2.header(); - inner2.elements()[0] = root.header(); - inner2.elements()[1] = inner2_element1.header(); - root->field2 = inner3.header(); - inner3->field1 = inner2.header(); - - auto stats = Mark({root, inner1, inner2}); - - EXPECT_MARKED(stats, inner3, inner2_element1); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - - TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) { Object root1; ObjectArray root2; @@ -603,44 +395,6 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForest) { auto stats = Mark({root1, root2, root3}); EXPECT_MARKED(stats, root1, root2, root3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentFirst) { - Object root1{BaseObject::Kind::kPermanent}; - ObjectArray root2; - Object root3; - - auto stats = Mark({root1, root2, root3}); - - EXPECT_MARKED(stats, root2, root3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentSecond) { - Object root1; - ObjectArray root2{BaseObject::Kind::kPermanent}; - Object root3; - - auto stats = Mark({root1, root2, root3}); - - EXPECT_MARKED(stats, root1, root3); - - EXPECT_THAT(stats.duplicateEntries, 0); -} - -TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithPermanentThird) { - Object root1; - ObjectArray root2; - Object root3{BaseObject::Kind::kPermanent}; - - auto stats = Mark({root1, root2, root3}); - - EXPECT_MARKED(stats, root1, root2); - - EXPECT_THAT(stats.duplicateEntries, 0); } TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) { @@ -655,6 +409,4 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkForestWithInterconnectedRoots) { auto stats = Mark({root1, root2, root3}); EXPECT_MARKED(stats, root1, root2, root3); - - EXPECT_THAT(stats.duplicateEntries, 2); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 2cf88ea7ea9..11bf1fdea04 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -23,17 +23,28 @@ using namespace kotlin; namespace { struct MarkTraits { - static bool IsMarked(ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); - return objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack; + using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue; + + static bool isEmpty(const MarkQueue& queue) noexcept { + return queue.empty(); } - static bool TryMark(ObjHeader* object) noexcept { + static void clear(MarkQueue& queue) noexcept { + queue.clear(); + } + + static ObjHeader* dequeue(MarkQueue& queue) noexcept { + auto top = queue.back(); + queue.pop_back(); + return top; + } + + static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { auto& objectData = mm::ObjectFactory::NodeRef::From(object).GCObjectData(); - if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return false; + if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return; objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); - return true; - }; + queue.push_back(object); + } }; struct SweepTraits { @@ -104,7 +115,7 @@ NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(Safepoi gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), gcScheduler_(gcScheduler) { - graySet_.reserve(1000); + markQueue_.reserve(1000); gcScheduler_.SetScheduleGC([]() { // TODO: CMS is also responsible for avoiding scheduling while GC hasn't started running. // Investigate, if it's possible to move this logic into the scheduler. @@ -143,17 +154,17 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogInfo( {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); - collectRootSet(graySet_); + gc::collectRootSet(markQueue_); auto timeRootSetUs = konan::getTimeMicros(); // Can be unsafe, because we've stopped the world. auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); RuntimeLogInfo( - {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", graySet_.size(), + {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(), timeRootSetUs - timeSuspendUs); - auto markStats = gc::Mark(graySet_); + auto markStats = gc::Mark(markQueue_); auto timeMarkUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds. Processed %zu duplicate entries in the gray set", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs, markStats.duplicateEntries); + RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs); scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); auto timeSweepExtraObjectsUs = konan::getTimeMicros(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 172b643f766..6f461f3b59c 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -25,6 +25,9 @@ namespace gc { // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads. class SameThreadMarkAndSweep : private Pinned { public: + // This implementation of mark queue allocates memory during collection. + using MarkQueue = KStdVector; + enum class SafepointFlag { kNone, kNeedsSuspend, @@ -86,7 +89,7 @@ private: mm::ObjectFactory& objectFactory_; GCScheduler& gcScheduler_; - KStdVector graySet_; + MarkQueue markQueue_; }; namespace internal {