diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index fcc76cbc296..60fbbf0733b 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -410,11 +410,17 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept { } } +GCHandle::GCSweepScope GCHandle::sweep() { return GCSweepScope(*this); } +GCHandle::GCSweepExtraObjectsScope GCHandle::sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); } +GCHandle::GCGlobalRootSetScope GCHandle::collectGlobalRoots() { return GCGlobalRootSetScope(*this); } +GCHandle::GCThreadRootSetScope GCHandle::collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); } GCHandle::GCMarkScope GCHandle::mark() { return GCMarkScope(*this); } +GCHandle::GCProcessWeaksScope GCHandle::processWeaks() noexcept { return GCProcessWeaksScope(*this); } -GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} +GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle handle) : GCStageScopeBase(handle) {} GCHandle::GCSweepScope::~GCSweepScope() { + if (!handle_.isValid()) return; handle_.swept(stats_, markedCount_); GCLogDebug( handle_.getEpoch(), @@ -423,9 +429,10 @@ GCHandle::GCSweepScope::~GCSweepScope() { stats_.sweptCount, getStageTime(), stats_.keptCount); } -GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} +GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle handle) : GCStageScopeBase(handle) {} GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() { + if (!handle_.isValid()) return; handle_.sweptExtraObjects(stats_); GCLogDebug( handle_.getEpoch(), @@ -434,49 +441,39 @@ GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() { stats_.sweptCount, getStageTime(), stats_.keptCount); } -GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} +GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle handle) : GCStageScopeBase(handle) {} GCHandle::GCGlobalRootSetScope::~GCGlobalRootSetScope(){ + if (!handle_.isValid()) return; handle_.globalRootSetCollected(globalRoots_, stableRoots_); - GCLogDebug(handle_.getEpoch(), "Collected global root set global=%" PRIu64 " stableRef=%" PRIu64 " in %" PRIu64" microseconds.", - globalRoots_, stableRoots_, getStageTime()); + GCLogDebug( + handle_.getEpoch(), "Collected global root set global=%" PRIu64 " stableRef=%" PRIu64 " in %" PRIu64 " microseconds.", + globalRoots_, stableRoots_, getStageTime()); } -GCHandle::GCThreadRootSetScope::GCThreadRootSetScope(kotlin::gc::GCHandle& handle, mm::ThreadData& threadData) : - handle_(handle), threadData_(threadData) {} +GCHandle::GCThreadRootSetScope::GCThreadRootSetScope(kotlin::gc::GCHandle handle, mm::ThreadData& threadData) : + GCStageScopeBase(handle), threadData_(threadData) {} GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){ + if (!handle_.isValid()) return; handle_.threadRootSetCollected(threadData_, threadLocalRoots_, stackRoots_); - GCLogDebug(handle_.getEpoch(), "Collected root set for thread #%d: stack=%" PRIu64 " tls=%" PRIu64 " in %" PRIu64" microseconds.", - threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime()); + GCLogDebug( + handle_.getEpoch(), "Collected root set for thread #%d: stack=%" PRIu64 " tls=%" PRIu64 " in %" PRIu64 " microseconds.", + threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime()); } -void GCHandle::GCMarkScope::swap(GCHandle::GCMarkScope& other) noexcept { - std::swap(handle_, other.handle_); - std::swap(startTime_, other.startTime_); -} - -GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} - -GCHandle::GCMarkScope::GCMarkScope(GCHandle::GCMarkScope&& that) noexcept { - swap(that); -} - -GCHandle::GCMarkScope& GCHandle::GCMarkScope::operator=(GCHandle::GCMarkScope that) noexcept { - swap(that); - return *this; -} +GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle handle) : GCStageScopeBase(handle) {} GCHandle::GCMarkScope::~GCMarkScope() { - if (handle_.isValid()) { - handle_.marked(stats_); - GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime()); - } + if (!handle_.isValid()) return; + handle_.marked(stats_); + GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime()); } -gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {} +GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle handle) noexcept : GCStageScopeBase(handle) {} -gc::GCHandle::GCProcessWeaksScope::~GCProcessWeaksScope() { +GCHandle::GCProcessWeaksScope::~GCProcessWeaksScope() { + if (!handle_.isValid()) return; GCLogDebug( handle_.getEpoch(), "Processed special refs in %" PRIu64 " microseconds. %" PRIu64 " are undisposed, %" PRIu64 " are alive, %" PRIu64 diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 82157d1d970..06259a0558f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -37,92 +37,15 @@ struct MarkStats { class GCHandle { public: - class GCStageScopeUsTimer { - protected: - uint64_t startTime_ = konan::getTimeMicros(); - uint64_t getStageTime() const { return (konan::getTimeMicros() - startTime_); } - }; - - class GCSweepScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - SweepStats stats_; - uint64_t markedCount_ = 0; - - public: - explicit GCSweepScope(GCHandle& handle); - ~GCSweepScope(); - - void addSweptObject() noexcept { stats_.sweptCount += 1; } - void addKeptObject() noexcept { stats_.keptCount += 1; } - // Custom allocator only. To be finalized objects are kept alive. - void addMarkedObject() noexcept { markedCount_ += 1; } - }; - - class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - SweepStats stats_; - - public: - explicit GCSweepExtraObjectsScope(GCHandle& handle); - ~GCSweepExtraObjectsScope(); - - void addSweptObject() noexcept { stats_.sweptCount += 1; } - void addKeptObject() noexcept { stats_.keptCount += 1; } - }; - - class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - uint64_t globalRoots_ = 0; - uint64_t stableRoots_ = 0; - - public: - explicit GCGlobalRootSetScope(GCHandle& handle); - ~GCGlobalRootSetScope(); - void addGlobalRoot() { globalRoots_++; } - void addStableRoot() { stableRoots_++; } - }; - - class GCThreadRootSetScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - mm::ThreadData& threadData_; - uint64_t stackRoots_ = 0; - uint64_t threadLocalRoots_ = 0; - - public: - explicit GCThreadRootSetScope(GCHandle& handle, mm::ThreadData& threadData); - ~GCThreadRootSetScope(); - void addStackRoot() { stackRoots_++; } - void addThreadLocalRoot() { threadLocalRoots_++; } - }; + class GCStageScopeBase; + class GCSweepScope; + class GCSweepExtraObjectsScope; + class GCGlobalRootSetScope; + class GCThreadRootSetScope; class GCMarkScope; + class GCProcessWeaksScope; - class GCProcessWeaksScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - uint64_t undisposedCount_ = 0; - uint64_t aliveCount_ = 0; - uint64_t nulledCount_ = 0; - - public: - explicit GCProcessWeaksScope(GCHandle& handle) noexcept; - ~GCProcessWeaksScope(); - - void addUndisposed() noexcept { ++undisposedCount_; } - void addAlive() noexcept { ++aliveCount_; } - void addNulled() noexcept { ++nulledCount_; } - }; - -private: - uint64_t epoch_; - explicit GCHandle(uint64_t epoch) : epoch_(epoch) {} - - void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences); - void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences); - void swept(SweepStats stats, uint64_t markedCount) noexcept; - void sweptExtraObjects(SweepStats stats) noexcept; - void marked(MarkStats stats); - -public: static GCHandle create(uint64_t epoch); static GCHandle createFakeForTests(); static GCHandle getByEpoch(uint64_t epoch); @@ -138,28 +61,134 @@ public: void suspensionRequested(); void threadsAreSuspended(); void threadsAreResumed(); - GCSweepScope sweep() { return GCSweepScope(*this); } - GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); } - GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); } - GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); } - GCMarkScope mark(); - GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); } + + [[nodiscard]] GCSweepScope sweep(); + [[nodiscard]] GCSweepExtraObjectsScope sweepExtraObjects(); + [[nodiscard]] GCGlobalRootSetScope collectGlobalRoots(); + [[nodiscard]] GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData); + [[nodiscard]] GCMarkScope mark(); + [[nodiscard]] GCProcessWeaksScope processWeaks() noexcept; MarkStats getMarked(); + +private: + uint64_t epoch_; + explicit GCHandle(uint64_t epoch) : epoch_(epoch) {} + + void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences); + void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences); + void swept(SweepStats stats, uint64_t markedCount) noexcept; + void sweptExtraObjects(SweepStats stats) noexcept; + void marked(MarkStats stats); }; -class GCHandle::GCMarkScope : GCStageScopeUsTimer { - GCHandle handle_ = GCHandle::invalid(); - MarkStats stats_; +class GCHandle::GCStageScopeBase : private MoveOnly { +public: + explicit GCStageScopeBase(GCHandle gcHandle) : handle_(gcHandle) {} - void swap(GCMarkScope& other) noexcept; + friend void swap(GCStageScopeBase& first, GCStageScopeBase& second) noexcept { + using std::swap; + swap(first.handle_, second.handle_); + swap(first.startTime_, second.startTime_); + } + GCStageScopeBase(GCStageScopeBase&& that) noexcept : GCStageScopeBase(GCHandle::invalid()) { + swap(*this, that); + } + GCStageScopeBase& operator=(GCStageScopeBase&& that) noexcept { + auto tmp = std::move(that); + swap(*this, tmp); + return *this; + } + +protected: + uint64_t getStageTime() const { return (konan::getTimeMicros() - startTime_); } + + GCHandle handle_; + uint64_t startTime_ = konan::getTimeMicros(); +}; + +class GCHandle::GCSweepScope : GCStageScopeBase { + SweepStats stats_; + uint64_t markedCount_ = 0; public: - explicit GCMarkScope(GCHandle& handle); - GCMarkScope(GCMarkScope&& that) noexcept; - GCMarkScope& operator=(GCMarkScope that) noexcept; + explicit GCSweepScope(GCHandle handle); + GCSweepScope(GCSweepScope&& that) = default; + GCSweepScope& operator=(GCSweepScope&& that) = default; + ~GCSweepScope(); + + void addSweptObject() noexcept { stats_.sweptCount += 1; } + void addKeptObject() noexcept { stats_.keptCount += 1; } + // Custom allocator only. To be finalized objects are kept alive. + void addMarkedObject() noexcept { markedCount_ += 1; } +}; + +class GCHandle::GCSweepExtraObjectsScope : private GCStageScopeBase { + SweepStats stats_; + +public: + explicit GCSweepExtraObjectsScope(GCHandle handle); + GCSweepExtraObjectsScope(GCSweepExtraObjectsScope&& that) = default; + GCSweepExtraObjectsScope& operator=(GCSweepExtraObjectsScope&& that) = default; + ~GCSweepExtraObjectsScope(); + + void addSweptObject() noexcept { stats_.sweptCount += 1; } + void addKeptObject() noexcept { stats_.keptCount += 1; } +}; + +class GCHandle::GCGlobalRootSetScope : private GCStageScopeBase { + uint64_t globalRoots_ = 0; + uint64_t stableRoots_ = 0; + +public: + explicit GCGlobalRootSetScope(GCHandle handle); + GCGlobalRootSetScope(GCGlobalRootSetScope&& that) = default; + GCGlobalRootSetScope& operator=(GCGlobalRootSetScope&& that) = default; + ~GCGlobalRootSetScope(); + void addGlobalRoot() { globalRoots_++; } + void addStableRoot() { stableRoots_++; } +}; + +class GCHandle::GCThreadRootSetScope : private GCStageScopeBase { + mm::ThreadData& threadData_; + uint64_t stackRoots_ = 0; + uint64_t threadLocalRoots_ = 0; + +public: + explicit GCThreadRootSetScope(GCHandle handle, mm::ThreadData& threadData); + GCThreadRootSetScope(GCThreadRootSetScope&& that) = default; + GCThreadRootSetScope& operator=(GCThreadRootSetScope&& that) = delete; + ~GCThreadRootSetScope(); + void addStackRoot() { stackRoots_++; } + void addThreadLocalRoot() { threadLocalRoots_++; } +}; + +class GCHandle::GCMarkScope : private GCStageScopeBase { + MarkStats stats_; + +public: + explicit GCMarkScope(GCHandle handle); + GCMarkScope(GCMarkScope&& that) = default; + GCMarkScope& operator=(GCMarkScope&& that) = default; ~GCMarkScope(); void addObject() noexcept { ++stats_.markedCount; } }; + +class GCHandle::GCProcessWeaksScope : private GCStageScopeBase { + uint64_t undisposedCount_ = 0; + uint64_t aliveCount_ = 0; + uint64_t nulledCount_ = 0; + +public: + explicit GCProcessWeaksScope(GCHandle handle) noexcept; + GCProcessWeaksScope(GCProcessWeaksScope&& that) = default; + GCProcessWeaksScope& operator=(GCProcessWeaksScope&& that) = default; + ~GCProcessWeaksScope(); + + void addUndisposed() noexcept { ++undisposedCount_; } + void addAlive() noexcept { ++aliveCount_; } + void addNulled() noexcept { ++nulledCount_; } +}; + }