From 095925537922c2b2874099098e61ec976f3dc4bd Mon Sep 17 00:00:00 2001 From: Pavel Kunyavskiy Date: Tue, 4 Oct 2022 14:57:21 +0200 Subject: [PATCH] [K/N] Refactor GC logging and statistics Same code is now used for GC logging in all gc versions. ^KT-53064 --- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 94 ++---- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 8 +- .../gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp | 4 +- .../runtime/src/gc/cms/cpp/GCImpl.cpp | 16 + .../runtime/src/gc/common/cpp/GC.hpp | 5 + .../src/gc/common/cpp/GCStatistics.cpp | 314 ++++++++++++++++++ .../src/gc/common/cpp/GCStatistics.hpp | 121 +++++++ .../src/gc/common/cpp/MarkAndSweepUtils.hpp | 49 ++- .../common/cpp/MarkAndSweepUtilsMarkTest.cpp | 14 +- .../common/cpp/MarkAndSweepUtilsSweepTest.cpp | 4 +- .../runtime/src/gc/noop/cpp/GCImpl.cpp | 16 + .../runtime/src/gc/stms/cpp/GCImpl.cpp | 16 + .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 60 +--- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 3 +- .../runtime/src/legacymm/cpp/Memory.cpp | 5 + .../runtime/src/main/cpp/Logging.cpp | 18 +- .../src/main/cpp/std_support/Optional.hpp | 18 + .../src/main/cpp/std_support/README.md | 1 + .../src/mm/cpp/ExtraObjectDataFactory.hpp | 1 + .../runtime/src/mm/cpp/ObjectFactory.hpp | 48 +-- .../runtime/src/mm/cpp/ObjectFactoryTest.cpp | 30 +- .../runtime/src/mm/cpp/ThreadSuspension.cpp | 2 +- 22 files changed, 646 insertions(+), 201 deletions(-) create mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp create mode 100644 kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp create mode 100644 kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 70df78d35f7..a5fc9b896be 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -20,6 +20,7 @@ #include "ThreadSuspension.hpp" #include "GCState.hpp" #include "FinalizerProcessor.hpp" +#include "GCStatistics.hpp" using namespace kotlin; @@ -27,6 +28,7 @@ namespace { [[clang::no_destroy]] std::mutex markingMutex; [[clang::no_destroy]] std::condition_variable markingCondVar; [[clang::no_destroy]] std::atomic markingRequested = false; + [[clang::no_destroy]] std::atomic markingEpoch = 0; struct SweepTraits { using ObjectFactory = mm::ObjectFactory; @@ -72,25 +74,28 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { std::unique_lock lock(markingMutex); - if (!markingRequested.load()) - return; + if (!markingRequested.load()) return; AutoReset scopedAssignMarking(&marking_, true); threadData_.Publish(); markingCondVar.wait(lock, []() { return !markingRequested.load(); }); // // Unlock while marking to allow mutliple threads to mark in parallel. lock.unlock(); - RuntimeLogDebug({kTagGC}, "Parallel marking in thread %d", konan::currentThreadId()); + uint64_t epoch = markingEpoch.load(); + GCLogDebug(epoch, "Parallel marking in thread %d", konan::currentThreadId()); MarkQueue markQueue; - gc::collectRootSetForThread(markQueue, threadData_); - MarkStats stats = gc::Mark(markQueue); - gc_.MergeMarkStats(stats); + auto handle = GCHandle::getByEpoch(epoch); + gc::collectRootSetForThread(handle, markQueue, threadData_); + gc::Mark(handle, markQueue); } gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( mm::ObjectFactory& objectFactory, GCScheduler& gcScheduler) noexcept : objectFactory_(objectFactory), gcScheduler_(gcScheduler), - finalizerProcessor_(std_support::make_unique([this](int64_t epoch) { state_.finalized(epoch); })) { + finalizerProcessor_(std_support::make_unique([this](int64_t epoch) { + state_.finalized(epoch); + GCHandle::getByEpoch(epoch).finalizersDone(); + })) { gcScheduler_.SetScheduleGC([this]() NO_INLINE { RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); // This call acquires a lock, so we need to ensure that we're in the safe state. @@ -135,76 +140,46 @@ void gc::ConcurrentMarkAndSweep::SetMarkingBehaviorForTests(MarkingBehavior mark } bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); - SetMarkingRequested(); - auto timeStartUs = konan::getTimeMicros(); + auto gcHandle = GCHandle::create(epoch); + SetMarkingRequested(epoch); bool didSuspend = mm::RequestThreadsSuspension(); RuntimeAssert(didSuspend, "Only GC thread can request suspension"); - RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId()); + gcHandle.suspensionRequested(); RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "Concurrent GC must run on unregistered thread"); WaitForThreadsReadyToMark(); - auto timeSuspendUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); - lastGCMarkStats_ = MarkStats(); + gcHandle.threadsAreSuspended(); auto& scheduler = gcScheduler_; scheduler.gcData().OnPerformFullGC(); state_.start(epoch); - RuntimeLogInfo( - {kTagGC}, "Started GC epoch %" PRId64 ". Time since last GC %" PRIu64 " microseconds", epoch, timeStartUs - lastGCTimestampUs_); - CollectRootSetAndStartMarking(); + CollectRootSetAndStartMarking(gcHandle); // Can be unsafe, because we've stopped the world. - auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); + gc::Mark(gcHandle, markQueue_); - auto markStats = gc::Mark(markQueue_); - MergeMarkStats(markStats); - - RuntimeLogDebug({kTagGC}, "Waiting for marking in threads"); mm::WaitForThreadsSuspension(); - auto timeMarkingUs = konan::getTimeMicros(); - RuntimeLogInfo({kTagGC}, "Collected root set of size %zu and marked %zu objects in all threads in %" PRIu64 " microseconds", lastGCMarkStats_.rootSetSize, lastGCMarkStats_.aliveHeapSet, timeMarkingUs - timeSuspendUs); + mm::ExtraObjectDataFactory& extraObjectDataFactory = mm::GlobalData::Instance().extraObjectDataFactory(); + auto markStats = gcHandle.getMarked(); + scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize); - scheduler.gcData().UpdateAliveSetBytes(lastGCMarkStats_.aliveHeapSetBytes); - - gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); - auto timeSweepExtraObjectsUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkingUs); + gc::SweepExtraObjects(gcHandle, extraObjectDataFactory); auto objectFactoryIterable = objectFactory_.LockForIter(); mm::ResumeThreads(); - auto timeResumeUs = konan::getTimeMicros(); + gcHandle.threadsAreResumed(); - RuntimeLogInfo( - {kTagGC}, "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %" PRIu64 " microseconds", - timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs); - - auto finalizerQueue = gc::Sweep(objectFactoryIterable); - auto timeSweepUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs); + auto finalizerQueue = gc::Sweep(gcHandle, objectFactoryIterable); kotlin::compactObjectPoolInMainThread(); - // Can be unsafe, because we have a lock in objectFactoryIterable - auto objectsCountAfter = objectFactory_.GetSizeUnsafe(); - auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); - - auto finalizersCount = finalizerQueue.size(); - auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount; - state_.finish(epoch); + gcHandle.finalizersScheduled(finalizerQueue.size()); + gcHandle.finished(); finalizerProcessor_->ScheduleTasks(std::move(finalizerQueue), epoch); - - RuntimeLogInfo( - {kTagGC}, - "Finished GC epoch %" PRId64 ". Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64 - " microseconds", - epoch, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeSweepUs - timeStartUs); - lastGCTimestampUs_ = timeResumeUs; return true; } @@ -234,8 +209,9 @@ namespace { } } // namespace -void gc::ConcurrentMarkAndSweep::SetMarkingRequested() noexcept { +void gc::ConcurrentMarkAndSweep::SetMarkingRequested(uint64_t epoch) noexcept { markingRequested = markingBehavior_ == MarkingBehavior::kMarkOwnStack; + markingEpoch = epoch; } void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept { @@ -244,16 +220,16 @@ void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept { } } -NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking() noexcept { +NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept { std::unique_lock lock(markingMutex); markingRequested = false; gc::collectRootSet( - markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); }); + gcHandle, + markQueue_, + [](mm::ThreadData& thread) { + return !thread.gc().impl().gc().marking_.load(); + } + ); RuntimeLogDebug({kTagGC}, "Requesting marking in threads"); markingCondVar.notify_all(); } - -void gc::ConcurrentMarkAndSweep::MergeMarkStats(gc::MarkStats stats) noexcept { - std::unique_lock lock(markingMutex); - lastGCMarkStats_.merge(stats); -} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 0ef4d0b289f..252876c6d1b 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -18,6 +18,7 @@ #include "Utils.hpp" #include "GCState.hpp" #include "std_support/Memory.hpp" +#include "GCStatistics.hpp" namespace kotlin { @@ -114,25 +115,22 @@ public: void StopFinalizerThreadIfRunning() noexcept; bool FinalizersThreadIsRunning() noexcept; void SetMarkingBehaviorForTests(MarkingBehavior markingBehavior) noexcept; - void SetMarkingRequested() noexcept; + void SetMarkingRequested(uint64_t epoch) noexcept; void WaitForThreadsReadyToMark() noexcept; - void CollectRootSetAndStartMarking() noexcept; + void CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept; private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). bool PerformFullGC(int64_t epoch) noexcept; - void MergeMarkStats(MarkStats stats) noexcept; mm::ObjectFactory& objectFactory_; GCScheduler& gcScheduler_; - uint64_t lastGCTimestampUs_ = 0; GCStateHolder state_; ScopedThread gcThread_; std_support::unique_ptr finalizerProcessor_; MarkQueue markQueue_; - MarkStats lastGCMarkStats_; MarkingBehavior markingBehavior_; }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 3c7becdb4ef..7c0140a1621 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -1172,7 +1172,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) { std_support::vector> gcFutures(kDefaultThreadCount); - mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested(); + mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested(0); for (int i = 0; i < kDefaultThreadCount; ++i) { gcFutures[i] = mutators[i] @@ -1181,7 +1181,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) { if (GetParam() == gc::ConcurrentMarkAndSweep::kMarkOwnStack) { mm::GlobalData::Instance().gc().impl().gc().WaitForThreadsReadyToMark(); - mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking(); + mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking(gc::GCHandle::createFakeForTests()); } for (int i = 0; i < kDefaultThreadCount; ++i) { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index f593272e76d..114efb334c9 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -6,6 +6,7 @@ #include "GCImpl.hpp" #include "GC.hpp" +#include "GCStatistics.hpp" #include "MarkAndSweepUtils.hpp" #include "ThreadSuspension.hpp" #include "std_support/Memory.hpp" @@ -74,6 +75,20 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { return mm::ObjectFactory::GetAllocatedHeapSize(object); } + +size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { + return impl_->objectFactory().GetObjectsCountUnsafe(); +} +size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { + return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); +} +size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); +} +size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } @@ -81,6 +96,7 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { void gc::GC::ClearForTests() noexcept { impl_->gc().StopFinalizerThreadIfRunning(); impl_->objectFactory().ClearForTests(); + GCHandle::ClearForTests(); } void gc::GC::StartFinalizerThreadIfNeeded() noexcept { diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 23df367fb0f..742dcb35fe9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -58,6 +58,11 @@ public: static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; + size_t GetHeapObjectsCountUnsafe() const noexcept; + size_t GetTotalHeapObjectsSizeUnsafe() const noexcept; + size_t GetExtraObjectsCountUnsafe() const noexcept; + size_t GetTotalExtraObjectsSizeUnsafe() const noexcept; + gc::GCSchedulerConfig& gcSchedulerConfig() noexcept; void ClearForTests() noexcept; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp new file mode 100644 index 00000000000..94c9b914067 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -0,0 +1,314 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +#include "GCStatistics.hpp" +#include "Mutex.hpp" +#include "Porting.h" + +#include "Types.h" +#include "Logging.hpp" +#include "ThreadData.hpp" +#include "std_support/Optional.hpp" +#include +#include + + +namespace { + +struct MemoryUsageMap { + std::optional heap; + std::optional extra; +}; + +struct RootSetStatistics { + KLong threadLocalReferences; + KLong stackReferences; + KLong globalReferences; + KLong stableReferences; + KLong total() const { return threadLocalReferences + stableReferences + globalReferences + stableReferences; } +}; + +struct GCInfo { + std::optional epoch; + std::optional startTime; // time since process start + std::optional endTime; + std::optional pauseStartTime; + std::optional pauseEndTime; + std::optional finalizersDoneTime; + std::optional rootSet; + std::optional markStats; + MemoryUsageMap memoryUsageBefore; + MemoryUsageMap memoryUsageAfter; +}; + +GCInfo last; +GCInfo current; +// This lock can be got by thread in runnable state making parallel mark, so +kotlin::SpinLock lock; + +GCInfo* statByEpoch(uint64_t epoch) { + if (current.epoch == epoch) return ¤t; + if (last.epoch == epoch) return &last; + return nullptr; +} + +} // namespace + +namespace kotlin::gc { +GCHandle GCHandle::create(uint64_t epoch) { + std::lock_guard guard(lock); + RuntimeAssert(statByEpoch(epoch) == nullptr, "Starting epoch, which already existed"); + if (current.epoch) { + last = current; + current = {}; + RuntimeLogWarning({kTagGC}, "Starting new GC epoch, while previous is not finished\n"); + } + current.epoch = static_cast(epoch); + current.startTime = static_cast(konan::getTimeNanos()); + if (last.endTime) { + GCLogInfo(epoch, "Started. Time since last GC %" PRIu64 " microseconds.", *current.startTime - *last.endTime); + } else { + GCLogInfo(epoch, "Started."); + } + return getByEpoch(epoch); +} +GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits::max()); } +GCHandle GCHandle::getByEpoch(uint64_t epoch) { + return GCHandle{epoch}; +} +void GCHandle::ClearForTests() { + std::lock_guard guard(lock); + current = {}; + last = {}; +} +void GCHandle::finished() { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->endTime = static_cast(konan::getTimeNanos()); + if (stat->rootSet) { + GCLogInfo( + epoch_, + "Root set: " + "%" PRIu64 " thread local references, " + "%" PRIu64 " stack references, " + "%" PRIu64 " global references, " + "%" PRIu64 " stable references. " + "In total %" PRIu64 " roots.", + stat->rootSet->threadLocalReferences, stat->rootSet->stackReferences, stat->rootSet->globalReferences, + stat->rootSet->stableReferences, stat->rootSet->total()); + } + if (stat->startTime) { + auto time = (*current.endTime - *current.startTime) / 1000; + GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time); + } + + if (stat == ¤t) { + last = current; + current = {}; + } + } +} +void GCHandle::suspensionRequested() { + std::lock_guard guard(lock); + GCLogDebug(epoch_, "Requested thread suspension by thread %d", konan::currentThreadId()); + if (auto* stat = statByEpoch(epoch_)) { + stat->pauseStartTime = static_cast(konan::getTimeNanos()); + } +} +void GCHandle::threadsAreSuspended() { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + if (stat->pauseStartTime) { + auto time = (konan::getTimeNanos() - *stat->pauseStartTime) / 1000; + GCLogInfo(epoch_, "Suspended all threads in %" PRIu64 " microseconds", time); + return; + } + } +} +void GCHandle::threadsAreResumed() { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->pauseEndTime = static_cast(konan::getTimeNanos()); + if (stat->pauseStartTime) { + auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000; + GCLogInfo(epoch_, "Resume all threads. Total pause time is %" PRId64 " microseconds.", time); + return; + } + } +} +void GCHandle::finalizersDone() { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->finalizersDoneTime = static_cast(konan::getTimeNanos()); + if (stat->endTime) { + auto time = (*stat->finalizersDoneTime - *stat->endTime) / 1000; + GCLogInfo(epoch_, "Finalization is done in %" PRId64 " microseconds after epoch end.", time); + return; + } + } + GCLogInfo(epoch_, "Finalization is done."); +} +void GCHandle::finalizersScheduled(uint64_t finalizersCount) { + GCLogInfo(epoch_, "Finalization is scheduled for %" PRIu64 " objects.", finalizersCount); +} +void GCHandle::threadRootSetCollected(mm::ThreadData &threadData, uint64_t threadLocalReferences, uint64_t stackReferences) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + if (!stat->rootSet) { + stat->rootSet = RootSetStatistics{0, 0, 0, 0}; + } + stat->rootSet->stackReferences += static_cast(stackReferences); + stat->rootSet->threadLocalReferences += static_cast(threadLocalReferences); + } +} +void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + if (!stat->rootSet) { + stat->rootSet = RootSetStatistics{0, 0, 0, 0}; + } + stat->rootSet->globalReferences += static_cast(globalReferences); + stat->rootSet->stableReferences += static_cast(stableReferences); + } +} + + +void GCHandle::heapUsageBefore(MemoryUsage usage) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->memoryUsageBefore.heap = usage; + } +} + +void GCHandle::marked(kotlin::gc::MemoryUsage usage) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + if (!stat->markStats) { + stat->markStats = MemoryUsage{0, 0}; + } + stat->markStats->totalObjectsSize += usage.totalObjectsSize; + stat->markStats->objectsCount += usage.objectsCount; + } +} + +MemoryUsage GCHandle::getMarked() { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + if (stat->markStats) { + return *stat->markStats; + } + } + return MemoryUsage{0, 0}; +} + +void GCHandle::heapUsageAfter(MemoryUsage usage) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->memoryUsageAfter.heap = usage; + if (stat->memoryUsageBefore.heap) { + GCLogInfo( + epoch_, + "Collected %" PRId64 " heap objects of total size %" PRId64 " bytes. " + "%" PRId64 " heap objects of total size %" PRId64 " bytes are still alive.", + stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount, + stat->memoryUsageBefore.heap->totalObjectsSize - stat->memoryUsageAfter.heap->totalObjectsSize, + stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->totalObjectsSize); + } + if (stat->markStats) { + RuntimeAssert( + stat->markStats->objectsCount == usage.objectsCount, + "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep", + stat->markStats->objectsCount, usage.objectsCount); + RuntimeAssert( + stat->markStats->totalObjectsSize == usage.totalObjectsSize, + "Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep", + stat->markStats->totalObjectsSize, usage.totalObjectsSize); + } + } +} + +void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->memoryUsageBefore.extra = usage; + } +} +void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) { + std::lock_guard guard(lock); + if (auto* stat = statByEpoch(epoch_)) { + stat->memoryUsageAfter.extra = usage; + if (stat->memoryUsageBefore.extra) { + GCLogInfo( + epoch_, + "Collected %" PRId64 " extra objects of total size %" PRId64 ". " + "%" PRId64 " extra objects of total size %" PRId64 " are still alive.", + stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount, + stat->memoryUsageBefore.extra->totalObjectsSize - stat->memoryUsageAfter.extra->totalObjectsSize, + stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->totalObjectsSize); + } + } +} + +MemoryUsage GCHandle::GCSweepScope::getUsage() { + return MemoryUsage{ + mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(), + mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(), + }; +} + +GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : + handle_(handle) { + handle_.heapUsageBefore(getUsage()); +} + +GCHandle::GCSweepScope::~GCSweepScope() { + GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime()); + handle_.heapUsageAfter(getUsage()); +} + +MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() { + return MemoryUsage{ + mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(), + mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(), + }; +} + +GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : + handle_(handle) { + handle_.extraObjectsUsageBefore(getUsage()); +} + +GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() { + GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime()); + handle_.extraObjectsUsageAfter(getUsage()); +} + +GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} + +GCHandle::GCGlobalRootSetScope::~GCGlobalRootSetScope(){ + handle_.globalRootSetCollected(globalRoots_, stableRoots_); + GCLogDebug(handle_.getEpoch(), "Collected global root set global=%" PRIu64 " stableRef=%" PRIu64 " in %" PRIu64" microseconds.", + globalRoots_, stableRoots_, getStageTime()); +} + +GCHandle::GCThreadRootSetScope::GCThreadRootSetScope(kotlin::gc::GCHandle& handle, mm::ThreadData& threadData) : + handle_(handle), threadData_(threadData) {} + +GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){ + handle_.threadRootSetCollected(threadData_, threadLocalRoots_, stackRoots_); + GCLogDebug(handle_.getEpoch(), "Collected root set for thread #%d: stack=%" PRIu64 " tls=%" PRIu64 " in %" PRIu64" microseconds.", + threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime()); +} + +GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){} + +GCHandle::GCMarkScope::~GCMarkScope() { + handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes}); + GCLogDebug(handle_.getEpoch(), + "Marked %" PRIu64 " objects in %" PRIu64 " microseconds in thread %d", + objectsCount, getStageTime(), konan::currentThreadId()); +} + +} // namespace kotlin::gc \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp new file mode 100644 index 00000000000..2cf6dd8d9c2 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -0,0 +1,121 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +#pragma once + +#include +#include +#include "Common.h" +#include "ThreadData.hpp" + +#define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__) +#define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__) + +namespace kotlin::gc { + +class GCHandle; + +struct MemoryUsage { + uint64_t objectsCount; + uint64_t totalObjectsSize; +}; + +class GCHandle { + class GCStageScopeUsTimer { + protected: + uint64_t startTime_ = konan::getTimeMicros(); + uint64_t getStageTime() const { return (konan::getTimeMicros() - startTime_); } + }; + + class GCSweepScope : GCStageScopeUsTimer, Pinned { + GCHandle& handle_; + MemoryUsage getUsage(); + + public: + explicit GCSweepScope(GCHandle& handle); + ~GCSweepScope(); + }; + + class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned { + GCHandle& handle_; + MemoryUsage getUsage(); + + public: + explicit GCSweepExtraObjectsScope(GCHandle& handle); + ~GCSweepExtraObjectsScope(); + }; + + class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned { + GCHandle& handle_; + uint64_t globalRoots_ = 0; + uint64_t stableRoots_ = 0; + + public: + explicit GCGlobalRootSetScope(GCHandle& handle); + ~GCGlobalRootSetScope(); + void addGlobalRoot() { globalRoots_++; } + void addStableRoot() { stableRoots_++; } + }; + + class GCThreadRootSetScope : GCStageScopeUsTimer, Pinned { + GCHandle& handle_; + mm::ThreadData& threadData_; + uint64_t stackRoots_ = 0; + uint64_t threadLocalRoots_ = 0; + + public: + explicit GCThreadRootSetScope(GCHandle& handle, mm::ThreadData& threadData); + ~GCThreadRootSetScope(); + void addStackRoot() { stackRoots_++; } + void addThreadLocalRoot() { threadLocalRoots_++; } + }; + + class GCMarkScope : GCStageScopeUsTimer, Pinned { + GCHandle& handle_; + uint64_t objectsCount = 0; + uint64_t totalObjectSizeBytes = 0; + + public: + explicit GCMarkScope(GCHandle& handle); + ~GCMarkScope(); + void addObject(uint64_t objectSize) { + totalObjectSizeBytes += objectSize; + objectsCount++; + } + }; + + uint64_t epoch_; + explicit GCHandle(uint64_t epoch) : epoch_(epoch) {} + + void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences); + void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences); + void heapUsageBefore(MemoryUsage usage); + void heapUsageAfter(MemoryUsage usage); + void extraObjectsUsageBefore(MemoryUsage usage); + void extraObjectsUsageAfter(MemoryUsage usage); + void marked(MemoryUsage usage); + +public: + static GCHandle create(uint64_t epoch); + static GCHandle createFakeForTests(); + static GCHandle getByEpoch(uint64_t epoch); + static void ClearForTests(); + + uint64_t getEpoch() { return epoch_; } + void finished(); + void finalizersDone(); + void finalizersScheduled(uint64_t finalizersCount); + void suspensionRequested(); + void threadsAreSuspended(); + void threadsAreResumed(); + GCSweepScope sweep() { return GCSweepScope(*this); } + GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); } + GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); } + GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); } + GCMarkScope mark() { return GCMarkScope(*this); } + + MemoryUsage getMarked(); +}; +} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 7eef2a1bbcb..d39ab318865 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -9,6 +9,7 @@ #include "ExtraObjectData.hpp" #include "FinalizerHooks.hpp" #include "GlobalData.hpp" +#include "GCStatistics.hpp" #include "Logging.hpp" #include "Memory.h" #include "ObjectOps.hpp" @@ -88,18 +89,15 @@ struct MarkStats { }; template -MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept { - MarkStats stats; - stats.rootSetSize = markQueue.size(); - auto timeStart = konan::getTimeMicros(); +void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept { + auto markHandle = handle.mark(); while (!Traits::isEmpty(markQueue)) { ObjHeader* top = Traits::dequeue(markQueue); RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top); RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local()); - stats.aliveHeapSet++; - stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top); + markHandle.addObject(mm::GetAllocatedHeapSize(top)); Traits::processInMark(markQueue, top); @@ -112,14 +110,12 @@ MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept { } } } - auto timeEnd = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds in thread %d", stats.aliveHeapSet, timeEnd - timeStart, konan::currentThreadId()); - return stats; } template -void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noexcept { +void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& objectFactory) noexcept { objectFactory.ProcessDeletions(); + auto sweepHandle = handle.sweepExtraObjects(); auto iter = objectFactory.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { auto &extraObject = *it; @@ -140,8 +136,9 @@ void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noex } template -typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept { +typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept { typename Traits::ObjectFactory::FinalizerQueue finalizerQueue; + auto sweepHandle = handle.sweep(); for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) { if (Traits::TryResetMark(*it)) { @@ -160,64 +157,60 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact } template -typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory& objectFactory) noexcept { +typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept { auto iter = objectFactory.LockForIter(); - return Sweep(iter); + return Sweep(handle, iter); } template -void collectRootSetForThread(typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { +void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { + auto handle = gcHandle.collectThreadRoots(thread); thread.gc().OnStoppedForGC(); - size_t stack = 0; - size_t tls = 0; // TODO: Remove useless mm::ThreadRootSet abstraction. for (auto value : mm::ThreadRootSet(thread)) { if (internal::collectRoot(markQueue, value.object)) { switch (value.source) { case mm::ThreadRootSet::Source::kStack: - ++stack; + handle.addStackRoot(); break; case mm::ThreadRootSet::Source::kTLS: - ++tls; + handle.addThreadLocalRoot(); break; } } } - RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls); } template -void collectRootSetGlobals(typename Traits::MarkQueue& markQueue) noexcept { +void collectRootSetGlobals(GCHandle gcHandle, typename Traits::MarkQueue& markQueue) noexcept { + auto handle = gcHandle.collectGlobalRoots(); mm::StableRefRegistry::Instance().ProcessDeletions(); - size_t global = 0; - size_t stableRef = 0; // TODO: Remove useless mm::GlobalRootSet abstraction. for (auto value : mm::GlobalRootSet()) { if (internal::collectRoot(markQueue, value.object)) { switch (value.source) { case mm::GlobalRootSet::Source::kGlobal: - ++global; + handle.addGlobalRoot(); break; case mm::GlobalRootSet::Source::kStableRef: - ++stableRef; + handle.addStableRoot(); break; } } } - RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef); } // TODO: This needs some tests now. template -void collectRootSet(typename Traits::MarkQueue& markQueue, F&& filter) noexcept { +void collectRootSet(GCHandle handle, typename Traits::MarkQueue& markQueue, F&& filter) noexcept { Traits::clear(markQueue); for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { if (!filter(thread)) continue; thread.Publish(); - collectRootSetForThread(markQueue, thread); + collectRootSetForThread(handle, markQueue, thread); } - collectRootSetGlobals(markQueue); + collectRootSetGlobals(handle, markQueue); } } // namespace gc diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index ad477a8f509..5da15cb0ab5 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -174,13 +174,19 @@ public: return testing::UnorderedElementsAreArray(objects); } - gc::MarkStats Mark(std::initializer_list> graySet) { + gc::MemoryUsage Mark(std::initializer_list> graySet) { std_support::vector objects; for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader()); - return gc::Mark(objects); + auto handle = gc::GCHandle::create(epoch_++); + gc::Mark(handle, objects); + handle.finished(); + return handle.getMarked(); } + ~MarkAndSweepUtilsMarkTest() { mm::GlobalData::Instance().gc().ClearForTests(); } + private: + uint64_t epoch_ = 0; kotlin::ScopedMemoryInit memoryInit; ScopedMarkTraits markTraits_; }; @@ -196,8 +202,8 @@ size_t GetObjectsSize(std::initializer_list> #define EXPECT_MARKED(stats, ...) \ do { \ std::initializer_list> objects = {__VA_ARGS__}; \ - EXPECT_THAT(stats.aliveHeapSet, objects.size()); \ - EXPECT_THAT(stats.aliveHeapSetBytes, GetObjectsSize(objects)); \ + EXPECT_THAT(stats.objectsCount, objects.size()); \ + EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \ EXPECT_THAT(marked(), MarkedMatcher(objects)); \ } while (false) diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index 68b1be572f0..f815e248a5f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -199,8 +199,8 @@ public: } std_support::vector Sweep() { - gc::SweepExtraObjects(extraObjectFactory_); - auto finalizers = gc::Sweep(objectFactory_); + gc::SweepExtraObjects(gc::GCHandle::getByEpoch(0), extraObjectFactory_); + auto finalizers = gc::Sweep(gc::GCHandle::getByEpoch(0), objectFactory_); std_support::vector objects; for (auto node : finalizers.IterForTests()) { objects.push_back(node.GetObjHeader()); diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index bf724db035d..dfca326cd84 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -7,6 +7,8 @@ #include "GC.hpp" #include "std_support/Memory.hpp" +#include "GlobalData.hpp" +#include "GCStatistics.hpp" using namespace kotlin; @@ -61,12 +63,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { return mm::ObjectFactory::GetAllocatedHeapSize(object); } +size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { + return impl_->objectFactory().GetObjectsCountUnsafe(); +} +size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { + return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); +} +size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); +} +size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } void gc::GC::ClearForTests() noexcept { impl_->objectFactory().ClearForTests(); + GCHandle::ClearForTests(); } void gc::GC::StartFinalizerThreadIfNeeded() noexcept {} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 338534bcae5..fbb33f6c974 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -8,6 +8,8 @@ #include "GC.hpp" #include "MarkAndSweepUtils.hpp" #include "std_support/Memory.hpp" +#include "GlobalData.hpp" +#include "GCStatistics.hpp" using namespace kotlin; @@ -74,12 +76,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { return mm::ObjectFactory::GetAllocatedHeapSize(object); } +size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { + return impl_->objectFactory().GetObjectsCountUnsafe(); +} +size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { + return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); +} +size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); +} +size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { + return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } void gc::GC::ClearForTests() noexcept { impl_->objectFactory().ClearForTests(); + GCHandle::ClearForTests(); } void gc::GC::StartFinalizerThreadIfNeeded() noexcept {} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 6687ed84f4d..ba229fed7d7 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -9,6 +9,7 @@ #include "CompilerConstants.hpp" #include "GlobalData.hpp" +#include "GCStatistics.hpp" #include "Logging.hpp" #include "MarkAndSweepUtils.hpp" #include "Memory.h" @@ -103,7 +104,6 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); - auto timeStartUs = konan::getTimeMicros(); bool didSuspend = mm::RequestThreadsSuspension(); if (!didSuspend) { RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId()); @@ -111,8 +111,9 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { // TODO: This breaks if suspension is used by something apart from GC. return false; } - RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId()); gSafepointFlag = SafepointFlag::kNeedsSuspend; + auto gcHandle = GCHandle::create(epoch_++); + gcHandle.suspensionRequested(); mm::ObjectFactory::FinalizerQueue finalizerQueue; { @@ -120,55 +121,27 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { ThreadStateGuard guard(ThreadState::kNative); mm::WaitForThreadsSuspension(); - auto timeSuspendUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); + gcHandle.threadsAreSuspended(); auto& scheduler = gcScheduler_; scheduler.gcData().OnPerformFullGC(); - RuntimeLogInfo( - {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); - gc::collectRootSet(markQueue_, [](mm::ThreadData&) { return true; }); - auto timeRootSetUs = konan::getTimeMicros(); - // Can be unsafe, because we've stopped the world. - auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); + gc::collectRootSet(gcHandle, markQueue_, [] (mm::ThreadData&) { return true; }); + auto& extraObjectsDataFactory = mm::GlobalData::Instance().extraObjectDataFactory(); - RuntimeLogInfo( - {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(), - timeRootSetUs - timeSuspendUs); - auto markStats = gc::Mark(markQueue_); - auto timeMarkUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs); - scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); - gc::SweepExtraObjects(mm::GlobalData::Instance().extraObjectDataFactory()); - auto timeSweepExtraObjectsUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs); - finalizerQueue = gc::Sweep(objectFactory_); - auto timeSweepUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs); + gc::Mark(gcHandle, markQueue_); + auto markStats = gcHandle.getMarked(); + scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize); + gc::SweepExtraObjects(gcHandle, extraObjectsDataFactory); + finalizerQueue = gc::Sweep(gcHandle, objectFactory_); kotlin::compactObjectPoolInMainThread(); - // Can be unsafe, because we've stopped the world. - auto objectsCountAfter = objectFactory_.GetSizeUnsafe(); - auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); - gSafepointFlag = SafepointFlag::kNone; mm::ResumeThreads(); - auto timeResumeUs = konan::getTimeMicros(); - - RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs); - - auto finalizersCount = finalizerQueue.size(); - auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount; - - RuntimeLogInfo( - {kTagGC}, - "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64 - " microseconds", - epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs); - ++epoch_; - lastGCTimestampUs_ = timeResumeUs; + gcHandle.threadsAreResumed(); + gcHandle.finalizersScheduled(finalizerQueue.size()); + gcHandle.finished(); } // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would @@ -177,11 +150,8 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { // TODO: These will actually need to be run on a separate thread. AssertThreadState(ThreadState::kRunnable); - RuntimeLogDebug({kTagGC}, "Starting to run finalizers"); - auto timeBeforeUs = konan::getTimeMicros(); finalizerQueue.Finalize(); - auto timeAfterUs = konan::getTimeMicros(); - RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs); + gcHandle.finalizersDone(); return true; } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index afe98620e6d..95e216e4c06 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -99,8 +99,7 @@ private: // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). bool PerformFullGC() noexcept; - size_t epoch_ = 0; - uint64_t lastGCTimestampUs_ = 0; + uint64_t epoch_ = 0; mm::ObjectFactory& objectFactory_; GCScheduler& gcScheduler_; diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index ef2f7569d5a..92106f59630 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3516,6 +3516,9 @@ void Kotlin_native_internal_GC_collect(KRef) { #endif } +extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) { +} + void Kotlin_native_internal_GC_collectCyclic(KRef) { #if USE_CYCLIC_GC if (g_hasCyclicCollector) @@ -3862,3 +3865,5 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {} bool kotlin::FinalizersThreadIsRunning() noexcept { return false; } + + diff --git a/kotlin-native/runtime/src/main/cpp/Logging.cpp b/kotlin-native/runtime/src/main/cpp/Logging.cpp index 867a5aab287..0818954b3d0 100644 --- a/kotlin-native/runtime/src/main/cpp/Logging.cpp +++ b/kotlin-native/runtime/src/main/cpp/Logging.cpp @@ -6,25 +6,15 @@ #include "Logging.hpp" #include -#if __has_include() -#include -#elif __has_include() -// TODO: Remove when wasm32 is gone. -#include -namespace std { -template -using optional = std::experimental::optional; -inline constexpr auto nullopt = std::experimental::nullopt; -} // namespace std -#else -#error "No " -#endif + + #include "Format.h" #include "KAssert.h" #include "Porting.h" #include "std_support/Map.hpp" #include "std_support/String.hpp" +#include "std_support/Optional.hpp" using namespace kotlin; @@ -115,7 +105,7 @@ private: class StderrLogger : public logging::internal::Logger { public: - void Log(logging::Level level, std_support::span tags, std::string_view message) const noexcept override { + NO_EXTERNAL_CALLS_CHECK void Log(logging::Level level, std_support::span tags, std::string_view message) const noexcept override { konan::consoleErrorUtf8(message.data(), message.size()); } }; diff --git a/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp b/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp new file mode 100644 index 00000000000..26b313784e8 --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp @@ -0,0 +1,18 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +#if __has_include() +#include +#elif __has_include() +// TODO: Remove when wasm32 is gone. +#include +namespace std { +template +using optional = std::experimental::optional; +inline constexpr auto nullopt = std::experimental::nullopt; +} // namespace std +#else +#error "No " +#endif \ No newline at end of file diff --git a/kotlin-native/runtime/src/main/cpp/std_support/README.md b/kotlin-native/runtime/src/main/cpp/std_support/README.md index 47babf6d681..1de7b8fca0e 100644 --- a/kotlin-native/runtime/src/main/cpp/std_support/README.md +++ b/kotlin-native/runtime/src/main/cpp/std_support/README.md @@ -25,3 +25,4 @@ Adjustments: `std_support::kdelete` as a replacement for operator `delete` for objects created with custom `new`. * `Deque.hpp`, `ForwardList.hpp`, `List.hpp`, `Map.hpp`, `Set.hpp`, `String.hpp`, `UnorderedMap.hpp`, `UnorderedSet.hpp`, `Vector.hpp` - standard containers and `std_support::string` that default to using `std_support::allocator`. +* `Optional.hpp` - wrapper choosing correct way to include on different platforms. \ No newline at end of file diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp index 0525f3f97ba..f4b438f9501 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp @@ -53,6 +53,7 @@ public: void ClearForTests() noexcept { extraObjects_.ClearForTests(); } size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); } + size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); } // requires LockForIter void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index d2b8c90c962..b3440e8a281 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -145,6 +145,7 @@ public: last_ = nodePtr; ++size_; + totalObjectsSizeBytes_ += Node::GetSizeForDataSize(dataSize); RuntimeAssert(root_ != nullptr, "Must not be empty"); AssertCorrect(); return *nodePtr; @@ -181,7 +182,9 @@ public: owner_.last_ = last_; last_ = nullptr; owner_.size_ += size_; + owner_.totalObjectsSizeBytes_ += totalObjectsSizeBytes_; size_ = 0; + totalObjectsSizeBytes_ = 0; RuntimeAssert(root_ == nullptr, "Must be empty"); AssertCorrect(); @@ -216,6 +219,7 @@ public: unique_ptr root_; Node* last_ = nullptr; size_t size_ = 0; + size_t totalObjectsSizeBytes_ = 0; }; class Iterator { @@ -348,13 +352,13 @@ public: Iterator begin() noexcept { return Iterator(nullptr, owner_.root_.get()); } Iterator end() noexcept { return Iterator(owner_.last_, nullptr); } - void EraseAndAdvance(Iterator& iterator) noexcept { - auto result = owner_.ExtractUnsafe(iterator.previousNode_); + void EraseAndAdvance(Iterator& iterator, size_t objectedSize) noexcept { + auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectedSize); iterator.node_ = result.second; } - void MoveAndAdvance(Consumer& consumer, Iterator& iterator) noexcept { - auto result = owner_.ExtractUnsafe(iterator.previousNode_); + void MoveAndAdvance(Consumer& consumer, Iterator& iterator, size_t objectSize) noexcept { + auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectSize); iterator.node_ = result.second; consumer.Insert(std::move(result.first)); } @@ -373,40 +377,32 @@ public: Iterable LockForIter() noexcept { return Iterable(*this); } size_t GetSizeUnsafe() const noexcept { return size_; } + size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; } void ClearForTests() { root_.reset(); last_ = nullptr; size_ = 0; + totalObjectsSizeBytes_ = 0; } private: // Expects `mutex_` to be held by the current thread. - std::pair, Node*> ExtractUnsafe(Node* previousNode) noexcept { + std::pair, Node*> ExtractUnsafe(Node* previousNode, size_t objectSize) noexcept { RuntimeAssert(root_ != nullptr, "Must not be empty"); AssertCorrectUnsafe(); - if (previousNode == nullptr) { - // Extracting the root. - auto node = std::move(root_); - root_ = std::move(node->next_); - if (!root_) { - last_ = nullptr; - } - --size_; - AssertCorrectUnsafe(); - return {std::move(node), root_.get()}; - } - - auto node = std::move(previousNode->next_); - previousNode->next_ = std::move(node->next_); - if (!previousNode->next_) { + unique_ptr &pointerToNext = (previousNode == nullptr) ? root_ : previousNode->next_; + auto node = std::move(pointerToNext); + pointerToNext = std::move(node->next_); + if (!pointerToNext) { last_ = previousNode; } --size_; + totalObjectsSizeBytes_ -= objectSize; AssertCorrectUnsafe(); - return {std::move(node), previousNode->next_.get()}; + return {std::move(node), pointerToNext.get()}; } // Expects `mutex_` to be held by the current thread. @@ -422,6 +418,7 @@ private: unique_ptr root_; Node* last_ = nullptr; size_t size_ = 0; + size_t totalObjectsSizeBytes_ = 0; SpinLock mutex_; }; @@ -656,10 +653,12 @@ public: Iterator begin() noexcept { return Iterator(iter_.begin()); } Iterator end() noexcept { return Iterator(iter_.end()); } - void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); } + void EraseAndAdvance(Iterator& iterator) noexcept { + iter_.EraseAndAdvance(iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader())); + } void MoveAndAdvance(FinalizerQueue& queue, Iterator& iterator) noexcept { - iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_); + iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader())); } private: @@ -672,7 +671,8 @@ public: // Lock ObjectFactory for safe iteration. Iterable LockForIter() noexcept { return Iterable(*this); } - size_t GetSizeUnsafe() const noexcept { return storage_.GetSizeUnsafe(); } + size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); } + size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); } void ClearForTests() { storage_.ClearForTests(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 81a957bfd9b..840dedc1580 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -240,7 +240,7 @@ TEST(ObjectFactoryStorageTest, EraseFirst) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 1) { - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -268,7 +268,7 @@ TEST(ObjectFactoryStorageTest, EraseMiddle) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 2) { - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -296,7 +296,7 @@ TEST(ObjectFactoryStorageTest, EraseLast) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 3) { - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -323,7 +323,7 @@ TEST(ObjectFactoryStorageTest, EraseAll) { { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } } @@ -345,7 +345,7 @@ TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) { { auto iter = storage.LockForIter(); auto it = iter.begin(); - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); EXPECT_THAT(it, iter.end()); } @@ -371,7 +371,7 @@ TEST(ObjectFactoryStorageTest, MoveFirst) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 1) { - iter.MoveAndAdvance(consumer, it); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -403,7 +403,7 @@ TEST(ObjectFactoryStorageTest, MoveMiddle) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 2) { - iter.MoveAndAdvance(consumer, it); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -435,7 +435,7 @@ TEST(ObjectFactoryStorageTest, MoveLast) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() == 3) { - iter.MoveAndAdvance(consumer, it); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -466,7 +466,7 @@ TEST(ObjectFactoryStorageTest, MoveAll) { { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { - iter.MoveAndAdvance(consumer, it); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } } @@ -499,7 +499,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->Data() % 2 == 0) { - iter.MoveAndAdvance(consumer1, it); + iter.MoveAndAdvance(consumer1, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } @@ -508,7 +508,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) { { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { - iter.MoveAndAdvance(consumer2, it); + iter.MoveAndAdvance(consumer2, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } } @@ -577,7 +577,7 @@ TEST(ObjectFactoryStorageTest, MoveTheOnlyElement) { { auto iter = storage.LockForIter(); auto it = iter.begin(); - iter.MoveAndAdvance(consumer, it); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); EXPECT_THAT(it, iter.end()); } @@ -612,8 +612,8 @@ TEST(ObjectFactoryStorageTest, MoveAndErase) { auto iter = storage.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { ++it; - iter.EraseAndAdvance(it); - iter.MoveAndAdvance(consumer, it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); + iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } } @@ -759,7 +759,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { for (auto it = iter.begin(); it != iter.end();) { if (it->Data() % 2 != 0) { - iter.EraseAndAdvance(it); + iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int))); } else { ++it; } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index d87fe7b0759..5d9cffefc8a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -81,7 +81,7 @@ NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::RequestThreadsSuspension() noexcept { } void kotlin::mm::WaitForThreadsSuspension() noexcept { - // Spin wating for threads to suspend. Ignore Native threads. + // Spin waiting for threads to suspend. Ignore Native threads. while(!allThreads(isSuspendedOrNative)) { yield(); }