[K/N] Refactor GC logging and statistics

Same code is now used for GC logging in all gc versions.

^KT-53064

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -20,6 +20,7 @@
 #include "ThreadSuspension.hpp"
 #include "GCState.hpp"
 #include "FinalizerProcessor.hpp"
+#include "GCStatistics.hpp"
 
 using namespace kotlin;
 
@@ -27,6 +28,7 @@ namespace {
     [[clang::no_destroy]] std::mutex markingMutex;
     [[clang::no_destroy]] std::condition_variable markingCondVar;
     [[clang::no_destroy]] std::atomic<bool> markingRequested = false;
+    [[clang::no_destroy]] std::atomic<uint64_t> markingEpoch = 0;
 
 struct SweepTraits {
     using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
@@ -72,25 +74,28 @@ void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
 
 NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
     std::unique_lock lock(markingMutex);
-    if (!markingRequested.load())
+    if (!markingRequested.load()) return;
-        return;
     AutoReset scopedAssignMarking(&marking_, true);
     threadData_.Publish();
     markingCondVar.wait(lock, []() { return !markingRequested.load(); });
     // // Unlock while marking to allow mutliple threads to mark in parallel.
     lock.unlock();
-    RuntimeLogDebug({kTagGC}, "Parallel marking in thread %d", konan::currentThreadId());
+    uint64_t epoch = markingEpoch.load();
+    GCLogDebug(epoch, "Parallel marking in thread %d", konan::currentThreadId());
     MarkQueue markQueue;
-    gc::collectRootSetForThread<internal::MarkTraits>(markQueue, threadData_);
+    auto handle = GCHandle::getByEpoch(epoch);
-    MarkStats stats = gc::Mark<internal::MarkTraits>(markQueue);
+    gc::collectRootSetForThread<internal::MarkTraits>(handle, markQueue, threadData_);
-    gc_.MergeMarkStats(stats);
+    gc::Mark<internal::MarkTraits>(handle, markQueue);
 }
 
 gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
         mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, GCScheduler& gcScheduler) noexcept :
     objectFactory_(objectFactory),
     gcScheduler_(gcScheduler),
-    finalizerProcessor_(std_support::make_unique<FinalizerProcessor>([this](int64_t epoch) { state_.finalized(epoch); })) {
+    finalizerProcessor_(std_support::make_unique<FinalizerProcessor>([this](int64_t epoch) {
+        state_.finalized(epoch);
+        GCHandle::getByEpoch(epoch).finalizersDone();
+    })) {
     gcScheduler_.SetScheduleGC([this]() NO_INLINE {
         RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId());
         // This call acquires a lock, so we need to ensure that we're in the safe state.
@@ -135,76 +140,46 @@ void gc::ConcurrentMarkAndSweep::SetMarkingBehaviorForTests(MarkingBehavior mark
 }
 
 bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
-    RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
+    auto gcHandle = GCHandle::create(epoch);
-    SetMarkingRequested();
+    SetMarkingRequested(epoch);
-    auto timeStartUs = konan::getTimeMicros();
     bool didSuspend = mm::RequestThreadsSuspension();
     RuntimeAssert(didSuspend, "Only GC thread can request suspension");
-    RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
+    gcHandle.suspensionRequested();
 
     RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "Concurrent GC must run on unregistered thread");
     WaitForThreadsReadyToMark();
-    auto timeSuspendUs = konan::getTimeMicros();
+    gcHandle.threadsAreSuspended();
-    RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
-    lastGCMarkStats_ = MarkStats();
 
     auto& scheduler = gcScheduler_;
     scheduler.gcData().OnPerformFullGC();
 
     state_.start(epoch);
-    RuntimeLogInfo(
-            {kTagGC}, "Started GC epoch %" PRId64 ". Time since last GC %" PRIu64 " microseconds", epoch, timeStartUs - lastGCTimestampUs_);
 
-    CollectRootSetAndStartMarking();
+    CollectRootSetAndStartMarking(gcHandle);
 
     // Can be unsafe, because we've stopped the world.
-    auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
+    gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
 
-    auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
-    MergeMarkStats(markStats);
-
-    RuntimeLogDebug({kTagGC}, "Waiting for marking in threads");
     mm::WaitForThreadsSuspension();
-    auto timeMarkingUs = konan::getTimeMicros();
+    mm::ExtraObjectDataFactory& extraObjectDataFactory = mm::GlobalData::Instance().extraObjectDataFactory();
-    RuntimeLogInfo({kTagGC}, "Collected root set of size %zu and marked %zu objects in all threads in %" PRIu64 " microseconds", lastGCMarkStats_.rootSetSize, lastGCMarkStats_.aliveHeapSet, timeMarkingUs - timeSuspendUs);
+    auto markStats = gcHandle.getMarked();
+    scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
 
-    scheduler.gcData().UpdateAliveSetBytes(lastGCMarkStats_.aliveHeapSetBytes);
+    gc::SweepExtraObjects<SweepTraits>(gcHandle, extraObjectDataFactory);
-
-    gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
-    auto timeSweepExtraObjectsUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkingUs);
 
     auto objectFactoryIterable = objectFactory_.LockForIter();
 
     mm::ResumeThreads();
-    auto timeResumeUs = konan::getTimeMicros();
+    gcHandle.threadsAreResumed();
 
-    RuntimeLogInfo(
+    auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, objectFactoryIterable);
-            {kTagGC}, "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %" PRIu64 " microseconds",
-            timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
-
-    auto finalizerQueue = gc::Sweep<SweepTraits>(objectFactoryIterable);
-    auto timeSweepUs = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs);
 
     kotlin::compactObjectPoolInMainThread();
 
-    // Can be unsafe, because we have a lock in objectFactoryIterable
-    auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
-    auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
-
-    auto finalizersCount = finalizerQueue.size();
-    auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
-
     state_.finish(epoch);
+    gcHandle.finalizersScheduled(finalizerQueue.size());
+    gcHandle.finished();
     finalizerProcessor_->ScheduleTasks(std::move(finalizerQueue), epoch);
-
-    RuntimeLogInfo(
-            {kTagGC},
-            "Finished GC epoch %" PRId64 ". Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
-            " microseconds",
-            epoch, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeSweepUs - timeStartUs);
-    lastGCTimestampUs_ = timeResumeUs;
     return true;
 }
 
@@ -234,8 +209,9 @@ namespace {
     }
 } // namespace
 
-void gc::ConcurrentMarkAndSweep::SetMarkingRequested() noexcept {
+void gc::ConcurrentMarkAndSweep::SetMarkingRequested(uint64_t epoch) noexcept {
     markingRequested = markingBehavior_ == MarkingBehavior::kMarkOwnStack;
+    markingEpoch = epoch;
 }
 
 void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept {
@@ -244,16 +220,16 @@ void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept {
     }
 }
 
-NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking() noexcept {
+NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept {
         std::unique_lock lock(markingMutex);
         markingRequested = false;
         gc::collectRootSet<internal::MarkTraits>(
-                markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); });
+                gcHandle,
+                markQueue_,
+                [](mm::ThreadData& thread) {
+                    return !thread.gc().impl().gc().marking_.load();
+                }
+            );
         RuntimeLogDebug({kTagGC}, "Requesting marking in threads");
         markingCondVar.notify_all();
 }
-
-void gc::ConcurrentMarkAndSweep::MergeMarkStats(gc::MarkStats stats) noexcept {
-    std::unique_lock lock(markingMutex);
-    lastGCMarkStats_.merge(stats);
-}

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -18,6 +18,7 @@
 #include "Utils.hpp"
 #include "GCState.hpp"
 #include "std_support/Memory.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin {
 
@@ -114,25 +115,22 @@ public:
     void StopFinalizerThreadIfRunning() noexcept;
     bool FinalizersThreadIsRunning() noexcept;
     void SetMarkingBehaviorForTests(MarkingBehavior markingBehavior) noexcept;
-    void SetMarkingRequested() noexcept;
+    void SetMarkingRequested(uint64_t epoch) noexcept;
     void WaitForThreadsReadyToMark() noexcept;
-    void CollectRootSetAndStartMarking() noexcept;
+    void CollectRootSetAndStartMarking(GCHandle gcHandle) noexcept;
 
 private:
     // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
     bool PerformFullGC(int64_t epoch) noexcept;
-    void MergeMarkStats(MarkStats stats) noexcept;
 
     mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
     GCScheduler& gcScheduler_;
 
-    uint64_t lastGCTimestampUs_ = 0;
     GCStateHolder state_;
     ScopedThread gcThread_;
     std_support::unique_ptr<FinalizerProcessor> finalizerProcessor_;
 
     MarkQueue markQueue_;
-    MarkStats lastGCMarkStats_;
     MarkingBehavior markingBehavior_;
 };
 

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -1172,7 +1172,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
 
     std_support::vector<std::future<void>> gcFutures(kDefaultThreadCount);
 
-    mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested();
+    mm::GlobalData::Instance().gc().impl().gc().SetMarkingRequested(0);
 
     for (int i = 0; i < kDefaultThreadCount; ++i) {
         gcFutures[i] = mutators[i]
@@ -1181,7 +1181,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MutatorsCanMarkOwnLocals) {
 
     if (GetParam() == gc::ConcurrentMarkAndSweep::kMarkOwnStack) {
         mm::GlobalData::Instance().gc().impl().gc().WaitForThreadsReadyToMark();
-        mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking();
+        mm::GlobalData::Instance().gc().impl().gc().CollectRootSetAndStartMarking(gc::GCHandle::createFakeForTests());
     }
 
     for (int i = 0; i < kDefaultThreadCount; ++i) {

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -6,6 +6,7 @@
 #include "GCImpl.hpp"
 
 #include "GC.hpp"
+#include "GCStatistics.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "ThreadSuspension.hpp"
 #include "std_support/Memory.hpp"
@@ -74,6 +75,20 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
     return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
 }
 
+
+size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
+    return impl_->objectFactory().GetObjectsCountUnsafe();
+}
+size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
+    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
+}
+size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
+}
+size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
+}
+
 gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
     return impl_->gcScheduler().config();
 }
@@ -81,6 +96,7 @@ gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
 void gc::GC::ClearForTests() noexcept {
     impl_->gc().StopFinalizerThreadIfRunning();
     impl_->objectFactory().ClearForTests();
+    GCHandle::ClearForTests();
 }
 
 void gc::GC::StartFinalizerThreadIfNeeded() noexcept {

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -58,6 +58,11 @@ public:
 
     static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
 
+    size_t GetHeapObjectsCountUnsafe() const noexcept;
+    size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
+    size_t GetExtraObjectsCountUnsafe() const noexcept;
+    size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
+
     gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
 
     void ClearForTests() noexcept;

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp

@@ -0,0 +1,314 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+#include "GCStatistics.hpp"
+#include "Mutex.hpp"
+#include "Porting.h"
+
+#include "Types.h"
+#include "Logging.hpp"
+#include "ThreadData.hpp"
+#include "std_support/Optional.hpp"
+#include <cinttypes>
+#include <mutex>
+
+
+namespace {
+
+struct MemoryUsageMap {
+    std::optional<kotlin::gc::MemoryUsage> heap;
+    std::optional<kotlin::gc::MemoryUsage> extra;
+};
+
+struct RootSetStatistics {
+    KLong threadLocalReferences;
+    KLong stackReferences;
+    KLong globalReferences;
+    KLong stableReferences;
+    KLong total() const { return threadLocalReferences + stableReferences + globalReferences + stableReferences; }
+};
+
+struct GCInfo {
+    std::optional<uint64_t> epoch;
+    std::optional<KLong> startTime; // time since process start
+    std::optional<KLong> endTime;
+    std::optional<KLong> pauseStartTime;
+    std::optional<KLong> pauseEndTime;
+    std::optional<KLong> finalizersDoneTime;
+    std::optional<RootSetStatistics> rootSet;
+    std::optional<kotlin::gc::MemoryUsage> markStats;
+    MemoryUsageMap memoryUsageBefore;
+    MemoryUsageMap memoryUsageAfter;
+};
+
+GCInfo last;
+GCInfo current;
+// This lock can be got by thread in runnable state making parallel mark, so
+kotlin::SpinLock<kotlin::MutexThreadStateHandling::kIgnore> lock;
+
+GCInfo* statByEpoch(uint64_t epoch) {
+    if (current.epoch == epoch) return &current;
+    if (last.epoch == epoch) return &last;
+    return nullptr;
+}
+
+} // namespace
+
+namespace kotlin::gc {
+GCHandle GCHandle::create(uint64_t epoch) {
+    std::lock_guard guard(lock);
+    RuntimeAssert(statByEpoch(epoch) == nullptr, "Starting epoch, which already existed");
+    if (current.epoch) {
+        last = current;
+        current = {};
+        RuntimeLogWarning({kTagGC}, "Starting new GC epoch, while previous is not finished\n");
+    }
+    current.epoch = static_cast<KLong>(epoch);
+    current.startTime = static_cast<KLong>(konan::getTimeNanos());
+    if (last.endTime) {
+        GCLogInfo(epoch, "Started. Time since last GC %" PRIu64 " microseconds.", *current.startTime - *last.endTime);
+    } else {
+        GCLogInfo(epoch, "Started.");
+    }
+    return getByEpoch(epoch);
+}
+GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits<uint64_t>::max()); }
+GCHandle GCHandle::getByEpoch(uint64_t epoch) {
+    return GCHandle{epoch};
+}
+void GCHandle::ClearForTests() {
+    std::lock_guard guard(lock);
+    current = {};
+    last = {};
+}
+void GCHandle::finished() {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->endTime = static_cast<KLong>(konan::getTimeNanos());
+        if (stat->rootSet) {
+            GCLogInfo(
+                    epoch_,
+                    "Root set: "
+                    "%" PRIu64 " thread local references, "
+                    "%" PRIu64 " stack references, "
+                    "%" PRIu64 " global references, "
+                    "%" PRIu64 " stable references. "
+                    "In total %" PRIu64 " roots.",
+                    stat->rootSet->threadLocalReferences, stat->rootSet->stackReferences, stat->rootSet->globalReferences,
+                    stat->rootSet->stableReferences, stat->rootSet->total());
+        }
+        if (stat->startTime) {
+            auto time = (*current.endTime - *current.startTime) / 1000;
+            GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time);
+        }
+
+        if (stat == &current) {
+            last = current;
+            current = {};
+        }
+    }
+}
+void GCHandle::suspensionRequested() {
+    std::lock_guard guard(lock);
+    GCLogDebug(epoch_, "Requested thread suspension by thread %d", konan::currentThreadId());
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->pauseStartTime = static_cast<KLong>(konan::getTimeNanos());
+    }
+}
+void GCHandle::threadsAreSuspended() {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        if (stat->pauseStartTime) {
+            auto time = (konan::getTimeNanos() - *stat->pauseStartTime) / 1000;
+            GCLogInfo(epoch_, "Suspended all threads in %" PRIu64 " microseconds", time);
+            return;
+        }
+    }
+}
+void GCHandle::threadsAreResumed() {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->pauseEndTime = static_cast<KLong>(konan::getTimeNanos());
+        if (stat->pauseStartTime) {
+            auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
+            GCLogInfo(epoch_, "Resume all threads. Total pause time is %" PRId64 " microseconds.", time);
+            return;
+        }
+    }
+}
+void GCHandle::finalizersDone() {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->finalizersDoneTime = static_cast<KLong>(konan::getTimeNanos());
+        if (stat->endTime) {
+            auto time = (*stat->finalizersDoneTime - *stat->endTime) / 1000;
+            GCLogInfo(epoch_, "Finalization is done in %" PRId64 " microseconds after epoch end.", time);
+            return;
+        }
+    }
+    GCLogInfo(epoch_, "Finalization is done.");
+}
+void GCHandle::finalizersScheduled(uint64_t finalizersCount) {
+    GCLogInfo(epoch_, "Finalization is scheduled for %" PRIu64 " objects.", finalizersCount);
+}
+void GCHandle::threadRootSetCollected(mm::ThreadData &threadData, uint64_t threadLocalReferences, uint64_t stackReferences) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        if (!stat->rootSet) {
+            stat->rootSet = RootSetStatistics{0, 0, 0, 0};
+        }
+        stat->rootSet->stackReferences += static_cast<KLong>(stackReferences);
+        stat->rootSet->threadLocalReferences += static_cast<KLong>(threadLocalReferences);
+    }
+}
+void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        if (!stat->rootSet) {
+            stat->rootSet = RootSetStatistics{0, 0, 0, 0};
+        }
+        stat->rootSet->globalReferences += static_cast<KLong>(globalReferences);
+        stat->rootSet->stableReferences += static_cast<KLong>(stableReferences);
+    }
+}
+
+
+void GCHandle::heapUsageBefore(MemoryUsage usage) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->memoryUsageBefore.heap = usage;
+    }
+}
+
+void GCHandle::marked(kotlin::gc::MemoryUsage usage) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        if (!stat->markStats) {
+            stat->markStats = MemoryUsage{0, 0};
+        }
+        stat->markStats->totalObjectsSize += usage.totalObjectsSize;
+        stat->markStats->objectsCount += usage.objectsCount;
+    }
+}
+
+MemoryUsage GCHandle::getMarked() {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        if (stat->markStats) {
+            return *stat->markStats;
+        }
+    }
+    return MemoryUsage{0, 0};
+}
+
+void GCHandle::heapUsageAfter(MemoryUsage usage) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->memoryUsageAfter.heap = usage;
+        if (stat->memoryUsageBefore.heap) {
+            GCLogInfo(
+                    epoch_,
+                    "Collected %" PRId64 " heap objects of total size %" PRId64 " bytes. "
+                    "%" PRId64 " heap objects of total size %" PRId64 " bytes are still alive.",
+                    stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount,
+                    stat->memoryUsageBefore.heap->totalObjectsSize - stat->memoryUsageAfter.heap->totalObjectsSize,
+                    stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->totalObjectsSize);
+        }
+        if (stat->markStats) {
+            RuntimeAssert(
+                    stat->markStats->objectsCount == usage.objectsCount,
+                    "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep",
+                    stat->markStats->objectsCount, usage.objectsCount);
+            RuntimeAssert(
+                    stat->markStats->totalObjectsSize == usage.totalObjectsSize,
+                    "Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep",
+                    stat->markStats->totalObjectsSize, usage.totalObjectsSize);
+        }
+    }
+}
+
+void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->memoryUsageBefore.extra = usage;
+    }
+}
+void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) {
+    std::lock_guard guard(lock);
+    if (auto* stat = statByEpoch(epoch_)) {
+        stat->memoryUsageAfter.extra = usage;
+        if (stat->memoryUsageBefore.extra) {
+            GCLogInfo(
+                    epoch_,
+                    "Collected %" PRId64 " extra objects of total size %" PRId64 ". "
+                    "%" PRId64 " extra objects of total size %" PRId64 " are still alive.",
+                    stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount,
+                    stat->memoryUsageBefore.extra->totalObjectsSize - stat->memoryUsageAfter.extra->totalObjectsSize,
+                    stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->totalObjectsSize);
+        }
+    }
+}
+
+MemoryUsage GCHandle::GCSweepScope::getUsage() {
+    return MemoryUsage{
+            mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(),
+            mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(),
+    };
+}
+
+GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) :
+    handle_(handle) {
+    handle_.heapUsageBefore(getUsage());
+}
+
+GCHandle::GCSweepScope::~GCSweepScope() {
+    GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime());
+    handle_.heapUsageAfter(getUsage());
+}
+
+MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() {
+    return MemoryUsage{
+            mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(),
+            mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(),
+    };
+}
+
+GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) :
+    handle_(handle) {
+    handle_.extraObjectsUsageBefore(getUsage());
+}
+
+GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() {
+    GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime());
+    handle_.extraObjectsUsageAfter(getUsage());
+}
+
+GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
+
+GCHandle::GCGlobalRootSetScope::~GCGlobalRootSetScope(){
+    handle_.globalRootSetCollected(globalRoots_, stableRoots_);
+    GCLogDebug(handle_.getEpoch(), "Collected global root set global=%" PRIu64 " stableRef=%" PRIu64 " in %" PRIu64" microseconds.",
+               globalRoots_, stableRoots_, getStageTime());
+}
+
+GCHandle::GCThreadRootSetScope::GCThreadRootSetScope(kotlin::gc::GCHandle& handle, mm::ThreadData& threadData) :
+    handle_(handle), threadData_(threadData) {}
+
+GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){
+    handle_.threadRootSetCollected(threadData_, threadLocalRoots_, stackRoots_);
+    GCLogDebug(handle_.getEpoch(), "Collected root set for thread #%d: stack=%" PRIu64 " tls=%" PRIu64 " in %" PRIu64" microseconds.",
+               threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime());
+}
+
+GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){}
+
+GCHandle::GCMarkScope::~GCMarkScope() {
+    handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes});
+    GCLogDebug(handle_.getEpoch(),
+               "Marked %" PRIu64 " objects in %" PRIu64 " microseconds in thread %d",
+               objectsCount, getStageTime(), konan::currentThreadId());
+}
+
+} // namespace kotlin::gc

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <pthread.h>
+#include "Common.h"
+#include "ThreadData.hpp"
+
+#define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
+#define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
+
+namespace kotlin::gc {
+
+class GCHandle;
+
+struct MemoryUsage {
+    uint64_t objectsCount;
+    uint64_t totalObjectsSize;
+};
+
+class GCHandle {
+    class GCStageScopeUsTimer {
+    protected:
+        uint64_t startTime_ = konan::getTimeMicros();
+        uint64_t getStageTime() const { return (konan::getTimeMicros() - startTime_); }
+    };
+
+    class GCSweepScope : GCStageScopeUsTimer, Pinned {
+        GCHandle& handle_;
+        MemoryUsage getUsage();
+
+    public:
+        explicit GCSweepScope(GCHandle& handle);
+        ~GCSweepScope();
+    };
+
+    class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned {
+        GCHandle& handle_;
+        MemoryUsage getUsage();
+
+    public:
+        explicit GCSweepExtraObjectsScope(GCHandle& handle);
+        ~GCSweepExtraObjectsScope();
+    };
+
+    class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned {
+        GCHandle& handle_;
+        uint64_t globalRoots_ = 0;
+        uint64_t stableRoots_ = 0;
+
+    public:
+        explicit GCGlobalRootSetScope(GCHandle& handle);
+        ~GCGlobalRootSetScope();
+        void addGlobalRoot() { globalRoots_++; }
+        void addStableRoot() { stableRoots_++; }
+    };
+
+    class GCThreadRootSetScope : GCStageScopeUsTimer, Pinned {
+        GCHandle& handle_;
+        mm::ThreadData& threadData_;
+        uint64_t stackRoots_ = 0;
+        uint64_t threadLocalRoots_ = 0;
+
+    public:
+        explicit GCThreadRootSetScope(GCHandle& handle, mm::ThreadData& threadData);
+        ~GCThreadRootSetScope();
+        void addStackRoot() { stackRoots_++; }
+        void addThreadLocalRoot() { threadLocalRoots_++; }
+    };
+
+    class GCMarkScope : GCStageScopeUsTimer, Pinned {
+        GCHandle& handle_;
+        uint64_t objectsCount = 0;
+        uint64_t totalObjectSizeBytes = 0;
+
+    public:
+        explicit GCMarkScope(GCHandle& handle);
+        ~GCMarkScope();
+        void addObject(uint64_t objectSize) {
+            totalObjectSizeBytes += objectSize;
+            objectsCount++;
+        }
+    };
+
+    uint64_t epoch_;
+    explicit GCHandle(uint64_t epoch) : epoch_(epoch) {}
+
+    void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences);
+    void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences);
+    void heapUsageBefore(MemoryUsage usage);
+    void heapUsageAfter(MemoryUsage usage);
+    void extraObjectsUsageBefore(MemoryUsage usage);
+    void extraObjectsUsageAfter(MemoryUsage usage);
+    void marked(MemoryUsage usage);
+
+public:
+    static GCHandle create(uint64_t epoch);
+    static GCHandle createFakeForTests();
+    static GCHandle getByEpoch(uint64_t epoch);
+    static void ClearForTests();
+
+    uint64_t getEpoch() { return epoch_; }
+    void finished();
+    void finalizersDone();
+    void finalizersScheduled(uint64_t finalizersCount);
+    void suspensionRequested();
+    void threadsAreSuspended();
+    void threadsAreResumed();
+    GCSweepScope sweep() { return GCSweepScope(*this); }
+    GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); }
+    GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); }
+    GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); }
+    GCMarkScope mark() { return GCMarkScope(*this); }
+
+    MemoryUsage getMarked();
+};
+}

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -9,6 +9,7 @@
 #include "ExtraObjectData.hpp"
 #include "FinalizerHooks.hpp"
 #include "GlobalData.hpp"
+#include "GCStatistics.hpp"
 #include "Logging.hpp"
 #include "Memory.h"
 #include "ObjectOps.hpp"
@@ -88,18 +89,15 @@ struct MarkStats {
 };
 
 template <typename Traits>
-MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
+void Mark(GCHandle handle, typename Traits::MarkQueue& markQueue) noexcept {
-    MarkStats stats;
+    auto markHandle = handle.mark();
-    stats.rootSetSize = markQueue.size();
-    auto timeStart = konan::getTimeMicros();
     while (!Traits::isEmpty(markQueue)) {
         ObjHeader* top = Traits::dequeue(markQueue);
 
         RuntimeAssert(!isNullOrMarker(top), "Got invalid reference %p in mark queue", top);
         RuntimeAssert(top->heap(), "Got non-heap reference %p in mark queue, permanent=%d stack=%d", top, top->permanent(), top->local());
 
-        stats.aliveHeapSet++;
+        markHandle.addObject(mm::GetAllocatedHeapSize(top));
-        stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
 
         Traits::processInMark(markQueue, top);
 
@@ -112,14 +110,12 @@ MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
             }
         }
     }
-    auto timeEnd = konan::getTimeMicros();
-    RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds in thread %d", stats.aliveHeapSet, timeEnd - timeStart, konan::currentThreadId());
-    return stats;
 }
 
 template <typename Traits>
-void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noexcept {
+void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory& objectFactory) noexcept {
     objectFactory.ProcessDeletions();
+    auto sweepHandle = handle.sweepExtraObjects();
     auto iter = objectFactory.LockForIter();
     for (auto it = iter.begin(); it != iter.end();) {
         auto &extraObject = *it;
@@ -140,8 +136,9 @@ void SweepExtraObjects(typename Traits::ExtraObjectsFactory& objectFactory) noex
 }
 
 template <typename Traits>
-typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
+typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory::Iterable& objectFactoryIter) noexcept {
     typename Traits::ObjectFactory::FinalizerQueue finalizerQueue;
+    auto sweepHandle = handle.sweep();
 
     for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
         if (Traits::TryResetMark(*it)) {
@@ -160,64 +157,60 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFact
 }
 
 template <typename Traits>
-typename Traits::ObjectFactory::FinalizerQueue Sweep(typename Traits::ObjectFactory& objectFactory) noexcept {
+typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename Traits::ObjectFactory& objectFactory) noexcept {
     auto iter = objectFactory.LockForIter();
-    return Sweep<Traits>(iter);
+    return Sweep<Traits>(handle, iter);
 }
 
 template <typename Traits>
-void collectRootSetForThread(typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
+void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
+    auto handle = gcHandle.collectThreadRoots(thread);
     thread.gc().OnStoppedForGC();
-    size_t stack = 0;
-    size_t tls = 0;
     // TODO: Remove useless mm::ThreadRootSet abstraction.
     for (auto value : mm::ThreadRootSet(thread)) {
         if (internal::collectRoot<Traits>(markQueue, value.object)) {
             switch (value.source) {
                 case mm::ThreadRootSet::Source::kStack:
-                    ++stack;
+                    handle.addStackRoot();
                     break;
                 case mm::ThreadRootSet::Source::kTLS:
-                    ++tls;
+                    handle.addThreadLocalRoot();
                     break;
             }
         }
     }
-    RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls);
 }
 
 template <typename Traits>
-void collectRootSetGlobals(typename Traits::MarkQueue& markQueue) noexcept {
+void collectRootSetGlobals(GCHandle gcHandle, typename Traits::MarkQueue& markQueue) noexcept {
+    auto handle = gcHandle.collectGlobalRoots();
     mm::StableRefRegistry::Instance().ProcessDeletions();
-    size_t global = 0;
-    size_t stableRef = 0;
     // TODO: Remove useless mm::GlobalRootSet abstraction.
     for (auto value : mm::GlobalRootSet()) {
         if (internal::collectRoot<Traits>(markQueue, value.object)) {
             switch (value.source) {
                 case mm::GlobalRootSet::Source::kGlobal:
-                    ++global;
+                    handle.addGlobalRoot();
                     break;
                 case mm::GlobalRootSet::Source::kStableRef:
-                    ++stableRef;
+                    handle.addStableRoot();
                     break;
             }
         }
     }
-    RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef);
 }
 
 // TODO: This needs some tests now.
 template <typename Traits, typename F>
-void collectRootSet(typename Traits::MarkQueue& markQueue, F&& filter) noexcept {
+void collectRootSet(GCHandle handle, typename Traits::MarkQueue& markQueue, F&& filter) noexcept {
     Traits::clear(markQueue);
     for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) {
         if (!filter(thread))
             continue;
         thread.Publish();
-        collectRootSetForThread<Traits>(markQueue, thread);
+        collectRootSetForThread<Traits>(handle, markQueue, thread);
     }
-    collectRootSetGlobals<Traits>(markQueue);
+    collectRootSetGlobals<Traits>(handle, markQueue);
 }
 
 } // namespace gc

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -174,13 +174,19 @@ public:
         return testing::UnorderedElementsAreArray(objects);
     }
 
-    gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
+    gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
         std_support::vector<ObjHeader*> objects;
         for (auto& object : graySet) ScopedMarkTraits::enqueue(objects, object.get().GetObjHeader());
-        return gc::Mark<ScopedMarkTraits>(objects);
+        auto handle = gc::GCHandle::create(epoch_++);
+        gc::Mark<ScopedMarkTraits>(handle, objects);
+        handle.finished();
+        return handle.getMarked();
     }
 
+    ~MarkAndSweepUtilsMarkTest() { mm::GlobalData::Instance().gc().ClearForTests(); }
+
 private:
+    uint64_t epoch_ = 0;
     kotlin::ScopedMemoryInit memoryInit;
     ScopedMarkTraits markTraits_;
 };
@@ -196,8 +202,8 @@ size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<BaseObject>>
 #define EXPECT_MARKED(stats, ...) \
     do { \
         std::initializer_list<std::reference_wrapper<BaseObject>> objects = {__VA_ARGS__}; \
-        EXPECT_THAT(stats.aliveHeapSet, objects.size()); \
+        EXPECT_THAT(stats.objectsCount, objects.size()); \
-        EXPECT_THAT(stats.aliveHeapSetBytes, GetObjectsSize(objects)); \
+        EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \
         EXPECT_THAT(marked(), MarkedMatcher(objects)); \
     } while (false)
 

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp

@@ -199,8 +199,8 @@ public:
     }
 
     std_support::vector<ObjHeader*> Sweep() {
-        gc::SweepExtraObjects<SweepTraits>(extraObjectFactory_);
+        gc::SweepExtraObjects<SweepTraits>(gc::GCHandle::getByEpoch(0), extraObjectFactory_);
-        auto finalizers = gc::Sweep<SweepTraits>(objectFactory_);
+        auto finalizers = gc::Sweep<SweepTraits>(gc::GCHandle::getByEpoch(0), objectFactory_);
         std_support::vector<ObjHeader*> objects;
         for (auto node : finalizers.IterForTests()) {
             objects.push_back(node.GetObjHeader());

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -7,6 +7,8 @@
 
 #include "GC.hpp"
 #include "std_support/Memory.hpp"
+#include "GlobalData.hpp"
+#include "GCStatistics.hpp"
 
 using namespace kotlin;
 
@@ -61,12 +63,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
     return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
 }
 
+size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
+    return impl_->objectFactory().GetObjectsCountUnsafe();
+}
+size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
+    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
+}
+size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
+}
+size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
+}
+
 gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
     return impl_->gcScheduler().config();
 }
 
 void gc::GC::ClearForTests() noexcept {
     impl_->objectFactory().ClearForTests();
+    GCHandle::ClearForTests();
 }
 
 void gc::GC::StartFinalizerThreadIfNeeded() noexcept {}

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -8,6 +8,8 @@
 #include "GC.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "std_support/Memory.hpp"
+#include "GlobalData.hpp"
+#include "GCStatistics.hpp"
 
 using namespace kotlin;
 
@@ -74,12 +76,26 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
     return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
 }
 
+size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
+    return impl_->objectFactory().GetObjectsCountUnsafe();
+}
+size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
+    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
+}
+size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
+}
+size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
+    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
+}
+
 gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
     return impl_->gcScheduler().config();
 }
 
 void gc::GC::ClearForTests() noexcept {
     impl_->objectFactory().ClearForTests();
+    GCHandle::ClearForTests();
 }
 
 void gc::GC::StartFinalizerThreadIfNeeded() noexcept {}

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -9,6 +9,7 @@
 
 #include "CompilerConstants.hpp"
 #include "GlobalData.hpp"
+#include "GCStatistics.hpp"
 #include "Logging.hpp"
 #include "MarkAndSweepUtils.hpp"
 #include "Memory.h"
@@ -103,7 +104,6 @@ gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
 
 bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
     RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId());
-    auto timeStartUs = konan::getTimeMicros();
     bool didSuspend = mm::RequestThreadsSuspension();
     if (!didSuspend) {
         RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId());
@@ -111,8 +111,9 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         // TODO: This breaks if suspension is used by something apart from GC.
         return false;
     }
-    RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId());
     gSafepointFlag = SafepointFlag::kNeedsSuspend;
+    auto gcHandle = GCHandle::create(epoch_++);
+    gcHandle.suspensionRequested();
 
     mm::ObjectFactory<gc::SameThreadMarkAndSweep>::FinalizerQueue finalizerQueue;
     {
@@ -120,55 +121,27 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         ThreadStateGuard guard(ThreadState::kNative);
 
         mm::WaitForThreadsSuspension();
-        auto timeSuspendUs = konan::getTimeMicros();
+        gcHandle.threadsAreSuspended();
-        RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs);
 
         auto& scheduler = gcScheduler_;
         scheduler.gcData().OnPerformFullGC();
 
-        RuntimeLogInfo(
+        gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [] (mm::ThreadData&) { return true; });
-                {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
+        auto& extraObjectsDataFactory = mm::GlobalData::Instance().extraObjectDataFactory();
-        gc::collectRootSet<internal::MarkTraits>(markQueue_, [](mm::ThreadData&) { return true; });
-        auto timeRootSetUs = konan::getTimeMicros();
-        // Can be unsafe, because we've stopped the world.
-        auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
 
-        RuntimeLogInfo(
+        gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
-                {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(),
+        auto markStats = gcHandle.getMarked();
-                timeRootSetUs - timeSuspendUs);
+        scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
-        auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
+        gc::SweepExtraObjects<SweepTraits>(gcHandle, extraObjectsDataFactory);
-        auto timeMarkUs = konan::getTimeMicros();
+        finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, objectFactory_);
-        RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs);
-        scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);
-        gc::SweepExtraObjects<SweepTraits>(mm::GlobalData::Instance().extraObjectDataFactory());
-        auto timeSweepExtraObjectsUs = konan::getTimeMicros();
-        RuntimeLogDebug({kTagGC}, "Sweeped extra objects in %" PRIu64 " microseconds", timeSweepExtraObjectsUs - timeMarkUs);
-        finalizerQueue = gc::Sweep<SweepTraits>(objectFactory_);
-        auto timeSweepUs = konan::getTimeMicros();
-        RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeSweepExtraObjectsUs);
 
         kotlin::compactObjectPoolInMainThread();
 
-        // Can be unsafe, because we've stopped the world.
-        auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
-        auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
-
         gSafepointFlag = SafepointFlag::kNone;
         mm::ResumeThreads();
-        auto timeResumeUs = konan::getTimeMicros();
+        gcHandle.threadsAreResumed();
-
+        gcHandle.finalizersScheduled(finalizerQueue.size());
-        RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs);
+        gcHandle.finished();
-
-        auto finalizersCount = finalizerQueue.size();
-        auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount;
-
-        RuntimeLogInfo(
-                {kTagGC},
-                "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects and %zd extra data objects remain. Total pause time %" PRIu64
-                " microseconds",
-                epoch_, collectedCount, finalizersCount, objectsCountAfter, extraObjectsCountAfter, timeResumeUs - timeStartUs);
-        ++epoch_;
-        lastGCTimestampUs_ = timeResumeUs;
     }
 
     // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would
@@ -177,11 +150,8 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
     // TODO: These will actually need to be run on a separate thread.
     AssertThreadState(ThreadState::kRunnable);
-    RuntimeLogDebug({kTagGC}, "Starting to run finalizers");
-    auto timeBeforeUs = konan::getTimeMicros();
     finalizerQueue.Finalize();
-    auto timeAfterUs = konan::getTimeMicros();
+    gcHandle.finalizersDone();
-    RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs);
 
     return true;
 }

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -99,8 +99,7 @@ private:
     // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads).
     bool PerformFullGC() noexcept;
 
-    size_t epoch_ = 0;
+    uint64_t epoch_ = 0;
-    uint64_t lastGCTimestampUs_ = 0;
 
     mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
     GCScheduler& gcScheduler_;

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -3516,6 +3516,9 @@ void Kotlin_native_internal_GC_collect(KRef) {
 #endif
 }
 
+extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) {
+}
+
 void Kotlin_native_internal_GC_collectCyclic(KRef) {
 #if USE_CYCLIC_GC
   if (g_hasCyclicCollector)
@@ -3862,3 +3865,5 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {}
 bool kotlin::FinalizersThreadIsRunning() noexcept {
     return false;
 }
+
+

kotlin-native/runtime/src/main/cpp/Logging.cpp

@@ -6,25 +6,15 @@
 #include "Logging.hpp"
 
 #include <array>
-#if __has_include(<optional>)
+
-#include <optional>
+
-#elif __has_include(<experimental/optional>)
-// TODO: Remove when wasm32 is gone.
-#include <experimental/optional>
-namespace std {
-template <typename T>
-using optional = std::experimental::optional<T>;
-inline constexpr auto nullopt = std::experimental::nullopt;
-} // namespace std
-#else
-#error "No <optional>"
-#endif
 
 #include "Format.h"
 #include "KAssert.h"
 #include "Porting.h"
 #include "std_support/Map.hpp"
 #include "std_support/String.hpp"
+#include "std_support/Optional.hpp"
 
 using namespace kotlin;
 
@@ -115,7 +105,7 @@ private:
 
 class StderrLogger : public logging::internal::Logger {
 public:
-    void Log(logging::Level level, std_support::span<const char* const> tags, std::string_view message) const noexcept override {
+    NO_EXTERNAL_CALLS_CHECK void Log(logging::Level level, std_support::span<const char* const> tags, std::string_view message) const noexcept override {
         konan::consoleErrorUtf8(message.data(), message.size());
     }
 };

kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+#if __has_include(<optional>)
+#include <optional>
+#elif __has_include(<experimental/optional>)
+// TODO: Remove when wasm32 is gone.
+#include <experimental/optional>
+namespace std {
+template <typename T>
+using optional = std::experimental::optional<T>;
+inline constexpr auto nullopt = std::experimental::nullopt;
+} // namespace std
+#else
+#error "No <optional>"
+#endif

kotlin-native/runtime/src/main/cpp/std_support/README.md

@@ -25,3 +25,4 @@ Adjustments:
   `std_support::kdelete` as a replacement for operator `delete` for objects created with custom `new`.
 * `Deque.hpp`, `ForwardList.hpp`, `List.hpp`, `Map.hpp`, `Set.hpp`, `String.hpp`, `UnorderedMap.hpp`, `UnorderedSet.hpp`, `Vector.hpp` -
   standard containers and `std_support::string` that default to using `std_support::allocator`.
+* `Optional.hpp` - wrapper choosing correct way to include on different platforms. 

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp

@@ -53,6 +53,7 @@ public:
     void ClearForTests() noexcept { extraObjects_.ClearForTests(); }
 
     size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); }
+    size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); }
 
     // requires LockForIter
     void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); }

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -145,6 +145,7 @@ public:
 
             last_ = nodePtr;
             ++size_;
+            totalObjectsSizeBytes_ += Node::GetSizeForDataSize(dataSize);
             RuntimeAssert(root_ != nullptr, "Must not be empty");
             AssertCorrect();
             return *nodePtr;
@@ -181,7 +182,9 @@ public:
             owner_.last_ = last_;
             last_ = nullptr;
             owner_.size_ += size_;
+            owner_.totalObjectsSizeBytes_ += totalObjectsSizeBytes_;
             size_ = 0;
+            totalObjectsSizeBytes_ = 0;
 
             RuntimeAssert(root_ == nullptr, "Must be empty");
             AssertCorrect();
@@ -216,6 +219,7 @@ public:
         unique_ptr<Node> root_;
         Node* last_ = nullptr;
         size_t size_ = 0;
+        size_t totalObjectsSizeBytes_ = 0;
     };
 
     class Iterator {
@@ -348,13 +352,13 @@ public:
         Iterator begin() noexcept { return Iterator(nullptr, owner_.root_.get()); }
         Iterator end() noexcept { return Iterator(owner_.last_, nullptr); }
 
-        void EraseAndAdvance(Iterator& iterator) noexcept {
+        void EraseAndAdvance(Iterator& iterator, size_t objectedSize) noexcept {
-            auto result = owner_.ExtractUnsafe(iterator.previousNode_);
+            auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectedSize);
             iterator.node_ = result.second;
         }
 
-        void MoveAndAdvance(Consumer& consumer, Iterator& iterator) noexcept {
+        void MoveAndAdvance(Consumer& consumer, Iterator& iterator, size_t objectSize) noexcept {
-            auto result = owner_.ExtractUnsafe(iterator.previousNode_);
+            auto result = owner_.ExtractUnsafe(iterator.previousNode_, objectSize);
             iterator.node_ = result.second;
             consumer.Insert(std::move(result.first));
         }
@@ -373,40 +377,32 @@ public:
     Iterable LockForIter() noexcept { return Iterable(*this); }
 
     size_t GetSizeUnsafe() const noexcept { return size_; }
+    size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; }
 
     void ClearForTests() {
         root_.reset();
         last_ = nullptr;
         size_ = 0;
+        totalObjectsSizeBytes_ = 0;
     }
 
 private:
     // Expects `mutex_` to be held by the current thread.
-    std::pair<unique_ptr<Node>, Node*> ExtractUnsafe(Node* previousNode) noexcept {
+    std::pair<unique_ptr<Node>, Node*> ExtractUnsafe(Node* previousNode, size_t objectSize) noexcept {
         RuntimeAssert(root_ != nullptr, "Must not be empty");
         AssertCorrectUnsafe();
 
-        if (previousNode == nullptr) {
+        unique_ptr<Node> &pointerToNext = (previousNode == nullptr) ? root_ : previousNode->next_;
-            // Extracting the root.
+        auto node = std::move(pointerToNext);
-            auto node = std::move(root_);
+        pointerToNext = std::move(node->next_);
-            root_ = std::move(node->next_);
+        if (!pointerToNext) {
-            if (!root_) {
-                last_ = nullptr;
-            }
-            --size_;
-            AssertCorrectUnsafe();
-            return {std::move(node), root_.get()};
-        }
-
-        auto node = std::move(previousNode->next_);
-        previousNode->next_ = std::move(node->next_);
-        if (!previousNode->next_) {
             last_ = previousNode;
         }
         --size_;
+        totalObjectsSizeBytes_ -= objectSize;
 
         AssertCorrectUnsafe();
-        return {std::move(node), previousNode->next_.get()};
+        return {std::move(node), pointerToNext.get()};
     }
 
     // Expects `mutex_` to be held by the current thread.
@@ -422,6 +418,7 @@ private:
     unique_ptr<Node> root_;
     Node* last_ = nullptr;
     size_t size_ = 0;
+    size_t totalObjectsSizeBytes_ = 0;
     SpinLock<MutexThreadStateHandling::kIgnore> mutex_;
 };
 
@@ -656,10 +653,12 @@ public:
         Iterator begin() noexcept { return Iterator(iter_.begin()); }
         Iterator end() noexcept { return Iterator(iter_.end()); }
 
-        void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); }
+        void EraseAndAdvance(Iterator& iterator) noexcept {
+            iter_.EraseAndAdvance(iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader()));
+        }
 
         void MoveAndAdvance(FinalizerQueue& queue, Iterator& iterator) noexcept {
-            iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_);
+            iter_.MoveAndAdvance(queue.consumer_, iterator.iterator_, GetAllocatedHeapSize(iterator->GetObjHeader()));
         }
 
     private:
@@ -672,7 +671,8 @@ public:
     // Lock ObjectFactory for safe iteration.
     Iterable LockForIter() noexcept { return Iterable(*this); }
 
-    size_t GetSizeUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
+    size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
+    size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); }
 
     void ClearForTests() { storage_.ClearForTests(); }
 

kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp

@@ -240,7 +240,7 @@ TEST(ObjectFactoryStorageTest, EraseFirst) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 1) {
-                iter.EraseAndAdvance(it);
+                iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -268,7 +268,7 @@ TEST(ObjectFactoryStorageTest, EraseMiddle) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 2) {
-                iter.EraseAndAdvance(it);
+                iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -296,7 +296,7 @@ TEST(ObjectFactoryStorageTest, EraseLast) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 3) {
-                iter.EraseAndAdvance(it);
+                iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -323,7 +323,7 @@ TEST(ObjectFactoryStorageTest, EraseAll) {
     {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
-            iter.EraseAndAdvance(it);
+            iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         }
     }
 
@@ -345,7 +345,7 @@ TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) {
     {
         auto iter = storage.LockForIter();
         auto it = iter.begin();
-        iter.EraseAndAdvance(it);
+        iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         EXPECT_THAT(it, iter.end());
     }
 
@@ -371,7 +371,7 @@ TEST(ObjectFactoryStorageTest, MoveFirst) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 1) {
-                iter.MoveAndAdvance(consumer, it);
+                iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -403,7 +403,7 @@ TEST(ObjectFactoryStorageTest, MoveMiddle) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 2) {
-                iter.MoveAndAdvance(consumer, it);
+                iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -435,7 +435,7 @@ TEST(ObjectFactoryStorageTest, MoveLast) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() == 3) {
-                iter.MoveAndAdvance(consumer, it);
+                iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -466,7 +466,7 @@ TEST(ObjectFactoryStorageTest, MoveAll) {
     {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
-            iter.MoveAndAdvance(consumer, it);
+            iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         }
     }
 
@@ -499,7 +499,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() % 2 == 0) {
-                iter.MoveAndAdvance(consumer1, it);
+                iter.MoveAndAdvance(consumer1, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }
@@ -508,7 +508,7 @@ TEST(ObjectFactoryStorageTest, MergeWith) {
     {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
-            iter.MoveAndAdvance(consumer2, it);
+            iter.MoveAndAdvance(consumer2, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         }
     }
 
@@ -577,7 +577,7 @@ TEST(ObjectFactoryStorageTest, MoveTheOnlyElement) {
     {
         auto iter = storage.LockForIter();
         auto it = iter.begin();
-        iter.MoveAndAdvance(consumer, it);
+        iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         EXPECT_THAT(it, iter.end());
     }
 
@@ -612,8 +612,8 @@ TEST(ObjectFactoryStorageTest, MoveAndErase) {
         auto iter = storage.LockForIter();
         for (auto it = iter.begin(); it != iter.end();) {
             ++it;
-            iter.EraseAndAdvance(it);
+            iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
-            iter.MoveAndAdvance(consumer, it);
+            iter.MoveAndAdvance(consumer, it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
         }
     }
 
@@ -759,7 +759,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) {
 
         for (auto it = iter.begin(); it != iter.end();) {
             if (it->Data<int>() % 2 != 0) {
-                iter.EraseAndAdvance(it);
+                iter.EraseAndAdvance(it, ObjectFactoryStorageRegular::Node::GetSizeForDataSize(sizeof(int)));
             } else {
                 ++it;
             }

kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp

@@ -81,7 +81,7 @@ NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::RequestThreadsSuspension() noexcept {
 }
 
 void kotlin::mm::WaitForThreadsSuspension() noexcept {
-    // Spin wating for threads to suspend. Ignore Native threads.
+    // Spin waiting for threads to suspend. Ignore Native threads.
     while(!allThreads(isSuspendedOrNative)) {
         yield();
     }