[K/N] Track object counts during sweep ^KT-55364

Previously object count was tracked via allocator. It adds additional
burden on every allocation. Tracking via sweeper is better because
it mostly happens on the GC thread during a concurrent sweep.

kotlin-native/backend.native/tests/runtime/memory/gcStats.kt

@@ -12,13 +12,11 @@ import kotlin.test.*
 @Test
 fun `nothing new collected`() {
     GC.collect()
-    GC.collect();
+    GC.collect()
     val stat = GC.lastGCInfo
     assertNotNull(stat)
-    assertEquals(stat.memoryUsageBefore.keys, stat.memoryUsageAfter.keys)
+    for (key in stat.sweepStatistics.keys) {
-    for (key in stat.memoryUsageBefore.keys) {
+        assertEquals(stat.sweepStatistics[key]!!.sweptCount, 0L)
-        assertEquals(stat.memoryUsageBefore[key]!!.objectsCount, stat.memoryUsageAfter[key]!!.objectsCount)
-        assertEquals(stat.memoryUsageBefore[key]!!.totalObjectsSizeBytes, stat.memoryUsageAfter[key]!!.totalObjectsSizeBytes)
     }
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp

@@ -45,7 +45,7 @@ mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept {
     return freeBlock->Data();
 }
 
-bool ExtraObjectPage::Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
+bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
     CustomAllocInfo("ExtraObjectPage(%p)::Sweep()", this);
     // `end` is after the last legal allocation of a block, but does not
     // necessarily match an actual block starting point.
@@ -58,15 +58,24 @@ bool ExtraObjectPage::Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexce
             nextFree = &cell->next_;
             continue;
         }
-        // If the current cell was marked, it's alive, and the whole page is alive.
+        auto status = SweepExtraObject(cell, finalizerQueue);
-        if (!SweepExtraObject(cell, finalizerQueue)) {
+        switch (status) {
-            alive = true;
+            case ExtraObjectStatus::TO_BE_FINALIZED:
-            continue;
+                sweepHandle.addMarkedObject();
+                // fallthrough
+            case ExtraObjectStatus::KEPT:
+                // If the current cell was marked, it's alive, and the whole page is alive.
+                alive = true;
+                sweepHandle.addKeptObject();
+                break;
+            case ExtraObjectStatus::SWEPT:
+                // Free the current block and insert it into the free list.
+                cell->next_ = *nextFree;
+                *nextFree = cell;
+                nextFree = &cell->next_;
+                sweepHandle.addSweptObject();
+                break;
         }
-        // Free the current block and insert it into the free list.
-        cell->next_ = *nextFree;
-        *nextFree = cell;
-        nextFree = &cell->next_;
     }
     return alive;
 }

kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp

@@ -11,6 +11,7 @@
 
 #include "AtomicStack.hpp"
 #include "ExtraObjectData.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::alloc {
 
@@ -34,7 +35,7 @@ public:
     // Tries to allocate in current page, returns null if no free block in page
     mm::ExtraObjectData* TryAllocate() noexcept;
 
-    bool Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
+    bool Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
 
 private:
     friend class AtomicStack<ExtraObjectPage>;

kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp

@@ -41,7 +41,9 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
 TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
     Page* page = Page::Create();
     Queue finalizerQueue;
-    EXPECT_FALSE(page->Sweep(finalizerQueue));
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweepExtraObjects();
+    EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
     EXPECT_EQ(finalizerQueue.size(), size_t(0));
     page->Destroy();
 }
@@ -56,7 +58,9 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
     }
     EXPECT_EQ(count, EXTRA_OBJECT_COUNT);
     Queue finalizerQueue;
-    EXPECT_FALSE(page->Sweep(finalizerQueue));
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweepExtraObjects();
+    EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
     EXPECT_EQ(finalizerQueue.size(), size_t(0));
     page->Destroy();
 }

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp

@@ -46,7 +46,7 @@ uint8_t* FixedBlockPage::TryAllocate() noexcept {
     return freeBlock->data;
 }
 
-bool FixedBlockPage::Sweep() noexcept {
+bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
     CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
     // `end` is after the last legal allocation of a block, but does not
     // necessarily match an actual block starting point.
@@ -62,6 +62,7 @@ bool FixedBlockPage::Sweep() noexcept {
         // If the current cell was marked, it's alive, and the whole page is alive.
         if (TryResetMark(cell)) {
             alive = true;
+            sweepHandle.addKeptObject();
             continue;
         }
         CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell);
@@ -69,6 +70,7 @@ bool FixedBlockPage::Sweep() noexcept {
         cell->nextFree = *nextFree;
         *nextFree = cell;
         nextFree = &cell->nextFree;
+        sweepHandle.addSweptObject();
     }
     return alive;
 }

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp

@@ -10,6 +10,7 @@
 #include <cstdint>
 
 #include "AtomicStack.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::alloc {
 
@@ -30,7 +31,7 @@ public:
     // Tries to allocate in current page, returns null if no free block in page
     uint8_t* TryAllocate() noexcept;
 
-    bool Sweep() noexcept;
+    bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
 
 private:
     friend class AtomicStack<FixedBlockPage>;

kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp

@@ -44,52 +44,62 @@ TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) {
 }
 
 TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
         FixedBlockPage* page = FixedBlockPage::Create(size);
-        EXPECT_FALSE(page->Sweep());
+        EXPECT_FALSE(page->Sweep(gcScope));
         page->Destroy();
     }
 }
 
 TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
         FixedBlockPage* page = FixedBlockPage::Create(size);
         uint32_t count = 0;
         while (alloc(page, size)) ++count;
         EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size);
-        EXPECT_FALSE(page->Sweep());
+        EXPECT_FALSE(page->Sweep(gcScope));
         page->Destroy();
     }
 }
 
 TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
         FixedBlockPage* page = FixedBlockPage::Create(size);
         uint8_t* ptr = alloc(page, size);
         mark(ptr);
-        EXPECT_TRUE(page->Sweep());
+        EXPECT_TRUE(page->Sweep(gcScope));
         page->Destroy();
     }
 }
 
 TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
         FixedBlockPage* page = FixedBlockPage::Create(size);
         uint8_t* ptr = alloc(page, size);
-        EXPECT_FALSE(page->Sweep());
+        EXPECT_FALSE(page->Sweep(gcScope));
         EXPECT_EQ(alloc(page, size), ptr);
         page->Destroy();
     }
 }
 
 TEST(CustomAllocTest, FixedBlockPageSweepReuse) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
         FixedBlockPage* page = FixedBlockPage::Create(size);
         uint8_t* ptr;
         for (int count = 0; (ptr = alloc(page, size)); ++count) {
             if (count % 2 == 0) mark(ptr);
         }
-        EXPECT_TRUE(page->Sweep());
+        EXPECT_TRUE(page->Sweep(gcScope));
         uint32_t count = 0;
         for (; (ptr = alloc(page, size)); ++count) {
             if (count % 2 == 0) mark(ptr);

kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp

@@ -46,22 +46,22 @@ static bool IsAlive(ObjHeader* baseObject) noexcept {
     return objectData.marked();
 }
 
-bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
+ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
     auto* extraObject = extraObjectCell->Data();
     if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
         CustomAllocDebug("SweepIsCollectable(%p): already finalized", extraObject);
-        return true;
+        return ExtraObjectStatus::SWEPT;
     }
     auto* baseObject = extraObject->GetBaseObject();
     RuntimeAssert(baseObject->heap(), "SweepIsCollectable on a non-heap object");
     if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
         CustomAllocDebug("SweepIsCollectable(%p): already in finalizer queue, keep base object (%p) alive", extraObject, baseObject);
         KeepAlive(baseObject);
-        return false;
+        return ExtraObjectStatus::TO_BE_FINALIZED;
     }
     if (IsAlive(baseObject)) {
         CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject);
-        return false;
+        return ExtraObjectStatus::KEPT;
     }
     extraObject->ClearRegularWeakReferenceImpl();
     if (extraObject->HasAssociatedObject()) {
@@ -69,18 +69,18 @@ bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectC
         finalizerQueue.Push(extraObjectCell);
         KeepAlive(baseObject);
         CustomAllocDebug("SweepIsCollectable(%p): add to finalizerQueue", extraObject);
-        return false;
+        return ExtraObjectStatus::TO_BE_FINALIZED;
     } else {
         if (HasFinalizers(baseObject)) {
             extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
             finalizerQueue.Push(extraObjectCell);
             KeepAlive(baseObject);
             CustomAllocDebug("SweepIsCollectable(%p): addings to finalizerQueue, keep base object (%p) alive", extraObject, baseObject);
-            return false;
+            return ExtraObjectStatus::TO_BE_FINALIZED;
         }
         extraObject->Uninstall();
         CustomAllocDebug("SweepIsCollectable(%p): uninstalled extraObject", extraObject);
-        return true;
+        return ExtraObjectStatus::SWEPT;
     }
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp

@@ -18,8 +18,13 @@ namespace kotlin::alloc {
 
 bool TryResetMark(void* ptr) noexcept;
 
-// Returns true if swept successfully, i.e., if the extraobject can be reclaimed now.
+enum class ExtraObjectStatus {
-bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
+    TO_BE_FINALIZED,
+    KEPT,
+    SWEPT,
+};
+
+ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
 
 void* SafeAlloc(uint64_t size) noexcept;
 void Free(void* ptr, size_t size) noexcept;

kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp

@@ -40,21 +40,23 @@ void Heap::PrepareForGC() noexcept {
     usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_));
 }
 
-void Heap::Sweep() noexcept {
+void Heap::Sweep(gc::GCHandle gcHandle) noexcept {
+    auto sweepHandle = gcHandle.sweep();
     CustomAllocDebug("Heap::Sweep()");
     for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
-        fixedBlockPages_[blockSize].Sweep();
+        fixedBlockPages_[blockSize].Sweep(sweepHandle);
     }
-    nextFitPages_.Sweep();
+    nextFitPages_.Sweep(sweepHandle);
-    singleObjectPages_.SweepAndFree();
+    singleObjectPages_.SweepAndFree(sweepHandle);
 }
 
 AtomicStack<ExtraObjectCell> Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept {
+    auto sweepHandle = gcHandle.sweepExtraObjects();
     CustomAllocDebug("Heap::SweepExtraObjects()");
     AtomicStack<ExtraObjectCell> finalizerQueue;
     ExtraObjectPage* page;
     while ((page = usedExtraObjectPages_.Pop())) {
-        if (!page->Sweep(finalizerQueue)) {
+        if (!page->Sweep(sweepHandle, finalizerQueue)) {
             CustomAllocInfo("SweepExtraObjects free(%p)", page);
             page->Destroy();
         } else {

kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp

@@ -30,7 +30,7 @@ public:
     // Sweep through all remaining pages, freeing those blocks where CanReclaim
     // returns true. If multiple sweepers are active, each page will only be
     // seen by one sweeper.
-    void Sweep() noexcept;
+    void Sweep(gc::GCHandle gcHandle) noexcept;
 
     AtomicStack<ExtraObjectCell> SweepExtraObjects(gc::GCHandle gcHandle) noexcept;
 

kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp

@@ -36,7 +36,8 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) {
         mark(obj); // to make the page survive a sweep
     }
     heap.PrepareForGC();
-    heap.Sweep();
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    heap.Sweep(gcHandle);
     for (int blocks = MIN; blocks < MAX; ++blocks) {
         EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks));
     }
@@ -49,7 +50,8 @@ TEST(CustomAllocTest, HeapReuseNextFitPages) {
     void* obj = page->TryAllocate(BLOCKSIZE);
     mark(obj); // to make the page survive a sweep
     heap.PrepareForGC();
-    heap.Sweep();
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    heap.Sweep(gcHandle);
     EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE));
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp

@@ -39,7 +39,7 @@ uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept {
     return curBlock_->TryAllocate(cellsNeeded);
 }
 
-bool NextFitPage::Sweep() noexcept {
+bool NextFitPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
     CustomAllocDebug("NextFitPage@%p::Sweep()", this);
     Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT;
     bool alive = false;
@@ -47,8 +47,10 @@ bool NextFitPage::Sweep() noexcept {
         if (block->isAllocated_) {
             if (TryResetMark(block->data_)) {
                 alive = true;
+                sweepHandle.addKeptObject();
             } else {
                 block->Deallocate();
+                sweepHandle.addSweptObject();
             }
         }
     }

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp

@@ -11,6 +11,7 @@
 
 #include "AtomicStack.hpp"
 #include "Cell.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::alloc {
 
@@ -23,7 +24,7 @@ public:
     // Tries to allocate in current page, returns null if no free block in page is big enough
     uint8_t* TryAllocate(uint32_t blockSize) noexcept;
 
-    bool Sweep() noexcept;
+    bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
 
     // Testing method
     bool CheckInvariants() noexcept;

kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp

@@ -52,7 +52,9 @@ TEST(CustomAllocTest, NextFitPageAlloc) {
 
 TEST(CustomAllocTest, NextFitPageSweepEmptyPage) {
     NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
-    EXPECT_FALSE(page->Sweep());
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
+    EXPECT_FALSE(page->Sweep(gcScope));
     page->Destroy();
 }
 
@@ -61,7 +63,9 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) {
         std::minstd_rand r(seed);
         NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
         while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {}
-        EXPECT_FALSE(page->Sweep());
+        auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+        auto gcScope = gcHandle.sweep();
+        EXPECT_FALSE(page->Sweep(gcScope));
         page->Destroy();
     }
 }
@@ -69,17 +73,21 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) {
 TEST(CustomAllocTest, NextFitPageSweepSingleMarked) {
     NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
     mark(alloc(page, MIN_BLOCK_SIZE));
-    EXPECT_TRUE(page->Sweep());
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
+    EXPECT_TRUE(page->Sweep(gcScope));
     page->Destroy();
 }
 
 TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
         std::minstd_rand r(seed);
         NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
         int count1 = 0;
         while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1;
-        EXPECT_FALSE(page->Sweep());
+        EXPECT_FALSE(page->Sweep(gcScope));
         r.seed(seed);
         int count2 = 0;
         while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count2;
@@ -89,6 +97,8 @@ TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
 }
 
 TEST(CustomAllocTest, NextFitPageSweepReuse) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
         std::minstd_rand r(seed);
         NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
@@ -102,7 +112,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
                 ++unmarked;
             }
         }
-        page->Sweep();
+        page->Sweep(gcScope);
         int freed = 0;
         while (alloc(page, MIN_BLOCK_SIZE)) ++freed;
         EXPECT_EQ(freed, unmarked);
@@ -111,9 +121,11 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
 }
 
 TEST(CustomAllocTest, NextFitPageSweepCoallesce) {
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
     NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
     EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1));
-    EXPECT_FALSE(page->Sweep());
+    EXPECT_FALSE(page->Sweep(gcScope));
     EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1));
     page->Destroy();
 }

kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp

@@ -9,6 +9,7 @@
 #include <atomic>
 
 #include "AtomicStack.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::alloc {
 
@@ -22,14 +23,15 @@ public:
         while ((page = empty_.Pop())) page->Destroy();
     }
 
-    void Sweep() noexcept {
+    void Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
-        while (SweepSingle(unswept_, ready_)) {}
+        while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_)) {
+        }
     }
 
-    void SweepAndFree() noexcept {
+    void SweepAndFree(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
         T* page;
         while ((page = unswept_.Pop())) {
-            if (page->Sweep()) {
+            if (page->Sweep(sweepHandle)) {
                 ready_.Push(page);
             } else {
                 page->Destroy();
@@ -39,8 +41,12 @@ public:
 
     T* GetPage(uint32_t cellCount) noexcept {
         T* page;
-        if ((page = SweepSingle(unswept_, used_))) {
+        if ((page = unswept_.Pop())) {
-            return page;
+            // If there're unswept_ pages, the GC is in progress.
+            auto sweepHandle = gc::GCHandle::currentEpoch()->sweep();
+            if ((page = SweepSingle(sweepHandle, page, unswept_, used_))) {
+                return page;
+            }
         }
         if ((page = ready_.Pop())) {
             used_.Push(page);
@@ -68,15 +74,17 @@ public:
     }
 
 private:
-    T* SweepSingle(AtomicStack<T>& from, AtomicStack<T>& to) noexcept {
+    T* SweepSingle(gc::GCHandle::GCSweepScope& sweepHandle, T* page, AtomicStack<T>& from, AtomicStack<T>& to) noexcept {
-        T* page;
+        if (!page) {
-        while ((page = from.Pop())) {
+            return nullptr;
-            if (page->Sweep()) {
+        }
+        do {
+            if (page->Sweep(sweepHandle)) {
                 to.Push(page);
                 return page;
             }
             empty_.Push(page);
-        }
+        } while ((page = from.Pop()));
         return nullptr;
     }
 

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp

@@ -37,12 +37,14 @@ uint8_t* SingleObjectPage::TryAllocate() noexcept {
     return Data();
 }
 
-bool SingleObjectPage::Sweep() noexcept {
+bool SingleObjectPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
     CustomAllocDebug("SingleObjectPage@%p::Sweep()", this);
     if (!TryResetMark(Data())) {
         isAllocated_ = false;
+        sweepHandle.addKeptObject();
         return false;
     }
+    sweepHandle.addSweptObject();
     return true;
 }
 

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp

@@ -10,6 +10,7 @@
 #include <cstdint>
 
 #include "AtomicStack.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::alloc {
 
@@ -23,7 +24,7 @@ public:
 
     uint8_t* TryAllocate() noexcept;
 
-    bool Sweep() noexcept;
+    bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
 
 private:
     friend class AtomicStack<SingleObjectPage>;

kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp

@@ -34,7 +34,9 @@ SingleObjectPage* alloc(uint64_t blockSize) {
 TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) {
     SingleObjectPage* page = alloc(MIN_BLOCK_SIZE);
     EXPECT_TRUE(page);
-    EXPECT_FALSE(page->Sweep());
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
+    EXPECT_FALSE(page->Sweep(gcScope));
     page->Destroy();
 }
 
@@ -43,7 +45,9 @@ TEST(CustomAllocTest, SingleObjectPageSweepFullPage) {
     EXPECT_TRUE(page);
     EXPECT_TRUE(page->Data());
     mark(page->Data());
-    EXPECT_TRUE(page->Sweep());
+    auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
+    auto gcScope = gcHandle.sweep();
+    EXPECT_TRUE(page->Sweep(gcScope));
     page->Destroy();
 }
 

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -187,7 +187,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     mm::WaitForThreadsSuspension();
     auto markStats = gcHandle.getMarked();
-    scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
+    scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
 
     gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 
@@ -207,7 +207,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     mm::ResumeThreads();
     gcHandle.threadsAreResumed();
-    heap_.Sweep();
+    heap_.Sweep(gcHandle);
 #endif
     state_.finish(epoch);
     gcHandle.finalizersScheduled(finalizerQueue.size());

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -95,36 +95,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
 #endif
 }
 
-
-size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactory().GetObjectsCountUnsafe();
-#else
-    return 0;
-#endif
-}
-size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
-#else
-    return 0;
-#endif
-}
-size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
-#else
-    return 0;
-#endif
-}
-size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
-#ifndef CUSTOM_ALLOCATOR
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
-#else
-    return 0;
-#endif
-}
-
 size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
     return allocatedBytes();
 }

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -59,10 +59,6 @@ public:
 
     static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
 
-    size_t GetHeapObjectsCountUnsafe() const noexcept;
-    size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
-    size_t GetExtraObjectsCountUnsafe() const noexcept;
-    size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
     size_t GetTotalHeapObjectsSizeBytes() const noexcept;
 
     gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp

@@ -26,26 +26,41 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu
 void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz,
                                                  KLong threadLocalReferences, KLong stackReferences,
                                                  KLong globalReferences, KLong stableReferences);
-void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize);
+void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount);
-void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize);
+void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount);
+void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong sizeBytes);
+void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong sizeBytes);
 }
 
 namespace {
 
+constexpr KNativePtr heapPoolKey = const_cast<KNativePtr>(static_cast<const void*>("heap"));
+constexpr KNativePtr extraPoolKey = const_cast<KNativePtr>(static_cast<const void*>("extra"));
+
+struct MemoryUsage {
+    uint64_t sizeBytes;
+};
+
 struct MemoryUsageMap {
-    std::optional<kotlin::gc::MemoryUsage> heap;
+    std::optional<MemoryUsage> heap;
-    std::optional<kotlin::gc::MemoryUsage> extra;
+
+    void build(KRef builder, void (*add)(KRef, KNativePtr, KLong)) {
+        if (heap) {
+            add(builder, heapPoolKey, static_cast<KLong>(heap->sizeBytes));
+        }
+    }
+};
+
+struct SweepStatsMap {
+    std::optional<gc::SweepStats> heap;
+    std::optional<gc::SweepStats> extra;
 
     void build(KRef builder, void (*add)(KRef, KNativePtr, KLong, KLong)) {
         if (heap) {
-            add(builder, const_cast<KNativePtr>(static_cast<const void*>("heap")),
+            add(builder, heapPoolKey, static_cast<KLong>(heap->keptCount), static_cast<KLong>(heap->sweptCount));
-                static_cast<KLong>(heap->objectsCount),
-                static_cast<KLong>(heap->totalObjectsSize));
         }
         if (extra) {
-            add(builder, const_cast<KNativePtr>(static_cast<const void*>("extra")),
+            add(builder, extraPoolKey, static_cast<KLong>(extra->keptCount), static_cast<KLong>(extra->sweptCount));
-                static_cast<KLong>(extra->objectsCount),
-                static_cast<KLong>(extra->totalObjectsSize));
         }
     }
 };
@@ -66,7 +81,8 @@ struct GCInfo {
     std::optional<KLong> pauseEndTime;
     std::optional<KLong> finalizersDoneTime;
     std::optional<RootSetStatistics> rootSet;
-    std::optional<kotlin::gc::MemoryUsage> markStats;
+    std::optional<kotlin::gc::MarkStats> markStats;
+    SweepStatsMap sweepStats;
     MemoryUsageMap memoryUsageBefore;
     MemoryUsageMap memoryUsageAfter;
 
@@ -82,6 +98,9 @@ struct GCInfo {
             Kotlin_Internal_GC_GCInfoBuilder_setRootSet(
                     builder, rootSet->threadLocalReferences, rootSet->stackReferences, rootSet->globalReferences,
                     rootSet->stableReferences);
+        if (markStats)
+            Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(builder, markStats->markedCount);
+        sweepStats.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setSweepStats);
         memoryUsageBefore.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore);
         memoryUsageAfter.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter);
     }
@@ -98,6 +117,12 @@ GCInfo* statByEpoch(uint64_t epoch) {
     return nullptr;
 }
 
+MemoryUsage currentHeapUsage() noexcept {
+    return MemoryUsage{
+            mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes(),
+    };
+}
+
 } // namespace
 
 extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) {
@@ -132,12 +157,23 @@ GCHandle GCHandle::create(uint64_t epoch) {
     } else {
         GCLogInfo(epoch, "Started.");
     }
+    current.memoryUsageBefore.heap = currentHeapUsage();
     return getByEpoch(epoch);
 }
 GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits<uint64_t>::max()); }
 GCHandle GCHandle::getByEpoch(uint64_t epoch) {
     return GCHandle{epoch};
 }
+
+// static
+std::optional<gc::GCHandle> gc::GCHandle::currentEpoch() noexcept {
+    std::lock_guard guard(lock);
+    if (auto epoch = current.epoch) {
+        return GCHandle::getByEpoch(*epoch);
+    }
+    return std::nullopt;
+}
+
 void GCHandle::ClearForTests() {
     std::lock_guard guard(lock);
     current = {};
@@ -147,6 +183,7 @@ void GCHandle::finished() {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
         stat->endTime = static_cast<KLong>(konan::getTimeNanos());
+        stat->memoryUsageAfter.heap = currentHeapUsage();
         if (stat->rootSet) {
             GCLogInfo(
                     epoch_,
@@ -160,20 +197,22 @@ void GCHandle::finished() {
                     stat->rootSet->stableReferences, stat->rootSet->total());
         }
         if (stat->markStats) {
-            GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->objectsCount);
+            GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->markedCount);
         }
-        if (stat->memoryUsageAfter.extra && stat->memoryUsageBefore.extra) {
+        if (auto stats = stat->sweepStats.extra) {
             GCLogInfo(
                     epoch_, "Sweep extra objects: swept %" PRIu64 " objects, kept %" PRIu64 " objects",
-                    stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->objectsCount);
+                    stats->sweptCount, stats->keptCount);
+        }
+        if (auto stats = stat->sweepStats.heap) {
+            GCLogInfo(
+                    epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects", stats->sweptCount,
+                    stats->keptCount);
         }
         if (stat->memoryUsageBefore.heap && stat->memoryUsageAfter.heap) {
             GCLogInfo(
-                    epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects",
+                    epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->sizeBytes,
-                    stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->objectsCount);
+                    stat->memoryUsageAfter.heap->sizeBytes);
-            GCLogInfo(
-                    epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->totalObjectsSize,
-                    stat->memoryUsageAfter.heap->totalObjectsSize);
         }
         if (stat->pauseStartTime && stat->pauseEndTime) {
             auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
@@ -184,18 +223,6 @@ void GCHandle::finished() {
             GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time);
         }
 
-
-        if (auto usage = stat->memoryUsageAfter.heap; usage && stat->markStats) {
-            RuntimeAssert(
-                    stat->markStats->objectsCount == usage->objectsCount,
-                    "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep",
-                    stat->markStats->objectsCount, usage->objectsCount);
-            RuntimeAssert(
-                    stat->markStats->totalObjectsSize == usage->totalObjectsSize,
-                    "Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep",
-                    stat->markStats->totalObjectsSize, usage->totalObjectsSize);
-        }
-
         if (stat == &current) {
             last = current;
             current = {};
@@ -218,6 +245,15 @@ void GCHandle::threadsAreSuspended() {
             return;
         }
     }
+    if (last.epoch) {
+        // Assisted sweeping from the last epoch must be completed before the check can be run.
+        if (last.markStats && last.sweepStats.heap) {
+            RuntimeAssert(
+                    last.markStats->markedCount == last.sweepStats.heap->keptCount,
+                    "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " are alive after sweep",
+                    last.markStats->markedCount, last.sweepStats.heap->keptCount);
+        }
+    }
 }
 void GCHandle::threadsAreResumed() {
     std::lock_guard guard(lock);
@@ -267,87 +303,73 @@ void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stable
     }
 }
 
-
+void GCHandle::marked(kotlin::gc::MarkStats stats) {
-void GCHandle::heapUsageBefore(MemoryUsage usage) {
-    std::lock_guard guard(lock);
-    if (auto* stat = statByEpoch(epoch_)) {
-        stat->memoryUsageBefore.heap = usage;
-    }
-}
-
-void GCHandle::marked(kotlin::gc::MemoryUsage usage) {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
         if (!stat->markStats) {
-            stat->markStats = MemoryUsage{0, 0};
+            stat->markStats = MarkStats{};
         }
-        stat->markStats->totalObjectsSize += usage.totalObjectsSize;
+        stat->markStats->markedSizeBytes += stats.markedSizeBytes;
-        stat->markStats->objectsCount += usage.objectsCount;
+        stat->markStats->markedCount += stats.markedCount;
     }
 }
 
-MemoryUsage GCHandle::getMarked() {
+MarkStats GCHandle::getMarked() {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
         if (stat->markStats) {
             return *stat->markStats;
         }
     }
-    return MemoryUsage{0, 0};
+    return MarkStats{};
 }
 
-void GCHandle::heapUsageAfter(MemoryUsage usage) {
+void GCHandle::swept(gc::SweepStats stats) noexcept {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
-        stat->memoryUsageAfter.heap = usage;
+        auto& heap = stat->sweepStats.heap;
+        if (!heap) {
+            heap = gc::SweepStats{};
+        }
+        heap->keptCount += stats.keptCount;
+        heap->sweptCount += stats.sweptCount;
     }
 }
 
-void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) {
+void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noexcept {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
-        stat->memoryUsageBefore.extra = usage;
+        auto& extra = stat->sweepStats.extra;
-    }
+        if (!extra) {
-}
+            extra = gc::SweepStats{};
-void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) {
+        }
-    std::lock_guard guard(lock);
+        extra->keptCount += stats.keptCount;
-    if (auto* stat = statByEpoch(epoch_)) {
+        extra->sweptCount += stats.sweptCount;
-        stat->memoryUsageAfter.extra = usage;
+        RuntimeAssert(static_cast<bool>(stat->markStats), "Mark must have already happened");
+        stat->markStats->markedCount += markedCount;
     }
 }
 
-MemoryUsage GCHandle::GCSweepScope::getUsage() {
+GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
-    return MemoryUsage{
-            mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(),
-            mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(),
-    };
-}
-
-GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) :
-    handle_(handle) {
-    handle_.heapUsageBefore(getUsage());
-}
 
 GCHandle::GCSweepScope::~GCSweepScope() {
-    GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime());
+    handle_.swept(stats_);
-    handle_.heapUsageAfter(getUsage());
+    GCLogDebug(
+            handle_.getEpoch(),
+            "Collected %" PRId64 " heap objects in %" PRIu64 " microseconds. "
+            "%" PRId64 " heap objects are kept alive.",
+            stats_.sweptCount, getStageTime(), stats_.keptCount);
 }
 
-MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() {
+GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
-    return MemoryUsage{
-            mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(),
-            mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(),
-    };
-}
-
-GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) :
-    handle_(handle) {
-    handle_.extraObjectsUsageBefore(getUsage());
-}
 
 GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() {
-    GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime());
+    handle_.sweptExtraObjects(stats_, markedCount_);
-    handle_.extraObjectsUsageAfter(getUsage());
+    GCLogDebug(
+            handle_.getEpoch(),
+            "Collected %" PRId64 " extra objects in %" PRIu64 " microseconds. "
+            "%" PRId64 " extra objects are kept alive.",
+            stats_.sweptCount, getStageTime(), stats_.keptCount);
 }
 
 GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
@@ -370,10 +392,8 @@ GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){
 GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){}
 
 GCHandle::GCMarkScope::~GCMarkScope() {
-    handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes});
+    handle_.marked(stats_);
-    GCLogDebug(handle_.getEpoch(),
+    GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime());
-               "Marked %" PRIu64 " objects in %" PRIu64 " microseconds",
-               objectsCount, getStageTime());
 }
 
 gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {}

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp

@@ -11,6 +11,7 @@
 #include "Common.h"
 #include "Porting.h"
 #include "Utils.hpp"
+#include "std_support/Optional.hpp"
 
 #define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
 #define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
@@ -24,9 +25,14 @@ namespace kotlin::gc {
 
 class GCHandle;
 
-struct MemoryUsage {
+struct SweepStats {
-    uint64_t objectsCount;
+    uint64_t sweptCount = 0;
-    uint64_t totalObjectsSize;
+    uint64_t keptCount = 0;
+};
+
+struct MarkStats {
+    uint64_t markedCount = 0;
+    uint64_t markedSizeBytes = 0;
 };
 
 class GCHandle {
@@ -39,20 +45,29 @@ public:
 
     class GCSweepScope : GCStageScopeUsTimer, Pinned {
         GCHandle& handle_;
-        MemoryUsage getUsage();
+        SweepStats stats_;
 
     public:
         explicit GCSweepScope(GCHandle& handle);
         ~GCSweepScope();
+
+        void addSweptObject() noexcept { stats_.sweptCount += 1; }
+        void addKeptObject() noexcept { stats_.keptCount += 1; }
     };
 
     class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned {
         GCHandle& handle_;
-        MemoryUsage getUsage();
+        SweepStats stats_;
+        uint64_t markedCount_ = 0;
 
     public:
         explicit GCSweepExtraObjectsScope(GCHandle& handle);
         ~GCSweepExtraObjectsScope();
+
+        void addSweptObject() noexcept { stats_.sweptCount += 1; }
+        void addKeptObject() noexcept { stats_.keptCount += 1; }
+        // Custom allocator only. To be finalized objects are kept alive.
+        void addMarkedObject() noexcept { markedCount_ += 1; }
     };
 
     class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned {
@@ -82,15 +97,15 @@ public:
 
     class GCMarkScope : GCStageScopeUsTimer, Pinned {
         GCHandle& handle_;
-        uint64_t objectsCount = 0;
+        MarkStats stats_;
-        uint64_t totalObjectSizeBytes = 0;
 
     public:
         explicit GCMarkScope(GCHandle& handle);
         ~GCMarkScope();
-        void addObject(uint64_t objectSize) {
+
-            totalObjectSizeBytes += objectSize;
+        void addObject(uint64_t objectSize) noexcept {
-            objectsCount++;
+            ++stats_.markedCount;
+            stats_.markedSizeBytes += objectSize;
         }
     };
 
@@ -115,16 +130,15 @@ private:
 
     void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences);
     void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences);
-    void heapUsageBefore(MemoryUsage usage);
+    void swept(SweepStats stats) noexcept;
-    void heapUsageAfter(MemoryUsage usage);
+    void sweptExtraObjects(SweepStats stats, uint64_t markedCount) noexcept;
-    void extraObjectsUsageBefore(MemoryUsage usage);
+    void marked(MarkStats stats);
-    void extraObjectsUsageAfter(MemoryUsage usage);
-    void marked(MemoryUsage usage);
 
 public:
     static GCHandle create(uint64_t epoch);
     static GCHandle createFakeForTests();
     static GCHandle getByEpoch(uint64_t epoch);
+    static std::optional<GCHandle> currentEpoch() noexcept;
     static void ClearForTests();
 
     uint64_t getEpoch() { return epoch_; }
@@ -141,6 +155,6 @@ public:
     GCMarkScope mark() { return GCMarkScope(*this); }
     GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); }
 
-    MemoryUsage getMarked();
+    MarkStats getMarked();
 };
 }

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -118,12 +118,15 @@ void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::It
             if (extraObject.HasAssociatedObject()) {
                 extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
                 ++it;
+                sweepHandle.addKeptObject();
             } else {
                 extraObject.Uninstall();
                 it.EraseAndAdvance();
+                sweepHandle.addSweptObject();
             }
         } else {
             ++it;
+            sweepHandle.addKeptObject();
         }
     }
 }
@@ -142,8 +145,10 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T
     for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
         if (Traits::TryResetMark(*it)) {
             ++it;
+            sweepHandle.addKeptObject();
             continue;
         }
+        sweepHandle.addSweptObject();
         auto* objHeader = it->GetObjHeader();
         if (HasFinalizers(objHeader)) {
             objectFactoryIter.MoveAndAdvance(finalizerQueue, it);

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -140,7 +140,7 @@ public:
         return testing::UnorderedElementsAreArray(objects);
     }
 
-    gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<test_support::Any>> graySet) {
+    gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<test_support::Any>> graySet) {
         std_support::vector<ObjHeader*> objects;
         for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().header());
         auto handle = gc::GCHandle::create(epoch_++);
@@ -168,8 +168,8 @@ size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<test_support:
 #define EXPECT_MARKED(stats, ...) \
     do { \
         std::initializer_list<std::reference_wrapper<test_support::Any>> objects = {__VA_ARGS__}; \
-        EXPECT_THAT(stats.objectsCount, objects.size()); \
+        EXPECT_THAT(stats.markedCount, objects.size()); \
-        EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \
+        EXPECT_THAT(stats.markedSizeBytes, GetObjectsSize(objects)); \
         EXPECT_THAT(marked(), MarkedMatcher(objects)); \
     } while (false)
 

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -67,19 +67,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
     return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
 }
 
-size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
-    return impl_->objectFactory().GetObjectsCountUnsafe();
-}
-size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
-    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
-}
-size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
-}
-size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
-}
-
 size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
     return allocatedBytes();
 }

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -80,19 +80,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
     return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
 }
 
-size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
-    return impl_->objectFactory().GetObjectsCountUnsafe();
-}
-size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
-    return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
-}
-size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
-}
-size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
-    return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
-}
-
 size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
     return allocatedBytes();
 }

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -137,7 +137,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
         gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
         auto markStats = gcHandle.getMarked();
-        scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
+        scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
 
         gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt

@@ -102,13 +102,13 @@ public class GCInfo(
                         },
                         info.memoryUsageBefore.mapValues { (_, v) ->
                             MemoryUsage(
-                                    v.objectsCount,
+                                    0L,
                                     v.totalObjectsSizeBytes,
                             )
                         },
                         info.memoryUsageAfter.mapValues { (_, v) ->
                             MemoryUsage(
-                                    v.objectsCount,
+                                    0L,
                                     v.totalObjectsSizeBytes,
                             )
                         }

kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt

@@ -16,7 +16,6 @@ import kotlin.system.*
 /**
  * This class represents statistics of memory usage in one memory pool.
  *
- * @property objectsCount The number of allocated objects.
  * @property totalObjectsSizeBytes The total size of allocated objects. System allocator overhead is not included,
  *                                 so it can not perfectly match the value received by os tools.
  *                                 All alignment and auxiliary object headers are included.
@@ -24,10 +23,22 @@ import kotlin.system.*
 @NativeRuntimeApi
 @SinceKotlin("1.9")
 public class MemoryUsage(
-        val objectsCount: Long,
         val totalObjectsSizeBytes: Long,
 )
 
+/**
+ * This class represents statistics of sweeping in one memory pool.
+ *
+ * @property sweptCount The of objects that were freed.
+ * @property keptCount The number of objects that were processed but kept alive.
+ */
+@NativeRuntimeApi
+@SinceKotlin("1.9")
+public class SweepStatistics(
+    val sweptCount: Long,
+    val keptCount: Long,
+)
+
 /**
  * This class represents statistics of the root set for garbage collector run, separated by root set pools.
  * These nodes are assumed to be used, even if there are no references for them.
@@ -64,12 +75,16 @@ public class RootSetStatistics(
  * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos].
  *                                If null, memory reclamation is still in progress.
  * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details.
+ * @property markedCount How many objects were processed during marking phase.
+ * @property sweepStatistics Sweeping statistics separated by memory pools.
+ *                           The set of memory pools depends on the collector implementation.
+ *                           Can be empty, if collection is in progress.
  * @property memoryUsageAfter Memory usage at the start of garbage collector run, separated by memory pools.
  *                            The set of memory pools depends on the collector implementation.
- *                            Can be empty, of colelction is in progress.
+ *                            Can be empty, if collection is in progress.
  * @property memoryUsageBefore Memory usage at the end of garbage collector run, separated by memory pools.
  *                            The set of memory pools depends on the collector implementation.
- *                            Can be empty, of colelction is in progress.
+ *                            Can be empty, if collection is in progress.
  */
 @NativeRuntimeApi
 @SinceKotlin("1.9")
@@ -81,6 +96,8 @@ public class GCInfo(
         val pauseEndTimeNs: Long,
         val postGcCleanupTimeNs: Long?,
         val rootSet: RootSetStatistics,
+        val markedCount: Long,
+        val sweepStatistics: Map<String, SweepStatistics>,
         val memoryUsageBefore: Map<String, MemoryUsage>,
         val memoryUsageAfter: Map<String, MemoryUsage>,
 ) {
@@ -102,6 +119,8 @@ private class GCInfoBuilder() {
     var pauseEndTimeNs: Long? = null
     var postGcCleanupTimeNs: Long? = null
     var rootSet: RootSetStatistics? = null
+    var markedCount: Long? = null
+    var sweepStatistics = mutableMapOf<String, SweepStatistics>()
     var memoryUsageBefore = mutableMapOf<String, MemoryUsage>()
     var memoryUsageAfter = mutableMapOf<String, MemoryUsage>()
 
@@ -140,17 +159,29 @@ private class GCInfoBuilder() {
         rootSet = RootSetStatistics(threadLocalReferences, stackReferences, globalReferences, stableReferences)
     }
 
-    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore")
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMarkStats")
-    private fun setMemoryUsageBefore(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) {
+    private fun setMarkStats(markedCount: Long) {
+        this.markedCount = markedCount
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSweepStats")
+    private fun setSweepStats(name: NativePtr, keptCount: Long, sweptCount: Long) {
         val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
-        val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize)
+        val stats = SweepStatistics(sweptCount, keptCount)
+        sweepStatistics[nameString] = stats
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore")
+    private fun setMemoryUsageBefore(name: NativePtr, totalObjectsSize: Long) {
+        val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
+        val memoryUsage = MemoryUsage(totalObjectsSize)
         memoryUsageBefore[nameString] = memoryUsage
     }
 
     @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter")
-    private fun setMemoryUsageAfter(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) {
+    private fun setMemoryUsageAfter(name: NativePtr, totalObjectsSize: Long) {
         val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
-        val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize)
+        val memoryUsage = MemoryUsage(totalObjectsSize)
         memoryUsageAfter[nameString] = memoryUsage
     }
 
@@ -163,6 +194,8 @@ private class GCInfoBuilder() {
                 pauseEndTimeNs ?: return null,
                 postGcCleanupTimeNs,
                 rootSet ?: return null,
+                markedCount ?: return null,
+                sweepStatistics.toMap(),
                 memoryUsageBefore.toMap(),
                 memoryUsageAfter.toMap()
         )

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp

@@ -50,7 +50,6 @@ public:
     void ClearForTests() noexcept { extraObjects_.ClearForTests(); }
 
     size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); }
-    size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); }
 
     // requires LockForIter
     void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); }

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -390,7 +390,6 @@ public:
     Iterable LockForIter() noexcept { return Iterable(*this); }
 
     size_t GetSizeUnsafe() const noexcept { return size_; }
-    size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; }
 
     void ClearForTests() {
         root_.reset();
@@ -701,9 +700,6 @@ public:
     // Lock ObjectFactory for safe iteration.
     Iterable LockForIter() noexcept { return Iterable(*this); }
 
-    size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
-    size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); }
-
     void ClearForTests() { storage_.ClearForTests(); }
 
     static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept {

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -249,6 +249,12 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu
 void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz, KLong threadLocalReferences, KLong stackReferences, KLong globalReferences, KLong stableReferences) {
     throw std::runtime_error("Not implemented for tests");
 }
+void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount) {
+    throw std::runtime_error("Not implemented for tests");
+}
+void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount) {
+    throw std::runtime_error("Not implemented for tests");
+}
 void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize) {
     throw std::runtime_error("Not implemented for tests");
 }