diff --git a/kotlin-native/backend.native/tests/runtime/memory/gcStats.kt b/kotlin-native/backend.native/tests/runtime/memory/gcStats.kt index 15364da9230..f457695bd2f 100644 --- a/kotlin-native/backend.native/tests/runtime/memory/gcStats.kt +++ b/kotlin-native/backend.native/tests/runtime/memory/gcStats.kt @@ -12,13 +12,11 @@ import kotlin.test.* @Test fun `nothing new collected`() { GC.collect() - GC.collect(); + GC.collect() val stat = GC.lastGCInfo assertNotNull(stat) - assertEquals(stat.memoryUsageBefore.keys, stat.memoryUsageAfter.keys) - for (key in stat.memoryUsageBefore.keys) { - assertEquals(stat.memoryUsageBefore[key]!!.objectsCount, stat.memoryUsageAfter[key]!!.objectsCount) - assertEquals(stat.memoryUsageBefore[key]!!.totalObjectsSizeBytes, stat.memoryUsageAfter[key]!!.totalObjectsSizeBytes) + for (key in stat.sweepStatistics.keys) { + assertEquals(stat.sweepStatistics[key]!!.sweptCount, 0L) } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp index 663bf5c7146..3899013c36a 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp @@ -45,7 +45,7 @@ mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept { return freeBlock->Data(); } -bool ExtraObjectPage::Sweep(AtomicStack& finalizerQueue) noexcept { +bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack& finalizerQueue) noexcept { CustomAllocInfo("ExtraObjectPage(%p)::Sweep()", this); // `end` is after the last legal allocation of a block, but does not // necessarily match an actual block starting point. @@ -58,15 +58,24 @@ bool ExtraObjectPage::Sweep(AtomicStack& finalizerQueue) noexce nextFree = &cell->next_; continue; } - // If the current cell was marked, it's alive, and the whole page is alive. - if (!SweepExtraObject(cell, finalizerQueue)) { - alive = true; - continue; + auto status = SweepExtraObject(cell, finalizerQueue); + switch (status) { + case ExtraObjectStatus::TO_BE_FINALIZED: + sweepHandle.addMarkedObject(); + // fallthrough + case ExtraObjectStatus::KEPT: + // If the current cell was marked, it's alive, and the whole page is alive. + alive = true; + sweepHandle.addKeptObject(); + break; + case ExtraObjectStatus::SWEPT: + // Free the current block and insert it into the free list. + cell->next_ = *nextFree; + *nextFree = cell; + nextFree = &cell->next_; + sweepHandle.addSweptObject(); + break; } - // Free the current block and insert it into the free list. - cell->next_ = *nextFree; - *nextFree = cell; - nextFree = &cell->next_; } return alive; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp index 4d55f7e1ba6..2a0e0368365 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp @@ -11,6 +11,7 @@ #include "AtomicStack.hpp" #include "ExtraObjectData.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { @@ -34,7 +35,7 @@ public: // Tries to allocate in current page, returns null if no free block in page mm::ExtraObjectData* TryAllocate() noexcept; - bool Sweep(AtomicStack& finalizerQueue) noexcept; + bool Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack& finalizerQueue) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp index e9408241887..87d06fa30f6 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp @@ -41,7 +41,9 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) { TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) { Page* page = Page::Create(); Queue finalizerQueue; - EXPECT_FALSE(page->Sweep(finalizerQueue)); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweepExtraObjects(); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); EXPECT_EQ(finalizerQueue.size(), size_t(0)); page->Destroy(); } @@ -56,7 +58,9 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) { } EXPECT_EQ(count, EXTRA_OBJECT_COUNT); Queue finalizerQueue; - EXPECT_FALSE(page->Sweep(finalizerQueue)); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweepExtraObjects(); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); EXPECT_EQ(finalizerQueue.size(), size_t(0)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp index 5c96ad00990..7fb918563c2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -46,7 +46,7 @@ uint8_t* FixedBlockPage::TryAllocate() noexcept { return freeBlock->data; } -bool FixedBlockPage::Sweep() noexcept { +bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this); // `end` is after the last legal allocation of a block, but does not // necessarily match an actual block starting point. @@ -62,6 +62,7 @@ bool FixedBlockPage::Sweep() noexcept { // If the current cell was marked, it's alive, and the whole page is alive. if (TryResetMark(cell)) { alive = true; + sweepHandle.addKeptObject(); continue; } CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell); @@ -69,6 +70,7 @@ bool FixedBlockPage::Sweep() noexcept { cell->nextFree = *nextFree; *nextFree = cell; nextFree = &cell->nextFree; + sweepHandle.addSweptObject(); } return alive; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp index 95a0b532f0d..81bf5a5cbb0 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp @@ -10,6 +10,7 @@ #include #include "AtomicStack.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { @@ -30,7 +31,7 @@ public: // Tries to allocate in current page, returns null if no free block in page uint8_t* TryAllocate() noexcept; - bool Sweep() noexcept; + bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp index 554d3ae1431..effa128faa0 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp @@ -44,52 +44,62 @@ TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) { } TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); - EXPECT_FALSE(page->Sweep()); + EXPECT_FALSE(page->Sweep(gcScope)); page->Destroy(); } } TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint32_t count = 0; while (alloc(page, size)) ++count; EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size); - EXPECT_FALSE(page->Sweep()); + EXPECT_FALSE(page->Sweep(gcScope)); page->Destroy(); } } TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); mark(ptr); - EXPECT_TRUE(page->Sweep()); + EXPECT_TRUE(page->Sweep(gcScope)); page->Destroy(); } } TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); - EXPECT_FALSE(page->Sweep()); + EXPECT_FALSE(page->Sweep(gcScope)); EXPECT_EQ(alloc(page, size), ptr); page->Destroy(); } } TEST(CustomAllocTest, FixedBlockPageSweepReuse) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr; for (int count = 0; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); } - EXPECT_TRUE(page->Sweep()); + EXPECT_TRUE(page->Sweep(gcScope)); uint32_t count = 0; for (; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 35bcb6d916c..b35338a0675 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -46,22 +46,22 @@ static bool IsAlive(ObjHeader* baseObject) noexcept { return objectData.marked(); } -bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept { +ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept { auto* extraObject = extraObjectCell->Data(); if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { CustomAllocDebug("SweepIsCollectable(%p): already finalized", extraObject); - return true; + return ExtraObjectStatus::SWEPT; } auto* baseObject = extraObject->GetBaseObject(); RuntimeAssert(baseObject->heap(), "SweepIsCollectable on a non-heap object"); if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { CustomAllocDebug("SweepIsCollectable(%p): already in finalizer queue, keep base object (%p) alive", extraObject, baseObject); KeepAlive(baseObject); - return false; + return ExtraObjectStatus::TO_BE_FINALIZED; } if (IsAlive(baseObject)) { CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject); - return false; + return ExtraObjectStatus::KEPT; } extraObject->ClearRegularWeakReferenceImpl(); if (extraObject->HasAssociatedObject()) { @@ -69,18 +69,18 @@ bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStacksetFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); finalizerQueue.Push(extraObjectCell); KeepAlive(baseObject); CustomAllocDebug("SweepIsCollectable(%p): addings to finalizerQueue, keep base object (%p) alive", extraObject, baseObject); - return false; + return ExtraObjectStatus::TO_BE_FINALIZED; } extraObject->Uninstall(); CustomAllocDebug("SweepIsCollectable(%p): uninstalled extraObject", extraObject); - return true; + return ExtraObjectStatus::SWEPT; } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index 9de000c8cba..61d5ff92860 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -18,8 +18,13 @@ namespace kotlin::alloc { bool TryResetMark(void* ptr) noexcept; -// Returns true if swept successfully, i.e., if the extraobject can be reclaimed now. -bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept; +enum class ExtraObjectStatus { + TO_BE_FINALIZED, + KEPT, + SWEPT, +}; + +ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept; void* SafeAlloc(uint64_t size) noexcept; void Free(void* ptr, size_t size) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index 22523a9eda4..bc0427b37bf 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -40,21 +40,23 @@ void Heap::PrepareForGC() noexcept { usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_)); } -void Heap::Sweep() noexcept { +void Heap::Sweep(gc::GCHandle gcHandle) noexcept { + auto sweepHandle = gcHandle.sweep(); CustomAllocDebug("Heap::Sweep()"); for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { - fixedBlockPages_[blockSize].Sweep(); + fixedBlockPages_[blockSize].Sweep(sweepHandle); } - nextFitPages_.Sweep(); - singleObjectPages_.SweepAndFree(); + nextFitPages_.Sweep(sweepHandle); + singleObjectPages_.SweepAndFree(sweepHandle); } AtomicStack Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept { + auto sweepHandle = gcHandle.sweepExtraObjects(); CustomAllocDebug("Heap::SweepExtraObjects()"); AtomicStack finalizerQueue; ExtraObjectPage* page; while ((page = usedExtraObjectPages_.Pop())) { - if (!page->Sweep(finalizerQueue)) { + if (!page->Sweep(sweepHandle, finalizerQueue)) { CustomAllocInfo("SweepExtraObjects free(%p)", page); page->Destroy(); } else { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp index 0325a0ad34c..5c82305ba75 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp @@ -30,7 +30,7 @@ public: // Sweep through all remaining pages, freeing those blocks where CanReclaim // returns true. If multiple sweepers are active, each page will only be // seen by one sweeper. - void Sweep() noexcept; + void Sweep(gc::GCHandle gcHandle) noexcept; AtomicStack SweepExtraObjects(gc::GCHandle gcHandle) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp index 9ed2e6e2502..5bd869bd949 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp @@ -36,7 +36,8 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) { mark(obj); // to make the page survive a sweep } heap.PrepareForGC(); - heap.Sweep(); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + heap.Sweep(gcHandle); for (int blocks = MIN; blocks < MAX; ++blocks) { EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks)); } @@ -49,7 +50,8 @@ TEST(CustomAllocTest, HeapReuseNextFitPages) { void* obj = page->TryAllocate(BLOCKSIZE); mark(obj); // to make the page survive a sweep heap.PrepareForGC(); - heap.Sweep(); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + heap.Sweep(gcHandle); EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE)); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index 56766e166b4..86b127fe850 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -39,7 +39,7 @@ uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept { return curBlock_->TryAllocate(cellsNeeded); } -bool NextFitPage::Sweep() noexcept { +bool NextFitPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { CustomAllocDebug("NextFitPage@%p::Sweep()", this); Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT; bool alive = false; @@ -47,8 +47,10 @@ bool NextFitPage::Sweep() noexcept { if (block->isAllocated_) { if (TryResetMark(block->data_)) { alive = true; + sweepHandle.addKeptObject(); } else { block->Deallocate(); + sweepHandle.addSweptObject(); } } } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp index b7a9563e60c..130fcba78c2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp @@ -11,6 +11,7 @@ #include "AtomicStack.hpp" #include "Cell.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { @@ -23,7 +24,7 @@ public: // Tries to allocate in current page, returns null if no free block in page is big enough uint8_t* TryAllocate(uint32_t blockSize) noexcept; - bool Sweep() noexcept; + bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; // Testing method bool CheckInvariants() noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp index 1bb8306b770..26bdff18b50 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp @@ -52,7 +52,9 @@ TEST(CustomAllocTest, NextFitPageAlloc) { TEST(CustomAllocTest, NextFitPageSweepEmptyPage) { NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); - EXPECT_FALSE(page->Sweep()); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + EXPECT_FALSE(page->Sweep(gcScope)); page->Destroy(); } @@ -61,7 +63,9 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {} - EXPECT_FALSE(page->Sweep()); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + EXPECT_FALSE(page->Sweep(gcScope)); page->Destroy(); } } @@ -69,17 +73,21 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) { TEST(CustomAllocTest, NextFitPageSweepSingleMarked) { NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); mark(alloc(page, MIN_BLOCK_SIZE)); - EXPECT_TRUE(page->Sweep()); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + EXPECT_TRUE(page->Sweep(gcScope)); page->Destroy(); } TEST(CustomAllocTest, NextFitPageSweepSingleReuse) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); int count1 = 0; while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1; - EXPECT_FALSE(page->Sweep()); + EXPECT_FALSE(page->Sweep(gcScope)); r.seed(seed); int count2 = 0; while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count2; @@ -89,6 +97,8 @@ TEST(CustomAllocTest, NextFitPageSweepSingleReuse) { } TEST(CustomAllocTest, NextFitPageSweepReuse) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); @@ -102,7 +112,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) { ++unmarked; } } - page->Sweep(); + page->Sweep(gcScope); int freed = 0; while (alloc(page, MIN_BLOCK_SIZE)) ++freed; EXPECT_EQ(freed, unmarked); @@ -111,9 +121,11 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) { } TEST(CustomAllocTest, NextFitPageSweepCoallesce) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1)); - EXPECT_FALSE(page->Sweep()); + EXPECT_FALSE(page->Sweep(gcScope)); EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp index aad373530ec..5ac2a103f67 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp @@ -9,6 +9,7 @@ #include #include "AtomicStack.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { @@ -22,14 +23,15 @@ public: while ((page = empty_.Pop())) page->Destroy(); } - void Sweep() noexcept { - while (SweepSingle(unswept_, ready_)) {} + void Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { + while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_)) { + } } - void SweepAndFree() noexcept { + void SweepAndFree(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { T* page; while ((page = unswept_.Pop())) { - if (page->Sweep()) { + if (page->Sweep(sweepHandle)) { ready_.Push(page); } else { page->Destroy(); @@ -39,8 +41,12 @@ public: T* GetPage(uint32_t cellCount) noexcept { T* page; - if ((page = SweepSingle(unswept_, used_))) { - return page; + if ((page = unswept_.Pop())) { + // If there're unswept_ pages, the GC is in progress. + auto sweepHandle = gc::GCHandle::currentEpoch()->sweep(); + if ((page = SweepSingle(sweepHandle, page, unswept_, used_))) { + return page; + } } if ((page = ready_.Pop())) { used_.Push(page); @@ -68,15 +74,17 @@ public: } private: - T* SweepSingle(AtomicStack& from, AtomicStack& to) noexcept { - T* page; - while ((page = from.Pop())) { - if (page->Sweep()) { + T* SweepSingle(gc::GCHandle::GCSweepScope& sweepHandle, T* page, AtomicStack& from, AtomicStack& to) noexcept { + if (!page) { + return nullptr; + } + do { + if (page->Sweep(sweepHandle)) { to.Push(page); return page; } empty_.Push(page); - } + } while ((page = from.Pop())); return nullptr; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp index 55a3b1e2f3d..14f23d58676 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -37,12 +37,14 @@ uint8_t* SingleObjectPage::TryAllocate() noexcept { return Data(); } -bool SingleObjectPage::Sweep() noexcept { +bool SingleObjectPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { CustomAllocDebug("SingleObjectPage@%p::Sweep()", this); if (!TryResetMark(Data())) { isAllocated_ = false; + sweepHandle.addKeptObject(); return false; } + sweepHandle.addSweptObject(); return true; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index 7781f1feb2c..4e94eb72f0d 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -10,6 +10,7 @@ #include #include "AtomicStack.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { @@ -23,7 +24,7 @@ public: uint8_t* TryAllocate() noexcept; - bool Sweep() noexcept; + bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp index fb6f3764d93..333e6cb1963 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp @@ -34,7 +34,9 @@ SingleObjectPage* alloc(uint64_t blockSize) { TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) { SingleObjectPage* page = alloc(MIN_BLOCK_SIZE); EXPECT_TRUE(page); - EXPECT_FALSE(page->Sweep()); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + EXPECT_FALSE(page->Sweep(gcScope)); page->Destroy(); } @@ -43,7 +45,9 @@ TEST(CustomAllocTest, SingleObjectPageSweepFullPage) { EXPECT_TRUE(page); EXPECT_TRUE(page->Data()); mark(page->Data()); - EXPECT_TRUE(page->Sweep()); + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + EXPECT_TRUE(page->Sweep(gcScope)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 4fc03d5d0e6..1a53554548e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -187,7 +187,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { mm::WaitForThreadsSuspension(); auto markStats = gcHandle.getMarked(); - scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize); + scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes); gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); @@ -207,7 +207,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { mm::ResumeThreads(); gcHandle.threadsAreResumed(); - heap_.Sweep(); + heap_.Sweep(gcHandle); #endif state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 049e173e819..194cf29d00b 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -95,36 +95,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { #endif } - -size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->objectFactory().GetObjectsCountUnsafe(); -#else - return 0; -#endif -} -size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { -#ifndef CUSTOM_ALLOCATOR - return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); -#else - return 0; -#endif -} -size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { -#ifndef CUSTOM_ALLOCATOR - return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); -#else - return 0; -#endif -} -size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { -#ifndef CUSTOM_ALLOCATOR - return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); -#else - return 0; -#endif -} - size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 1f8ce32ece9..1b68ad2978f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -59,10 +59,6 @@ public: static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; - size_t GetHeapObjectsCountUnsafe() const noexcept; - size_t GetTotalHeapObjectsSizeUnsafe() const noexcept; - size_t GetExtraObjectsCountUnsafe() const noexcept; - size_t GetTotalExtraObjectsSizeUnsafe() const noexcept; size_t GetTotalHeapObjectsSizeBytes() const noexcept; gc::GCSchedulerConfig& gcSchedulerConfig() noexcept; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index 6c20295f8c1..2ab36124102 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -26,26 +26,41 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz, KLong threadLocalReferences, KLong stackReferences, KLong globalReferences, KLong stableReferences); -void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize); -void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize); +void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount); +void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount); +void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong sizeBytes); +void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong sizeBytes); } namespace { +constexpr KNativePtr heapPoolKey = const_cast(static_cast("heap")); +constexpr KNativePtr extraPoolKey = const_cast(static_cast("extra")); + +struct MemoryUsage { + uint64_t sizeBytes; +}; + struct MemoryUsageMap { - std::optional heap; - std::optional extra; + std::optional heap; + + void build(KRef builder, void (*add)(KRef, KNativePtr, KLong)) { + if (heap) { + add(builder, heapPoolKey, static_cast(heap->sizeBytes)); + } + } +}; + +struct SweepStatsMap { + std::optional heap; + std::optional extra; void build(KRef builder, void (*add)(KRef, KNativePtr, KLong, KLong)) { if (heap) { - add(builder, const_cast(static_cast("heap")), - static_cast(heap->objectsCount), - static_cast(heap->totalObjectsSize)); + add(builder, heapPoolKey, static_cast(heap->keptCount), static_cast(heap->sweptCount)); } if (extra) { - add(builder, const_cast(static_cast("extra")), - static_cast(extra->objectsCount), - static_cast(extra->totalObjectsSize)); + add(builder, extraPoolKey, static_cast(extra->keptCount), static_cast(extra->sweptCount)); } } }; @@ -66,7 +81,8 @@ struct GCInfo { std::optional pauseEndTime; std::optional finalizersDoneTime; std::optional rootSet; - std::optional markStats; + std::optional markStats; + SweepStatsMap sweepStats; MemoryUsageMap memoryUsageBefore; MemoryUsageMap memoryUsageAfter; @@ -82,6 +98,9 @@ struct GCInfo { Kotlin_Internal_GC_GCInfoBuilder_setRootSet( builder, rootSet->threadLocalReferences, rootSet->stackReferences, rootSet->globalReferences, rootSet->stableReferences); + if (markStats) + Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(builder, markStats->markedCount); + sweepStats.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setSweepStats); memoryUsageBefore.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore); memoryUsageAfter.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter); } @@ -98,6 +117,12 @@ GCInfo* statByEpoch(uint64_t epoch) { return nullptr; } +MemoryUsage currentHeapUsage() noexcept { + return MemoryUsage{ + mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes(), + }; +} + } // namespace extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) { @@ -132,12 +157,23 @@ GCHandle GCHandle::create(uint64_t epoch) { } else { GCLogInfo(epoch, "Started."); } + current.memoryUsageBefore.heap = currentHeapUsage(); return getByEpoch(epoch); } GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits::max()); } GCHandle GCHandle::getByEpoch(uint64_t epoch) { return GCHandle{epoch}; } + +// static +std::optional gc::GCHandle::currentEpoch() noexcept { + std::lock_guard guard(lock); + if (auto epoch = current.epoch) { + return GCHandle::getByEpoch(*epoch); + } + return std::nullopt; +} + void GCHandle::ClearForTests() { std::lock_guard guard(lock); current = {}; @@ -147,6 +183,7 @@ void GCHandle::finished() { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { stat->endTime = static_cast(konan::getTimeNanos()); + stat->memoryUsageAfter.heap = currentHeapUsage(); if (stat->rootSet) { GCLogInfo( epoch_, @@ -160,20 +197,22 @@ void GCHandle::finished() { stat->rootSet->stableReferences, stat->rootSet->total()); } if (stat->markStats) { - GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->objectsCount); + GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->markedCount); } - if (stat->memoryUsageAfter.extra && stat->memoryUsageBefore.extra) { + if (auto stats = stat->sweepStats.extra) { GCLogInfo( epoch_, "Sweep extra objects: swept %" PRIu64 " objects, kept %" PRIu64 " objects", - stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->objectsCount); + stats->sweptCount, stats->keptCount); + } + if (auto stats = stat->sweepStats.heap) { + GCLogInfo( + epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects", stats->sweptCount, + stats->keptCount); } if (stat->memoryUsageBefore.heap && stat->memoryUsageAfter.heap) { GCLogInfo( - epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects", - stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->objectsCount); - GCLogInfo( - epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->totalObjectsSize, - stat->memoryUsageAfter.heap->totalObjectsSize); + epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->sizeBytes, + stat->memoryUsageAfter.heap->sizeBytes); } if (stat->pauseStartTime && stat->pauseEndTime) { auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000; @@ -184,18 +223,6 @@ void GCHandle::finished() { GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time); } - - if (auto usage = stat->memoryUsageAfter.heap; usage && stat->markStats) { - RuntimeAssert( - stat->markStats->objectsCount == usage->objectsCount, - "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep", - stat->markStats->objectsCount, usage->objectsCount); - RuntimeAssert( - stat->markStats->totalObjectsSize == usage->totalObjectsSize, - "Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep", - stat->markStats->totalObjectsSize, usage->totalObjectsSize); - } - if (stat == ¤t) { last = current; current = {}; @@ -218,6 +245,15 @@ void GCHandle::threadsAreSuspended() { return; } } + if (last.epoch) { + // Assisted sweeping from the last epoch must be completed before the check can be run. + if (last.markStats && last.sweepStats.heap) { + RuntimeAssert( + last.markStats->markedCount == last.sweepStats.heap->keptCount, + "Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " are alive after sweep", + last.markStats->markedCount, last.sweepStats.heap->keptCount); + } + } } void GCHandle::threadsAreResumed() { std::lock_guard guard(lock); @@ -267,87 +303,73 @@ void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stable } } - -void GCHandle::heapUsageBefore(MemoryUsage usage) { - std::lock_guard guard(lock); - if (auto* stat = statByEpoch(epoch_)) { - stat->memoryUsageBefore.heap = usage; - } -} - -void GCHandle::marked(kotlin::gc::MemoryUsage usage) { +void GCHandle::marked(kotlin::gc::MarkStats stats) { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { if (!stat->markStats) { - stat->markStats = MemoryUsage{0, 0}; + stat->markStats = MarkStats{}; } - stat->markStats->totalObjectsSize += usage.totalObjectsSize; - stat->markStats->objectsCount += usage.objectsCount; + stat->markStats->markedSizeBytes += stats.markedSizeBytes; + stat->markStats->markedCount += stats.markedCount; } } -MemoryUsage GCHandle::getMarked() { +MarkStats GCHandle::getMarked() { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { if (stat->markStats) { return *stat->markStats; } } - return MemoryUsage{0, 0}; + return MarkStats{}; } -void GCHandle::heapUsageAfter(MemoryUsage usage) { +void GCHandle::swept(gc::SweepStats stats) noexcept { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { - stat->memoryUsageAfter.heap = usage; + auto& heap = stat->sweepStats.heap; + if (!heap) { + heap = gc::SweepStats{}; + } + heap->keptCount += stats.keptCount; + heap->sweptCount += stats.sweptCount; } } -void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) { +void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noexcept { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { - stat->memoryUsageBefore.extra = usage; - } -} -void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) { - std::lock_guard guard(lock); - if (auto* stat = statByEpoch(epoch_)) { - stat->memoryUsageAfter.extra = usage; + auto& extra = stat->sweepStats.extra; + if (!extra) { + extra = gc::SweepStats{}; + } + extra->keptCount += stats.keptCount; + extra->sweptCount += stats.sweptCount; + RuntimeAssert(static_cast(stat->markStats), "Mark must have already happened"); + stat->markStats->markedCount += markedCount; } } -MemoryUsage GCHandle::GCSweepScope::getUsage() { - return MemoryUsage{ - mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(), - mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(), - }; -} - -GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : - handle_(handle) { - handle_.heapUsageBefore(getUsage()); -} +GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} GCHandle::GCSweepScope::~GCSweepScope() { - GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime()); - handle_.heapUsageAfter(getUsage()); + handle_.swept(stats_); + GCLogDebug( + handle_.getEpoch(), + "Collected %" PRId64 " heap objects in %" PRIu64 " microseconds. " + "%" PRId64 " heap objects are kept alive.", + stats_.sweptCount, getStageTime(), stats_.keptCount); } -MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() { - return MemoryUsage{ - mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(), - mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(), - }; -} - -GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : - handle_(handle) { - handle_.extraObjectsUsageBefore(getUsage()); -} +GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() { - GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime()); - handle_.extraObjectsUsageAfter(getUsage()); + handle_.sweptExtraObjects(stats_, markedCount_); + GCLogDebug( + handle_.getEpoch(), + "Collected %" PRId64 " extra objects in %" PRIu64 " microseconds. " + "%" PRId64 " extra objects are kept alive.", + stats_.sweptCount, getStageTime(), stats_.keptCount); } GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} @@ -370,10 +392,8 @@ GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){ GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){} GCHandle::GCMarkScope::~GCMarkScope() { - handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes}); - GCLogDebug(handle_.getEpoch(), - "Marked %" PRIu64 " objects in %" PRIu64 " microseconds", - objectsCount, getStageTime()); + handle_.marked(stats_); + GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime()); } gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 9d335986302..7b7cab24d78 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -11,6 +11,7 @@ #include "Common.h" #include "Porting.h" #include "Utils.hpp" +#include "std_support/Optional.hpp" #define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__) #define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__) @@ -24,9 +25,14 @@ namespace kotlin::gc { class GCHandle; -struct MemoryUsage { - uint64_t objectsCount; - uint64_t totalObjectsSize; +struct SweepStats { + uint64_t sweptCount = 0; + uint64_t keptCount = 0; +}; + +struct MarkStats { + uint64_t markedCount = 0; + uint64_t markedSizeBytes = 0; }; class GCHandle { @@ -39,20 +45,29 @@ public: class GCSweepScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; - MemoryUsage getUsage(); + SweepStats stats_; public: explicit GCSweepScope(GCHandle& handle); ~GCSweepScope(); + + void addSweptObject() noexcept { stats_.sweptCount += 1; } + void addKeptObject() noexcept { stats_.keptCount += 1; } }; class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; - MemoryUsage getUsage(); + SweepStats stats_; + uint64_t markedCount_ = 0; public: explicit GCSweepExtraObjectsScope(GCHandle& handle); ~GCSweepExtraObjectsScope(); + + void addSweptObject() noexcept { stats_.sweptCount += 1; } + void addKeptObject() noexcept { stats_.keptCount += 1; } + // Custom allocator only. To be finalized objects are kept alive. + void addMarkedObject() noexcept { markedCount_ += 1; } }; class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned { @@ -82,15 +97,15 @@ public: class GCMarkScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; - uint64_t objectsCount = 0; - uint64_t totalObjectSizeBytes = 0; + MarkStats stats_; public: explicit GCMarkScope(GCHandle& handle); ~GCMarkScope(); - void addObject(uint64_t objectSize) { - totalObjectSizeBytes += objectSize; - objectsCount++; + + void addObject(uint64_t objectSize) noexcept { + ++stats_.markedCount; + stats_.markedSizeBytes += objectSize; } }; @@ -115,16 +130,15 @@ private: void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences); void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences); - void heapUsageBefore(MemoryUsage usage); - void heapUsageAfter(MemoryUsage usage); - void extraObjectsUsageBefore(MemoryUsage usage); - void extraObjectsUsageAfter(MemoryUsage usage); - void marked(MemoryUsage usage); + void swept(SweepStats stats) noexcept; + void sweptExtraObjects(SweepStats stats, uint64_t markedCount) noexcept; + void marked(MarkStats stats); public: static GCHandle create(uint64_t epoch); static GCHandle createFakeForTests(); static GCHandle getByEpoch(uint64_t epoch); + static std::optional currentEpoch() noexcept; static void ClearForTests(); uint64_t getEpoch() { return epoch_; } @@ -141,6 +155,6 @@ public: GCMarkScope mark() { return GCMarkScope(*this); } GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); } - MemoryUsage getMarked(); + MarkStats getMarked(); }; } diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 5aacbe0e4ca..fd9f1f66d7a 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -118,12 +118,15 @@ void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::It if (extraObject.HasAssociatedObject()) { extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); ++it; + sweepHandle.addKeptObject(); } else { extraObject.Uninstall(); it.EraseAndAdvance(); + sweepHandle.addSweptObject(); } } else { ++it; + sweepHandle.addKeptObject(); } } } @@ -142,8 +145,10 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) { if (Traits::TryResetMark(*it)) { ++it; + sweepHandle.addKeptObject(); continue; } + sweepHandle.addSweptObject(); auto* objHeader = it->GetObjHeader(); if (HasFinalizers(objHeader)) { objectFactoryIter.MoveAndAdvance(finalizerQueue, it); diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index 3cfeeb49f3b..7b78ee307fa 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -140,7 +140,7 @@ public: return testing::UnorderedElementsAreArray(objects); } - gc::MemoryUsage Mark(std::initializer_list> graySet) { + gc::MarkStats Mark(std::initializer_list> graySet) { std_support::vector objects; for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().header()); auto handle = gc::GCHandle::create(epoch_++); @@ -168,8 +168,8 @@ size_t GetObjectsSize(std::initializer_list> objects = {__VA_ARGS__}; \ - EXPECT_THAT(stats.objectsCount, objects.size()); \ - EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \ + EXPECT_THAT(stats.markedCount, objects.size()); \ + EXPECT_THAT(stats.markedSizeBytes, GetObjectsSize(objects)); \ EXPECT_THAT(marked(), MarkedMatcher(objects)); \ } while (false) diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 05f3bb57834..f60b61263e4 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -67,19 +67,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { return mm::ObjectFactory::GetAllocatedHeapSize(object); } -size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { - return impl_->objectFactory().GetObjectsCountUnsafe(); -} -size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { - return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); -} -size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { - return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); -} -size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { - return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); -} - size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 48fd63ea14f..5acb3ef266b 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -80,19 +80,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { return mm::ObjectFactory::GetAllocatedHeapSize(object); } -size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept { - return impl_->objectFactory().GetObjectsCountUnsafe(); -} -size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept { - return impl_->objectFactory().GetTotalObjectsSizeUnsafe(); -} -size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept { - return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); -} -size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { - return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); -} - size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { return allocatedBytes(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 916e4302b0b..d27016ca6f1 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -137,7 +137,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { gc::Mark(gcHandle, markQueue_); auto markStats = gcHandle.getMarked(); - scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize); + scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes); gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt index 72a84ded15b..5c435d69742 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt @@ -102,13 +102,13 @@ public class GCInfo( }, info.memoryUsageBefore.mapValues { (_, v) -> MemoryUsage( - v.objectsCount, + 0L, v.totalObjectsSizeBytes, ) }, info.memoryUsageAfter.mapValues { (_, v) -> MemoryUsage( - v.objectsCount, + 0L, v.totalObjectsSizeBytes, ) } diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt index 189ecf8886b..1ea9764c492 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt @@ -16,7 +16,6 @@ import kotlin.system.* /** * This class represents statistics of memory usage in one memory pool. * - * @property objectsCount The number of allocated objects. * @property totalObjectsSizeBytes The total size of allocated objects. System allocator overhead is not included, * so it can not perfectly match the value received by os tools. * All alignment and auxiliary object headers are included. @@ -24,10 +23,22 @@ import kotlin.system.* @NativeRuntimeApi @SinceKotlin("1.9") public class MemoryUsage( - val objectsCount: Long, val totalObjectsSizeBytes: Long, ) +/** + * This class represents statistics of sweeping in one memory pool. + * + * @property sweptCount The of objects that were freed. + * @property keptCount The number of objects that were processed but kept alive. + */ +@NativeRuntimeApi +@SinceKotlin("1.9") +public class SweepStatistics( + val sweptCount: Long, + val keptCount: Long, +) + /** * This class represents statistics of the root set for garbage collector run, separated by root set pools. * These nodes are assumed to be used, even if there are no references for them. @@ -64,12 +75,16 @@ public class RootSetStatistics( * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos]. * If null, memory reclamation is still in progress. * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details. + * @property markedCount How many objects were processed during marking phase. + * @property sweepStatistics Sweeping statistics separated by memory pools. + * The set of memory pools depends on the collector implementation. + * Can be empty, if collection is in progress. * @property memoryUsageAfter Memory usage at the start of garbage collector run, separated by memory pools. * The set of memory pools depends on the collector implementation. - * Can be empty, of colelction is in progress. + * Can be empty, if collection is in progress. * @property memoryUsageBefore Memory usage at the end of garbage collector run, separated by memory pools. * The set of memory pools depends on the collector implementation. - * Can be empty, of colelction is in progress. + * Can be empty, if collection is in progress. */ @NativeRuntimeApi @SinceKotlin("1.9") @@ -81,6 +96,8 @@ public class GCInfo( val pauseEndTimeNs: Long, val postGcCleanupTimeNs: Long?, val rootSet: RootSetStatistics, + val markedCount: Long, + val sweepStatistics: Map, val memoryUsageBefore: Map, val memoryUsageAfter: Map, ) { @@ -102,6 +119,8 @@ private class GCInfoBuilder() { var pauseEndTimeNs: Long? = null var postGcCleanupTimeNs: Long? = null var rootSet: RootSetStatistics? = null + var markedCount: Long? = null + var sweepStatistics = mutableMapOf() var memoryUsageBefore = mutableMapOf() var memoryUsageAfter = mutableMapOf() @@ -140,17 +159,29 @@ private class GCInfoBuilder() { rootSet = RootSetStatistics(threadLocalReferences, stackReferences, globalReferences, stableReferences) } - @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore") - private fun setMemoryUsageBefore(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) { + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMarkStats") + private fun setMarkStats(markedCount: Long) { + this.markedCount = markedCount + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSweepStats") + private fun setSweepStats(name: NativePtr, keptCount: Long, sweptCount: Long) { val nameString = interpretCPointer(name)!!.toKString() - val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize) + val stats = SweepStatistics(sweptCount, keptCount) + sweepStatistics[nameString] = stats + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore") + private fun setMemoryUsageBefore(name: NativePtr, totalObjectsSize: Long) { + val nameString = interpretCPointer(name)!!.toKString() + val memoryUsage = MemoryUsage(totalObjectsSize) memoryUsageBefore[nameString] = memoryUsage } @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter") - private fun setMemoryUsageAfter(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) { + private fun setMemoryUsageAfter(name: NativePtr, totalObjectsSize: Long) { val nameString = interpretCPointer(name)!!.toKString() - val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize) + val memoryUsage = MemoryUsage(totalObjectsSize) memoryUsageAfter[nameString] = memoryUsage } @@ -163,6 +194,8 @@ private class GCInfoBuilder() { pauseEndTimeNs ?: return null, postGcCleanupTimeNs, rootSet ?: return null, + markedCount ?: return null, + sweepStatistics.toMap(), memoryUsageBefore.toMap(), memoryUsageAfter.toMap() ) diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp index 5cc3c00cd3c..50079455565 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp @@ -50,7 +50,6 @@ public: void ClearForTests() noexcept { extraObjects_.ClearForTests(); } size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); } - size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); } // requires LockForIter void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index ba66c436abb..0f316a21ae5 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -390,7 +390,6 @@ public: Iterable LockForIter() noexcept { return Iterable(*this); } size_t GetSizeUnsafe() const noexcept { return size_; } - size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; } void ClearForTests() { root_.reset(); @@ -458,7 +457,7 @@ class ObjectFactory : private Pinned { size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); } - + static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped // at about half of uint64_t max. @@ -466,7 +465,7 @@ class ObjectFactory : private Pinned { // Note: array body is aligned, but for size computation it is enough to align the sum. return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); } - + struct DataSizeProvider { static size_t GetDataSize(void* data) noexcept { ObjHeader* object = &static_cast(data)->object; @@ -701,9 +700,6 @@ public: // Lock ObjectFactory for safe iteration. Iterable LockForIter() noexcept { return Iterable(*this); } - size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); } - size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); } - void ClearForTests() { storage_.ClearForTests(); } static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index 952baeb567a..de869abd725 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -249,6 +249,12 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz, KLong threadLocalReferences, KLong stackReferences, KLong globalReferences, KLong stableReferences) { throw std::runtime_error("Not implemented for tests"); } +void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount) { + throw std::runtime_error("Not implemented for tests"); +} +void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount) { + throw std::runtime_error("Not implemented for tests"); +} void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize) { throw std::runtime_error("Not implemented for tests"); }