azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Track object counts during sweep ^KT-55364

Browse Source 
Previously object count was tracked via allocator. It adds additional
burden on every allocation. Tracking via sweeper is better because
it mostly happens on the GC thread during a concurrent sweep.
This commit is contained in:
Alexander Shabalin
2023-01-30 13:56:42 +01:00
committed by Space Cloud
parent 63231f7b73
commit 9fd05632f0
34 changed files with 326 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/backend.native/tests/runtime/memory/gcStats.kt
+3 -5
View File
@@ -12,13 +12,11 @@ import kotlin.test.*
@Test
fun `nothing new collected`() {
GC.collect()
GC.collect();
GC.collect()
val stat = GC.lastGCInfo
assertNotNull(stat)
assertEquals(stat.memoryUsageBefore.keys, stat.memoryUsageAfter.keys)
for (key in stat.memoryUsageBefore.keys) {
assertEquals(stat.memoryUsageBefore[key]!!.objectsCount, stat.memoryUsageAfter[key]!!.objectsCount)
assertEquals(stat.memoryUsageBefore[key]!!.totalObjectsSizeBytes, stat.memoryUsageAfter[key]!!.totalObjectsSizeBytes)
for (key in stat.sweepStatistics.keys) {
assertEquals(stat.sweepStatistics[key]!!.sweptCount, 0L)
}
}
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp
+18 -9
View File
@@ -45,7 +45,7 @@ mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept {
return freeBlock->Data();
}
bool ExtraObjectPage::Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
CustomAllocInfo("ExtraObjectPage(%p)::Sweep()", this);
// `end` is after the last legal allocation of a block, but does not
// necessarily match an actual block starting point.
@@ -58,15 +58,24 @@ bool ExtraObjectPage::Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexce
nextFree = &cell->next_;
continue;
}
// If the current cell was marked, it's alive, and the whole page is alive.
if (!SweepExtraObject(cell, finalizerQueue)) {
alive = true;
continue;
auto status = SweepExtraObject(cell, finalizerQueue);
switch (status) {
case ExtraObjectStatus::TO_BE_FINALIZED:
sweepHandle.addMarkedObject();
// fallthrough
case ExtraObjectStatus::KEPT:
// If the current cell was marked, it's alive, and the whole page is alive.
alive = true;
sweepHandle.addKeptObject();
break;
case ExtraObjectStatus::SWEPT:
// Free the current block and insert it into the free list.
cell->next_ = *nextFree;
*nextFree = cell;
nextFree = &cell->next_;
sweepHandle.addSweptObject();
break;
}
// Free the current block and insert it into the free list.
cell->next_ = *nextFree;
*nextFree = cell;
nextFree = &cell->next_;
}
return alive;
}
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp
+2 -1
View File
@@ -11,6 +11,7 @@
#include "AtomicStack.hpp"
#include "ExtraObjectData.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -34,7 +35,7 @@ public:
// Tries to allocate in current page, returns null if no free block in page
mm::ExtraObjectData* TryAllocate() noexcept;
bool Sweep(AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
bool Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
private:
friend class AtomicStack<ExtraObjectPage>;
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp
+6 -2
View File
@@ -41,7 +41,9 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
Page* page = Page::Create();
Queue finalizerQueue;
EXPECT_FALSE(page->Sweep(finalizerQueue));
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweepExtraObjects();
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
EXPECT_EQ(finalizerQueue.size(), size_t(0));
page->Destroy();
}
@@ -56,7 +58,9 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
}
EXPECT_EQ(count, EXTRA_OBJECT_COUNT);
Queue finalizerQueue;
EXPECT_FALSE(page->Sweep(finalizerQueue));
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweepExtraObjects();
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
EXPECT_EQ(finalizerQueue.size(), size_t(0));
page->Destroy();
}
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp
+3 -1
View File
@@ -46,7 +46,7 @@ uint8_t* FixedBlockPage::TryAllocate() noexcept {
return freeBlock->data;
}
bool FixedBlockPage::Sweep() noexcept {
bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
// `end` is after the last legal allocation of a block, but does not
// necessarily match an actual block starting point.
@@ -62,6 +62,7 @@ bool FixedBlockPage::Sweep() noexcept {
// If the current cell was marked, it's alive, and the whole page is alive.
if (TryResetMark(cell)) {
alive = true;
sweepHandle.addKeptObject();
continue;
}
CustomAllocInfo("FixedBlockPage(%p)::Sweep: reclaim %p", this, cell);
@@ -69,6 +70,7 @@ bool FixedBlockPage::Sweep() noexcept {
cell->nextFree = *nextFree;
*nextFree = cell;
nextFree = &cell->nextFree;
sweepHandle.addSweptObject();
}
return alive;
}
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp
+2 -1
View File
@@ -10,6 +10,7 @@
#include <cstdint>
#include "AtomicStack.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -30,7 +31,7 @@ public:
// Tries to allocate in current page, returns null if no free block in page
uint8_t* TryAllocate() noexcept;
bool Sweep() noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
private:
friend class AtomicStack<FixedBlockPage>;
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp
+15 -5
View File
@@ -44,52 +44,62 @@ TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) {
}
TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
EXPECT_FALSE(page->Sweep());
EXPECT_FALSE(page->Sweep(gcScope));
page->Destroy();
}
}
TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint32_t count = 0;
while (alloc(page, size)) ++count;
EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size);
EXPECT_FALSE(page->Sweep());
EXPECT_FALSE(page->Sweep(gcScope));
page->Destroy();
}
}
TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr = alloc(page, size);
mark(ptr);
EXPECT_TRUE(page->Sweep());
EXPECT_TRUE(page->Sweep(gcScope));
page->Destroy();
}
}
TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr = alloc(page, size);
EXPECT_FALSE(page->Sweep());
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_EQ(alloc(page, size), ptr);
page->Destroy();
}
}
TEST(CustomAllocTest, FixedBlockPageSweepReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr;
for (int count = 0; (ptr = alloc(page, size)); ++count) {
if (count % 2 == 0) mark(ptr);
}
EXPECT_TRUE(page->Sweep());
EXPECT_TRUE(page->Sweep(gcScope));
uint32_t count = 0;
for (; (ptr = alloc(page, size)); ++count) {
if (count % 2 == 0) mark(ptr);
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+7 -7
View File
@@ -46,22 +46,22 @@ static bool IsAlive(ObjHeader* baseObject) noexcept {
return objectData.marked();
}
bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
auto* extraObject = extraObjectCell->Data();
if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
CustomAllocDebug("SweepIsCollectable(%p): already finalized", extraObject);
return true;
return ExtraObjectStatus::SWEPT;
}
auto* baseObject = extraObject->GetBaseObject();
RuntimeAssert(baseObject->heap(), "SweepIsCollectable on a non-heap object");
if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepIsCollectable(%p): already in finalizer queue, keep base object (%p) alive", extraObject, baseObject);
KeepAlive(baseObject);
return false;
return ExtraObjectStatus::TO_BE_FINALIZED;
}
if (IsAlive(baseObject)) {
CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject);
return false;
return ExtraObjectStatus::KEPT;
}
extraObject->ClearRegularWeakReferenceImpl();
if (extraObject->HasAssociatedObject()) {
@@ -69,18 +69,18 @@ bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectC
finalizerQueue.Push(extraObjectCell);
KeepAlive(baseObject);
CustomAllocDebug("SweepIsCollectable(%p): add to finalizerQueue", extraObject);
return false;
return ExtraObjectStatus::TO_BE_FINALIZED;
} else {
if (HasFinalizers(baseObject)) {
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
finalizerQueue.Push(extraObjectCell);
KeepAlive(baseObject);
CustomAllocDebug("SweepIsCollectable(%p): addings to finalizerQueue, keep base object (%p) alive", extraObject, baseObject);
return false;
return ExtraObjectStatus::TO_BE_FINALIZED;
}
extraObject->Uninstall();
CustomAllocDebug("SweepIsCollectable(%p): uninstalled extraObject", extraObject);
return true;
return ExtraObjectStatus::SWEPT;
}
}
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp
+7 -2
View File
@@ -18,8 +18,13 @@ namespace kotlin::alloc {
bool TryResetMark(void* ptr) noexcept;
// Returns true if swept successfully, i.e., if the extraobject can be reclaimed now.
bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
enum class ExtraObjectStatus {
TO_BE_FINALIZED,
KEPT,
SWEPT,
};
ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
void* SafeAlloc(uint64_t size) noexcept;
void Free(void* ptr, size_t size) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp
+7 -5
View File
@@ -40,21 +40,23 @@ void Heap::PrepareForGC() noexcept {
usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_));
}
void Heap::Sweep() noexcept {
void Heap::Sweep(gc::GCHandle gcHandle) noexcept {
auto sweepHandle = gcHandle.sweep();
CustomAllocDebug("Heap::Sweep()");
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
fixedBlockPages_[blockSize].Sweep();
fixedBlockPages_[blockSize].Sweep(sweepHandle);
}
nextFitPages_.Sweep();
singleObjectPages_.SweepAndFree();
nextFitPages_.Sweep(sweepHandle);
singleObjectPages_.SweepAndFree(sweepHandle);
}
AtomicStack<ExtraObjectCell> Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept {
auto sweepHandle = gcHandle.sweepExtraObjects();
CustomAllocDebug("Heap::SweepExtraObjects()");
AtomicStack<ExtraObjectCell> finalizerQueue;
ExtraObjectPage* page;
while ((page = usedExtraObjectPages_.Pop())) {
if (!page->Sweep(finalizerQueue)) {
if (!page->Sweep(sweepHandle, finalizerQueue)) {
CustomAllocInfo("SweepExtraObjects free(%p)", page);
page->Destroy();
} else {
kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp
+1 -1
View File
@@ -30,7 +30,7 @@ public:
// Sweep through all remaining pages, freeing those blocks where CanReclaim
// returns true. If multiple sweepers are active, each page will only be
// seen by one sweeper.
void Sweep() noexcept;
void Sweep(gc::GCHandle gcHandle) noexcept;
AtomicStack<ExtraObjectCell> SweepExtraObjects(gc::GCHandle gcHandle) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp
+4 -2
View File
@@ -36,7 +36,8 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) {
mark(obj); // to make the page survive a sweep
}
heap.PrepareForGC();
heap.Sweep();
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
heap.Sweep(gcHandle);
for (int blocks = MIN; blocks < MAX; ++blocks) {
EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks));
}
@@ -49,7 +50,8 @@ TEST(CustomAllocTest, HeapReuseNextFitPages) {
void* obj = page->TryAllocate(BLOCKSIZE);
mark(obj); // to make the page survive a sweep
heap.PrepareForGC();
heap.Sweep();
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
heap.Sweep(gcHandle);
EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE));
}
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp
+3 -1
View File
@@ -39,7 +39,7 @@ uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept {
return curBlock_->TryAllocate(cellsNeeded);
}
bool NextFitPage::Sweep() noexcept {
bool NextFitPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
CustomAllocDebug("NextFitPage@%p::Sweep()", this);
Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT;
bool alive = false;
@@ -47,8 +47,10 @@ bool NextFitPage::Sweep() noexcept {
if (block->isAllocated_) {
if (TryResetMark(block->data_)) {
alive = true;
sweepHandle.addKeptObject();
} else {
block->Deallocate();
sweepHandle.addSweptObject();
}
}
}
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp
+2 -1
View File
@@ -11,6 +11,7 @@
#include "AtomicStack.hpp"
#include "Cell.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -23,7 +24,7 @@ public:
// Tries to allocate in current page, returns null if no free block in page is big enough
uint8_t* TryAllocate(uint32_t blockSize) noexcept;
bool Sweep() noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
// Testing method
bool CheckInvariants() noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp
+18 -6
View File
@@ -52,7 +52,9 @@ TEST(CustomAllocTest, NextFitPageAlloc) {
TEST(CustomAllocTest, NextFitPageSweepEmptyPage) {
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
EXPECT_FALSE(page->Sweep());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
page->Destroy();
}
@@ -61,7 +63,9 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {}
EXPECT_FALSE(page->Sweep());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
page->Destroy();
}
}
@@ -69,17 +73,21 @@ TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) {
TEST(CustomAllocTest, NextFitPageSweepSingleMarked) {
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
mark(alloc(page, MIN_BLOCK_SIZE));
EXPECT_TRUE(page->Sweep());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_TRUE(page->Sweep(gcScope));
page->Destroy();
}
TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
int count1 = 0;
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1;
EXPECT_FALSE(page->Sweep());
EXPECT_FALSE(page->Sweep(gcScope));
r.seed(seed);
int count2 = 0;
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count2;
@@ -89,6 +97,8 @@ TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
}
TEST(CustomAllocTest, NextFitPageSweepReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
@@ -102,7 +112,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
++unmarked;
}
}
page->Sweep();
page->Sweep(gcScope);
int freed = 0;
while (alloc(page, MIN_BLOCK_SIZE)) ++freed;
EXPECT_EQ(freed, unmarked);
@@ -111,9 +121,11 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
}
TEST(CustomAllocTest, NextFitPageSweepCoallesce) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1));
EXPECT_FALSE(page->Sweep());
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1));
page->Destroy();
}
kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp
+19 -11
View File
@@ -9,6 +9,7 @@
#include <atomic>
#include "AtomicStack.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -22,14 +23,15 @@ public:
while ((page = empty_.Pop())) page->Destroy();
}
void Sweep() noexcept {
while (SweepSingle(unswept_, ready_)) {}
void Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_)) {
}
}
void SweepAndFree() noexcept {
void SweepAndFree(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
T* page;
while ((page = unswept_.Pop())) {
if (page->Sweep()) {
if (page->Sweep(sweepHandle)) {
ready_.Push(page);
} else {
page->Destroy();
@@ -39,8 +41,12 @@ public:
T* GetPage(uint32_t cellCount) noexcept {
T* page;
if ((page = SweepSingle(unswept_, used_))) {
return page;
if ((page = unswept_.Pop())) {
// If there're unswept_ pages, the GC is in progress.
auto sweepHandle = gc::GCHandle::currentEpoch()->sweep();
if ((page = SweepSingle(sweepHandle, page, unswept_, used_))) {
return page;
}
}
if ((page = ready_.Pop())) {
used_.Push(page);
@@ -68,15 +74,17 @@ public:
}
private:
T* SweepSingle(AtomicStack<T>& from, AtomicStack<T>& to) noexcept {
T* page;
while ((page = from.Pop())) {
if (page->Sweep()) {
T* SweepSingle(gc::GCHandle::GCSweepScope& sweepHandle, T* page, AtomicStack<T>& from, AtomicStack<T>& to) noexcept {
if (!page) {
return nullptr;
}
do {
if (page->Sweep(sweepHandle)) {
to.Push(page);
return page;
}
empty_.Push(page);
}
} while ((page = from.Pop()));
return nullptr;
}
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp
+3 -1
View File
@@ -37,12 +37,14 @@ uint8_t* SingleObjectPage::TryAllocate() noexcept {
return Data();
}
bool SingleObjectPage::Sweep() noexcept {
bool SingleObjectPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
CustomAllocDebug("SingleObjectPage@%p::Sweep()", this);
if (!TryResetMark(Data())) {
isAllocated_ = false;
sweepHandle.addKeptObject();
return false;
}
sweepHandle.addSweptObject();
return true;
}
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp
+2 -1
View File
@@ -10,6 +10,7 @@
#include <cstdint>
#include "AtomicStack.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -23,7 +24,7 @@ public:
uint8_t* TryAllocate() noexcept;
bool Sweep() noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
private:
friend class AtomicStack<SingleObjectPage>;
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp
+6 -2
View File
@@ -34,7 +34,9 @@ SingleObjectPage* alloc(uint64_t blockSize) {
TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) {
SingleObjectPage* page = alloc(MIN_BLOCK_SIZE);
EXPECT_TRUE(page);
EXPECT_FALSE(page->Sweep());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
page->Destroy();
}
@@ -43,7 +45,9 @@ TEST(CustomAllocTest, SingleObjectPageSweepFullPage) {
EXPECT_TRUE(page);
EXPECT_TRUE(page->Data());
mark(page->Data());
EXPECT_TRUE(page->Sweep());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_TRUE(page->Sweep(gcScope));
page->Destroy();
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+2 -2
View File
@@ -187,7 +187,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
mm::WaitForThreadsSuspension();
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
@@ -207,7 +207,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
mm::ResumeThreads();
gcHandle.threadsAreResumed();
heap_.Sweep();
heap_.Sweep(gcHandle);
#endif
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
-30
View File
@@ -95,36 +95,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#endif
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactory().GetObjectsCountUnsafe();
#else
return 0;
#endif
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
#else
return 0;
#endif
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
#ifndef CUSTOM_ALLOCATOR
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
#else
return 0;
#endif
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
#ifndef CUSTOM_ALLOCATOR
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
#else
return 0;
#endif
}
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
return allocatedBytes();
}
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
-4
View File
@@ -59,10 +59,6 @@ public:
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
size_t GetHeapObjectsCountUnsafe() const noexcept;
size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
size_t GetExtraObjectsCountUnsafe() const noexcept;
size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
size_t GetTotalHeapObjectsSizeBytes() const noexcept;
gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp
+105 -85
View File
@@ -26,26 +26,41 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu
void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz,
KLong threadLocalReferences, KLong stackReferences,
KLong globalReferences, KLong stableReferences);
void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize);
void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize);
void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount);
void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount);
void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong sizeBytes);
void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter(KRef thiz, KNativePtr name, KLong sizeBytes);
}
namespace {
constexpr KNativePtr heapPoolKey = const_cast<KNativePtr>(static_cast<const void*>("heap"));
constexpr KNativePtr extraPoolKey = const_cast<KNativePtr>(static_cast<const void*>("extra"));
struct MemoryUsage {
uint64_t sizeBytes;
};
struct MemoryUsageMap {
std::optional<kotlin::gc::MemoryUsage> heap;
std::optional<kotlin::gc::MemoryUsage> extra;
std::optional<MemoryUsage> heap;
void build(KRef builder, void (*add)(KRef, KNativePtr, KLong)) {
if (heap) {
add(builder, heapPoolKey, static_cast<KLong>(heap->sizeBytes));
}
}
};
struct SweepStatsMap {
std::optional<gc::SweepStats> heap;
std::optional<gc::SweepStats> extra;
void build(KRef builder, void (*add)(KRef, KNativePtr, KLong, KLong)) {
if (heap) {
add(builder, const_cast<KNativePtr>(static_cast<const void*>("heap")),
static_cast<KLong>(heap->objectsCount),
static_cast<KLong>(heap->totalObjectsSize));
add(builder, heapPoolKey, static_cast<KLong>(heap->keptCount), static_cast<KLong>(heap->sweptCount));
}
if (extra) {
add(builder, const_cast<KNativePtr>(static_cast<const void*>("extra")),
static_cast<KLong>(extra->objectsCount),
static_cast<KLong>(extra->totalObjectsSize));
add(builder, extraPoolKey, static_cast<KLong>(extra->keptCount), static_cast<KLong>(extra->sweptCount));
}
}
};
@@ -66,7 +81,8 @@ struct GCInfo {
std::optional<KLong> pauseEndTime;
std::optional<KLong> finalizersDoneTime;
std::optional<RootSetStatistics> rootSet;
std::optional<kotlin::gc::MemoryUsage> markStats;
std::optional<kotlin::gc::MarkStats> markStats;
SweepStatsMap sweepStats;
MemoryUsageMap memoryUsageBefore;
MemoryUsageMap memoryUsageAfter;
@@ -82,6 +98,9 @@ struct GCInfo {
Kotlin_Internal_GC_GCInfoBuilder_setRootSet(
builder, rootSet->threadLocalReferences, rootSet->stackReferences, rootSet->globalReferences,
rootSet->stableReferences);
if (markStats)
Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(builder, markStats->markedCount);
sweepStats.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setSweepStats);
memoryUsageBefore.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore);
memoryUsageAfter.build(builder, Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter);
}
@@ -98,6 +117,12 @@ GCInfo* statByEpoch(uint64_t epoch) {
return nullptr;
}
MemoryUsage currentHeapUsage() noexcept {
return MemoryUsage{
mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeBytes(),
};
}
} // namespace
extern "C" void Kotlin_Internal_GC_GCInfoBuilder_Fill(KRef builder, int id) {
@@ -132,12 +157,23 @@ GCHandle GCHandle::create(uint64_t epoch) {
} else {
GCLogInfo(epoch, "Started.");
}
current.memoryUsageBefore.heap = currentHeapUsage();
return getByEpoch(epoch);
}
GCHandle GCHandle::createFakeForTests() { return getByEpoch(std::numeric_limits<uint64_t>::max()); }
GCHandle GCHandle::getByEpoch(uint64_t epoch) {
return GCHandle{epoch};
}
// static
std::optional<gc::GCHandle> gc::GCHandle::currentEpoch() noexcept {
std::lock_guard guard(lock);
if (auto epoch = current.epoch) {
return GCHandle::getByEpoch(*epoch);
}
return std::nullopt;
}
void GCHandle::ClearForTests() {
std::lock_guard guard(lock);
current = {};
@@ -147,6 +183,7 @@ void GCHandle::finished() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->endTime = static_cast<KLong>(konan::getTimeNanos());
stat->memoryUsageAfter.heap = currentHeapUsage();
if (stat->rootSet) {
GCLogInfo(
epoch_,
@@ -160,20 +197,22 @@ void GCHandle::finished() {
stat->rootSet->stableReferences, stat->rootSet->total());
}
if (stat->markStats) {
GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->objectsCount);
GCLogInfo(epoch_, "Mark: %" PRIu64 " objects.", stat->markStats->markedCount);
}
if (stat->memoryUsageAfter.extra && stat->memoryUsageBefore.extra) {
if (auto stats = stat->sweepStats.extra) {
GCLogInfo(
epoch_, "Sweep extra objects: swept %" PRIu64 " objects, kept %" PRIu64 " objects",
stat->memoryUsageBefore.extra->objectsCount - stat->memoryUsageAfter.extra->objectsCount, stat->memoryUsageAfter.extra->objectsCount);
stats->sweptCount, stats->keptCount);
}
if (auto stats = stat->sweepStats.heap) {
GCLogInfo(
epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects", stats->sweptCount,
stats->keptCount);
}
if (stat->memoryUsageBefore.heap && stat->memoryUsageAfter.heap) {
GCLogInfo(
epoch_, "Sweep: swept %" PRIu64 " objects, kept %" PRIu64 " objects",
stat->memoryUsageBefore.heap->objectsCount - stat->memoryUsageAfter.heap->objectsCount, stat->memoryUsageAfter.heap->objectsCount);
GCLogInfo(
epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->totalObjectsSize,
stat->memoryUsageAfter.heap->totalObjectsSize);
epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->sizeBytes,
stat->memoryUsageAfter.heap->sizeBytes);
}
if (stat->pauseStartTime && stat->pauseEndTime) {
auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
@@ -184,18 +223,6 @@ void GCHandle::finished() {
GCLogInfo(epoch_, "Finished. Total GC epoch time is %" PRId64" microseconds.", time);
}
if (auto usage = stat->memoryUsageAfter.heap; usage && stat->markStats) {
RuntimeAssert(
stat->markStats->objectsCount == usage->objectsCount,
"Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " is alive after sweep",
stat->markStats->objectsCount, usage->objectsCount);
RuntimeAssert(
stat->markStats->totalObjectsSize == usage->totalObjectsSize,
"Mismatch in statistics: total marked size is %" PRId64 " bytes, while %" PRId64 " bytes is alive after sweep",
stat->markStats->totalObjectsSize, usage->totalObjectsSize);
}
if (stat == &current) {
last = current;
current = {};
@@ -218,6 +245,15 @@ void GCHandle::threadsAreSuspended() {
return;
}
}
if (last.epoch) {
// Assisted sweeping from the last epoch must be completed before the check can be run.
if (last.markStats && last.sweepStats.heap) {
RuntimeAssert(
last.markStats->markedCount == last.sweepStats.heap->keptCount,
"Mismatch in statistics: marked %" PRId64 " objects, while %" PRId64 " are alive after sweep",
last.markStats->markedCount, last.sweepStats.heap->keptCount);
}
}
}
void GCHandle::threadsAreResumed() {
std::lock_guard guard(lock);
@@ -267,87 +303,73 @@ void GCHandle::globalRootSetCollected(uint64_t globalReferences, uint64_t stable
}
}
void GCHandle::heapUsageBefore(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageBefore.heap = usage;
}
}
void GCHandle::marked(kotlin::gc::MemoryUsage usage) {
void GCHandle::marked(kotlin::gc::MarkStats stats) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (!stat->markStats) {
stat->markStats = MemoryUsage{0, 0};
stat->markStats = MarkStats{};
}
stat->markStats->totalObjectsSize += usage.totalObjectsSize;
stat->markStats->objectsCount += usage.objectsCount;
stat->markStats->markedSizeBytes += stats.markedSizeBytes;
stat->markStats->markedCount += stats.markedCount;
}
}
MemoryUsage GCHandle::getMarked() {
MarkStats GCHandle::getMarked() {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
if (stat->markStats) {
return *stat->markStats;
}
}
return MemoryUsage{0, 0};
return MarkStats{};
}
void GCHandle::heapUsageAfter(MemoryUsage usage) {
void GCHandle::swept(gc::SweepStats stats) noexcept {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageAfter.heap = usage;
auto& heap = stat->sweepStats.heap;
if (!heap) {
heap = gc::SweepStats{};
}
heap->keptCount += stats.keptCount;
heap->sweptCount += stats.sweptCount;
}
}
void GCHandle::extraObjectsUsageBefore(MemoryUsage usage) {
void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noexcept {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageBefore.extra = usage;
}
}
void GCHandle::extraObjectsUsageAfter(MemoryUsage usage) {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
stat->memoryUsageAfter.extra = usage;
auto& extra = stat->sweepStats.extra;
if (!extra) {
extra = gc::SweepStats{};
}
extra->keptCount += stats.keptCount;
extra->sweptCount += stats.sweptCount;
RuntimeAssert(static_cast<bool>(stat->markStats), "Mark must have already happened");
stat->markStats->markedCount += markedCount;
}
}
MemoryUsage GCHandle::GCSweepScope::getUsage() {
return MemoryUsage{
mm::GlobalData::Instance().gc().GetHeapObjectsCountUnsafe(),
mm::GlobalData::Instance().gc().GetTotalHeapObjectsSizeUnsafe(),
};
}
GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) :
handle_(handle) {
handle_.heapUsageBefore(getUsage());
}
GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
GCHandle::GCSweepScope::~GCSweepScope() {
GCLogDebug(handle_.getEpoch(), "Swept is done in %" PRIu64 " microseconds.", getStageTime());
handle_.heapUsageAfter(getUsage());
handle_.swept(stats_);
GCLogDebug(
handle_.getEpoch(),
"Collected %" PRId64 " heap objects in %" PRIu64 " microseconds. "
"%" PRId64 " heap objects are kept alive.",
stats_.sweptCount, getStageTime(), stats_.keptCount);
}
MemoryUsage GCHandle::GCSweepExtraObjectsScope::getUsage() {
return MemoryUsage{
mm::GlobalData::Instance().gc().GetExtraObjectsCountUnsafe(),
mm::GlobalData::Instance().gc().GetTotalExtraObjectsSizeUnsafe(),
};
}
GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) :
handle_(handle) {
handle_.extraObjectsUsageBefore(getUsage());
}
GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() {
GCLogDebug(handle_.getEpoch(), "Swept extra objects is done in %" PRIu64 " microseconds", getStageTime());
handle_.extraObjectsUsageAfter(getUsage());
handle_.sweptExtraObjects(stats_, markedCount_);
GCLogDebug(
handle_.getEpoch(),
"Collected %" PRId64 " extra objects in %" PRIu64 " microseconds. "
"%" PRId64 " extra objects are kept alive.",
stats_.sweptCount, getStageTime(), stats_.keptCount);
}
GCHandle::GCGlobalRootSetScope::GCGlobalRootSetScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
@@ -370,10 +392,8 @@ GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){
GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){}
GCHandle::GCMarkScope::~GCMarkScope() {
handle_.marked(MemoryUsage{objectsCount, totalObjectSizeBytes});
GCLogDebug(handle_.getEpoch(),
"Marked %" PRIu64 " objects in %" PRIu64 " microseconds",
objectsCount, getStageTime());
handle_.marked(stats_);
GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime());
}
gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {}
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp
+30 -16
View File
@@ -11,6 +11,7 @@
#include "Common.h"
#include "Porting.h"
#include "Utils.hpp"
#include "std_support/Optional.hpp"
#define GCLogInfo(epoch, format, ...) RuntimeLogInfo({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
#define GCLogDebug(epoch, format, ...) RuntimeLogDebug({kTagGC}, "Epoch #%" PRIu64 ": " format, epoch, ##__VA_ARGS__)
@@ -24,9 +25,14 @@ namespace kotlin::gc {
class GCHandle;
struct MemoryUsage {
uint64_t objectsCount;
uint64_t totalObjectsSize;
struct SweepStats {
uint64_t sweptCount = 0;
uint64_t keptCount = 0;
};
struct MarkStats {
uint64_t markedCount = 0;
uint64_t markedSizeBytes = 0;
};
class GCHandle {
@@ -39,20 +45,29 @@ public:
class GCSweepScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
MemoryUsage getUsage();
SweepStats stats_;
public:
explicit GCSweepScope(GCHandle& handle);
~GCSweepScope();
void addSweptObject() noexcept { stats_.sweptCount += 1; }
void addKeptObject() noexcept { stats_.keptCount += 1; }
};
class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
MemoryUsage getUsage();
SweepStats stats_;
uint64_t markedCount_ = 0;
public:
explicit GCSweepExtraObjectsScope(GCHandle& handle);
~GCSweepExtraObjectsScope();
void addSweptObject() noexcept { stats_.sweptCount += 1; }
void addKeptObject() noexcept { stats_.keptCount += 1; }
// Custom allocator only. To be finalized objects are kept alive.
void addMarkedObject() noexcept { markedCount_ += 1; }
};
class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned {
@@ -82,15 +97,15 @@ public:
class GCMarkScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
uint64_t objectsCount = 0;
uint64_t totalObjectSizeBytes = 0;
MarkStats stats_;
public:
explicit GCMarkScope(GCHandle& handle);
~GCMarkScope();
void addObject(uint64_t objectSize) {
totalObjectSizeBytes += objectSize;
objectsCount++;
void addObject(uint64_t objectSize) noexcept {
++stats_.markedCount;
stats_.markedSizeBytes += objectSize;
}
};
@@ -115,16 +130,15 @@ private:
void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences);
void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences);
void heapUsageBefore(MemoryUsage usage);
void heapUsageAfter(MemoryUsage usage);
void extraObjectsUsageBefore(MemoryUsage usage);
void extraObjectsUsageAfter(MemoryUsage usage);
void marked(MemoryUsage usage);
void swept(SweepStats stats) noexcept;
void sweptExtraObjects(SweepStats stats, uint64_t markedCount) noexcept;
void marked(MarkStats stats);
public:
static GCHandle create(uint64_t epoch);
static GCHandle createFakeForTests();
static GCHandle getByEpoch(uint64_t epoch);
static std::optional<GCHandle> currentEpoch() noexcept;
static void ClearForTests();
uint64_t getEpoch() { return epoch_; }
@@ -141,6 +155,6 @@ public:
GCMarkScope mark() { return GCMarkScope(*this); }
GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); }
MemoryUsage getMarked();
MarkStats getMarked();
};
}
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+5
View File
@@ -118,12 +118,15 @@ void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::It
if (extraObject.HasAssociatedObject()) {
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
++it;
sweepHandle.addKeptObject();
} else {
extraObject.Uninstall();
it.EraseAndAdvance();
sweepHandle.addSweptObject();
}
} else {
++it;
sweepHandle.addKeptObject();
}
}
}
@@ -142,8 +145,10 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T
for (auto it = objectFactoryIter.begin(); it != objectFactoryIter.end();) {
if (Traits::TryResetMark(*it)) {
++it;
sweepHandle.addKeptObject();
continue;
}
sweepHandle.addSweptObject();
auto* objHeader = it->GetObjHeader();
if (HasFinalizers(objHeader)) {
objectFactoryIter.MoveAndAdvance(finalizerQueue, it);
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp
+3 -3
View File
@@ -140,7 +140,7 @@ public:
return testing::UnorderedElementsAreArray(objects);
}
gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<test_support::Any>> graySet) {
gc::MarkStats Mark(std::initializer_list<std::reference_wrapper<test_support::Any>> graySet) {
std_support::vector<ObjHeader*> objects;
for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().header());
auto handle = gc::GCHandle::create(epoch_++);
@@ -168,8 +168,8 @@ size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<test_support:
#define EXPECT_MARKED(stats, ...) \
do { \
std::initializer_list<std::reference_wrapper<test_support::Any>> objects = {__VA_ARGS__}; \
EXPECT_THAT(stats.objectsCount, objects.size()); \
EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \
EXPECT_THAT(stats.markedCount, objects.size()); \
EXPECT_THAT(stats.markedSizeBytes, GetObjectsSize(objects)); \
EXPECT_THAT(marked(), MarkedMatcher(objects)); \
} while (false)
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
-13
View File
@@ -67,19 +67,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
return impl_->objectFactory().GetObjectsCountUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
return allocatedBytes();
}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
-13
View File
@@ -80,19 +80,6 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
}
size_t gc::GC::GetHeapObjectsCountUnsafe() const noexcept {
return impl_->objectFactory().GetObjectsCountUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeUnsafe() const noexcept {
return impl_->objectFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetExtraObjectsCountUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();
}
size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
}
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
return allocatedBytes();
}
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+1 -1
View File
@@ -137,7 +137,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
auto markStats = gcHandle.getMarked();
scheduler.gcData().UpdateAliveSetBytes(markStats.totalObjectsSize);
scheduler.gcData().UpdateAliveSetBytes(markStats.markedSizeBytes);
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt
+2 -2
View File
@@ -102,13 +102,13 @@ public class GCInfo(
},
info.memoryUsageBefore.mapValues { (_, v) ->
MemoryUsage(
v.objectsCount,
0L,
v.totalObjectsSizeBytes,
)
},
info.memoryUsageAfter.mapValues { (_, v) ->
MemoryUsage(
v.objectsCount,
0L,
v.totalObjectsSizeBytes,
)
}
kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt
+42 -9
View File
@@ -16,7 +16,6 @@ import kotlin.system.*
/**
* This class represents statistics of memory usage in one memory pool.
*
* @property objectsCount The number of allocated objects.
* @property totalObjectsSizeBytes The total size of allocated objects. System allocator overhead is not included,
* so it can not perfectly match the value received by os tools.
* All alignment and auxiliary object headers are included.
@@ -24,10 +23,22 @@ import kotlin.system.*
@NativeRuntimeApi
@SinceKotlin("1.9")
public class MemoryUsage(
val objectsCount: Long,
val totalObjectsSizeBytes: Long,
)
/**
* This class represents statistics of sweeping in one memory pool.
*
* @property sweptCount The of objects that were freed.
* @property keptCount The number of objects that were processed but kept alive.
*/
@NativeRuntimeApi
@SinceKotlin("1.9")
public class SweepStatistics(
val sweptCount: Long,
val keptCount: Long,
)
/**
* This class represents statistics of the root set for garbage collector run, separated by root set pools.
* These nodes are assumed to be used, even if there are no references for them.
@@ -64,12 +75,16 @@ public class RootSetStatistics(
* @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos].
* If null, memory reclamation is still in progress.
* @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details.
* @property markedCount How many objects were processed during marking phase.
* @property sweepStatistics Sweeping statistics separated by memory pools.
* The set of memory pools depends on the collector implementation.
* Can be empty, if collection is in progress.
* @property memoryUsageAfter Memory usage at the start of garbage collector run, separated by memory pools.
* The set of memory pools depends on the collector implementation.
* Can be empty, of colelction is in progress.
* Can be empty, if collection is in progress.
* @property memoryUsageBefore Memory usage at the end of garbage collector run, separated by memory pools.
* The set of memory pools depends on the collector implementation.
* Can be empty, of colelction is in progress.
* Can be empty, if collection is in progress.
*/
@NativeRuntimeApi
@SinceKotlin("1.9")
@@ -81,6 +96,8 @@ public class GCInfo(
val pauseEndTimeNs: Long,
val postGcCleanupTimeNs: Long?,
val rootSet: RootSetStatistics,
val markedCount: Long,
val sweepStatistics: Map<String, SweepStatistics>,
val memoryUsageBefore: Map<String, MemoryUsage>,
val memoryUsageAfter: Map<String, MemoryUsage>,
) {
@@ -102,6 +119,8 @@ private class GCInfoBuilder() {
var pauseEndTimeNs: Long? = null
var postGcCleanupTimeNs: Long? = null
var rootSet: RootSetStatistics? = null
var markedCount: Long? = null
var sweepStatistics = mutableMapOf<String, SweepStatistics>()
var memoryUsageBefore = mutableMapOf<String, MemoryUsage>()
var memoryUsageAfter = mutableMapOf<String, MemoryUsage>()
@@ -140,17 +159,29 @@ private class GCInfoBuilder() {
rootSet = RootSetStatistics(threadLocalReferences, stackReferences, globalReferences, stableReferences)
}
@ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore")
private fun setMemoryUsageBefore(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) {
@ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMarkStats")
private fun setMarkStats(markedCount: Long) {
this.markedCount = markedCount
}
@ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSweepStats")
private fun setSweepStats(name: NativePtr, keptCount: Long, sweptCount: Long) {
val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize)
val stats = SweepStatistics(sweptCount, keptCount)
sweepStatistics[nameString] = stats
}
@ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore")
private fun setMemoryUsageBefore(name: NativePtr, totalObjectsSize: Long) {
val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
val memoryUsage = MemoryUsage(totalObjectsSize)
memoryUsageBefore[nameString] = memoryUsage
}
@ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageAfter")
private fun setMemoryUsageAfter(name: NativePtr, objectsCount: Long, totalObjectsSize: Long) {
private fun setMemoryUsageAfter(name: NativePtr, totalObjectsSize: Long) {
val nameString = interpretCPointer<ByteVar>(name)!!.toKString()
val memoryUsage = MemoryUsage(objectsCount, totalObjectsSize)
val memoryUsage = MemoryUsage(totalObjectsSize)
memoryUsageAfter[nameString] = memoryUsage
}
@@ -163,6 +194,8 @@ private class GCInfoBuilder() {
pauseEndTimeNs ?: return null,
postGcCleanupTimeNs,
rootSet ?: return null,
markedCount ?: return null,
sweepStatistics.toMap(),
memoryUsageBefore.toMap(),
memoryUsageAfter.toMap()
)
kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp
-1
View File
@@ -50,7 +50,6 @@ public:
void ClearForTests() noexcept { extraObjects_.ClearForTests(); }
size_t GetSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe(); }
size_t GetTotalObjectsSizeUnsafe() noexcept { return extraObjects_.GetSizeUnsafe() * sizeof(ExtraObjectData); }
// requires LockForIter
void EraseAndAdvance(Iterator &it) { extraObjects_.EraseAndAdvance(it); }
kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp
+2 -6
View File
@@ -390,7 +390,6 @@ public:
Iterable LockForIter() noexcept { return Iterable(*this); }
size_t GetSizeUnsafe() const noexcept { return size_; }
size_t GetTotalObjectsSizeUnsafe() const noexcept { return totalObjectsSizeBytes_; }
void ClearForTests() {
root_.reset();
@@ -458,7 +457,7 @@ class ObjectFactory : private Pinned {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
}
static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
@@ -466,7 +465,7 @@ class ObjectFactory : private Pinned {
// Note: array body is aligned, but for size computation it is enough to align the sum.
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
}
struct DataSizeProvider {
static size_t GetDataSize(void* data) noexcept {
ObjHeader* object = &static_cast<HeapObjHeader*>(data)->object;
@@ -701,9 +700,6 @@ public:
// Lock ObjectFactory for safe iteration.
Iterable LockForIter() noexcept { return Iterable(*this); }
size_t GetObjectsCountUnsafe() const noexcept { return storage_.GetSizeUnsafe(); }
size_t GetTotalObjectsSizeUnsafe() const noexcept { return storage_.GetTotalObjectsSizeUnsafe(); }
void ClearForTests() { storage_.ClearForTests(); }
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept {
kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp
+6
View File
@@ -249,6 +249,12 @@ void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong valu
void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz, KLong threadLocalReferences, KLong stackReferences, KLong globalReferences, KLong stableReferences) {
throw std::runtime_error("Not implemented for tests");
}
void Kotlin_Internal_GC_GCInfoBuilder_setMarkStats(KRef thiz, KLong markedCount) {
throw std::runtime_error("Not implemented for tests");
}
void Kotlin_Internal_GC_GCInfoBuilder_setSweepStats(KRef thiz, KNativePtr name, KLong sweptCount, KLong keptCount) {
throw std::runtime_error("Not implemented for tests");
}
void Kotlin_Internal_GC_GCInfoBuilder_setMemoryUsageBefore(KRef thiz, KNativePtr name, KLong objectsCount, KLong totalObjectsSize) {
throw std::runtime_error("Not implemented for tests");
}