From 918a247bcce0828a793a1e9ea5ed5be274eae6ca Mon Sep 17 00:00:00 2001 From: Troels Bjerre Lund Date: Tue, 9 May 2023 16:37:58 +0000 Subject: [PATCH] [K/N] custom_alloc: sweep extraobjects outside stw ^KT-55364 Co-authored-by: Troels Lund Merge-request: KOTLIN-MR-677 Merged-by: Alexander Shabalin --- .../src/custom_alloc/cpp/CustomAllocator.cpp | 29 +++--- .../src/custom_alloc/cpp/CustomAllocator.hpp | 3 + .../custom_alloc/cpp/CustomAllocatorTest.cpp | 10 +-- .../cpp/CustomFinalizerProcessor.cpp | 1 - .../src/custom_alloc/cpp/CustomLogging.hpp | 9 +- .../src/custom_alloc/cpp/ExtraObjectPage.cpp | 32 +++---- .../src/custom_alloc/cpp/ExtraObjectPage.hpp | 15 +++- .../custom_alloc/cpp/ExtraObjectPageTest.cpp | 8 +- .../src/custom_alloc/cpp/FixedBlockPage.cpp | 6 +- .../src/custom_alloc/cpp/FixedBlockPage.hpp | 7 +- .../custom_alloc/cpp/FixedBlockPageTest.cpp | 62 +++++++++++-- .../runtime/src/custom_alloc/cpp/GCApi.cpp | 90 +++++++------------ .../runtime/src/custom_alloc/cpp/GCApi.hpp | 25 ++++-- .../runtime/src/custom_alloc/cpp/Heap.cpp | 63 +++++-------- .../runtime/src/custom_alloc/cpp/Heap.hpp | 17 ++-- .../runtime/src/custom_alloc/cpp/HeapTest.cpp | 11 ++- .../src/custom_alloc/cpp/NextFitPage.cpp | 4 +- .../src/custom_alloc/cpp/NextFitPage.hpp | 7 +- .../src/custom_alloc/cpp/NextFitPageTest.cpp | 25 ++++-- .../src/custom_alloc/cpp/PageStore.hpp | 21 +++-- .../src/custom_alloc/cpp/SingleObjectPage.cpp | 12 ++- .../src/custom_alloc/cpp/SingleObjectPage.hpp | 7 +- .../custom_alloc/cpp/SingleObjectPageTest.cpp | 9 +- .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 6 +- .../src/gc/common/cpp/GCStatistics.cpp | 12 +-- .../src/gc/common/cpp/GCStatistics.hpp | 10 +-- .../runtime/src/mm/cpp/ExtraObjectData.cpp | 2 +- .../runtime/src/mm/cpp/ExtraObjectData.hpp | 2 +- kotlin-native/runtime/src/mm/cpp/Memory.cpp | 4 +- 29 files changed, 279 insertions(+), 230 deletions(-) diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp index e6dc7c7b731..319910fcc69 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp @@ -22,24 +22,11 @@ #include "Memory.h" #include "FixedBlockPage.hpp" #include "GCImpl.hpp" +#include "GCApi.hpp" #include "TypeInfo.h" namespace kotlin::alloc { -using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData; - -struct HeapObjHeader { - ObjectData gcData; - alignas(kObjectAlignment) ObjHeader object; -}; - -// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible -// with `ObjHeader`: the former can always be casted to the other. -struct HeapArrayHeader { - ObjectData gcData; - alignas(kObjectAlignment) ArrayHeader array; -}; - size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); @@ -67,6 +54,8 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { if (typeInfo->flags_ & TF_HAS_FINALIZER) { auto* extraObject = CreateExtraObject(); object->typeInfoOrMeta_ = reinterpret_cast(new (extraObject) mm::ExtraObjectData(object, typeInfo)); + CustomAllocDebug("CustomAllocator: %p gets extraObject %p", object, extraObject); + CustomAllocDebug("CustomAllocator: %p->BaseObject == %p", extraObject, extraObject->GetBaseObject()); } else { object->typeInfoOrMeta_ = const_cast(typeInfo); } @@ -94,7 +83,7 @@ mm::ExtraObjectData* CustomAllocator::CreateExtraObject() noexcept { } } CustomAllocDebug("Failed to allocate in current ExtraObjectPage"); - while ((page = heap_.GetExtraObjectPage())) { + while ((page = heap_.GetExtraObjectPage(finalizerQueue_))) { mm::ExtraObjectData* block = page->TryAllocate(); if (block) { extraObjectPage_ = page; @@ -112,6 +101,10 @@ mm::ExtraObjectData& CustomAllocator::CreateExtraObjectDataForObject( return *new (extraObject) mm::ExtraObjectData(baseObject, info); } +FinalizerQueue CustomAllocator::ExtractFinalizerQueue() noexcept { + return std::move(finalizerQueue_); +} + void CustomAllocator::PrepareForGC() noexcept { CustomAllocInfo("CustomAllocator@%p::PrepareForGC()", this); nextFitPage_ = nullptr; @@ -148,7 +141,7 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept { uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept { CustomAllocDebug("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount); - uint8_t* block = heap_.GetSingleObjectPage(cellCount)->TryAllocate(); + uint8_t* block = heap_.GetSingleObjectPage(cellCount, finalizerQueue_)->TryAllocate(); return block; } @@ -160,7 +153,7 @@ uint8_t* CustomAllocator::AllocateInNextFitPage(uint32_t cellCount) noexcept { } CustomAllocDebug("Failed to allocate in curPage"); while (true) { - nextFitPage_ = heap_.GetNextFitPage(cellCount); + nextFitPage_ = heap_.GetNextFitPage(cellCount, finalizerQueue_); uint8_t* block = nextFitPage_->TryAllocate(cellCount); if (block) return block; } @@ -174,7 +167,7 @@ uint8_t* CustomAllocator::AllocateInFixedBlockPage(uint32_t cellCount) noexcept if (block) return block; } CustomAllocDebug("Failed to allocate in current FixedBlockPage"); - while ((page = heap_.GetFixedBlockPage(cellCount))) { + while ((page = heap_.GetFixedBlockPage(cellCount, finalizerQueue_))) { uint8_t* block = page->TryAllocate(); if (block) { fixedBlockPages_[cellCount] = page; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp index cd0a73673a4..807ab7b24f8 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp @@ -34,6 +34,8 @@ public: void PrepareForGC() noexcept; + FinalizerQueue ExtractFinalizerQueue() noexcept; + static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept; private: @@ -47,6 +49,7 @@ private: NextFitPage* nextFitPage_; FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; ExtraObjectPage* extraObjectPage_; + FinalizerQueue finalizerQueue_; }; } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp index c9e402e0996..aa386fba3e9 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp @@ -25,7 +25,7 @@ TEST(CustomAllocTest, SmallAllocNonNull) { const int N = 200; TypeInfo fakeTypes[N]; for (int i = 1; i < N; ++i) { - fakeTypes[i] = {.instanceSize_ = 8 * i, .flags_ = 0}; + fakeTypes[i] = {.typeInfo_ = &fakeTypes[i], .instanceSize_ = 8 * i, .flags_ = 0}; } Heap heap; kotlin::gc::GCSchedulerConfig config; @@ -46,7 +46,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) { kotlin::gc::GCSchedulerConfig config; kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {}); CustomAllocator ca(heap, schedulerData); - TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; + TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; uint8_t* first = reinterpret_cast(ca.CreateObject(&fakeType)); for (int i = 1; i < N; ++i) { uint8_t* obj = reinterpret_cast(ca.CreateObject(&fakeType)); @@ -64,7 +64,7 @@ TEST(CustomAllocTest, FixedBlockPageThreshold) { const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10; const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { - TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; + TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; ca.CreateObject(&fakeType); } } @@ -77,7 +77,7 @@ TEST(CustomAllocTest, NextFitPageThreshold) { const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10; const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10; for (int blocks = FROM; blocks <= TO; ++blocks) { - TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; + TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; ca.CreateObject(&fakeType); } } @@ -90,7 +90,7 @@ TEST(CustomAllocTest, TwoAllocatorsDifferentPages) { kotlin::gc::GCSchedulerThreadData schedulerData2(config, [](auto&) {}); CustomAllocator ca1(heap, schedulerData1); CustomAllocator ca2(heap, schedulerData2); - TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0}; + TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0}; uint8_t* obj1 = reinterpret_cast(ca1.CreateObject(&fakeType)); uint8_t* obj2 = reinterpret_cast(ca2.CreateObject(&fakeType)); uint64_t dist = abs(obj2 - obj1); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp index 49094d16543..3aa5bb17ccb 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp @@ -50,7 +50,6 @@ void CustomFinalizerProcessor::StartFinalizerThreadIfNone() noexcept { auto* extraObject = cell->Data(); auto* baseObject = extraObject->GetBaseObject(); RunFinalizers(baseObject); - extraObject->setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); } } epochDoneCallback_(finalizersEpoch); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp index 4c3d378a286..6df19cbff88 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp @@ -7,11 +7,10 @@ #define CUSTOM_ALLOC_CPP_CUSTOMLOGGING_HPP_ #include "Logging.hpp" -#include "Porting.h" -#define CustomAllocInfo(format, ...) RuntimeLogInfo({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__) -#define CustomAllocDebug(format, ...) RuntimeLogDebug({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__) -#define CustomAllocWarning(format, ...) RuntimeLogWarning({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__) -#define CustomAllocError(format, ...) RuntimeLogError({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__) +#define CustomAllocInfo(format, ...) RuntimeLogInfo({"alloc"}, format, ##__VA_ARGS__) +#define CustomAllocDebug(format, ...) RuntimeLogDebug({"alloc"}, format, ##__VA_ARGS__) +#define CustomAllocWarning(format, ...) RuntimeLogWarning({"alloc"}, format, ##__VA_ARGS__) +#define CustomAllocError(format, ...) RuntimeLogError({"alloc"}, format, ##__VA_ARGS__) #endif diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp index 3899013c36a..e01ffa12df2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp @@ -6,6 +6,7 @@ #include "ExtraObjectPage.hpp" #include +#include #include #include @@ -17,7 +18,7 @@ namespace kotlin::alloc { -ExtraObjectPage* ExtraObjectPage::Create() noexcept { +ExtraObjectPage* ExtraObjectPage::Create(uint32_t ignored) noexcept { CustomAllocInfo("ExtraObjectPage::Create()"); return new (SafeAlloc(EXTRA_OBJECT_PAGE_SIZE)) ExtraObjectPage(); } @@ -41,11 +42,11 @@ mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept { } ExtraObjectCell* freeBlock = nextFree_; nextFree_ = freeBlock->next_; - CustomAllocDebug("ExtraObjectPage(%p)::TryAllocate() = %p", this, freeBlock); + CustomAllocDebug("ExtraObjectPage(%p)::TryAllocate() = %p", this, freeBlock->Data()); return freeBlock->Data(); } -bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack& finalizerQueue) noexcept { +bool ExtraObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { CustomAllocInfo("ExtraObjectPage(%p)::Sweep()", this); // `end` is after the last legal allocation of a block, but does not // necessarily match an actual block starting point. @@ -58,23 +59,14 @@ bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, nextFree = &cell->next_; continue; } - auto status = SweepExtraObject(cell, finalizerQueue); - switch (status) { - case ExtraObjectStatus::TO_BE_FINALIZED: - sweepHandle.addMarkedObject(); - // fallthrough - case ExtraObjectStatus::KEPT: - // If the current cell was marked, it's alive, and the whole page is alive. - alive = true; - sweepHandle.addKeptObject(); - break; - case ExtraObjectStatus::SWEPT: - // Free the current block and insert it into the free list. - cell->next_ = *nextFree; - *nextFree = cell; - nextFree = &cell->next_; - sweepHandle.addSweptObject(); - break; + if (SweepExtraObject(cell->Data(), sweepHandle)) { + // If the current cell was marked, it's alive, and the whole page is alive. + alive = true; + } else { + // Free the current block and insert it into the free list. + cell->next_ = *nextFree; + *nextFree = cell; + nextFree = &cell->next_; } } return alive; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp index 2a0e0368365..d666277d4be 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp @@ -7,6 +7,7 @@ #define CUSTOM_ALLOC_CPP_EXTRA_OBJECTPAGE_HPP_ #include +#include #include #include "AtomicStack.hpp" @@ -24,18 +25,28 @@ struct ExtraObjectCell { struct alignas(mm::ExtraObjectData) { uint8_t data_[sizeof(mm::ExtraObjectData)]; }; + + static ExtraObjectCell* fromExtraObject(mm::ExtraObjectData* extraObjectData) { + return reinterpret_cast(reinterpret_cast(extraObjectData) - offsetof(ExtraObjectCell, data_)); + } }; +using FinalizerQueue = AtomicStack; + class alignas(8) ExtraObjectPage { public: - static ExtraObjectPage* Create() noexcept; + using GCSweepScope = gc::GCHandle::GCSweepExtraObjectsScope; + + static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweepExtraObjects(); } + + static ExtraObjectPage* Create(uint32_t ignored) noexcept; void Destroy() noexcept; // Tries to allocate in current page, returns null if no free block in page mm::ExtraObjectData* TryAllocate() noexcept; - bool Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack& finalizerQueue) noexcept; + bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp index 87d06fa30f6..e795053c518 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp @@ -28,7 +28,7 @@ Data* alloc(Page* page) { } TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) { - Page* page = Page::Create(); + Page* page = Page::Create(0); uint8_t* prev = reinterpret_cast(alloc(page)); uint8_t* cur; while ((cur = reinterpret_cast(alloc(page)))) { @@ -39,7 +39,7 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) { } TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) { - Page* page = Page::Create(); + Page* page = Page::Create(0); Queue finalizerQueue; auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweepExtraObjects(); @@ -49,11 +49,11 @@ TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) { } TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) { - Page* page = Page::Create(); + Page* page = Page::Create(0); int count = 0; Data* ptr; while ((ptr = alloc(page))) { - ptr->setFlag(Data::FLAGS_FINALIZED); + ptr->setFlag(Data::FLAGS_SWEEPABLE); ++count; } EXPECT_EQ(count, EXTRA_OBJECT_COUNT); diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp index 7fb918563c2..1d52a2c743f 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -42,11 +42,11 @@ uint8_t* FixedBlockPage::TryAllocate() noexcept { return nullptr; } nextFree_ = freeBlock->nextFree; - CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock); + CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock->data); return freeBlock->data; } -bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { +bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this); // `end` is after the last legal allocation of a block, but does not // necessarily match an actual block starting point. @@ -60,7 +60,7 @@ bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { continue; } // If the current cell was marked, it's alive, and the whole page is alive. - if (TryResetMark(cell)) { + if (SweepObject(cell->data, finalizerQueue, sweepHandle)) { alive = true; sweepHandle.addKeptObject(); continue; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp index 81bf5a5cbb0..cfd629f2070 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp @@ -10,6 +10,7 @@ #include #include "AtomicStack.hpp" +#include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" namespace kotlin::alloc { @@ -24,6 +25,10 @@ struct alignas(8) FixedBlockCell { class alignas(8) FixedBlockPage { public: + using GCSweepScope = gc::GCHandle::GCSweepScope; + + static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); } + static FixedBlockPage* Create(uint32_t blockSize) noexcept; void Destroy() noexcept; @@ -31,7 +36,7 @@ public: // Tries to allocate in current page, returns null if no free block in page uint8_t* TryAllocate() noexcept; - bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; + bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp index effa128faa0..70ee0d91be9 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp @@ -4,9 +4,14 @@ */ #include +#include +#include +#include +#include #include "Cell.hpp" #include "CustomAllocConstants.hpp" +#include "ExtraObjectPage.hpp" #include "gtest/gtest.h" #include "FixedBlockPage.hpp" #include "TypeInfo.h" @@ -15,7 +20,7 @@ namespace { using FixedBlockPage = typename kotlin::alloc::FixedBlockPage; -TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer +TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer void mark(void* obj) { reinterpret_cast(obj)[0] = 1; @@ -46,9 +51,10 @@ TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) { TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } } @@ -56,12 +62,13 @@ TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) { TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint32_t count = 0; while (alloc(page, size)) ++count; EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size); - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } } @@ -69,11 +76,12 @@ TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) { TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); mark(ptr); - EXPECT_TRUE(page->Sweep(gcScope)); + EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } } @@ -81,10 +89,11 @@ TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) { TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr = alloc(page, size); - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); EXPECT_EQ(alloc(page, size), ptr); page->Destroy(); } @@ -93,13 +102,14 @@ TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) { TEST(CustomAllocTest, FixedBlockPageSweepReuse) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { FixedBlockPage* page = FixedBlockPage::Create(size); uint8_t* ptr; for (int count = 0; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); } - EXPECT_TRUE(page->Sweep(gcScope)); + EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue)); uint32_t count = 0; for (; (ptr = alloc(page, size)); ++count) { if (count % 2 == 0) mark(ptr); @@ -108,4 +118,44 @@ TEST(CustomAllocTest, FixedBlockPageSweepReuse) { page->Destroy(); } } + +TEST(CustomAllocTest, FixedBlockPageRandomExercise) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; + std::minstd_rand r(42); + uint8_t* ptr; + for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) { + FixedBlockPage* page = FixedBlockPage::Create(size); + uint32_t BLOCK_COUNT = FIXED_BLOCK_PAGE_CELL_COUNT / size; + std::vector seen; + while ((ptr = alloc(page, size))) seen.push_back(ptr); + EXPECT_EQ(seen.size(), BLOCK_COUNT); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); + std::unordered_set live; + for (int gc = 0; gc < 10; gc++) { + int createCount = r() % BLOCK_COUNT; + while (createCount-- > 0) { + ptr = alloc(page, size); + if (!ptr) break; + EXPECT_TRUE(live.insert(ptr).second); + } + for (auto obj : seen) { + if (live.find(obj) != live.end()) { + if (r() % 2) { + mark(obj); + } else { + live.erase(obj); + } + } + } + EXPECT_EQ(page->Sweep(gcScope, finalizerQueue), !live.empty()); + } + while ((ptr = alloc(page, size))) live.insert(ptr); + EXPECT_EQ(live.size(), BLOCK_COUNT); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); + page->Destroy(); + } +} + } // namespace diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index b35338a0675..4dcf0be196b 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -10,7 +10,10 @@ #include "ConcurrentMarkAndSweep.hpp" #include "CustomLogging.hpp" +#include "ExtraObjectData.hpp" +#include "ExtraObjectPage.hpp" #include "FinalizerHooks.hpp" +#include "GCStatistics.hpp" #include "KAssert.h" #include "ObjectFactory.hpp" @@ -22,66 +25,41 @@ std::atomic allocatedBytesCounter; namespace kotlin::alloc { -bool TryResetMark(void* ptr) noexcept { - using Node = typename kotlin::mm::ObjectFactory::Storage::Node; - using NodeRef = typename kotlin::mm::ObjectFactory::NodeRef; - Node& node = Node::FromData(ptr); - NodeRef ref = NodeRef(node); - auto& objectData = ref.ObjectData(); - bool reset = objectData.tryResetMark(); - CustomAllocDebug("TryResetMark(%p) = %d", ptr, reset); - return reset; -} - -using ObjectFactory = mm::ObjectFactory; -using ExtraObjectsFactory = mm::ExtraObjectDataFactory; - -static void KeepAlive(ObjHeader* baseObject) noexcept { - auto& objectData = ObjectFactory::NodeRef::From(baseObject).ObjectData(); - objectData.tryMark(); -} - -static bool IsAlive(ObjHeader* baseObject) noexcept { - auto& objectData = ObjectFactory::NodeRef::From(baseObject).ObjectData(); - return objectData.marked(); -} - -ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept { - auto* extraObject = extraObjectCell->Data(); - if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) { - CustomAllocDebug("SweepIsCollectable(%p): already finalized", extraObject); - return ExtraObjectStatus::SWEPT; +bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { + HeapObjHeader* objHeader = reinterpret_cast(object); + if (objHeader->gcData.tryResetMark()) { + CustomAllocDebug("SweepObject(%p): still alive", object); + gcHandle.addKeptObject(); + return true; } - auto* baseObject = extraObject->GetBaseObject(); - RuntimeAssert(baseObject->heap(), "SweepIsCollectable on a non-heap object"); - if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { - CustomAllocDebug("SweepIsCollectable(%p): already in finalizer queue, keep base object (%p) alive", extraObject, baseObject); - KeepAlive(baseObject); - return ExtraObjectStatus::TO_BE_FINALIZED; - } - if (IsAlive(baseObject)) { - CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject); - return ExtraObjectStatus::KEPT; - } - extraObject->ClearRegularWeakReferenceImpl(); - if (extraObject->HasAssociatedObject()) { - extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); - finalizerQueue.Push(extraObjectCell); - KeepAlive(baseObject); - CustomAllocDebug("SweepIsCollectable(%p): add to finalizerQueue", extraObject); - return ExtraObjectStatus::TO_BE_FINALIZED; - } else { - if (HasFinalizers(baseObject)) { + auto* extraObject = mm::ExtraObjectData::Get(&objHeader->object); + if (extraObject) { + if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { + CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", object, extraObject); extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); - finalizerQueue.Push(extraObjectCell); - KeepAlive(baseObject); - CustomAllocDebug("SweepIsCollectable(%p): addings to finalizerQueue, keep base object (%p) alive", extraObject, baseObject); - return ExtraObjectStatus::TO_BE_FINALIZED; + CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject)); + finalizerQueue.Push(ExtraObjectCell::fromExtraObject(extraObject)); + gcHandle.addMarkedObject(); + return true; + } + if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE)) { + CustomAllocDebug("SweepObject(%p): already waiting to be finalized", object); + gcHandle.addMarkedObject(); + return true; } - extraObject->Uninstall(); - CustomAllocDebug("SweepIsCollectable(%p): uninstalled extraObject", extraObject); - return ExtraObjectStatus::SWEPT; } + CustomAllocDebug("SweepObject(%p): can be reclaimed", object); + gcHandle.addSweptObject(); + return false; +} + +bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& gcHandle) noexcept { + if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE)) { + CustomAllocDebug("SweepExtraObject(%p): can be reclaimed", extraObject); + return false; + } + CustomAllocDebug("SweepExtraObject(%p): is still needed", extraObject); + return true; } void* SafeAlloc(uint64_t size) noexcept { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index 61d5ff92860..f920e7f51be 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -12,19 +12,32 @@ #include #include "AtomicStack.hpp" +#include "ConcurrentMarkAndSweep.hpp" +#include "ExtraObjectData.hpp" #include "ExtraObjectPage.hpp" +#include "GCStatistics.hpp" namespace kotlin::alloc { -bool TryResetMark(void* ptr) noexcept; +// copied over from ObjectFactory -enum class ExtraObjectStatus { - TO_BE_FINALIZED, - KEPT, - SWEPT, +using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData; + +struct HeapObjHeader { + ObjectData gcData; + alignas(kObjectAlignment) ObjHeader object; }; -ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept; +struct HeapArrayHeader { + ObjectData gcData; + alignas(kObjectAlignment) ArrayHeader array; +}; + +// Returns `true` if the `object` must be kept alive still. +bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept; + +// Returns `true` if the `extraObject` must be kept alive still +bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& sweepScope) noexcept; void* SafeAlloc(uint64_t size) noexcept; void Free(void* ptr, size_t size) noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index bc0427b37bf..7d25a73519f 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -20,12 +20,6 @@ namespace kotlin::alloc { -Heap::~Heap() noexcept { - ExtraObjectPage* page; - while ((page = extraObjectPages_.Pop())) page->Destroy(); - while ((page = usedExtraObjectPages_.Pop())) page->Destroy(); -} - void Heap::PrepareForGC() noexcept { CustomAllocDebug("Heap::PrepareForGC()"); for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { @@ -37,58 +31,49 @@ void Heap::PrepareForGC() noexcept { for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { fixedBlockPages_[blockSize].PrepareForGC(); } - usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_)); + extraObjectPages_.PrepareForGC(); } -void Heap::Sweep(gc::GCHandle gcHandle) noexcept { - auto sweepHandle = gcHandle.sweep(); +FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept { + FinalizerQueue finalizerQueue; CustomAllocDebug("Heap::Sweep()"); - for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { - fixedBlockPages_[blockSize].Sweep(sweepHandle); - } - nextFitPages_.Sweep(sweepHandle); - singleObjectPages_.SweepAndFree(sweepHandle); -} - -AtomicStack Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept { - auto sweepHandle = gcHandle.sweepExtraObjects(); - CustomAllocDebug("Heap::SweepExtraObjects()"); - AtomicStack finalizerQueue; - ExtraObjectPage* page; - while ((page = usedExtraObjectPages_.Pop())) { - if (!page->Sweep(sweepHandle, finalizerQueue)) { - CustomAllocInfo("SweepExtraObjects free(%p)", page); - page->Destroy(); - } else { - extraObjectPages_.Push(page); + { + auto sweepHandle = gcHandle.sweep(); + for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { + fixedBlockPages_[blockSize].Sweep(sweepHandle, finalizerQueue); } + nextFitPages_.Sweep(sweepHandle, finalizerQueue); + singleObjectPages_.SweepAndFree(sweepHandle, finalizerQueue); } + { + auto sweepHandle = gcHandle.sweepExtraObjects(); + extraObjectPages_.Sweep(sweepHandle, finalizerQueue); + } + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue()); + } + CustomAllocDebug("Heap::Sweep done"); return finalizerQueue; } -NextFitPage* Heap::GetNextFitPage(uint32_t cellCount) noexcept { +NextFitPage* Heap::GetNextFitPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept { CustomAllocDebug("Heap::GetNextFitPage()"); - return nextFitPages_.GetPage(cellCount); + return nextFitPages_.GetPage(cellCount, finalizerQueue); } -FixedBlockPage* Heap::GetFixedBlockPage(uint32_t cellCount) noexcept { +FixedBlockPage* Heap::GetFixedBlockPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept { CustomAllocDebug("Heap::GetFixedBlockPage()"); - return fixedBlockPages_[cellCount].GetPage(cellCount); + return fixedBlockPages_[cellCount].GetPage(cellCount, finalizerQueue); } -SingleObjectPage* Heap::GetSingleObjectPage(uint64_t cellCount) noexcept { +SingleObjectPage* Heap::GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept { CustomAllocInfo("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount); return singleObjectPages_.NewPage(cellCount); } -ExtraObjectPage* Heap::GetExtraObjectPage() noexcept { +ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept { CustomAllocInfo("CustomAllocator::GetExtraObjectPage()"); - ExtraObjectPage* page = extraObjectPages_.Pop(); - if (page == nullptr) { - page = ExtraObjectPage::Create(); - } - usedExtraObjectPages_.Push(page); - return page; + return extraObjectPages_.GetPage(0, finalizerQueue); } } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp index 5c82305ba75..a3c206c0cda 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp @@ -22,29 +22,24 @@ namespace kotlin::alloc { class Heap { public: - ~Heap() noexcept; - // Called once by the GC thread after all mutators have been suspended void PrepareForGC() noexcept; // Sweep through all remaining pages, freeing those blocks where CanReclaim // returns true. If multiple sweepers are active, each page will only be // seen by one sweeper. - void Sweep(gc::GCHandle gcHandle) noexcept; + FinalizerQueue Sweep(gc::GCHandle gcHandle) noexcept; - AtomicStack SweepExtraObjects(gc::GCHandle gcHandle) noexcept; - - FixedBlockPage* GetFixedBlockPage(uint32_t cellCount) noexcept; - NextFitPage* GetNextFitPage(uint32_t cellCount) noexcept; - SingleObjectPage* GetSingleObjectPage(uint64_t cellCount) noexcept; - ExtraObjectPage* GetExtraObjectPage() noexcept; + FixedBlockPage* GetFixedBlockPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept; + NextFitPage* GetNextFitPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept; + SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept; + ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept; private: PageStore fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1]; PageStore nextFitPages_; PageStore singleObjectPages_; - AtomicStack extraObjectPages_; - AtomicStack usedExtraObjectPages_; + PageStore extraObjectPages_; }; } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp index 5bd869bd949..4b614cf69ea 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp @@ -7,6 +7,7 @@ #include #include +#include "ExtraObjectPage.hpp" #include "SingleObjectPage.hpp" #include "gtest/gtest.h" #include "Heap.hpp" @@ -30,8 +31,9 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) { const int MIN = MIN_BLOCK_SIZE; const int MAX = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1; FixedBlockPage* pages[MAX]; + kotlin::alloc::FinalizerQueue finalizerQueue; for (int blocks = MIN; blocks < MAX; ++blocks) { - pages[blocks] = heap.GetFixedBlockPage(blocks); + pages[blocks] = heap.GetFixedBlockPage(blocks, finalizerQueue); void* obj = pages[blocks]->TryAllocate(); mark(obj); // to make the page survive a sweep } @@ -39,20 +41,21 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); heap.Sweep(gcHandle); for (int blocks = MIN; blocks < MAX; ++blocks) { - EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks)); + EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks, finalizerQueue)); } } TEST(CustomAllocTest, HeapReuseNextFitPages) { Heap heap; const uint32_t BLOCKSIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 42; - NextFitPage* page = heap.GetNextFitPage(BLOCKSIZE); + kotlin::alloc::FinalizerQueue finalizerQueue; + NextFitPage* page = heap.GetNextFitPage(BLOCKSIZE, finalizerQueue); void* obj = page->TryAllocate(BLOCKSIZE); mark(obj); // to make the page survive a sweep heap.PrepareForGC(); auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); heap.Sweep(gcHandle); - EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE)); + EXPECT_EQ(page, heap.GetNextFitPage(0, finalizerQueue)); } } // namespace diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index 86b127fe850..12b13e40078 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -39,13 +39,13 @@ uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept { return curBlock_->TryAllocate(cellsNeeded); } -bool NextFitPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { +bool NextFitPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { CustomAllocDebug("NextFitPage@%p::Sweep()", this); Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT; bool alive = false; for (Cell* block = cells_ + 1; block != end; block = block->Next()) { if (block->isAllocated_) { - if (TryResetMark(block->data_)) { + if (SweepObject(block->data_, finalizerQueue, sweepHandle)) { alive = true; sweepHandle.addKeptObject(); } else { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp index 130fcba78c2..9f1277d7836 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp @@ -11,12 +11,17 @@ #include "AtomicStack.hpp" #include "Cell.hpp" +#include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" namespace kotlin::alloc { class alignas(8) NextFitPage { public: + using GCSweepScope = gc::GCHandle::GCSweepScope; + + static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); } + static NextFitPage* Create(uint32_t cellCount) noexcept; void Destroy() noexcept; @@ -24,7 +29,7 @@ public: // Tries to allocate in current page, returns null if no free block in page is big enough uint8_t* TryAllocate(uint32_t blockSize) noexcept; - bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; + bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; // Testing method bool CheckInvariants() noexcept; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp index 26bdff18b50..aba608e1512 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp @@ -8,6 +8,7 @@ #include "Cell.hpp" #include "CustomAllocConstants.hpp" +#include "ExtraObjectPage.hpp" #include "gtest/gtest.h" #include "NextFitPage.hpp" #include "TypeInfo.h" @@ -17,7 +18,7 @@ namespace { using NextFitPage = typename kotlin::alloc::NextFitPage; using Cell = typename kotlin::alloc::Cell; -TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer +TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer inline constexpr const size_t MIN_BLOCK_SIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1; @@ -54,18 +55,20 @@ TEST(CustomAllocTest, NextFitPageSweepEmptyPage) { NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); - EXPECT_FALSE(page->Sweep(gcScope)); + kotlin::alloc::FinalizerQueue finalizerQueue; + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) { + auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); + auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {} - auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); - auto gcScope = gcHandle.sweep(); - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } } @@ -75,19 +78,21 @@ TEST(CustomAllocTest, NextFitPageSweepSingleMarked) { mark(alloc(page, MIN_BLOCK_SIZE)); auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); - EXPECT_TRUE(page->Sweep(gcScope)); + kotlin::alloc::FinalizerQueue finalizerQueue; + EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } TEST(CustomAllocTest, NextFitPageSweepSingleReuse) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); int count1 = 0; while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1; - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); r.seed(seed); int count2 = 0; while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count2; @@ -99,6 +104,7 @@ TEST(CustomAllocTest, NextFitPageSweepSingleReuse) { TEST(CustomAllocTest, NextFitPageSweepReuse) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) { std::minstd_rand r(seed); NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); @@ -112,7 +118,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) { ++unmarked; } } - page->Sweep(gcScope); + page->Sweep(gcScope, finalizerQueue); int freed = 0; while (alloc(page, MIN_BLOCK_SIZE)) ++freed; EXPECT_EQ(freed, unmarked); @@ -123,9 +129,10 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) { TEST(CustomAllocTest, NextFitPageSweepCoallesce) { auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); + kotlin::alloc::FinalizerQueue finalizerQueue; NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE); EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1)); - EXPECT_FALSE(page->Sweep(gcScope)); + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp index 5ac2a103f67..31176f8b588 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp @@ -9,6 +9,7 @@ #include #include "AtomicStack.hpp" +#include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" namespace kotlin::alloc { @@ -16,6 +17,8 @@ namespace kotlin::alloc { template class PageStore { public: + using GCSweepScope = typename T::GCSweepScope; + void PrepareForGC() noexcept { unswept_.TransferAllFrom(std::move(ready_)); unswept_.TransferAllFrom(std::move(used_)); @@ -23,15 +26,15 @@ public: while ((page = empty_.Pop())) page->Destroy(); } - void Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { - while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_)) { + void Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { + while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_, finalizerQueue)) { } } - void SweepAndFree(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { + void SweepAndFree(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { T* page; while ((page = unswept_.Pop())) { - if (page->Sweep(sweepHandle)) { + if (page->Sweep(sweepHandle, finalizerQueue)) { ready_.Push(page); } else { page->Destroy(); @@ -39,12 +42,12 @@ public: } } - T* GetPage(uint32_t cellCount) noexcept { + T* GetPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept { T* page; if ((page = unswept_.Pop())) { // If there're unswept_ pages, the GC is in progress. - auto sweepHandle = gc::GCHandle::currentEpoch()->sweep(); - if ((page = SweepSingle(sweepHandle, page, unswept_, used_))) { + GCSweepScope sweepHandle = T::currentGCSweepScope(); + if ((page = SweepSingle(sweepHandle, page, unswept_, used_, finalizerQueue))) { return page; } } @@ -74,12 +77,12 @@ public: } private: - T* SweepSingle(gc::GCHandle::GCSweepScope& sweepHandle, T* page, AtomicStack& from, AtomicStack& to) noexcept { + T* SweepSingle(GCSweepScope& sweepHandle, T* page, AtomicStack& from, AtomicStack& to, FinalizerQueue& finalizerQueue) noexcept { if (!page) { return nullptr; } do { - if (page->Sweep(sweepHandle)) { + if (page->Sweep(sweepHandle, finalizerQueue)) { to.Push(page); return page; } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp index 14f23d58676..1c6f60a60cd 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -37,15 +37,13 @@ uint8_t* SingleObjectPage::TryAllocate() noexcept { return Data(); } -bool SingleObjectPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept { +bool SingleObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept { CustomAllocDebug("SingleObjectPage@%p::Sweep()", this); - if (!TryResetMark(Data())) { - isAllocated_ = false; - sweepHandle.addKeptObject(); - return false; + if (SweepObject(Data(), finalizerQueue, sweepHandle)) { + return true; } - sweepHandle.addSweptObject(); - return true; + isAllocated_ = false; + return false; } } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index 4e94eb72f0d..a0eef0a81e5 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -10,12 +10,17 @@ #include #include "AtomicStack.hpp" +#include "ExtraObjectPage.hpp" #include "GCStatistics.hpp" namespace kotlin::alloc { class alignas(8) SingleObjectPage { public: + using GCSweepScope = gc::GCHandle::GCSweepScope; + + static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); } + static SingleObjectPage* Create(uint64_t cellCount) noexcept; void Destroy() noexcept; @@ -24,7 +29,7 @@ public: uint8_t* TryAllocate() noexcept; - bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept; + bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept; private: friend class AtomicStack; diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp index 333e6cb1963..5f137851827 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp @@ -7,6 +7,7 @@ #include #include "CustomAllocConstants.hpp" +#include "ExtraObjectPage.hpp" #include "gtest/gtest.h" #include "SingleObjectPage.hpp" #include "TypeInfo.h" @@ -15,7 +16,7 @@ namespace { using SingleObjectPage = typename kotlin::alloc::SingleObjectPage; -TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer +TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer #define MIN_BLOCK_SIZE NEXT_FIT_PAGE_CELL_COUNT @@ -36,7 +37,8 @@ TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) { EXPECT_TRUE(page); auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); - EXPECT_FALSE(page->Sweep(gcScope)); + kotlin::alloc::FinalizerQueue finalizerQueue; + EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } @@ -47,7 +49,8 @@ TEST(CustomAllocTest, SingleObjectPageSweepFullPage) { mark(page->Data()); auto gcHandle = kotlin::gc::GCHandle::createFakeForTests(); auto gcScope = gcHandle.sweep(); - EXPECT_TRUE(page->Sweep(gcScope)); + kotlin::alloc::FinalizerQueue finalizerQueue; + EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue)); page->Destroy(); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 0e24294f902..8e28911df73 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -206,11 +206,11 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { objectFactoryIterable = std::nullopt; kotlin::compactObjectPoolInMainThread(); #else - auto finalizerQueue = heap_.SweepExtraObjects(gcHandle); - mm::ResumeThreads(); gcHandle.threadsAreResumed(); - heap_.Sweep(gcHandle); + + // also sweeps extraObjects + auto finalizerQueue = heap_.Sweep(gcHandle); #endif state_.finish(epoch); gcHandle.finalizersScheduled(finalizerQueue.size()); diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index 2ab36124102..1990dd39b88 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -324,7 +324,7 @@ MarkStats GCHandle::getMarked() { return MarkStats{}; } -void GCHandle::swept(gc::SweepStats stats) noexcept { +void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { auto& heap = stat->sweepStats.heap; @@ -333,10 +333,12 @@ void GCHandle::swept(gc::SweepStats stats) noexcept { } heap->keptCount += stats.keptCount; heap->sweptCount += stats.sweptCount; + RuntimeAssert(static_cast(stat->markStats), "Mark must have already happened"); + stat->markStats->markedCount += markedCount; } } -void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noexcept { +void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { auto& extra = stat->sweepStats.extra; @@ -345,15 +347,13 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noe } extra->keptCount += stats.keptCount; extra->sweptCount += stats.sweptCount; - RuntimeAssert(static_cast(stat->markStats), "Mark must have already happened"); - stat->markStats->markedCount += markedCount; } } GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} GCHandle::GCSweepScope::~GCSweepScope() { - handle_.swept(stats_); + handle_.swept(stats_, markedCount_); GCLogDebug( handle_.getEpoch(), "Collected %" PRId64 " heap objects in %" PRIu64 " microseconds. " @@ -364,7 +364,7 @@ GCHandle::GCSweepScope::~GCSweepScope() { GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() { - handle_.sweptExtraObjects(stats_, markedCount_); + handle_.sweptExtraObjects(stats_); GCLogDebug( handle_.getEpoch(), "Collected %" PRId64 " extra objects in %" PRIu64 " microseconds. " diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index 7b7cab24d78..585db9865b0 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -46,6 +46,7 @@ public: class GCSweepScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; SweepStats stats_; + uint64_t markedCount_ = 0; public: explicit GCSweepScope(GCHandle& handle); @@ -53,12 +54,13 @@ public: void addSweptObject() noexcept { stats_.sweptCount += 1; } void addKeptObject() noexcept { stats_.keptCount += 1; } + // Custom allocator only. To be finalized objects are kept alive. + void addMarkedObject() noexcept { markedCount_ += 1; } }; class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; SweepStats stats_; - uint64_t markedCount_ = 0; public: explicit GCSweepExtraObjectsScope(GCHandle& handle); @@ -66,8 +68,6 @@ public: void addSweptObject() noexcept { stats_.sweptCount += 1; } void addKeptObject() noexcept { stats_.keptCount += 1; } - // Custom allocator only. To be finalized objects are kept alive. - void addMarkedObject() noexcept { markedCount_ += 1; } }; class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned { @@ -130,8 +130,8 @@ private: void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences); void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences); - void swept(SweepStats stats) noexcept; - void sweptExtraObjects(SweepStats stats, uint64_t markedCount) noexcept; + void swept(SweepStats stats, uint64_t markedCount) noexcept; + void sweptExtraObjects(SweepStats stats) noexcept; void marked(MarkStats stats); public: diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index b8e381dcd95..f42c3181bfd 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -38,7 +38,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast(&data))) { // Somebody else created `mm::ExtraObjectData` for this object. - data.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); + data.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); #else auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo); diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp index 1b70fe67c88..b5a36d01832 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp @@ -26,7 +26,7 @@ public: FLAGS_FROZEN = 0, FLAGS_NEVER_FROZEN = 1, FLAGS_IN_FINALIZER_QUEUE = 2, - FLAGS_FINALIZED = 3, + FLAGS_SWEEPABLE = 3, FLAGS_RELEASE_ON_MAIN_QUEUE = 4, }; diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index c17cc940883..ee262403426 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -79,7 +79,9 @@ void ObjHeader::destroyMetaObject(ObjHeader* object) { RuntimeAssert(object->has_meta_object(), "Object must have a meta object set"); auto &extraObject = *mm::ExtraObjectData::Get(object); extraObject.Uninstall(); -#ifndef CUSTOM_ALLOCATOR +#ifdef CUSTOM_ALLOCATOR + extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE); +#else auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, extraObject); #endif