azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] custom_alloc: sweep extraobjects outside stw ^KT-55364

Browse Source 
Co-authored-by: Troels Lund <troels@google.com>

Merge-request: KOTLIN-MR-677
Merged-by: Alexander Shabalin <alexander.shabalin@jetbrains.com>
This commit is contained in:
Troels Bjerre Lund
2023-05-09 16:37:58 +00:00
committed by Space Cloud
parent 57707180c6
commit 918a247bcc
29 changed files with 279 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp
+11 -18
View File
@@ -22,24 +22,11 @@
#include "Memory.h"
#include "FixedBlockPage.hpp"
#include "GCImpl.hpp"
#include "GCApi.hpp"
#include "TypeInfo.h"
namespace kotlin::alloc {
using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData;
struct HeapObjHeader {
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object;
};
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader {
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array;
};
size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
@@ -67,6 +54,8 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
if (typeInfo->flags_ & TF_HAS_FINALIZER) {
auto* extraObject = CreateExtraObject();
object->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(new (extraObject) mm::ExtraObjectData(object, typeInfo));
CustomAllocDebug("CustomAllocator: %p gets extraObject %p", object, extraObject);
CustomAllocDebug("CustomAllocator: %p->BaseObject == %p", extraObject, extraObject->GetBaseObject());
} else {
object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
}
@@ -94,7 +83,7 @@ mm::ExtraObjectData* CustomAllocator::CreateExtraObject() noexcept {
}
}
CustomAllocDebug("Failed to allocate in current ExtraObjectPage");
while ((page = heap_.GetExtraObjectPage())) {
while ((page = heap_.GetExtraObjectPage(finalizerQueue_))) {
mm::ExtraObjectData* block = page->TryAllocate();
if (block) {
extraObjectPage_ = page;
@@ -112,6 +101,10 @@ mm::ExtraObjectData& CustomAllocator::CreateExtraObjectDataForObject(
return *new (extraObject) mm::ExtraObjectData(baseObject, info);
}
FinalizerQueue CustomAllocator::ExtractFinalizerQueue() noexcept {
return std::move(finalizerQueue_);
}
void CustomAllocator::PrepareForGC() noexcept {
CustomAllocInfo("CustomAllocator@%p::PrepareForGC()", this);
nextFitPage_ = nullptr;
@@ -148,7 +141,7 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept {
CustomAllocDebug("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount);
uint8_t* block = heap_.GetSingleObjectPage(cellCount)->TryAllocate();
uint8_t* block = heap_.GetSingleObjectPage(cellCount, finalizerQueue_)->TryAllocate();
return block;
}
@@ -160,7 +153,7 @@ uint8_t* CustomAllocator::AllocateInNextFitPage(uint32_t cellCount) noexcept {
}
CustomAllocDebug("Failed to allocate in curPage");
while (true) {
nextFitPage_ = heap_.GetNextFitPage(cellCount);
nextFitPage_ = heap_.GetNextFitPage(cellCount, finalizerQueue_);
uint8_t* block = nextFitPage_->TryAllocate(cellCount);
if (block) return block;
}
@@ -174,7 +167,7 @@ uint8_t* CustomAllocator::AllocateInFixedBlockPage(uint32_t cellCount) noexcept
if (block) return block;
}
CustomAllocDebug("Failed to allocate in current FixedBlockPage");
while ((page = heap_.GetFixedBlockPage(cellCount))) {
while ((page = heap_.GetFixedBlockPage(cellCount, finalizerQueue_))) {
uint8_t* block = page->TryAllocate();
if (block) {
fixedBlockPages_[cellCount] = page;
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp
+3
View File
@@ -34,6 +34,8 @@ public:
void PrepareForGC() noexcept;
FinalizerQueue ExtractFinalizerQueue() noexcept;
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
private:
@@ -47,6 +49,7 @@ private:
NextFitPage* nextFitPage_;
FixedBlockPage* fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1];
ExtraObjectPage* extraObjectPage_;
FinalizerQueue finalizerQueue_;
};
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocatorTest.cpp
+5 -5
View File
@@ -25,7 +25,7 @@ TEST(CustomAllocTest, SmallAllocNonNull) {
const int N = 200;
TypeInfo fakeTypes[N];
for (int i = 1; i < N; ++i) {
fakeTypes[i] = {.instanceSize_ = 8 * i, .flags_ = 0};
fakeTypes[i] = {.typeInfo_ = &fakeTypes[i], .instanceSize_ = 8 * i, .flags_ = 0};
}
Heap heap;
kotlin::gc::GCSchedulerConfig config;
@@ -46,7 +46,7 @@ TEST(CustomAllocTest, SmallAllocSameFixedBlockPage) {
kotlin::gc::GCSchedulerConfig config;
kotlin::gc::GCSchedulerThreadData schedulerData(config, [](auto&) {});
CustomAllocator ca(heap, schedulerData);
TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0};
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
uint8_t* first = reinterpret_cast<uint8_t*>(ca.CreateObject(&fakeType));
for (int i = 1; i < N; ++i) {
uint8_t* obj = reinterpret_cast<uint8_t*>(ca.CreateObject(&fakeType));
@@ -64,7 +64,7 @@ TEST(CustomAllocTest, FixedBlockPageThreshold) {
const int FROM = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE - 10;
const int TO = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 10;
for (int blocks = FROM; blocks <= TO; ++blocks) {
TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0};
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
ca.CreateObject(&fakeType);
}
}
@@ -77,7 +77,7 @@ TEST(CustomAllocTest, NextFitPageThreshold) {
const int FROM = NEXT_FIT_PAGE_MAX_BLOCK_SIZE - 10;
const int TO = NEXT_FIT_PAGE_MAX_BLOCK_SIZE + 10;
for (int blocks = FROM; blocks <= TO; ++blocks) {
TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0};
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
ca.CreateObject(&fakeType);
}
}
@@ -90,7 +90,7 @@ TEST(CustomAllocTest, TwoAllocatorsDifferentPages) {
kotlin::gc::GCSchedulerThreadData schedulerData2(config, [](auto&) {});
CustomAllocator ca1(heap, schedulerData1);
CustomAllocator ca2(heap, schedulerData2);
TypeInfo fakeType = {.instanceSize_ = 8 * blocks, .flags_ = 0};
TypeInfo fakeType = {.typeInfo_ = &fakeType, .instanceSize_ = 8 * blocks, .flags_ = 0};
uint8_t* obj1 = reinterpret_cast<uint8_t*>(ca1.CreateObject(&fakeType));
uint8_t* obj2 = reinterpret_cast<uint8_t*>(ca2.CreateObject(&fakeType));
uint64_t dist = abs(obj2 - obj1);
kotlin-native/runtime/src/custom_alloc/cpp/CustomFinalizerProcessor.cpp
-1
View File
@@ -50,7 +50,6 @@ void CustomFinalizerProcessor::StartFinalizerThreadIfNone() noexcept {
auto* extraObject = cell->Data();
auto* baseObject = extraObject->GetBaseObject();
RunFinalizers(baseObject);
extraObject->setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
}
}
epochDoneCallback_(finalizersEpoch);
kotlin-native/runtime/src/custom_alloc/cpp/CustomLogging.hpp
+4 -5
View File
@@ -7,11 +7,10 @@
#define CUSTOM_ALLOC_CPP_CUSTOMLOGGING_HPP_
#include "Logging.hpp"
#include "Porting.h"
#define CustomAllocInfo(format, ...) RuntimeLogInfo({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__)
#define CustomAllocDebug(format, ...) RuntimeLogDebug({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__)
#define CustomAllocWarning(format, ...) RuntimeLogWarning({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__)
#define CustomAllocError(format, ...) RuntimeLogError({"alloc"}, "t%u " format, konan::currentThreadId(), ##__VA_ARGS__)
#define CustomAllocInfo(format, ...) RuntimeLogInfo({"alloc"}, format, ##__VA_ARGS__)
#define CustomAllocDebug(format, ...) RuntimeLogDebug({"alloc"}, format, ##__VA_ARGS__)
#define CustomAllocWarning(format, ...) RuntimeLogWarning({"alloc"}, format, ##__VA_ARGS__)
#define CustomAllocError(format, ...) RuntimeLogError({"alloc"}, format, ##__VA_ARGS__)
#endif
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp
+12 -20
View File
@@ -6,6 +6,7 @@
#include "ExtraObjectPage.hpp"
#include <atomic>
#include <cstdint>
#include <cstring>
#include <random>
@@ -17,7 +18,7 @@
namespace kotlin::alloc {
ExtraObjectPage* ExtraObjectPage::Create() noexcept {
ExtraObjectPage* ExtraObjectPage::Create(uint32_t ignored) noexcept {
CustomAllocInfo("ExtraObjectPage::Create()");
return new (SafeAlloc(EXTRA_OBJECT_PAGE_SIZE)) ExtraObjectPage();
}
@@ -41,11 +42,11 @@ mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept {
}
ExtraObjectCell* freeBlock = nextFree_;
nextFree_ = freeBlock->next_;
CustomAllocDebug("ExtraObjectPage(%p)::TryAllocate() = %p", this, freeBlock);
CustomAllocDebug("ExtraObjectPage(%p)::TryAllocate() = %p", this, freeBlock->Data());
return freeBlock->Data();
}
bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
bool ExtraObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocInfo("ExtraObjectPage(%p)::Sweep()", this);
// `end` is after the last legal allocation of a block, but does not
// necessarily match an actual block starting point.
@@ -58,23 +59,14 @@ bool ExtraObjectPage::Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle,
nextFree = &cell->next_;
continue;
}
auto status = SweepExtraObject(cell, finalizerQueue);
switch (status) {
case ExtraObjectStatus::TO_BE_FINALIZED:
sweepHandle.addMarkedObject();
// fallthrough
case ExtraObjectStatus::KEPT:
// If the current cell was marked, it's alive, and the whole page is alive.
alive = true;
sweepHandle.addKeptObject();
break;
case ExtraObjectStatus::SWEPT:
// Free the current block and insert it into the free list.
cell->next_ = *nextFree;
*nextFree = cell;
nextFree = &cell->next_;
sweepHandle.addSweptObject();
break;
if (SweepExtraObject(cell->Data(), sweepHandle)) {
// If the current cell was marked, it's alive, and the whole page is alive.
alive = true;
} else {
// Free the current block and insert it into the free list.
cell->next_ = *nextFree;
*nextFree = cell;
nextFree = &cell->next_;
}
}
return alive;
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.hpp
+13 -2
View File
@@ -7,6 +7,7 @@
#define CUSTOM_ALLOC_CPP_EXTRA_OBJECTPAGE_HPP_
#include <atomic>
#include <cstddef>
#include <cstdint>
#include "AtomicStack.hpp"
@@ -24,18 +25,28 @@ struct ExtraObjectCell {
struct alignas(mm::ExtraObjectData) {
uint8_t data_[sizeof(mm::ExtraObjectData)];
};
static ExtraObjectCell* fromExtraObject(mm::ExtraObjectData* extraObjectData) {
return reinterpret_cast<ExtraObjectCell*>(reinterpret_cast<uint8_t*>(extraObjectData) - offsetof(ExtraObjectCell, data_));
}
};
using FinalizerQueue = AtomicStack<ExtraObjectCell>;
class alignas(8) ExtraObjectPage {
public:
static ExtraObjectPage* Create() noexcept;
using GCSweepScope = gc::GCHandle::GCSweepExtraObjectsScope;
static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweepExtraObjects(); }
static ExtraObjectPage* Create(uint32_t ignored) noexcept;
void Destroy() noexcept;
// Tries to allocate in current page, returns null if no free block in page
mm::ExtraObjectData* TryAllocate() noexcept;
bool Sweep(gc::GCHandle::GCSweepExtraObjectsScope& sweepHandle, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
private:
friend class AtomicStack<ExtraObjectPage>;
kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp
+4 -4
View File
@@ -28,7 +28,7 @@ Data* alloc(Page* page) {
}
TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
Page* page = Page::Create();
Page* page = Page::Create(0);
uint8_t* prev = reinterpret_cast<uint8_t*>(alloc(page));
uint8_t* cur;
while ((cur = reinterpret_cast<uint8_t*>(alloc(page)))) {
@@ -39,7 +39,7 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
}
TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
Page* page = Page::Create();
Page* page = Page::Create(0);
Queue finalizerQueue;
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweepExtraObjects();
@@ -49,11 +49,11 @@ TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
}
TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
Page* page = Page::Create();
Page* page = Page::Create(0);
int count = 0;
Data* ptr;
while ((ptr = alloc(page))) {
ptr->setFlag(Data::FLAGS_FINALIZED);
ptr->setFlag(Data::FLAGS_SWEEPABLE);
++count;
}
EXPECT_EQ(count, EXTRA_OBJECT_COUNT);
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp
+3 -3
View File
@@ -42,11 +42,11 @@ uint8_t* FixedBlockPage::TryAllocate() noexcept {
return nullptr;
}
nextFree_ = freeBlock->nextFree;
CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock);
CustomAllocDebug("FixedBlockPage(%p){%u}::TryAllocate() = %p", this, blockSize_, freeBlock->data);
return freeBlock->data;
}
bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
// `end` is after the last legal allocation of a block, but does not
// necessarily match an actual block starting point.
@@ -60,7 +60,7 @@ bool FixedBlockPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
continue;
}
// If the current cell was marked, it's alive, and the whole page is alive.
if (TryResetMark(cell)) {
if (SweepObject(cell->data, finalizerQueue, sweepHandle)) {
alive = true;
sweepHandle.addKeptObject();
continue;
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp
+6 -1
View File
@@ -10,6 +10,7 @@
#include <cstdint>
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -24,6 +25,10 @@ struct alignas(8) FixedBlockCell {
class alignas(8) FixedBlockPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); }
static FixedBlockPage* Create(uint32_t blockSize) noexcept;
void Destroy() noexcept;
@@ -31,7 +36,7 @@ public:
// Tries to allocate in current page, returns null if no free block in page
uint8_t* TryAllocate() noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
private:
friend class AtomicStack<FixedBlockPage>;
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp
+56 -6
View File
@@ -4,9 +4,14 @@
*/
#include <cstdint>
#include <random>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "Cell.hpp"
#include "CustomAllocConstants.hpp"
#include "ExtraObjectPage.hpp"
#include "gtest/gtest.h"
#include "FixedBlockPage.hpp"
#include "TypeInfo.h"
@@ -15,7 +20,7 @@ namespace {
using FixedBlockPage = typename kotlin::alloc::FixedBlockPage;
TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer
TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer
void mark(void* obj) {
reinterpret_cast<uint64_t*>(obj)[0] = 1;
@@ -46,9 +51,10 @@ TEST(CustomAllocTest, FixedBlockPageConsequtiveAlloc) {
TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
}
@@ -56,12 +62,13 @@ TEST(CustomAllocTest, FixedBlockPageSweepEmptyPage) {
TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint32_t count = 0;
while (alloc(page, size)) ++count;
EXPECT_EQ(count, FIXED_BLOCK_PAGE_CELL_COUNT / size);
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
}
@@ -69,11 +76,12 @@ TEST(CustomAllocTest, FixedBlockPageSweepFullUnmarkedPage) {
TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr = alloc(page, size);
mark(ptr);
EXPECT_TRUE(page->Sweep(gcScope));
EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
}
@@ -81,10 +89,11 @@ TEST(CustomAllocTest, FixedBlockPageSweepSingleMarked) {
TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr = alloc(page, size);
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
EXPECT_EQ(alloc(page, size), ptr);
page->Destroy();
}
@@ -93,13 +102,14 @@ TEST(CustomAllocTest, FixedBlockPageSweepSingleReuse) {
TEST(CustomAllocTest, FixedBlockPageSweepReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint8_t* ptr;
for (int count = 0; (ptr = alloc(page, size)); ++count) {
if (count % 2 == 0) mark(ptr);
}
EXPECT_TRUE(page->Sweep(gcScope));
EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue));
uint32_t count = 0;
for (; (ptr = alloc(page, size)); ++count) {
if (count % 2 == 0) mark(ptr);
@@ -108,4 +118,44 @@ TEST(CustomAllocTest, FixedBlockPageSweepReuse) {
page->Destroy();
}
}
TEST(CustomAllocTest, FixedBlockPageRandomExercise) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
std::minstd_rand r(42);
uint8_t* ptr;
for (uint32_t size = 2; size <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++size) {
FixedBlockPage* page = FixedBlockPage::Create(size);
uint32_t BLOCK_COUNT = FIXED_BLOCK_PAGE_CELL_COUNT / size;
std::vector<uint8_t*> seen;
while ((ptr = alloc(page, size))) seen.push_back(ptr);
EXPECT_EQ(seen.size(), BLOCK_COUNT);
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
std::unordered_set<uint8_t*> live;
for (int gc = 0; gc < 10; gc++) {
int createCount = r() % BLOCK_COUNT;
while (createCount-- > 0) {
ptr = alloc(page, size);
if (!ptr) break;
EXPECT_TRUE(live.insert(ptr).second);
}
for (auto obj : seen) {
if (live.find(obj) != live.end()) {
if (r() % 2) {
mark(obj);
} else {
live.erase(obj);
}
}
}
EXPECT_EQ(page->Sweep(gcScope, finalizerQueue), !live.empty());
}
while ((ptr = alloc(page, size))) live.insert(ptr);
EXPECT_EQ(live.size(), BLOCK_COUNT);
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
}
} // namespace
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+34 -56
View File
@@ -10,7 +10,10 @@
#include "ConcurrentMarkAndSweep.hpp"
#include "CustomLogging.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "FinalizerHooks.hpp"
#include "GCStatistics.hpp"
#include "KAssert.h"
#include "ObjectFactory.hpp"
@@ -22,66 +25,41 @@ std::atomic<size_t> allocatedBytesCounter;
namespace kotlin::alloc {
bool TryResetMark(void* ptr) noexcept {
using Node = typename kotlin::mm::ObjectFactory<kotlin::gc::ConcurrentMarkAndSweep>::Storage::Node;
using NodeRef = typename kotlin::mm::ObjectFactory<kotlin::gc::ConcurrentMarkAndSweep>::NodeRef;
Node& node = Node::FromData(ptr);
NodeRef ref = NodeRef(node);
auto& objectData = ref.ObjectData();
bool reset = objectData.tryResetMark();
CustomAllocDebug("TryResetMark(%p) = %d", ptr, reset);
return reset;
}
using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static void KeepAlive(ObjHeader* baseObject) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(baseObject).ObjectData();
objectData.tryMark();
}
static bool IsAlive(ObjHeader* baseObject) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(baseObject).ObjectData();
return objectData.marked();
}
ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept {
auto* extraObject = extraObjectCell->Data();
if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
CustomAllocDebug("SweepIsCollectable(%p): already finalized", extraObject);
return ExtraObjectStatus::SWEPT;
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
HeapObjHeader* objHeader = reinterpret_cast<HeapObjHeader*>(object);
if (objHeader->gcData.tryResetMark()) {
CustomAllocDebug("SweepObject(%p): still alive", object);
gcHandle.addKeptObject();
return true;
}
auto* baseObject = extraObject->GetBaseObject();
RuntimeAssert(baseObject->heap(), "SweepIsCollectable on a non-heap object");
if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepIsCollectable(%p): already in finalizer queue, keep base object (%p) alive", extraObject, baseObject);
KeepAlive(baseObject);
return ExtraObjectStatus::TO_BE_FINALIZED;
}
if (IsAlive(baseObject)) {
CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject);
return ExtraObjectStatus::KEPT;
}
extraObject->ClearRegularWeakReferenceImpl();
if (extraObject->HasAssociatedObject()) {
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
finalizerQueue.Push(extraObjectCell);
KeepAlive(baseObject);
CustomAllocDebug("SweepIsCollectable(%p): add to finalizerQueue", extraObject);
return ExtraObjectStatus::TO_BE_FINALIZED;
} else {
if (HasFinalizers(baseObject)) {
auto* extraObject = mm::ExtraObjectData::Get(&objHeader->object);
if (extraObject) {
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", object, extraObject);
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
finalizerQueue.Push(extraObjectCell);
KeepAlive(baseObject);
CustomAllocDebug("SweepIsCollectable(%p): addings to finalizerQueue, keep base object (%p) alive", extraObject, baseObject);
return ExtraObjectStatus::TO_BE_FINALIZED;
CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject));
finalizerQueue.Push(ExtraObjectCell::fromExtraObject(extraObject));
gcHandle.addMarkedObject();
return true;
}
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE)) {
CustomAllocDebug("SweepObject(%p): already waiting to be finalized", object);
gcHandle.addMarkedObject();
return true;
}
extraObject->Uninstall();
CustomAllocDebug("SweepIsCollectable(%p): uninstalled extraObject", extraObject);
return ExtraObjectStatus::SWEPT;
}
CustomAllocDebug("SweepObject(%p): can be reclaimed", object);
gcHandle.addSweptObject();
return false;
}
bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& gcHandle) noexcept {
if (extraObject->getFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE)) {
CustomAllocDebug("SweepExtraObject(%p): can be reclaimed", extraObject);
return false;
}
CustomAllocDebug("SweepExtraObject(%p): is still needed", extraObject);
return true;
}
void* SafeAlloc(uint64_t size) noexcept {
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp
+19 -6
View File
@@ -12,19 +12,32 @@
#include <stdlib.h>
#include "AtomicStack.hpp"
#include "ConcurrentMarkAndSweep.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
bool TryResetMark(void* ptr) noexcept;
// copied over from ObjectFactory
enum class ExtraObjectStatus {
TO_BE_FINALIZED,
KEPT,
SWEPT,
using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData;
struct HeapObjHeader {
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object;
};
ExtraObjectStatus SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
struct HeapArrayHeader {
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array;
};
// Returns `true` if the `object` must be kept alive still.
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept;
// Returns `true` if the `extraObject` must be kept alive still
bool SweepExtraObject(mm::ExtraObjectData* extraObject, gc::GCHandle::GCSweepExtraObjectsScope& sweepScope) noexcept;
void* SafeAlloc(uint64_t size) noexcept;
void Free(void* ptr, size_t size) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp
+24 -39
View File
@@ -20,12 +20,6 @@
namespace kotlin::alloc {
Heap::~Heap() noexcept {
ExtraObjectPage* page;
while ((page = extraObjectPages_.Pop())) page->Destroy();
while ((page = usedExtraObjectPages_.Pop())) page->Destroy();
}
void Heap::PrepareForGC() noexcept {
CustomAllocDebug("Heap::PrepareForGC()");
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
@@ -37,58 +31,49 @@ void Heap::PrepareForGC() noexcept {
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
fixedBlockPages_[blockSize].PrepareForGC();
}
usedExtraObjectPages_.TransferAllFrom(std::move(extraObjectPages_));
extraObjectPages_.PrepareForGC();
}
void Heap::Sweep(gc::GCHandle gcHandle) noexcept {
auto sweepHandle = gcHandle.sweep();
FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept {
FinalizerQueue finalizerQueue;
CustomAllocDebug("Heap::Sweep()");
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
fixedBlockPages_[blockSize].Sweep(sweepHandle);
}
nextFitPages_.Sweep(sweepHandle);
singleObjectPages_.SweepAndFree(sweepHandle);
}
AtomicStack<ExtraObjectCell> Heap::SweepExtraObjects(gc::GCHandle gcHandle) noexcept {
auto sweepHandle = gcHandle.sweepExtraObjects();
CustomAllocDebug("Heap::SweepExtraObjects()");
AtomicStack<ExtraObjectCell> finalizerQueue;
ExtraObjectPage* page;
while ((page = usedExtraObjectPages_.Pop())) {
if (!page->Sweep(sweepHandle, finalizerQueue)) {
CustomAllocInfo("SweepExtraObjects free(%p)", page);
page->Destroy();
} else {
extraObjectPages_.Push(page);
{
auto sweepHandle = gcHandle.sweep();
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
fixedBlockPages_[blockSize].Sweep(sweepHandle, finalizerQueue);
}
nextFitPages_.Sweep(sweepHandle, finalizerQueue);
singleObjectPages_.SweepAndFree(sweepHandle, finalizerQueue);
}
{
auto sweepHandle = gcHandle.sweepExtraObjects();
extraObjectPages_.Sweep(sweepHandle, finalizerQueue);
}
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
}
CustomAllocDebug("Heap::Sweep done");
return finalizerQueue;
}
NextFitPage* Heap::GetNextFitPage(uint32_t cellCount) noexcept {
NextFitPage* Heap::GetNextFitPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocDebug("Heap::GetNextFitPage()");
return nextFitPages_.GetPage(cellCount);
return nextFitPages_.GetPage(cellCount, finalizerQueue);
}
FixedBlockPage* Heap::GetFixedBlockPage(uint32_t cellCount) noexcept {
FixedBlockPage* Heap::GetFixedBlockPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocDebug("Heap::GetFixedBlockPage()");
return fixedBlockPages_[cellCount].GetPage(cellCount);
return fixedBlockPages_[cellCount].GetPage(cellCount, finalizerQueue);
}
SingleObjectPage* Heap::GetSingleObjectPage(uint64_t cellCount) noexcept {
SingleObjectPage* Heap::GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocInfo("CustomAllocator::AllocateInSingleObjectPage(%" PRIu64 ")", cellCount);
return singleObjectPages_.NewPage(cellCount);
}
ExtraObjectPage* Heap::GetExtraObjectPage() noexcept {
ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept {
CustomAllocInfo("CustomAllocator::GetExtraObjectPage()");
ExtraObjectPage* page = extraObjectPages_.Pop();
if (page == nullptr) {
page = ExtraObjectPage::Create();
}
usedExtraObjectPages_.Push(page);
return page;
return extraObjectPages_.GetPage(0, finalizerQueue);
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp
+6 -11
View File
@@ -22,29 +22,24 @@ namespace kotlin::alloc {
class Heap {
public:
~Heap() noexcept;
// Called once by the GC thread after all mutators have been suspended
void PrepareForGC() noexcept;
// Sweep through all remaining pages, freeing those blocks where CanReclaim
// returns true. If multiple sweepers are active, each page will only be
// seen by one sweeper.
void Sweep(gc::GCHandle gcHandle) noexcept;
FinalizerQueue Sweep(gc::GCHandle gcHandle) noexcept;
AtomicStack<ExtraObjectCell> SweepExtraObjects(gc::GCHandle gcHandle) noexcept;
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount) noexcept;
NextFitPage* GetNextFitPage(uint32_t cellCount) noexcept;
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount) noexcept;
ExtraObjectPage* GetExtraObjectPage() noexcept;
FixedBlockPage* GetFixedBlockPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept;
NextFitPage* GetNextFitPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept;
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept;
ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept;
private:
PageStore<FixedBlockPage> fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1];
PageStore<NextFitPage> nextFitPages_;
PageStore<SingleObjectPage> singleObjectPages_;
AtomicStack<ExtraObjectPage> extraObjectPages_;
AtomicStack<ExtraObjectPage> usedExtraObjectPages_;
PageStore<ExtraObjectPage> extraObjectPages_;
};
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/HeapTest.cpp
+7 -4
View File
@@ -7,6 +7,7 @@
#include <cstdint>
#include <random>
#include "ExtraObjectPage.hpp"
#include "SingleObjectPage.hpp"
#include "gtest/gtest.h"
#include "Heap.hpp"
@@ -30,8 +31,9 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) {
const int MIN = MIN_BLOCK_SIZE;
const int MAX = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1;
FixedBlockPage* pages[MAX];
kotlin::alloc::FinalizerQueue finalizerQueue;
for (int blocks = MIN; blocks < MAX; ++blocks) {
pages[blocks] = heap.GetFixedBlockPage(blocks);
pages[blocks] = heap.GetFixedBlockPage(blocks, finalizerQueue);
void* obj = pages[blocks]->TryAllocate();
mark(obj); // to make the page survive a sweep
}
@@ -39,20 +41,21 @@ TEST(CustomAllocTest, HeapReuseFixedBlockPages) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
heap.Sweep(gcHandle);
for (int blocks = MIN; blocks < MAX; ++blocks) {
EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks));
EXPECT_EQ(pages[blocks], heap.GetFixedBlockPage(blocks, finalizerQueue));
}
}
TEST(CustomAllocTest, HeapReuseNextFitPages) {
Heap heap;
const uint32_t BLOCKSIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 42;
NextFitPage* page = heap.GetNextFitPage(BLOCKSIZE);
kotlin::alloc::FinalizerQueue finalizerQueue;
NextFitPage* page = heap.GetNextFitPage(BLOCKSIZE, finalizerQueue);
void* obj = page->TryAllocate(BLOCKSIZE);
mark(obj); // to make the page survive a sweep
heap.PrepareForGC();
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
heap.Sweep(gcHandle);
EXPECT_EQ(page, heap.GetNextFitPage(BLOCKSIZE));
EXPECT_EQ(page, heap.GetNextFitPage(0, finalizerQueue));
}
} // namespace
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp
+2 -2
View File
@@ -39,13 +39,13 @@ uint8_t* NextFitPage::TryAllocate(uint32_t blockSize) noexcept {
return curBlock_->TryAllocate(cellsNeeded);
}
bool NextFitPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
bool NextFitPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocDebug("NextFitPage@%p::Sweep()", this);
Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT;
bool alive = false;
for (Cell* block = cells_ + 1; block != end; block = block->Next()) {
if (block->isAllocated_) {
if (TryResetMark(block->data_)) {
if (SweepObject(block->data_, finalizerQueue, sweepHandle)) {
alive = true;
sweepHandle.addKeptObject();
} else {
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp
+6 -1
View File
@@ -11,12 +11,17 @@
#include "AtomicStack.hpp"
#include "Cell.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
class alignas(8) NextFitPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); }
static NextFitPage* Create(uint32_t cellCount) noexcept;
void Destroy() noexcept;
@@ -24,7 +29,7 @@ public:
// Tries to allocate in current page, returns null if no free block in page is big enough
uint8_t* TryAllocate(uint32_t blockSize) noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
// Testing method
bool CheckInvariants() noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp
+16 -9
View File
@@ -8,6 +8,7 @@
#include "Cell.hpp"
#include "CustomAllocConstants.hpp"
#include "ExtraObjectPage.hpp"
#include "gtest/gtest.h"
#include "NextFitPage.hpp"
#include "TypeInfo.h"
@@ -17,7 +18,7 @@ namespace {
using NextFitPage = typename kotlin::alloc::NextFitPage;
using Cell = typename kotlin::alloc::Cell;
TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer
TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer
inline constexpr const size_t MIN_BLOCK_SIZE = FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1;
@@ -54,18 +55,20 @@ TEST(CustomAllocTest, NextFitPageSweepEmptyPage) {
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
kotlin::alloc::FinalizerQueue finalizerQueue;
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
TEST(CustomAllocTest, NextFitPageSweepFullUnmarkedPage) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) {}
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
}
@@ -75,19 +78,21 @@ TEST(CustomAllocTest, NextFitPageSweepSingleMarked) {
mark(alloc(page, MIN_BLOCK_SIZE));
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_TRUE(page->Sweep(gcScope));
kotlin::alloc::FinalizerQueue finalizerQueue;
EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
int count1 = 0;
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count1;
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
r.seed(seed);
int count2 = 0;
while (alloc(page, MIN_BLOCK_SIZE + r() % 100)) ++count2;
@@ -99,6 +104,7 @@ TEST(CustomAllocTest, NextFitPageSweepSingleReuse) {
TEST(CustomAllocTest, NextFitPageSweepReuse) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
for (uint32_t seed = 0xC0FFEE0; seed <= 0xC0FFEEF; ++seed) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
@@ -112,7 +118,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
++unmarked;
}
}
page->Sweep(gcScope);
page->Sweep(gcScope, finalizerQueue);
int freed = 0;
while (alloc(page, MIN_BLOCK_SIZE)) ++freed;
EXPECT_EQ(freed, unmarked);
@@ -123,9 +129,10 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
TEST(CustomAllocTest, NextFitPageSweepCoallesce) {
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
kotlin::alloc::FinalizerQueue finalizerQueue;
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) / 2 - 1));
EXPECT_FALSE(page->Sweep(gcScope));
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
EXPECT_TRUE(alloc(page, (NEXT_FIT_PAGE_CELL_COUNT-1) - 1));
page->Destroy();
}
kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp
+12 -9
View File
@@ -9,6 +9,7 @@
#include <atomic>
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
@@ -16,6 +17,8 @@ namespace kotlin::alloc {
template <class T>
class PageStore {
public:
using GCSweepScope = typename T::GCSweepScope;
void PrepareForGC() noexcept {
unswept_.TransferAllFrom(std::move(ready_));
unswept_.TransferAllFrom(std::move(used_));
@@ -23,15 +26,15 @@ public:
while ((page = empty_.Pop())) page->Destroy();
}
void Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_)) {
void Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
while (SweepSingle(sweepHandle, unswept_.Pop(), unswept_, ready_, finalizerQueue)) {
}
}
void SweepAndFree(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
void SweepAndFree(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
T* page;
while ((page = unswept_.Pop())) {
if (page->Sweep(sweepHandle)) {
if (page->Sweep(sweepHandle, finalizerQueue)) {
ready_.Push(page);
} else {
page->Destroy();
@@ -39,12 +42,12 @@ public:
}
}
T* GetPage(uint32_t cellCount) noexcept {
T* GetPage(uint32_t cellCount, FinalizerQueue& finalizerQueue) noexcept {
T* page;
if ((page = unswept_.Pop())) {
// If there're unswept_ pages, the GC is in progress.
auto sweepHandle = gc::GCHandle::currentEpoch()->sweep();
if ((page = SweepSingle(sweepHandle, page, unswept_, used_))) {
GCSweepScope sweepHandle = T::currentGCSweepScope();
if ((page = SweepSingle(sweepHandle, page, unswept_, used_, finalizerQueue))) {
return page;
}
}
@@ -74,12 +77,12 @@ public:
}
private:
T* SweepSingle(gc::GCHandle::GCSweepScope& sweepHandle, T* page, AtomicStack<T>& from, AtomicStack<T>& to) noexcept {
T* SweepSingle(GCSweepScope& sweepHandle, T* page, AtomicStack<T>& from, AtomicStack<T>& to, FinalizerQueue& finalizerQueue) noexcept {
if (!page) {
return nullptr;
}
do {
if (page->Sweep(sweepHandle)) {
if (page->Sweep(sweepHandle, finalizerQueue)) {
to.Push(page);
return page;
}
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp
+5 -7
View File
@@ -37,15 +37,13 @@ uint8_t* SingleObjectPage::TryAllocate() noexcept {
return Data();
}
bool SingleObjectPage::Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept {
bool SingleObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept {
CustomAllocDebug("SingleObjectPage@%p::Sweep()", this);
if (!TryResetMark(Data())) {
isAllocated_ = false;
sweepHandle.addKeptObject();
return false;
if (SweepObject(Data(), finalizerQueue, sweepHandle)) {
return true;
}
sweepHandle.addSweptObject();
return true;
isAllocated_ = false;
return false;
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp
+6 -1
View File
@@ -10,12 +10,17 @@
#include <cstdint>
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
namespace kotlin::alloc {
class alignas(8) SingleObjectPage {
public:
using GCSweepScope = gc::GCHandle::GCSweepScope;
static GCSweepScope currentGCSweepScope() noexcept { return gc::GCHandle::currentEpoch()->sweep(); }
static SingleObjectPage* Create(uint64_t cellCount) noexcept;
void Destroy() noexcept;
@@ -24,7 +29,7 @@ public:
uint8_t* TryAllocate() noexcept;
bool Sweep(gc::GCHandle::GCSweepScope& sweepHandle) noexcept;
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
private:
friend class AtomicStack<SingleObjectPage>;
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPageTest.cpp
+6 -3
View File
@@ -7,6 +7,7 @@
#include <random>
#include "CustomAllocConstants.hpp"
#include "ExtraObjectPage.hpp"
#include "gtest/gtest.h"
#include "SingleObjectPage.hpp"
#include "TypeInfo.h"
@@ -15,7 +16,7 @@ namespace {
using SingleObjectPage = typename kotlin::alloc::SingleObjectPage;
TypeInfo fakeType = {.flags_ = 0}; // a type without a finalizer
TypeInfo fakeType = {.typeInfo_ = &fakeType, .flags_ = 0}; // a type without a finalizer
#define MIN_BLOCK_SIZE NEXT_FIT_PAGE_CELL_COUNT
@@ -36,7 +37,8 @@ TEST(CustomAllocTest, SingleObjectPageSweepEmptyPage) {
EXPECT_TRUE(page);
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_FALSE(page->Sweep(gcScope));
kotlin::alloc::FinalizerQueue finalizerQueue;
EXPECT_FALSE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
@@ -47,7 +49,8 @@ TEST(CustomAllocTest, SingleObjectPageSweepFullPage) {
mark(page->Data());
auto gcHandle = kotlin::gc::GCHandle::createFakeForTests();
auto gcScope = gcHandle.sweep();
EXPECT_TRUE(page->Sweep(gcScope));
kotlin::alloc::FinalizerQueue finalizerQueue;
EXPECT_TRUE(page->Sweep(gcScope, finalizerQueue));
page->Destroy();
}
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+3 -3
View File
@@ -206,11 +206,11 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread();
#else
auto finalizerQueue = heap_.SweepExtraObjects(gcHandle);
mm::ResumeThreads();
gcHandle.threadsAreResumed();
heap_.Sweep(gcHandle);
// also sweeps extraObjects
auto finalizerQueue = heap_.Sweep(gcHandle);
#endif
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp
+6 -6
View File
@@ -324,7 +324,7 @@ MarkStats GCHandle::getMarked() {
return MarkStats{};
}
void GCHandle::swept(gc::SweepStats stats) noexcept {
void GCHandle::swept(gc::SweepStats stats, uint64_t markedCount) noexcept {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
auto& heap = stat->sweepStats.heap;
@@ -333,10 +333,12 @@ void GCHandle::swept(gc::SweepStats stats) noexcept {
}
heap->keptCount += stats.keptCount;
heap->sweptCount += stats.sweptCount;
RuntimeAssert(static_cast<bool>(stat->markStats), "Mark must have already happened");
stat->markStats->markedCount += markedCount;
}
}
void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noexcept {
void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept {
std::lock_guard guard(lock);
if (auto* stat = statByEpoch(epoch_)) {
auto& extra = stat->sweepStats.extra;
@@ -345,15 +347,13 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats, uint64_t markedCount) noe
}
extra->keptCount += stats.keptCount;
extra->sweptCount += stats.sweptCount;
RuntimeAssert(static_cast<bool>(stat->markStats), "Mark must have already happened");
stat->markStats->markedCount += markedCount;
}
}
GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
GCHandle::GCSweepScope::~GCSweepScope() {
handle_.swept(stats_);
handle_.swept(stats_, markedCount_);
GCLogDebug(
handle_.getEpoch(),
"Collected %" PRId64 " heap objects in %" PRIu64 " microseconds. "
@@ -364,7 +364,7 @@ GCHandle::GCSweepScope::~GCSweepScope() {
GCHandle::GCSweepExtraObjectsScope::GCSweepExtraObjectsScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
GCHandle::GCSweepExtraObjectsScope::~GCSweepExtraObjectsScope() {
handle_.sweptExtraObjects(stats_, markedCount_);
handle_.sweptExtraObjects(stats_);
GCLogDebug(
handle_.getEpoch(),
"Collected %" PRId64 " extra objects in %" PRIu64 " microseconds. "
kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp
+5 -5
View File
@@ -46,6 +46,7 @@ public:
class GCSweepScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
SweepStats stats_;
uint64_t markedCount_ = 0;
public:
explicit GCSweepScope(GCHandle& handle);
@@ -53,12 +54,13 @@ public:
void addSweptObject() noexcept { stats_.sweptCount += 1; }
void addKeptObject() noexcept { stats_.keptCount += 1; }
// Custom allocator only. To be finalized objects are kept alive.
void addMarkedObject() noexcept { markedCount_ += 1; }
};
class GCSweepExtraObjectsScope : GCStageScopeUsTimer, Pinned {
GCHandle& handle_;
SweepStats stats_;
uint64_t markedCount_ = 0;
public:
explicit GCSweepExtraObjectsScope(GCHandle& handle);
@@ -66,8 +68,6 @@ public:
void addSweptObject() noexcept { stats_.sweptCount += 1; }
void addKeptObject() noexcept { stats_.keptCount += 1; }
// Custom allocator only. To be finalized objects are kept alive.
void addMarkedObject() noexcept { markedCount_ += 1; }
};
class GCGlobalRootSetScope : GCStageScopeUsTimer, Pinned {
@@ -130,8 +130,8 @@ private:
void threadRootSetCollected(mm::ThreadData& threadData, uint64_t threadLocalReferences, uint64_t stackReferences);
void globalRootSetCollected(uint64_t globalReferences, uint64_t stableReferences);
void swept(SweepStats stats) noexcept;
void sweptExtraObjects(SweepStats stats, uint64_t markedCount) noexcept;
void swept(SweepStats stats, uint64_t markedCount) noexcept;
void sweptExtraObjects(SweepStats stats) noexcept;
void marked(MarkStats stats);
public:
kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp
+1 -1
View File
@@ -38,7 +38,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
// Somebody else created `mm::ExtraObjectData` for this object.
data.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
data.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
#else
auto& data = mm::ExtraObjectDataFactory::Instance().CreateExtraObjectDataForObject(threadData, object, typeInfo);
kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp
+1 -1
View File
@@ -26,7 +26,7 @@ public:
FLAGS_FROZEN = 0,
FLAGS_NEVER_FROZEN = 1,
FLAGS_IN_FINALIZER_QUEUE = 2,
FLAGS_FINALIZED = 3,
FLAGS_SWEEPABLE = 3,
FLAGS_RELEASE_ON_MAIN_QUEUE = 4,
};
kotlin-native/runtime/src/mm/cpp/Memory.cpp
+3 -1
View File
@@ -79,7 +79,9 @@ void ObjHeader::destroyMetaObject(ObjHeader* object) {
RuntimeAssert(object->has_meta_object(), "Object must have a meta object set");
auto &extraObject = *mm::ExtraObjectData::Get(object);
extraObject.Uninstall();
#ifndef CUSTOM_ALLOCATOR
#ifdef CUSTOM_ALLOCATOR
extraObject.setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
#else
auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
mm::ExtraObjectDataFactory::Instance().DestroyExtraObjectData(threadData, extraObject);
#endif