azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Custom allocator for all GCs ^KT-55364

Browse Source 
Also enables the integration tests for custom allocator and the
currently available GCs.

Co-authored-by: Troels Bjerre Lund <troels@google.com>

Merge-request: KT-MR-11199
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>
This commit is contained in:
Troels Bjerre Lund
2023-07-24 16:28:47 +00:00
committed by Space Team
parent 9864c5c386
commit 2818957689
41 changed files with 552 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt
+13 -25
View File
@@ -232,15 +232,11 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
val shouldCoverSources = configuration.getBoolean(KonanConfigKeys.COVERAGE)
private val shouldCoverLibraries = !configuration.getList(KonanConfigKeys.LIBRARIES_TO_COVER).isNullOrEmpty()
private val defaultAllocationMode get() = when {
gc == GC.PARALLEL_MARK_CONCURRENT_SWEEP && sanitizer == null -> {
private val defaultAllocationMode get() =
if (sanitizer == null)
AllocationMode.CUSTOM
}
target.supportsMimallocAllocator() && sanitizer == null -> {
AllocationMode.MIMALLOC
}
else -> AllocationMode.STD
}
else
AllocationMode.STD
val allocationMode by lazy {
when (configuration.get(KonanConfigKeys.ALLOCATION_MODE)) {
@@ -262,13 +258,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
if (sanitizer != null) {
configuration.report(CompilerMessageSeverity.STRONG_WARNING, "Sanitizers are useful only with the std allocator")
}
if (gc == GC.PARALLEL_MARK_CONCURRENT_SWEEP) {
AllocationMode.CUSTOM
} else {
configuration.report(CompilerMessageSeverity.STRONG_WARNING,
"Custom allocator is currently only integrated with concurrent mark and sweep gc. Using default mode.")
defaultAllocationMode
}
AllocationMode.CUSTOM
}
}
}
@@ -293,19 +283,17 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
}
if (allocationMode == AllocationMode.CUSTOM) {
add("experimental_memory_manager_custom.bc")
add("concurrent_ms_gc_custom.bc")
when (gc) {
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc_custom.bc")
GC.NOOP -> add("noop_gc_custom.bc")
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc_custom.bc")
}
} else {
add("experimental_memory_manager.bc")
when (gc) {
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> {
add("same_thread_ms_gc.bc")
}
GC.NOOP -> {
add("noop_gc.bc")
}
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> {
add("concurrent_ms_gc.bc")
}
GC.STOP_THE_WORLD_MARK_AND_SWEEP -> add("same_thread_ms_gc.bc")
GC.NOOP -> add("noop_gc.bc")
GC.PARALLEL_MARK_CONCURRENT_SWEEP -> add("concurrent_ms_gc.bc")
}
}
if (shouldCoverLibraries || shouldCoverSources) add("profileRuntime.bc")
kotlin-native/runtime/build.gradle.kts
+44 -1
View File
@@ -158,7 +158,7 @@ bitcode {
}
module("custom_alloc") {
headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp", "src/gc/cms/cpp"))
headersDirs.from(files("src/main/cpp", "src/mm/cpp", "src/gc/common/cpp", "src/gcScheduler/common/cpp"))
sourceSets {
main {}
test {}
@@ -195,6 +195,7 @@ bitcode {
onlyIf { target.supportsCoreSymbolication() }
}
module("source_info_libbacktrace") {
srcRoot.set(layout.projectDirectory.dir("src/source_info/libbacktrace"))
headersDirs.from(files("src/main/cpp", "src/libbacktrace/c/include"))
@@ -299,6 +300,19 @@ bitcode {
onlyIf { target.supportsThreads() }
}
module("noop_gc_custom") {
srcRoot.set(layout.projectDirectory.dir("src/gc/noop"))
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
sourceSets {
main {}
testFixtures {}
}
compilerArgs.add("-DCUSTOM_ALLOCATOR")
onlyIf { target.supportsThreads() }
}
module("same_thread_ms_gc") {
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
@@ -311,6 +325,20 @@ bitcode {
onlyIf { target.supportsThreads() }
}
module("same_thread_ms_gc_custom") {
srcRoot.set(layout.projectDirectory.dir("src/gc/stms"))
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp", "src/custom_alloc/cpp"))
sourceSets {
main {}
testFixtures {}
test {}
}
compilerArgs.add("-DCUSTOM_ALLOCATOR")
onlyIf { target.supportsThreads() }
}
module("concurrent_ms_gc") {
srcRoot.set(layout.projectDirectory.dir("src/gc/cms"))
headersDirs.from(files("src/gcScheduler/common/cpp", "src/gc/common/cpp", "src/mm/cpp", "src/main/cpp"))
@@ -394,6 +422,11 @@ bitcode {
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "same_thread_ms_gc", "std_alloc", "objc")
}
testsGroup("experimentalMM_custom_alloc_runtime_tests") {
testedModules.addAll("experimental_memory_manager_custom", "same_thread_ms_gc_custom")
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
}
testsGroup("experimentalMM_cms_mimalloc_runtime_tests") {
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "mimalloc", "opt_alloc", "objc")
}
@@ -402,6 +435,11 @@ bitcode {
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "concurrent_ms_gc", "std_alloc", "objc")
}
testsGroup("experimentalMM_cms_custom_alloc_runtime_tests") {
testedModules.addAll("experimental_memory_manager_custom", "concurrent_ms_gc_custom")
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
}
testsGroup("experimentalMM_noop_mimalloc_runtime_tests") {
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "mimalloc", "opt_alloc", "objc")
}
@@ -410,6 +448,11 @@ bitcode {
testedModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "manual_gcScheduler", "noop_gc", "std_alloc", "objc")
}
testsGroup("experimentalMM_noop_custom_alloc_runtime_tests") {
testedModules.addAll("experimental_memory_manager_custom", "noop_gc_custom")
testSupportModules.addAll("main", "common_gc", "common_gcScheduler", "manual_gcScheduler", "custom_alloc", "objc")
}
testsGroup("aggressive_gcScheduler_runtime_tests") {
testedModules.addAll("aggressive_gcScheduler")
testSupportModules.addAll("main", "experimental_memory_manager", "common_gc", "common_gcScheduler", "noop_gc", "std_alloc", "objc")
kotlin-native/runtime/src/custom_alloc/cpp/AtomicStack.hpp
+12
View File
@@ -10,6 +10,7 @@
#include "KAssert.h"
#include "Utils.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -82,6 +83,17 @@ public:
RuntimeAssert(isEmpty(), "AtomicStack must be empty on destruction");
}
// Test method
std_support::vector<T*> GetElements() {
std_support::vector<T*> elements;
T* elm = stack_.load();
while (elm) {
elements.push_back(elm);
elm = elm->next_;
}
return elements;
}
private:
std::atomic<T*> stack_{nullptr};
};
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp
+10 -11
View File
@@ -12,18 +12,17 @@
#include <cstring>
#include <new>
#include "ConcurrentMarkAndSweep.hpp"
#include "CustomAllocConstants.hpp"
#include "CustomLogging.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp"
#include "KAssert.h"
#include "SingleObjectPage.hpp"
#include "NextFitPage.hpp"
#include "Memory.h"
#include "FixedBlockPage.hpp"
#include "GCImpl.hpp"
#include "GCApi.hpp"
#include "TypeInfo.h"
@@ -31,7 +30,7 @@ namespace kotlin::alloc {
size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
return AlignUp(heapObjectHeaderSize + membersSize, kObjectAlignment);
}
uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
@@ -39,7 +38,7 @@ uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexce
// at about half of uint64_t max.
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
// Note: array body is aligned, but for size computation it is enough to align the sum.
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
return AlignUp<uint64_t>(heapArrayHeaderSize + membersSize, kObjectAlignment);
}
CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData& gcScheduler) noexcept :
@@ -51,8 +50,8 @@ CustomAllocator::CustomAllocator(Heap& heap, gcScheduler::GCSchedulerThreadData&
ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
size_t allocSize = ObjectAllocatedDataSize(typeInfo);
auto* heapObject = new (Allocate(allocSize)) HeapObjHeader();
auto* object = &heapObject->object;
uint8_t* heapObject = Allocate(allocSize);
auto* object = reinterpret_cast<ObjHeader*>(heapObject + gcDataSize);
if (typeInfo->flags_ & TF_HAS_FINALIZER) {
auto* extraObject = CreateExtraObject();
object->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(new (extraObject) mm::ExtraObjectData(object, typeInfo));
@@ -65,10 +64,11 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
}
ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count);
RuntimeAssert(typeInfo->IsArray(), "Must be an array");
auto allocSize = ArrayAllocatedDataSize(typeInfo, count);
auto* heapArray = new (Allocate(allocSize)) HeapArrayHeader();
auto* array = &heapArray->array;
uint8_t* heapArray = Allocate(allocSize);
auto* array = reinterpret_cast<ArrayHeader*>(heapArray + gcDataSize);
array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
array->count_ = count;
return array;
@@ -96,10 +96,9 @@ mm::ExtraObjectData* CustomAllocator::CreateExtraObject() noexcept {
return nullptr;
}
// static
mm::ExtraObjectData& CustomAllocator::CreateExtraObjectDataForObject(
mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept {
mm::ExtraObjectData* extraObject = threadData->gc().impl().alloc().CreateExtraObject();
ObjHeader* baseObject, const TypeInfo* info) noexcept {
mm::ExtraObjectData* extraObject = CreateExtraObject();
return *new (extraObject) mm::ExtraObjectData(baseObject, info);
}
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.hpp
+6 -2
View File
@@ -29,8 +29,8 @@ public:
mm::ExtraObjectData* CreateExtraObject() noexcept;
static mm::ExtraObjectData& CreateExtraObjectDataForObject(
mm::ThreadData* threadData, ObjHeader* baseObject, const TypeInfo* info) noexcept;
mm::ExtraObjectData& CreateExtraObjectDataForObject(
ObjHeader* baseObject, const TypeInfo* info) noexcept;
void PrepareForGC() noexcept;
@@ -38,6 +38,10 @@ public:
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;
Heap& heap() noexcept {
return heap_;
}
private:
uint8_t* Allocate(uint64_t cellCount) noexcept;
uint8_t* AllocateInSingleObjectPage(uint64_t cellCount) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp
+18
View File
@@ -86,4 +86,22 @@ bool FixedBlockPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQ
return nextFree_.first > 0 || nextFree_.last < end_;
}
std_support::vector<uint8_t*> FixedBlockPage::GetAllocatedBlocks() noexcept {
std_support::vector<uint8_t*> allocated;
CustomAllocInfo("FixedBlockPage(%p)::Sweep()", this);
FixedCellRange nextFree = nextFree_; // Accessing the previous free list structure.
for (uint32_t cell = 0 ; cell < end_ ; cell += blockSize_) {
for (; cell < nextFree.first ; cell += blockSize_) {
allocated.push_back(cells_[cell].data);
}
if (nextFree.last >= end_) {
break;
}
cell = nextFree.last;
nextFree = cells_[cell].nextFree;
}
return allocated;
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.hpp
+4
View File
@@ -12,6 +12,7 @@
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -43,6 +44,9 @@ public:
bool Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalizerQueue) noexcept;
// Testing method
std_support::vector<uint8_t*> GetAllocatedBlocks() noexcept;
private:
explicit FixedBlockPage(uint32_t blockSize) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPageTest.cpp
+9
View File
@@ -150,6 +150,15 @@ TEST(CustomAllocTest, FixedBlockPageRandomExercise) {
}
}
EXPECT_EQ(page->Sweep(gcScope, finalizerQueue), !live.empty());
uint8_t* prev = nullptr;
uint32_t allocCount = 0;
for (auto* obj : page->GetAllocatedBlocks()) {
EXPECT_LT(prev, obj);
prev = obj;
++allocCount;
EXPECT_NE(live.find(obj), live.end());
}
EXPECT_EQ(allocCount, live.size());
}
while ((ptr = alloc(page, size))) live.insert(ptr);
EXPECT_EQ(live.size(), BLOCK_COUNT);
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+13 -10
View File
@@ -7,20 +7,22 @@
#include <atomic>
#include <cstdint>
#include <cstring>
#include <limits>
#ifndef KONAN_WINDOWS
#include <sys/mman.h>
#endif
#include "ConcurrentMarkAndSweep.hpp"
#include "CompilerConstants.hpp"
#include "CustomLogging.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "FinalizerHooks.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "KAssert.h"
#include "Memory.h"
#include "ObjectFactory.hpp"
namespace {
@@ -31,18 +33,17 @@ std::atomic<size_t> allocatedBytesCounter;
namespace kotlin::alloc {
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
HeapObjHeader* objHeader = reinterpret_cast<HeapObjHeader*>(object);
if (objHeader->gcData.tryResetMark()) {
CustomAllocDebug("SweepObject(%p): still alive", object);
bool SweepObject(uint8_t* heapObjHeader, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
if (gc::GC::SweepObject(heapObjHeader)) {
CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader);
gcHandle.addKeptObject();
return true;
}
auto* baseObject = &objHeader->object;
auto* extraObject = mm::ExtraObjectData::Get(baseObject);
auto* objHeader = reinterpret_cast<ObjHeader*>(heapObjHeader + gcDataSize);
auto* extraObject = mm::ExtraObjectData::Get(objHeader);
if (extraObject) {
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", object, extraObject);
CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject);
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
extraObject->ClearRegularWeakReferenceImpl();
CustomAllocDebug("SweepObject: fromExtraObject(%p) = %p", extraObject, ExtraObjectCell::fromExtraObject(extraObject));
@@ -51,14 +52,14 @@ bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::
return true;
}
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
CustomAllocDebug("SweepObject(%p): already waiting to be finalized", object);
CustomAllocDebug("SweepObject(%p): already waiting to be finalized", heapObjHeader);
gcHandle.addMarkedObject();
return true;
}
extraObject->UnlinkFromBaseObject();
extraObject->setFlag(mm::ExtraObjectData::FLAGS_SWEEPABLE);
}
CustomAllocDebug("SweepObject(%p): can be reclaimed", object);
CustomAllocDebug("SweepObject(%p): can be reclaimed", heapObjHeader);
gcHandle.addSweptObject();
return false;
}
@@ -98,10 +99,12 @@ void* SafeAlloc(uint64_t size) noexcept {
konan::abort();
}
allocatedBytesCounter.fetch_add(static_cast<size_t>(size), std::memory_order_relaxed);
CustomAllocDebug("SafeAlloc(%zu) = %p", static_cast<size_t>(size), memory);
return memory;
}
void Free(void* ptr, size_t size) noexcept {
CustomAllocDebug("Free(%p, %zu)", ptr, size);
if (compiler::disableMmap()) {
free(ptr);
} else {
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp
+6 -14
View File
@@ -11,27 +11,19 @@
#include <cstdlib>
#include <limits>
#include "Alignment.hpp"
#include "AtomicStack.hpp"
#include "ConcurrentMarkAndSweep.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "Memory.h"
#include "GC.hpp"
namespace kotlin::alloc {
// copied over from ObjectFactory
using ObjectData = gc::ConcurrentMarkAndSweep::ObjectData;
struct HeapObjHeader {
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object;
};
struct HeapArrayHeader {
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array;
};
const size_t gcDataSize = AlignUp(gc::GC::objectDataSize, kObjectAlignment);
const size_t heapObjectHeaderSize = AlignUp(gcDataSize + sizeof(ObjHeader), kObjectAlignment);
const size_t heapArrayHeaderSize = AlignUp(gcDataSize + sizeof(ArrayHeader), kObjectAlignment);
// Returns `true` if the `object` must be kept alive still.
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp
+43 -8
View File
@@ -14,18 +14,17 @@
#include "CustomAllocConstants.hpp"
#include "AtomicStack.hpp"
#include "CustomLogging.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GCApi.hpp"
#include "Memory.h"
#include "ThreadRegistry.hpp"
#include "GCImpl.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
void Heap::PrepareForGC() noexcept {
CustomAllocDebug("Heap::PrepareForGC()");
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
thread.gc().impl().alloc().PrepareForGC();
}
nextFitPages_.PrepareForGC();
singleObjectPages_.PrepareForGC();
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
@@ -36,6 +35,7 @@ void Heap::PrepareForGC() noexcept {
FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept {
FinalizerQueue finalizerQueue;
CustomAllocDebug("Heap: before sweep FinalizerQueue size == %zu", finalizerQueue.size());
CustomAllocDebug("Heap::Sweep()");
{
auto sweepHandle = gcHandle.sweep();
@@ -45,13 +45,11 @@ FinalizerQueue Heap::Sweep(gc::GCHandle gcHandle) noexcept {
nextFitPages_.Sweep(sweepHandle, finalizerQueue);
singleObjectPages_.SweepAndFree(sweepHandle, finalizerQueue);
}
CustomAllocDebug("Heap: before extra sweep FinalizerQueue size == %zu", finalizerQueue.size());
{
auto sweepHandle = gcHandle.sweepExtraObjects();
extraObjectPages_.Sweep(sweepHandle, finalizerQueue);
}
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.gc().impl().alloc().ExtractFinalizerQueue());
}
CustomAllocDebug("Heap::Sweep done");
return finalizerQueue;
}
@@ -76,4 +74,41 @@ ExtraObjectPage* Heap::GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexce
return extraObjectPages_.GetPage(0, finalizerQueue);
}
std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
std_support::vector<ObjHeader*> allocated;
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
for (auto* page : fixedBlockPages_[blockSize].GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
}
}
}
for (auto* page : nextFitPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
}
}
for (auto* page : singleObjectPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
}
}
std_support::vector<ObjHeader*> unfinalized;
for (auto* block: allocated) {
if (!block->has_meta_object() || !mm::ExtraObjectData::Get(block)->getFlag(mm::ExtraObjectData::FLAGS_FINALIZED)) {
unfinalized.push_back(block);
}
}
return unfinalized;
}
void Heap::ClearForTests() noexcept {
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
fixedBlockPages_[blockSize].ClearForTests();
}
nextFitPages_.ClearForTests();
singleObjectPages_.ClearForTests();
extraObjectPages_.ClearForTests();
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/Heap.hpp
+5
View File
@@ -13,6 +13,7 @@
#include "CustomAllocConstants.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "Memory.h"
#include "SingleObjectPage.hpp"
#include "NextFitPage.hpp"
#include "PageStore.hpp"
@@ -35,6 +36,10 @@ public:
SingleObjectPage* GetSingleObjectPage(uint64_t cellCount, FinalizerQueue& finalizerQueue) noexcept;
ExtraObjectPage* GetExtraObjectPage(FinalizerQueue& finalizerQueue) noexcept;
// Test method
std_support::vector<ObjHeader*> GetAllocatedObjects() noexcept;
void ClearForTests() noexcept;
private:
PageStore<FixedBlockPage> fixedBlockPages_[FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE + 1];
PageStore<NextFitPage> nextFitPages_;
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp
+12
View File
@@ -11,6 +11,7 @@
#include "CustomLogging.hpp"
#include "CustomAllocConstants.hpp"
#include "GCApi.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -104,4 +105,15 @@ bool NextFitPage::CheckInvariants() noexcept {
}
}
std_support::vector<uint8_t*> NextFitPage::GetAllocatedBlocks() noexcept {
std_support::vector<uint8_t*> allocated;
Cell* end = cells_ + NEXT_FIT_PAGE_CELL_COUNT;
for (Cell* block = cells_ + 1; block != end; block = block->Next()) {
if (block->isAllocated_) {
allocated.push_back(block->data_);
}
}
return allocated;
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.hpp
+4
View File
@@ -13,6 +13,7 @@
#include "Cell.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -34,6 +35,9 @@ public:
// Testing method
bool CheckInvariants() noexcept;
// Testing method
std_support::vector<uint8_t*> GetAllocatedBlocks() noexcept;
private:
explicit NextFitPage(uint32_t cellCount) noexcept;
kotlin-native/runtime/src/custom_alloc/cpp/NextFitPageTest.cpp
+1 -3
View File
@@ -109,9 +109,7 @@ TEST(CustomAllocTest, NextFitPageSweepReuse) {
std::minstd_rand r(seed);
NextFitPage* page = NextFitPage::Create(MIN_BLOCK_SIZE);
int unmarked = 0;
while (true) {
uint8_t* ptr = alloc(page, MIN_BLOCK_SIZE);
if (ptr == nullptr) break;
while (uint8_t* ptr = alloc(page, MIN_BLOCK_SIZE)) {
if (r() & 1) {
mark(ptr);
} else {
kotlin-native/runtime/src/custom_alloc/cpp/PageStore.hpp
+20
View File
@@ -7,10 +7,12 @@
#define CUSTOM_ALLOC_CPP_PAGESTORE_HPP_
#include <atomic>
#include <cstdint>
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -78,6 +80,8 @@ public:
}
private:
friend class Heap;
T* SweepSingle(GCSweepScope& sweepHandle, T* page, AtomicStack<T>& from, AtomicStack<T>& to, FinalizerQueue& finalizerQueue) noexcept {
if (!page) {
return nullptr;
@@ -92,6 +96,22 @@ private:
return nullptr;
}
// Testing method
std_support::vector<T*> GetPages() noexcept {
std_support::vector<T*> pages;
for (T* page : ready_.GetElements()) pages.push_back(page);
for (T* page : used_.GetElements()) pages.push_back(page);
for (T* page : unswept_.GetElements()) pages.push_back(page);
return pages;
}
void ClearForTests() noexcept {
while (T* page = empty_.Pop()) page->Destroy();
while (T* page = ready_.Pop()) page->Destroy();
while (T* page = used_.Pop()) page->Destroy();
while (T* page = unswept_.Pop()) page->Destroy();
}
AtomicStack<T> empty_;
AtomicStack<T> ready_;
AtomicStack<T> used_;
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp
+9
View File
@@ -11,6 +11,7 @@
#include "CustomLogging.hpp"
#include "CustomAllocConstants.hpp"
#include "GCApi.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -46,4 +47,12 @@ bool SingleObjectPage::Sweep(GCSweepScope& sweepHandle, FinalizerQueue& finalize
return false;
}
std_support::vector<uint8_t*> SingleObjectPage::GetAllocatedBlocks() noexcept {
std_support::vector<uint8_t*> allocated;
if (isAllocated_) {
allocated.push_back(data_);
}
return allocated;
}
} // namespace kotlin::alloc
kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp
+5
View File
@@ -12,6 +12,7 @@
#include "AtomicStack.hpp"
#include "ExtraObjectPage.hpp"
#include "GCStatistics.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::alloc {
@@ -33,9 +34,13 @@ public:
private:
friend class AtomicStack<SingleObjectPage>;
friend class Heap;
explicit SingleObjectPage(size_t size) noexcept;
// Testing method
std_support::vector<uint8_t*> GetAllocatedBlocks() noexcept;
SingleObjectPage* next_;
bool isAllocated_ = false;
size_t size_;
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+7
View File
@@ -167,6 +167,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gcHandle.threadsAreSuspended();
#ifdef CUSTOM_ALLOCATOR
// This should really be done by each individual thread while waiting
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
thread.gc().Allocator().PrepareForGC();
}
heap_.PrepareForGC();
#endif
@@ -218,6 +222,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
#else
// also sweeps extraObjects
auto finalizerQueue = heap_.Sweep(gcHandle);
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
finalizerQueue.TransferAllFrom(thread.gc().Allocator().ExtractFinalizerQueue());
}
#endif
state_.finish(epoch);
gcHandle.finalizersScheduled(finalizerQueue.size());
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+25 -2
View File
@@ -179,6 +179,9 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
}
std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#ifdef CUSTOM_ALLOCATOR
return threadData.gc().impl().alloc().heap().GetAllocatedObjects();
#else
std_support::vector<ObjHeader*> objects;
for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
objects.push_back(node.GetObjHeader());
@@ -187,6 +190,7 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
objects.push_back(node.GetObjHeader());
}
return objects;
#endif
}
bool IsMarked(ObjHeader* objHeader) {
@@ -215,7 +219,9 @@ public:
~ConcurrentMarkAndSweepTest() {
mm::GlobalsRegistry::Instance().ClearForTests();
mm::SpecialRefRegistry::instance().clearForTests();
#ifndef CUSTOM_ALLOCATOR
mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
#endif
mm::GlobalData::Instance().gc().ClearForTests();
}
@@ -1016,6 +1022,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) {
}
}
TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
std_support::vector<Mutator> mutators(kDefaultThreadCount);
std_support::vector<ObjHeader*> globals(2 * kDefaultThreadCount);
@@ -1085,6 +1092,7 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
future.wait();
}
#ifndef CUSTOM_ALLOCATOR
// Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything).
std_support::vector<ObjHeader*> expectedAlive;
@@ -1107,9 +1115,24 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]);
EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread));
}
#else
// Custom allocator does not have a notion of objects alive only for some thread
std_support::vector<ObjHeader*> expectedAlive;
for (int i = 0; i < kDefaultThreadCount; ++i) {
expectedAlive.push_back(globals[i]);
expectedAlive.push_back(locals[i]);
expectedAlive.push_back(reachables[i]);
expectedAlive.push_back(globals[kDefaultThreadCount + i]);
expectedAlive.push_back(locals[kDefaultThreadCount + i]);
expectedAlive.push_back(reachables[kDefaultThreadCount + i]);
// Unreachables for new threads were not collected.
expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
}
// All threads see the same alive objects with the custom alloctor, enough to check a single mutator.
EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));
#endif // CUSTOM_ALLOCATOR
}
TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
std_support::vector<Mutator> mutators(2);
std::atomic<test_support::Object<Payload>*> object1 = nullptr;
@@ -1138,7 +1161,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header()));
done = true;
});
auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) {
while (object1.load() == nullptr) {}
ObjHolder holder;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+19
View File
@@ -5,6 +5,7 @@
#include "GCImpl.hpp"
#include "ConcurrentMarkAndSweep.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "MarkAndSweepUtils.hpp"
@@ -40,6 +41,8 @@ void gc::GC::ThreadData::Publish() noexcept {
void gc::GC::ThreadData::ClearForTests() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactoryThreadQueue().ClearForTests();
#else
impl_->alloc().PrepareForGC();
#endif
}
@@ -59,6 +62,12 @@ ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeI
#endif
}
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
return impl_->alloc();
}
#endif
void gc::GC::ThreadData::OnSuspendForGC() noexcept {
impl_->gc().OnSuspendForGC();
}
@@ -88,6 +97,8 @@ void gc::GC::ClearForTests() noexcept {
impl_->gc().StopFinalizerThreadIfRunning();
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactory().ClearForTests();
#else
impl_->gc().heap().ClearForTests();
#endif
GCHandle::ClearForTests();
}
@@ -135,3 +146,11 @@ bool gc::isMarked(ObjHeader* object) noexcept {
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_RESULT_OF(gc::WeakRefRead, object);
}
// static
const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData);
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<ConcurrentMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
}
kotlin-native/runtime/src/gc/cms/cpp/GCImplTestSupport.cpp
+4
View File
@@ -26,6 +26,10 @@ auto collectCopy(T& iterable) {
} // namespace
void gc::AssertClear(GC& gc) noexcept {
#ifdef CUSTOM_ALLOCATOR
auto objects = gc.impl().gc().heap().GetAllocatedObjects();
#else
auto objects = gc.impl().objectFactory().LockForIter();
#endif
EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
}
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+11 -1
View File
@@ -9,10 +9,13 @@
#include "GCScheduler.hpp"
#include "Memory.h"
#include "Types.h"
#include "Utils.hpp"
#include "std_support/Memory.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#endif
namespace kotlin {
namespace mm {
@@ -44,6 +47,10 @@ public:
ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& Allocator() noexcept;
#endif
void OnSuspendForGC() noexcept;
void safePoint() noexcept;
@@ -76,6 +83,9 @@ public:
void WaitFinalizers(int64_t epoch) noexcept;
void ScheduleAndWaitFullGCWithFinalizers() noexcept { WaitFinalizers(Schedule()); }
static const size_t objectDataSize;
static bool SweepObject(void* objectData) noexcept;
private:
std_support::unique_ptr<Impl> impl_;
};
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+43 -2
View File
@@ -5,16 +5,21 @@
#include "GCImpl.hpp"
#include "Common.h"
#include "GC.hpp"
#include "NoOpGC.hpp"
#include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
#include "ObjectOps.hpp"
using namespace kotlin;
gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData&, mm::ThreadData& threadData) noexcept :
gc::GC::ThreadData::ThreadData(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
#ifdef CUSTOM_ALLOCATOR
impl_(std_support::make_unique<Impl>(gc, gcScheduler, threadData)) {}
#else
impl_(std_support::make_unique<Impl>(gc, threadData)) {}
#endif
gc::GC::ThreadData::~ThreadData() = default;
@@ -31,21 +36,41 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
}
void gc::GC::ThreadData::Publish() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactoryThreadQueue().Publish();
#endif
}
void gc::GC::ThreadData::ClearForTests() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactoryThreadQueue().ClearForTests();
#else
impl_->alloc().PrepareForGC();
#endif
}
ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
#else
return impl_->alloc().CreateObject(typeInfo);
#endif
}
ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
#else
return impl_->alloc().CreateArray(typeInfo, elements);
#endif
}
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
return impl_->alloc();
}
#endif
void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
void gc::GC::ThreadData::safePoint() noexcept {}
@@ -56,7 +81,11 @@ gc::GC::~GC() = default;
// static
size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifndef CUSTOM_ALLOCATOR
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
#else
return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#endif
}
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
@@ -64,7 +93,11 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
}
void gc::GC::ClearForTests() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactory().ClearForTests();
#else
impl_->gc().heap().ClearForTests();
#endif
GCHandle::ClearForTests();
}
@@ -98,3 +131,11 @@ bool gc::isMarked(ObjHeader* object) noexcept {
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed));
}
// static
const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no_unique_address]]
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return true;
}
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp
+23
View File
@@ -10,6 +10,11 @@
#include "NoOpGC.hpp"
#include "ObjectFactory.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "GCScheduler.hpp"
#endif
namespace kotlin {
namespace gc {
@@ -19,25 +24,43 @@ class GC::Impl : private Pinned {
public:
Impl() noexcept = default;
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
#endif
GCImpl& gc() noexcept { return gc_; }
private:
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_;
#endif
GCImpl gc_;
};
class GC::ThreadData::Impl : private Pinned {
public:
#ifdef CUSTOM_ALLOCATOR
Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
alloc_(gc.impl_->gc().heap(), gcScheduler) {}
#else
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
#endif
GCImpl::ThreadData& gc() noexcept { return gc_; }
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#else
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
#endif
private:
GCImpl::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_;
/* gcScheduler::GCSchedulerThreadData; */
#else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
#endif
};
} // namespace gc
kotlin-native/runtime/src/gc/noop/cpp/GCImplTestSupport.cpp
+4
View File
@@ -26,6 +26,10 @@ auto collectCopy(T& iterable) {
} // namespace
void gc::AssertClear(GC& gc) noexcept {
#ifdef CUSTOM_ALLOCATOR
auto objects = gc.impl().gc().heap().GetAllocatedObjects();
#else
auto objects = gc.impl().objectFactory().LockForIter();
#endif
EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
}
kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp
+11
View File
@@ -12,6 +12,10 @@
#include "Logging.hpp"
#include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "Heap.hpp"
#endif
namespace kotlin {
namespace mm {
@@ -51,7 +55,14 @@ public:
NoOpGC() noexcept { RuntimeLogInfo({kTagGC}, "No-op GC initialized"); }
~NoOpGC() = default;
#ifdef CUSTOM_ALLOCATOR
alloc::Heap& heap() noexcept { return heap_; }
#endif
private:
#ifdef CUSTOM_ALLOCATOR
alloc::Heap heap_;
#endif
};
} // namespace gc
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+37
View File
@@ -7,6 +7,7 @@
#include "GC.hpp"
#include "MarkAndSweepUtils.hpp"
#include "SameThreadMarkAndSweep.hpp"
#include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
@@ -32,21 +33,41 @@ void gc::GC::ThreadData::ScheduleAndWaitFullGCWithFinalizers() noexcept {
}
void gc::GC::ThreadData::Publish() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactoryThreadQueue().Publish();
#endif
}
void gc::GC::ThreadData::ClearForTests() noexcept {
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactoryThreadQueue().ClearForTests();
#else
impl_->alloc().PrepareForGC();
#endif
}
ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
#else
return impl_->alloc().CreateObject(typeInfo);
#endif
}
ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
#ifndef CUSTOM_ALLOCATOR
return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
#else
return impl_->alloc().CreateArray(typeInfo, elements);
#endif
}
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& gc::GC::ThreadData::Allocator() noexcept {
return impl_->alloc();
}
#endif
void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
void gc::GC::ThreadData::safePoint() noexcept {}
@@ -57,7 +78,11 @@ gc::GC::~GC() = default;
// static
size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifdef CUSTOM_ALLOCATOR
return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#else
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
#endif
}
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
@@ -66,7 +91,11 @@ size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
void gc::GC::ClearForTests() noexcept {
impl_->gc().StopFinalizerThreadIfRunning();
#ifndef CUSTOM_ALLOCATOR
impl_->objectFactory().ClearForTests();
#else
impl_->gc().heap().ClearForTests();
#endif
GCHandle::ClearForTests();
}
@@ -113,3 +142,11 @@ bool gc::isMarked(ObjHeader* object) noexcept {
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed));
}
// static
const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData);
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<SameThreadMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp
+20 -1
View File
@@ -16,27 +16,46 @@ using GCImpl = SameThreadMarkAndSweep;
class GC::Impl : private Pinned {
public:
#ifdef CUSTOM_ALLOCATOR
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(gcScheduler) {}
#else
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, gcScheduler) {}
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
#endif
GCImpl& gc() noexcept { return gc_; }
private:
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_;
#endif
GCImpl gc_;
};
class GC::ThreadData::Impl : private Pinned {
public:
Impl(GC& gc, gcScheduler::GCSchedulerThreadData& gcScheduler, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData, gcScheduler), objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
gc_(gc.impl_->gc(), threadData, gcScheduler),
#ifdef CUSTOM_ALLOCATOR
alloc_(gc.impl_->gc().heap(), gcScheduler) {}
#else
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()) {}
#endif
GCImpl::ThreadData& gc() noexcept { return gc_; }
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
#else
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#endif
private:
GCImpl::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_;
#else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
#endif
};
} // namespace gc
kotlin-native/runtime/src/gc/stms/cpp/GCImplTestSupport.cpp
+4
View File
@@ -26,6 +26,10 @@ auto collectCopy(T& iterable) {
} // namespace
void gc::AssertClear(GC& gc) noexcept {
#ifdef CUSTOM_ALLOCATOR
auto objects = gc.impl().gc().heap().GetAllocatedObjects();
#else
auto objects = gc.impl().objectFactory().LockForIter();
#endif
EXPECT_THAT(collectCopy(objects), testing::UnorderedElementsAre());
}
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+26 -2
View File
@@ -19,6 +19,10 @@
#include "ThreadRegistry.hpp"
#include "ThreadSuspension.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomFinalizerProcessor.hpp"
#endif
using namespace kotlin;
namespace {
@@ -82,9 +86,17 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
ScheduleAndWaitFullGC();
}
#ifdef CUSTOM_ALLOCATOR
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory), gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
gcScheduler::GCScheduler& gcScheduler) noexcept :
#else
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory,
gcScheduler::GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory),
#endif
gcScheduler_(gcScheduler), finalizerProcessor_([this](int64_t epoch) noexcept {
GCHandle::getByEpoch(epoch).finalizersDone();
state_.finalized(epoch);
}) {
@@ -135,6 +147,14 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
state_.start(epoch);
#ifdef CUSTOM_ALLOCATOR
// This should really be done by each individual thread while waiting
for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
thread.gc().Allocator().PrepareForGC();
}
heap_.PrepareForGC();
#endif
gc::collectRootSet<internal::MarkTraits>(gcHandle, markQueue_, [](mm::ThreadData&) { return true; });
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
@@ -143,6 +163,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
#ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread
// would not publish into the global state at an unexpected time.
std::optional extraObjectFactoryIterable = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
@@ -153,6 +174,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, *objectFactoryIterable);
objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread();
#else
auto finalizerQueue = heap_.Sweep(gcHandle);
#endif
mm::ResumeThreads();
gcHandle.threadsAreResumed();
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+24
View File
@@ -17,6 +17,14 @@
#include "Types.h"
#include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "ExtraObjectPage.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
#endif
namespace kotlin {
namespace mm {
@@ -92,10 +100,18 @@ public:
using Allocator = ThreadData::Allocator;
#ifdef CUSTOM_ALLOCATOR
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept;
#else
using FinalizerQueue = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueue;
using FinalizerQueueTraits = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueueTraits;
SameThreadMarkAndSweep(mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, gcScheduler::GCScheduler& gcScheduler) noexcept;
#endif
~SameThreadMarkAndSweep();
void StartFinalizerThreadIfNeeded() noexcept;
@@ -105,10 +121,18 @@ public:
int64_t Schedule() noexcept { return state_.schedule(); }
void WaitFinalized(int64_t epoch) noexcept { state_.waitEpochFinalized(epoch); }
#ifdef CUSTOM_ALLOCATOR
alloc::Heap& heap() noexcept { return heap_; }
#endif
private:
void PerformFullGC(int64_t epoch) noexcept;
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
#else
alloc::Heap heap_;
#endif
gcScheduler::GCScheduler& gcScheduler_;
GCStateHolder state_;
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+27 -1
View File
@@ -180,6 +180,9 @@ test_support::Object<Payload>& AllocateObjectWithFinalizer(mm::ThreadData& threa
}
std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#ifdef CUSTOM_ALLOCATOR
return threadData.gc().impl().alloc().heap().GetAllocatedObjects();
#else
std_support::vector<ObjHeader*> objects;
for (auto node : threadData.gc().impl().objectFactoryThreadQueue()) {
objects.push_back(node.GetObjHeader());
@@ -188,6 +191,7 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
objects.push_back(node.GetObjHeader());
}
return objects;
#endif
}
bool IsMarked(ObjHeader* objHeader) {
@@ -211,7 +215,12 @@ public:
~SameThreadMarkAndSweepTest() {
mm::GlobalsRegistry::Instance().ClearForTests();
mm::SpecialRefRegistry::instance().clearForTests();
#ifndef CUSTOM_ALLOCATOR
mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
mm::GlobalData::Instance().gc().impl().objectFactory().ClearForTests();
#else
mm::GlobalData::Instance().gc().impl().gc().heap().ClearForTests();
#endif
mm::GlobalData::Instance().gc().ClearForTests();
}
@@ -1081,6 +1090,7 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
future.wait();
}
#ifndef CUSTOM_ALLOCATOR
// Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything).
std_support::vector<ObjHeader*> expectedAlive;
@@ -1103,6 +1113,22 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]);
EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread));
}
#else
// Custom allocator does not have a notion of objects alive only for some thread
std_support::vector<ObjHeader*> expectedAlive;
for (int i = 0; i < kDefaultThreadCount; ++i) {
expectedAlive.push_back(globals[i]);
expectedAlive.push_back(locals[i]);
expectedAlive.push_back(reachables[i]);
expectedAlive.push_back(globals[kDefaultThreadCount + i]);
expectedAlive.push_back(locals[kDefaultThreadCount + i]);
expectedAlive.push_back(reachables[kDefaultThreadCount + i]);
// Unreachables for new threads were not collected.
expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
}
// All threads see the same alive objects with the custom alloctor, enough to check a single mutator.
EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));
#endif // CUSTOM_ALLOCATOR
}
@@ -1134,7 +1160,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global1.header()));
done = true;
});
auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) {
while (object1.load() == nullptr) {}
ObjHolder holder;
kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp
+1 -1
View File
@@ -34,7 +34,7 @@ mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
auto *threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
#ifdef CUSTOM_ALLOCATOR
auto& data = alloc::CustomAllocator::CreateExtraObjectDataForObject(threadData, object, typeInfo);
auto& data = threadData->gc().Allocator().CreateExtraObjectDataForObject(object, typeInfo);
if (!compareExchange(object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(&data))) {
// Somebody else created `mm::ExtraObjectData` for this object.
kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.cpp
+2
View File
@@ -10,6 +10,7 @@
using namespace kotlin;
#ifndef CUSTOM_ALLOCATOR
// static
mm::ExtraObjectDataFactory& mm::ExtraObjectDataFactory::Instance() noexcept {
return GlobalData::Instance().extraObjectDataFactory();
@@ -41,3 +42,4 @@ void mm::ExtraObjectDataFactory::ProcessThread(mm::ThreadData* threadData) noexc
mm::ExtraObjectDataFactory::ExtraObjectDataFactory() = default;
mm::ExtraObjectDataFactory::~ExtraObjectDataFactory() = default;
#endif
kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp
+2
View File
@@ -29,7 +29,9 @@ public:
~ExtraObjectDataTest() {
mm::GlobalsRegistry::Instance().ClearForTests();
#ifndef CUSTOM_ALLOCATOR
mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
#endif
mm::GlobalData::Instance().gc().ClearForTests();
}
};
kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
+8 -1
View File
@@ -27,7 +27,9 @@ public:
ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; }
GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; }
SpecialRefRegistry& specialRefRegistry() noexcept { return specialRefRegistry_; }
#ifndef CUSTOM_ALLOCATOR
ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif
gcScheduler::GCScheduler& gcScheduler() noexcept { return gcScheduler_; }
gc::GC& gc() noexcept { return gc_; }
AppStateTracking& appStateTracking() noexcept { return appStateTracking_; }
@@ -43,9 +45,14 @@ private:
AppStateTracking appStateTracking_;
GlobalsRegistry globalsRegistry_;
SpecialRefRegistry specialRefRegistry_;
ExtraObjectDataFactory extraObjectDataFactory_;
gcScheduler::GCScheduler gcScheduler_;
gc::GC gc_{gcScheduler_};
#ifndef CUSTOM_ALLOCATOR
// by being last, ommiting it will not affect the offsets of the other
// members, and we avoid having to have _custom versions of the gcScheduler
// modules.
ExtraObjectDataFactory extraObjectDataFactory_;
#endif
};
} // namespace mm
kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp
+2
View File
@@ -442,6 +442,7 @@ class ObjectFactory : private Pinned {
using Allocator = typename Traits::Allocator;
struct HeapObjHeader {
[[no_unique_address]] // to account for GCs with empty ObjectData
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object;
};
@@ -449,6 +450,7 @@ class ObjectFactory : private Pinned {
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader {
[[no_unique_address]]
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array;
};
kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp
+3
View File
@@ -21,6 +21,8 @@
#include "std_support/CStdlib.hpp"
#include "std_support/Vector.hpp"
// ObjectFactory is not used by custom allocator
#ifndef CUSTOM_ALLOCATOR
using namespace kotlin;
using testing::_;
@@ -1130,3 +1132,4 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount);
}
#endif
kotlin-native/runtime/src/mm/cpp/TestSupport.cpp
+5 -2
View File
@@ -45,12 +45,15 @@ auto collectPointers(T& iterable) {
extern "C" void Kotlin_TestSupport_AssertClearGlobalState() {
// Validate that global registries are empty.
auto globals = mm::GlobalsRegistry::Instance().LockForIter();
auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
auto specialRefs = mm::SpecialRefRegistry::instance().lockForIter();
auto threads = mm::ThreadRegistry::Instance().LockForIter();
EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre());
#ifndef CUSTOM_ALLOCATOR
auto extraObjects = mm::GlobalData::Instance().extraObjectDataFactory().LockForIter();
EXPECT_THAT(collectPointers(extraObjects), testing::UnorderedElementsAre());
#endif
EXPECT_THAT(collectCopy(globals), testing::UnorderedElementsAre());
EXPECT_THAT(collectPointers(specialRefs), testing::UnorderedElementsAre());
EXPECT_THAT(collectPointers(threads), testing::UnorderedElementsAre());
gc::AssertClear(mm::GlobalData::Instance().gc());
kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
+10
View File
@@ -34,7 +34,9 @@ public:
threadId_(threadId),
globalsThreadQueue_(GlobalsRegistry::Instance()),
specialRefRegistry_(SpecialRefRegistry::instance()),
#ifndef CUSTOM_ALLOCATOR
extraObjectDataThreadQueue_(ExtraObjectDataFactory::Instance()),
#endif
gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()),
gc_(GlobalData::Instance().gc(), gcScheduler_, *this),
suspensionData_(ThreadState::kNative, *this) {}
@@ -49,7 +51,9 @@ public:
SpecialRefRegistry::ThreadQueue& specialRefRegistry() noexcept { return specialRefRegistry_; }
#ifndef CUSTOM_ALLOCATOR
ExtraObjectDataFactory::ThreadQueue& extraObjectDataThreadQueue() noexcept { return extraObjectDataThreadQueue_; }
#endif
ThreadState state() noexcept { return suspensionData_.state(); }
@@ -69,14 +73,18 @@ public:
// TODO: These use separate locks, which is inefficient.
globalsThreadQueue_.Publish();
specialRefRegistry_.publish();
#ifndef CUSTOM_ALLOCATOR
extraObjectDataThreadQueue_.Publish();
#endif
gc_.Publish();
}
void ClearForTests() noexcept {
globalsThreadQueue_.ClearForTests();
specialRefRegistry_.clearForTests();
#ifndef CUSTOM_ALLOCATOR
extraObjectDataThreadQueue_.ClearForTests();
#endif
gc_.ClearForTests();
}
@@ -85,7 +93,9 @@ private:
GlobalsRegistry::ThreadQueue globalsThreadQueue_;
ThreadLocalStorage tls_;
SpecialRefRegistry::ThreadQueue specialRefRegistry_;
#ifndef CUSTOM_ALLOCATOR
ExtraObjectDataFactory::ThreadQueue extraObjectDataThreadQueue_;
#endif
ShadowStack shadowStack_;
gcScheduler::GCSchedulerThreadData gcScheduler_;
gc::GC::ThreadData gc_;