azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Decouple ObjectFactory from concrete ObjectData ^KT-60928

Browse Source 
* Use type_layout to declaratively express heap object headers in both
  custom allocator and ObjectFactory.
* Invoke constructor (w/o invoking Kotin constructors) for created
  objects and arrays from both custom allocator and ObjectFactory.
  Previously:
  - custom allocator only checked body for nullability (now this is
    performed in body constructor)
  - ObjectFactory only constructed ObjectData
* In each GC have a AllocatorImpl.hpp and ObjectData.hpp headers
  the first encapsulating allocator-specific types, the second
  containing specific ObjectData implementation.
* In each GC have a separate ObjectFactoryTraits that does not
  actually depend on the specific GC anymore.
* Each GC now expose ObjectData (as undefined type) and its descriptor,
  the latter being used by the custom allocator and ObjectFactory.
* Descriptors for ObjectBody and ArrayBody now live in Memory.h and the
  code calculating size is now shared. Their constructors check that the
  memory is zeroed (Kotlin constructors will expect this).
This commit is contained in:
Alexander Shabalin
2023-07-28 15:08:15 +02:00
committed by Space Team
parent d956a5504d
commit 83a70ddf8b
37 changed files with 901 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp
+11 -27
View File
@@ -28,19 +28,6 @@
namespace kotlin::alloc { namespace kotlin::alloc {
size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(heapObjectHeaderSize + membersSize, kObjectAlignment);
}
uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
// Note: array body is aligned, but for size computation it is enough to align the sum.
return AlignUp<uint64_t>(heapArrayHeaderSize + membersSize, kObjectAlignment);
}
CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) { CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) {
CustomAllocInfo("CustomAllocator::CustomAllocator(heap)"); CustomAllocInfo("CustomAllocator::CustomAllocator(heap)");
memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_)); memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_));
@@ -52,9 +39,9 @@ CustomAllocator::~CustomAllocator() {
ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept { ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
size_t allocSize = ObjectAllocatedDataSize(typeInfo); auto descriptor = HeapObject::make_descriptor(typeInfo);
uint8_t* heapObject = Allocate(allocSize); auto& heapObject = *descriptor.construct(Allocate(descriptor.size()));
auto* object = reinterpret_cast<ObjHeader*>(heapObject + gcDataSize); ObjHeader* object = heapObject.header(descriptor).object();
if (typeInfo->flags_ & TF_HAS_FINALIZER) { if (typeInfo->flags_ & TF_HAS_FINALIZER) {
auto* extraObject = CreateExtraObject(); auto* extraObject = CreateExtraObject();
object->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(new (extraObject) mm::ExtraObjectData(object, typeInfo)); object->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(new (extraObject) mm::ExtraObjectData(object, typeInfo));
@@ -69,9 +56,9 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count); CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count);
RuntimeAssert(typeInfo->IsArray(), "Must be an array"); RuntimeAssert(typeInfo->IsArray(), "Must be an array");
auto allocSize = ArrayAllocatedDataSize(typeInfo, count); auto descriptor = HeapArray::make_descriptor(typeInfo, count);
uint8_t* heapArray = Allocate(allocSize); auto& heapArray = *descriptor.construct(Allocate(descriptor.size()));
auto* array = reinterpret_cast<ArrayHeader*>(heapArray + gcDataSize); ArrayHeader* array = heapArray.header(descriptor).array();
array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo); array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
array->count_ = count; array->count_ = count;
return array; return array;
@@ -121,9 +108,9 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object"); RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info(); const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) { if (typeInfo->IsArray()) {
return ArrayAllocatedDataSize(typeInfo, object->array()->count_); return HeapArray::make_descriptor(typeInfo, object->array()->count_).size();
} else { } else {
return ObjectAllocatedDataSize(typeInfo); return HeapObject::make_descriptor(typeInfo).size();
} }
} }
@@ -131,16 +118,13 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes"); RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes");
CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size); CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size);
uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell); uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell);
uint8_t* ptr;
if (cellCount <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE) { if (cellCount <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE) {
ptr = AllocateInFixedBlockPage(cellCount); return AllocateInFixedBlockPage(cellCount);
} else if (cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE) { } else if (cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE) {
ptr = AllocateInSingleObjectPage(cellCount); return AllocateInSingleObjectPage(cellCount);
} else { } else {
ptr = AllocateInNextFitPage(cellCount); return AllocateInNextFitPage(cellCount);
} }
RuntimeAssert(ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0, "CustomAllocator::Allocate: memory not zero!");
return ptr;
} }
uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept { uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept {
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+4 -5
View File
@@ -23,7 +23,6 @@
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "KAssert.h" #include "KAssert.h"
#include "Memory.h" #include "Memory.h"
#include "ObjectFactory.hpp"
namespace { namespace {
@@ -33,14 +32,14 @@ std::atomic<size_t> allocatedBytesCounter;
namespace kotlin::alloc { namespace kotlin::alloc {
bool SweepObject(uint8_t* heapObjHeader, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept { bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
if (gc::GC::SweepObject(heapObjHeader)) { auto* heapObjHeader = reinterpret_cast<HeapObjHeader*>(object);
if (gc::tryResetMark(heapObjHeader->objectData())) {
CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader); CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader);
gcHandle.addKeptObject(); gcHandle.addKeptObject();
return true; return true;
} }
auto* objHeader = reinterpret_cast<ObjHeader*>(heapObjHeader + gcDataSize); auto* extraObject = mm::ExtraObjectData::Get(heapObjHeader->object());
auto* extraObject = mm::ExtraObjectData::Get(objHeader);
if (extraObject) { if (extraObject) {
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) { if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject); CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject);
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp
+71 -4
View File
@@ -15,15 +15,74 @@
#include "AtomicStack.hpp" #include "AtomicStack.hpp"
#include "ExtraObjectData.hpp" #include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp" #include "ExtraObjectPage.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "Memory.h" #include "Memory.h"
#include "GC.hpp" #include "TypeLayout.hpp"
namespace kotlin::alloc { namespace kotlin::alloc {
const size_t gcDataSize = AlignUp(gc::GC::objectDataSize, kObjectAlignment); struct HeapObjHeader {
const size_t heapObjectHeaderSize = AlignUp(gcDataSize + sizeof(ObjHeader), kObjectAlignment); using descriptor = type_layout::Composite<HeapObjHeader, gc::GC::ObjectData, ObjHeader>;
const size_t heapArrayHeaderSize = AlignUp(gcDataSize + sizeof(ArrayHeader), kObjectAlignment);
static HeapObjHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); }
static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().fromField<1>(object); }
gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; }
ObjHeader* object() noexcept { return descriptor().field<1>(this).second; }
private:
HeapObjHeader() = delete;
~HeapObjHeader() = delete;
};
struct HeapObject {
using descriptor = type_layout::Composite<HeapObject, HeapObjHeader, ObjectBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept {
return descriptor{{}, type_layout::descriptor_t<ObjectBody>{typeInfo}};
}
HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; }
private:
HeapObject() = delete;
~HeapObject() = delete;
};
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader {
using descriptor = type_layout::Composite<HeapArrayHeader, gc::GC::ObjectData, ArrayHeader>;
static HeapArrayHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); }
static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().fromField<1>(array); }
gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; }
ArrayHeader* array() noexcept { return descriptor().field<1>(this).second; }
private:
HeapArrayHeader() = delete;
~HeapArrayHeader() = delete;
};
struct HeapArray {
using descriptor = type_layout::Composite<HeapArray, HeapArrayHeader, ArrayBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept {
return descriptor{{}, type_layout::descriptor_t<ArrayBody>{typeInfo, size}};
}
HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; }
private:
HeapArray() = delete;
~HeapArray() = delete;
};
// Returns `true` if the `object` must be kept alive still. // Returns `true` if the `object` must be kept alive still.
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept; bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept;
@@ -37,6 +96,14 @@ void Free(void* ptr, size_t size) noexcept;
size_t GetAllocatedBytes() noexcept; size_t GetAllocatedBytes() noexcept;
inline gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return HeapObjHeader::from(object).objectData();
}
inline ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
return HeapObjHeader::from(objectData).object();
}
} // namespace kotlin::alloc } // namespace kotlin::alloc
#endif #endif
kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp
+3 -3
View File
@@ -93,18 +93,18 @@ std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) { for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
for (auto* page : fixedBlockPages_[blockSize].GetPages()) { for (auto* page : fixedBlockPages_[blockSize].GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) { for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize)); allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
} }
} }
} }
for (auto* page : nextFitPages_.GetPages()) { for (auto* page : nextFitPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) { for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize)); allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
} }
} }
for (auto* page : singleObjectPages_.GetPages()) { for (auto* page : singleObjectPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) { for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize)); allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
} }
} }
std_support::vector<ObjHeader*> unfinalized; std_support::vector<ObjHeader*> unfinalized;
kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
namespace kotlin::gc {
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return kotlin::alloc::objectDataForObject(object);
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return kotlin::alloc::objectForObjectData(objectData);
}
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
} // namespace kotlin::gc
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
void OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return ObjectFactory::NodeRef::From(object).ObjectData();
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
}
using FinalizerQueue = ObjectFactory::FinalizerQueue;
using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+11 -44
View File
@@ -20,40 +20,12 @@
#include "GCState.hpp" #include "GCState.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "Heap.hpp"
#endif
using namespace kotlin; using namespace kotlin;
namespace { namespace {
[[clang::no_destroy]] std::mutex gcMutex; [[clang::no_destroy]] std::mutex gcMutex;
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
return objectData.tryResetMark();
}
};
struct ProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(obj).ObjectData();
return objectData.marked();
}
};
template<typename Body> template<typename Body>
ScopedThread createGCThread(const char* name, Body&& body) { ScopedThread createGCThread(const char* name, Body&& body) {
return ScopedThread(ScopedThread::attributes().name(name), [name, body] { return ScopedThread(ScopedThread::attributes().name(name), [name, body] {
@@ -63,8 +35,9 @@ ScopedThread createGCThread(const char* name, Body&& body) {
}); });
} }
#ifndef CUSTOM_ALLOCATOR
// TODO move to common // TODO move to common
[[maybe_unused]] inline void checkMarkCorrectness(mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::Iterable& heap) { [[maybe_unused]] inline void checkMarkCorrectness(gc::ObjectFactory::Iterable& heap) {
if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return; if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return;
for (auto objRef: heap) { for (auto objRef: heap) {
auto obj = objRef.GetObjHeader(); auto obj = objRef.GetObjHeader();
@@ -72,23 +45,17 @@ ScopedThread createGCThread(const char* name, Body&& body) {
if (objData.marked()) { if (objData.marked()) {
traverseReferredObjects(obj, [obj](ObjHeader* field) { traverseReferredObjects(obj, [obj](ObjHeader* field) {
if (field->heap()) { if (field->heap()) {
auto& fieldObjData = auto& fieldObjData = gc::ObjectFactory::NodeRef::From(field).ObjectData();
mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(field).ObjectData();
RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj); RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj);
} }
}); });
} }
} }
} }
#endif
} // namespace } // namespace
void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
CallsCheckerIgnoreGuard guard; CallsCheckerIgnoreGuard guard;
@@ -133,10 +100,11 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler, gcScheduler::GCScheduler& gcScheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept : bool mutatorsCooperate,
std::size_t auxGCThreads) noexcept :
objectFactory_(objectFactory), objectFactory_(objectFactory),
extraObjectDataFactory_(extraObjectDataFactory), extraObjectDataFactory_(extraObjectDataFactory),
#else #else
@@ -150,8 +118,7 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
state_.finalized(epoch); state_.finalized(epoch);
}), }),
markDispatcher_(mutatorsCooperate), markDispatcher_(mutatorsCooperate),
mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) {
{
for (std::size_t i = 0; i < auxGCThreads; ++i) { for (std::size_t i = 0; i < auxGCThreads; ++i) {
auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); })); auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); }));
} }
@@ -250,7 +217,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gcHandle.threadsAreResumed(); gcHandle.threadsAreResumed();
} }
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance()); gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
if (compiler::concurrentWeakSweep()) { if (compiler::concurrentWeakSweep()) {
// Expected to happen outside STW. // Expected to happen outside STW.
@@ -261,9 +228,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
} }
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
gc::SweepExtraObjects<SweepTraits>(gcHandle, *extraObjectFactoryIterable); gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
extraObjectFactoryIterable = std::nullopt; extraObjectFactoryIterable = std::nullopt;
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, *objectFactoryIterable); auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
objectFactoryIterable = std::nullopt; objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread(); kotlin::compactObjectPoolInMainThread();
#else #else
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+8 -36
View File
@@ -8,29 +8,21 @@
#include <atomic> #include <atomic>
#include <cstddef> #include <cstddef>
#include "Allocator.hpp" #include "AllocatorImpl.hpp"
#include "Barriers.hpp" #include "Barriers.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerProcessor.hpp" #include "FinalizerProcessor.hpp"
#include "GCScheduler.hpp" #include "GCScheduler.hpp"
#include "GCState.hpp" #include "GCState.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "IntrusiveList.hpp" #include "IntrusiveList.hpp"
#include "MarkAndSweepUtils.hpp" #include "MarkAndSweepUtils.hpp"
#include "ObjectFactory.hpp" #include "ObjectData.hpp"
#include "ParallelMark.hpp"
#include "ScopedThread.hpp" #include "ScopedThread.hpp"
#include "ThreadData.hpp" #include "ThreadData.hpp"
#include "Types.h" #include "Types.h"
#include "Utils.hpp" #include "Utils.hpp"
#include "std_support/Memory.hpp" #include "std_support/Memory.hpp"
#include "MarkStack.hpp"
#include "ParallelMark.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "Heap.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace gc { namespace gc {
@@ -39,25 +31,15 @@ namespace gc {
// TODO: Also make marking run concurrently with Kotlin threads. // TODO: Also make marking run concurrently with Kotlin threads.
class ConcurrentMarkAndSweep : private Pinned { class ConcurrentMarkAndSweep : private Pinned {
public: public:
class ThreadData : private Pinned { class ThreadData : private Pinned {
public: public:
using ObjectData = mark::ObjectData; explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept
: gc_(gc), threadData_(threadData) {}
~ThreadData() = default; ~ThreadData() = default;
void OnOOM(size_t size) noexcept;
void OnSuspendForGC() noexcept; void OnSuspendForGC() noexcept;
void safePoint() noexcept { barriers_.onCheckpoint(); } void safePoint() noexcept { barriers_.onCheckpoint(); }
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
BarriersThreadData& barriers() noexcept { return barriers_; } BarriersThreadData& barriers() noexcept { return barriers_; }
bool tryLockRootSet(); bool tryLockRootSet();
@@ -80,26 +62,16 @@ public:
std::atomic<bool> cooperative_ = false; std::atomic<bool> cooperative_ = false;
}; };
using ObjectData = ThreadData::ObjectData;
using Allocator = ThreadData::Allocator;
#ifndef CUSTOM_ALLOCATOR
using FinalizerQueue = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueue;
using FinalizerQueueTraits = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueueTraits;
#else
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
#endif
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler, explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; bool mutatorsCooperate, std::size_t auxGCThreads) noexcept;
#else #else
ConcurrentMarkAndSweep( ConcurrentMarkAndSweep(
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory, ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& scheduler, gcScheduler::GCScheduler& scheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept; bool mutatorsCooperate,
std::size_t auxGCThreads) noexcept;
#endif #endif
~ConcurrentMarkAndSweep(); ~ConcurrentMarkAndSweep();
@@ -121,7 +93,7 @@ private:
void PerformFullGC(int64_t epoch) noexcept; void PerformFullGC(int64_t epoch) noexcept;
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_; ObjectFactory& objectFactory_;
mm::ExtraObjectDataFactory& extraObjectDataFactory_; mm::ExtraObjectDataFactory& extraObjectDataFactory_;
#else #else
alloc::Heap heap_; alloc::Heap heap_;
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+104 -109
View File
@@ -193,11 +193,6 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#endif #endif
} }
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().marked();
}
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
@@ -263,12 +258,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false); ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false); ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false); ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false); ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -276,12 +271,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false); EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false); EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false); EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false); EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false); EXPECT_THAT(gc::isMarked(stack3.header()), false);
}); });
} }
@@ -309,12 +304,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false); ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false); ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false); ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false); ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -322,12 +317,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false); EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false); EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false); EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false); EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false); EXPECT_THAT(gc::isMarked(stack3.header()), false);
}); });
} }
@@ -337,8 +332,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData); auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -352,8 +347,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData); auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header())); EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -372,8 +367,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
})(); })();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header()); ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header())); EXPECT_CALL(finalizerHook(), Call(weak1.header()));
@@ -390,14 +385,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header()); ASSERT_THAT(weak1.get(), object1.header());
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false); EXPECT_THAT(gc::isMarked(weak1.header()), false);
EXPECT_THAT(weak1.get(), nullptr); EXPECT_THAT(weak1.get(), nullptr);
}); });
} }
@@ -420,12 +415,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -433,12 +428,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -467,14 +462,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -482,12 +477,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -516,14 +511,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header())); EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -533,12 +528,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -555,18 +550,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header(); object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
}); });
} }
@@ -595,14 +590,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -611,12 +606,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -633,12 +628,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header(); global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
}); });
} }
@@ -652,14 +647,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false); EXPECT_THAT(gc::isMarked(object.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false); EXPECT_THAT(gc::isMarked(object.header()), false);
}); });
} }
@@ -1165,9 +1160,9 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false); ASSERT_THAT(gc::isMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false); ASSERT_THAT(gc::isMarked(weak.load()->header()), false);
ASSERT_THAT(weak.load()->get(), object1_local.header()); ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr; global1->field1 = nullptr;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+20 -11
View File
@@ -9,9 +9,8 @@
#include "GC.hpp" #include "GC.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "MarkAndSweepUtils.hpp" #include "MarkAndSweepUtils.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp" #include "ObjectOps.hpp"
#include "ThreadSuspension.hpp"
#include "MarkStack.hpp"
#include "std_support/Memory.hpp" #include "std_support/Memory.hpp"
using namespace kotlin; using namespace kotlin;
@@ -86,7 +85,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
return alloc::CustomAllocator::GetAllocatedHeapSize(object); return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#else #else
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object); return ObjectFactory::GetAllocatedHeapSize(object);
#endif #endif
} }
@@ -145,20 +144,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
} }
bool gc::isMarked(ObjHeader* object) noexcept { bool gc::isMarked(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData(); return objectDataForObject(object).marked();
return objectData.marked();
} }
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept { ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_RESULT_OF(gc::WeakRefRead, object); RETURN_RESULT_OF(gc::WeakRefRead, object);
} }
// static ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData); return objectData.tryResetMark();
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<ConcurrentMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
} }
// static // static
@@ -172,3 +166,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif #endif
} }
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return sizeof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return alignof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return new (ptr) gc::GC::ObjectData();
}
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp
+11 -17
View File
@@ -7,19 +7,12 @@
#include "GC.hpp" #include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "ConcurrentMarkAndSweep.hpp" #include "ConcurrentMarkAndSweep.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace gc { namespace gc {
using GCImpl = ConcurrentMarkAndSweep;
class GC::Impl : private Pinned { class GC::Impl : private Pinned {
public: public:
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
@@ -29,17 +22,17 @@ public:
#endif #endif
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; } ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif #endif
GCImpl& gc() noexcept { return gc_; } ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
private: private:
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_; ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif #endif
GCImpl gc_; ConcurrentMarkAndSweep gc_;
}; };
class GC::ThreadData::Impl : private Pinned { class GC::ThreadData::Impl : private Pinned {
@@ -47,7 +40,7 @@ public:
Impl(GC& gc, mm::ThreadData& threadData) noexcept : Impl(GC& gc, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData), gc_(gc.impl_->gc(), threadData),
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
} }
#else #else
@@ -55,20 +48,21 @@ public:
} }
#endif #endif
GCImpl::ThreadData& gc() noexcept { return gc_; } ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& alloc() noexcept { return alloc_; } alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#else #else
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#endif #endif
private: private:
GCImpl::ThreadData gc_; ConcurrentMarkAndSweep::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_; alloc::CustomAllocator alloc_;
#else #else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_; [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif #endif
}; };
kotlin-native/runtime/src/gc/cms/cpp/MarkStack.hpp → kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp
+14 -23
View File
@@ -1,28 +1,23 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <cstddef> #include <type_traits>
#include "Allocator.hpp" #include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "IntrusiveList.hpp" #include "IntrusiveList.hpp"
#include "ObjectFactory.hpp" #include "KAssert.h"
#include "Types.h"
#include "Utils.hpp"
#include "std_support/Memory.hpp"
namespace kotlin::gc::mark { namespace kotlin::gc {
class ObjectData { class GC::ObjectData {
struct ObjectFactoryTraits {
using ObjectData = ObjectData;
class Allocator;
};
public: public:
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>; bool tryMark() noexcept { return trySetNext(reinterpret_cast<ObjectData*>(1)); }
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
bool marked() const noexcept { return next() != nullptr; } bool marked() const noexcept { return next() != nullptr; }
@@ -32,10 +27,6 @@ public:
return true; return true;
} }
ObjHeader* objHeader() noexcept { // FIXME const
return ObjectFactory::NodeRef::From(*this).GetObjHeader();
}
private: private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>; friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
@@ -52,6 +43,6 @@ private:
std::atomic<ObjectData*> next_ = nullptr; std::atomic<ObjectData*> next_ = nullptr;
}; };
static_assert(std::is_trivially_destructible_v<GC::ObjectData>);
} // namespace kotlin::gc
} // namespace kotlin::gc::mark
kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp
-1
View File
@@ -1,6 +1,5 @@
#include "ParallelMark.hpp" #include "ParallelMark.hpp"
#include "MarkStack.hpp"
#include "MarkAndSweepUtils.hpp" #include "MarkAndSweepUtils.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "Utils.hpp" #include "Utils.hpp"
kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp
+8 -10
View File
@@ -4,12 +4,12 @@
#include <condition_variable> #include <condition_variable>
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
#include "MarkStack.hpp" #include "ManuallyScoped.hpp"
#include "std_support/Vector.hpp" #include "ObjectData.hpp"
#include "ParallelProcessor.hpp"
#include "ThreadRegistry.hpp" #include "ThreadRegistry.hpp"
#include "Utils.hpp" #include "Utils.hpp"
#include "ParallelProcessor.hpp" #include "std_support/Vector.hpp"
#include "ManuallyScoped.hpp"
namespace kotlin::gc::mark { namespace kotlin::gc::mark {
@@ -70,14 +70,13 @@ private:
* Mark workers are able to balance work between each other through sharing/stealing. * Mark workers are able to balance work between each other through sharing/stealing.
*/ */
class ParallelMark : private Pinned { class ParallelMark : private Pinned {
using MarkStackImpl = intrusive_forward_list<ObjectData>; using MarkStackImpl = intrusive_forward_list<GC::ObjectData>;
// work balancing parameters were chosen pretty arbitrary // work balancing parameters were chosen pretty arbitrary
using ParallelProcessor = ParallelProcessor<MarkStackImpl, 512, 4096>; using ParallelProcessor = ParallelProcessor<MarkStackImpl, 512, 4096>;
public: public:
class MarkTraits { class MarkTraits {
public: public:
using MarkQueue = ParallelProcessor::Worker; using MarkQueue = ParallelProcessor::Worker;
using ObjectFactory = ObjectData::ObjectFactory;
static void clear(MarkQueue& queue) noexcept { static void clear(MarkQueue& queue) noexcept {
RuntimeAssert(queue.localEmpty(), "Mark queue must be empty"); RuntimeAssert(queue.localEmpty(), "Mark queue must be empty");
@@ -86,19 +85,18 @@ public:
static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept { static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop(); auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop();
if (obj) { if (obj) {
auto node = ObjectFactory::NodeRef::From(*obj); return objectForObjectData(*obj);
return node->GetObjHeader();
} }
return nullptr; return nullptr;
} }
static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData(); auto& objectData = objectDataForObject(object);
return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData); return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData);
} }
static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept { static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData(); auto& objectData = objectDataForObject(object);
return objectData.tryMark(); return objectData.tryMark();
} }
kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp
-1
View File
@@ -13,7 +13,6 @@
#include "KAssert.h" #include "KAssert.h"
#include "Memory.h" #include "Memory.h"
#include "ObjectFactory.hpp"
#include "Runtime.h" #include "Runtime.h"
#include "ScopedThread.hpp" #include "ScopedThread.hpp"
#include "Utils.hpp" #include "Utils.hpp"
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+25 -3
View File
@@ -51,6 +51,14 @@ public:
std_support::unique_ptr<Impl> impl_; std_support::unique_ptr<Impl> impl_;
}; };
// Header to be placed before each heap object. GC will use this to keep its data if needed.
// This is used via `type_layout::descriptor_t`, which is specialized below.
// If GC doesn't need any data, it can make `size()` return 0 and `alignment()`
// return 1.
// Note: GC does not deinitialize `ObjectData`, so the implementations must ensure that
// the destructor is a trivial one.
class ObjectData;
explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept; explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept;
~GC(); ~GC();
@@ -75,9 +83,6 @@ public:
void WaitFinished(int64_t epoch) noexcept; void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept; void WaitFinalizers(int64_t epoch) noexcept;
static const size_t objectDataSize;
static bool SweepObject(void* objectData) noexcept;
static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept; static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
private: private:
@@ -87,7 +92,24 @@ private:
bool isMarked(ObjHeader* object) noexcept; bool isMarked(ObjHeader* object) noexcept;
OBJ_GETTER(tryRef, std::atomic<ObjHeader*>& object) noexcept; OBJ_GETTER(tryRef, std::atomic<ObjHeader*>& object) noexcept;
// This will drop the mark bit if it was set and return `true`.
// If the mark bit was unset, this will return `false`.
bool tryResetMark(GC::ObjectData& objectData) noexcept;
inline constexpr bool kSupportsMultipleMutators = true; inline constexpr bool kSupportsMultipleMutators = true;
} // namespace gc } // namespace gc
template <>
struct type_layout::descriptor<gc::GC::ObjectData> {
struct type {
using value_type = gc::GC::ObjectData;
static uint64_t size() noexcept;
static size_t alignment() noexcept;
static value_type* construct(uint8_t* ptr) noexcept;
};
};
} // namespace kotlin } // namespace kotlin
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+19
View File
@@ -7,6 +7,7 @@
#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H #define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
#include "ExtraObjectData.hpp" #include "ExtraObjectData.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerHooks.hpp" #include "FinalizerHooks.hpp"
#include "GlobalData.hpp" #include "GlobalData.hpp"
#include "GCStatistics.hpp" #include "GCStatistics.hpp"
@@ -164,6 +165,20 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T
return Sweep<Traits>(handle, iter); return Sweep<Traits>(handle, iter);
} }
template <typename T>
struct DefaultSweepTraits {
using ObjectFactory = T;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
auto* baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
return gc::isMarked(baseObject);
}
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
};
template <typename Traits> template <typename Traits>
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) { void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
auto handle = gcHandle.collectThreadRoots(thread); auto handle = gcHandle.collectThreadRoots(thread);
@@ -235,6 +250,10 @@ void processWeaks(GCHandle gcHandle, mm::SpecialRefRegistry& registry) noexcept
} }
} }
struct DefaultProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept { return gc::isMarked(obj); }
};
} // namespace gc } // namespace gc
} // namespace kotlin } // namespace kotlin
kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "Heap.hpp"
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = Allocator;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(); }
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+21 -8
View File
@@ -7,11 +7,12 @@
#include "Common.h" #include "Common.h"
#include "GC.hpp" #include "GC.hpp"
#include "GCStatistics.hpp"
#include "NoOpGC.hpp" #include "NoOpGC.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp"
#include "ThreadData.hpp" #include "ThreadData.hpp"
#include "std_support/Memory.hpp" #include "std_support/Memory.hpp"
#include "GCStatistics.hpp"
#include "ObjectOps.hpp"
using namespace kotlin; using namespace kotlin;
@@ -79,7 +80,7 @@ gc::GC::~GC() = default;
// static // static
size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept { size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object); return ObjectFactory::GetAllocatedHeapSize(object);
#else #else
return alloc::CustomAllocator::GetAllocatedHeapSize(object); return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#endif #endif
@@ -133,11 +134,8 @@ ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed)); RETURN_OBJ(object.load(std::memory_order_relaxed));
} }
// static ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no_unique_address]] RuntimeAssert(false, "Should not reach here");
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return true; return true;
} }
@@ -152,3 +150,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif #endif
} }
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return 0;
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return 1;
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return reinterpret_cast<gc::GC::ObjectData*>(ptr);
}
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp
+11 -18
View File
@@ -7,36 +7,28 @@
#include "GC.hpp" #include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "NoOpGC.hpp" #include "NoOpGC.hpp"
#include "ObjectFactory.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace gc { namespace gc {
using GCImpl = NoOpGC;
class GC::Impl : private Pinned { class GC::Impl : private Pinned {
public: public:
Impl() noexcept = default; Impl() noexcept = default;
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; } ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif #endif
GCImpl& gc() noexcept { return gc_; } NoOpGC& gc() noexcept { return gc_; }
private: private:
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_; ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif #endif
GCImpl gc_; NoOpGC gc_;
}; };
class GC::ThreadData::Impl : private Pinned { class GC::ThreadData::Impl : private Pinned {
@@ -45,24 +37,25 @@ public:
Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {} Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {}
#else #else
Impl(GC& gc, mm::ThreadData& threadData) noexcept : Impl(GC& gc, mm::ThreadData& threadData) noexcept :
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {} extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {}
#endif #endif
GCImpl::ThreadData& gc() noexcept { return gc_; } NoOpGC::ThreadData& gc() noexcept { return gc_; }
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& alloc() noexcept { return alloc_; } alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#else #else
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#endif #endif
private: private:
GCImpl::ThreadData gc_; NoOpGC::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_; alloc::CustomAllocator alloc_;
#else #else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_; [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif #endif
}; };
kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp
+2 -15
View File
@@ -8,14 +8,11 @@
#include <cstddef> #include <cstddef>
#include "Allocator.hpp" #include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "Logging.hpp" #include "Logging.hpp"
#include "Utils.hpp" #include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "Heap.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace mm { namespace mm {
@@ -28,21 +25,11 @@ namespace gc {
// TODO: It can be made more efficient. // TODO: It can be made more efficient.
class NoOpGC : private Pinned { class NoOpGC : private Pinned {
public: public:
class ObjectData {};
using Allocator = gc::Allocator;
class ThreadData : private Pinned { class ThreadData : private Pinned {
public: public:
using ObjectData = NoOpGC::ObjectData;
ThreadData() noexcept {} ThreadData() noexcept {}
~ThreadData() = default; ~ThreadData() = default;
void OnOOM(size_t size) noexcept {}
Allocator CreateAllocator() noexcept { return Allocator(); }
private: private:
}; };
kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
namespace kotlin::gc {
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return kotlin::alloc::objectDataForObject(object);
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return kotlin::alloc::objectForObjectData(objectData);
}
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
} // namespace kotlin::gc
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
void OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return ObjectFactory::NodeRef::From(object).ObjectData();
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
}
using FinalizerQueue = ObjectFactory::FinalizerQueue;
using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+23 -12
View File
@@ -6,12 +6,13 @@
#include "GCImpl.hpp" #include "GCImpl.hpp"
#include "GC.hpp" #include "GC.hpp"
#include "GCStatistics.hpp"
#include "GlobalData.hpp"
#include "MarkAndSweepUtils.hpp" #include "MarkAndSweepUtils.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp"
#include "SameThreadMarkAndSweep.hpp" #include "SameThreadMarkAndSweep.hpp"
#include "std_support/Memory.hpp" #include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
#include "ObjectOps.hpp"
using namespace kotlin; using namespace kotlin;
@@ -81,7 +82,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
return alloc::CustomAllocator::GetAllocatedHeapSize(object); return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#else #else
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object); return ObjectFactory::GetAllocatedHeapSize(object);
#endif #endif
} }
@@ -140,20 +141,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
} }
bool gc::isMarked(ObjHeader* object) noexcept { bool gc::isMarked(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData(); return objectDataForObject(object).marked();
return objectData.marked();
} }
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept { ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed)); RETURN_OBJ(object.load(std::memory_order_relaxed));
} }
// static ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData); return objectData.tryResetMark();
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<SameThreadMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
} }
// static // static
@@ -167,3 +163,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED); extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif #endif
} }
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return sizeof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return alignof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return new (ptr) gc::GC::ObjectData();
}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp
+11 -17
View File
@@ -7,19 +7,12 @@
#include "GC.hpp" #include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "SameThreadMarkAndSweep.hpp" #include "SameThreadMarkAndSweep.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace gc { namespace gc {
using GCImpl = SameThreadMarkAndSweep;
class GC::Impl : private Pinned { class GC::Impl : private Pinned {
public: public:
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
@@ -27,17 +20,17 @@ public:
#else #else
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {} explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; } ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; } mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif #endif
GCImpl& gc() noexcept { return gc_; } SameThreadMarkAndSweep& gc() noexcept { return gc_; }
private: private:
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_; ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_; mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif #endif
GCImpl gc_; SameThreadMarkAndSweep gc_;
}; };
class GC::ThreadData::Impl : private Pinned { class GC::ThreadData::Impl : private Pinned {
@@ -48,25 +41,26 @@ public:
alloc_(gc.impl_->gc().heap()) { alloc_(gc.impl_->gc().heap()) {
} }
#else #else
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()), objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) { extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
} }
#endif #endif
GCImpl::ThreadData& gc() noexcept { return gc_; } SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; } mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#else #else
alloc::CustomAllocator& alloc() noexcept { return alloc_; } alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#endif #endif
private: private:
GCImpl::ThreadData gc_; SameThreadMarkAndSweep::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_; alloc::CustomAllocator alloc_;
#else #else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_; [[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_; mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif #endif
}; };
kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <type_traits>
#include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "IntrusiveList.hpp"
#include "KAssert.h"
namespace kotlin::gc {
class GC::ObjectData {
public:
bool tryMark() noexcept { return trySetNext(reinterpret_cast<ObjectData*>(1)); }
bool marked() const noexcept { return next_ != nullptr; }
bool tryResetMark() noexcept {
if (next_ == nullptr) return false;
next_ = nullptr;
return true;
}
private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_; }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_ != nullptr) {
return false;
}
next_ = next;
return true;
}
ObjectData* next_ = nullptr;
};
static_assert(std::is_trivially_destructible_v<GC::ObjectData>);
} // namespace kotlin::gc
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+5 -48
View File
@@ -14,64 +14,21 @@
#include "Logging.hpp" #include "Logging.hpp"
#include "MarkAndSweepUtils.hpp" #include "MarkAndSweepUtils.hpp"
#include "Memory.h" #include "Memory.h"
#include "ObjectAlloc.hpp"
#include "RootSet.hpp" #include "RootSet.hpp"
#include "Runtime.h" #include "Runtime.h"
#include "ThreadData.hpp" #include "ThreadData.hpp"
#include "ThreadRegistry.hpp" #include "ThreadRegistry.hpp"
#include "ThreadSuspension.hpp" #include "ThreadSuspension.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomFinalizerProcessor.hpp"
#endif
using namespace kotlin; using namespace kotlin;
namespace {
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
return objectData.tryResetMark();
}
};
struct FinalizeTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
};
struct ProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(obj).ObjectData();
return objectData.marked();
}
};
} // namespace
void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
gcScheduler::GCScheduler& gcScheduler) noexcept : gcScheduler::GCScheduler& gcScheduler) noexcept :
#else #else
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep( gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory), objectFactory_(objectFactory),
extraObjectDataFactory_(extraObjectDataFactory), extraObjectDataFactory_(extraObjectDataFactory),
@@ -139,7 +96,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_); gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance()); gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread // Taking the locks before the pause is completed. So that any destroying thread
@@ -147,9 +104,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
std::optional objectFactoryIterable = objectFactory_.LockForIter(); std::optional objectFactoryIterable = objectFactory_.LockForIter();
gc::SweepExtraObjects<SweepTraits>(gcHandle, *extraObjectFactoryIterable); gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
extraObjectFactoryIterable = std::nullopt; extraObjectFactoryIterable = std::nullopt;
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, *objectFactoryIterable); auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
objectFactoryIterable = std::nullopt; objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread(); kotlin::compactObjectPoolInMainThread();
#else #else
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+10 -74
View File
@@ -8,24 +8,17 @@
#include <cstddef> #include <cstddef>
#include "Allocator.hpp" #include "AllocatorImpl.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerProcessor.hpp" #include "FinalizerProcessor.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp" #include "GCScheduler.hpp"
#include "GCState.hpp" #include "GCState.hpp"
#include "GlobalData.hpp"
#include "IntrusiveList.hpp" #include "IntrusiveList.hpp"
#include "ObjectFactory.hpp" #include "ObjectData.hpp"
#include "Types.h" #include "Types.h"
#include "Utils.hpp" #include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "ExtraObjectPage.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
#endif
namespace kotlin { namespace kotlin {
namespace mm { namespace mm {
@@ -38,70 +31,20 @@ namespace gc {
// TODO: Rename to StopTheWorldMarkAndSweep. // TODO: Rename to StopTheWorldMarkAndSweep.
class SameThreadMarkAndSweep : private Pinned { class SameThreadMarkAndSweep : private Pinned {
public: public:
class ObjectData { using MarkQueue = intrusive_forward_list<GC::ObjectData>;
public:
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
bool marked() const noexcept { return next_ != nullptr; }
bool tryResetMark() noexcept {
if (next_ == nullptr) return false;
next_ = nullptr;
return true;
}
private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_; }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_ != nullptr) {
return false;
}
next_ = next;
return true;
}
ObjectData* next_ = nullptr;
};
using MarkQueue = intrusive_forward_list<ObjectData>;
class ThreadData : private Pinned { class ThreadData : private Pinned {
public: public:
using ObjectData = SameThreadMarkAndSweep::ObjectData;
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {} ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {}
~ThreadData() = default; ~ThreadData() = default;
void OnOOM(size_t size) noexcept;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
private: private:
}; };
using Allocator = ThreadData::Allocator;
#ifdef CUSTOM_ALLOCATOR #ifdef CUSTOM_ALLOCATOR
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept; SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept;
#else #else
using FinalizerQueue = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueue;
using FinalizerQueueTraits = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueueTraits;
SameThreadMarkAndSweep( SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory, ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler) noexcept; gcScheduler::GCScheduler& gcScheduler) noexcept;
#endif #endif
@@ -122,7 +65,7 @@ private:
void PerformFullGC(int64_t epoch) noexcept; void PerformFullGC(int64_t epoch) noexcept;
#ifndef CUSTOM_ALLOCATOR #ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_; ObjectFactory& objectFactory_;
mm::ExtraObjectDataFactory& extraObjectDataFactory_; mm::ExtraObjectDataFactory& extraObjectDataFactory_;
#else #else
alloc::Heap heap_; alloc::Heap heap_;
@@ -145,21 +88,14 @@ struct MarkTraits {
static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
if (auto* top = queue.try_pop_front()) { if (auto* top = queue.try_pop_front()) {
auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(*top); return objectForObjectData(*top);
return node->GetObjHeader();
} }
return nullptr; return nullptr;
} }
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { return queue.try_push_front(objectDataForObject(object)); }
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return queue.try_push_front(objectData);
}
static bool tryMark(ObjHeader* object) noexcept { static bool tryMark(ObjHeader* object) noexcept { return objectDataForObject(object).tryMark(); }
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.tryMark();
}
static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
auto process = object->type_info()->processObjectInMark; auto process = object->type_info()->processObjectInMark;
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+104 -109
View File
@@ -194,11 +194,6 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#endif #endif
} }
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().marked();
}
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
@@ -239,12 +234,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false); ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false); ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false); ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false); ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -252,12 +247,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false); EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false); EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false); EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false); EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false); EXPECT_THAT(gc::isMarked(stack3.header()), false);
}); });
} }
@@ -285,12 +280,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false); ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false); ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false); ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false); ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false); ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -298,12 +293,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header())); global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false); EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false); EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false); EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false); EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false); EXPECT_THAT(gc::isMarked(stack3.header()), false);
}); });
} }
@@ -313,8 +308,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData); auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -328,8 +323,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData); auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header())); EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header())); EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -348,8 +343,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
})(); })();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header()); ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header())); EXPECT_CALL(finalizerHook(), Call(weak1.header()));
@@ -366,14 +361,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false); ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header()); ASSERT_THAT(weak1.get(), object1.header());
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false); EXPECT_THAT(gc::isMarked(weak1.header()), false);
EXPECT_THAT(weak1.get(), nullptr); EXPECT_THAT(weak1.get(), nullptr);
}); });
} }
@@ -396,12 +391,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -409,12 +404,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -443,14 +438,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -458,12 +453,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -492,14 +487,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header())); EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header())); EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -509,12 +504,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -531,18 +526,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header(); object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
}); });
} }
@@ -571,14 +566,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(), global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header())); object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false); ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false); ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false); ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false); ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false); ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false); ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false); ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -587,12 +582,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
Alive(threadData), Alive(threadData),
testing::UnorderedElementsAre( testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header())); global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false); EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false); EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false); EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false); EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false); EXPECT_THAT(gc::isMarked(object4.header()), false);
}); });
} }
@@ -609,12 +604,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header(); global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false); EXPECT_THAT(gc::isMarked(global2.header()), false);
}); });
} }
@@ -628,14 +623,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header()); ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false); EXPECT_THAT(gc::isMarked(object.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header())); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false); EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false); EXPECT_THAT(gc::isMarked(object.header()), false);
}); });
} }
@@ -1141,9 +1136,9 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header())); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false); ASSERT_THAT(gc::isMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false); ASSERT_THAT(gc::isMarked(weak.load()->header()), false);
ASSERT_THAT(weak.load()->get(), object1_local.header()); ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr; global1->field1 = nullptr;
kotlin-native/runtime/src/main/cpp/Memory.h
+58
View File
@@ -19,9 +19,11 @@
#include <utility> #include <utility>
#include "Alignment.hpp"
#include "KAssert.h" #include "KAssert.h"
#include "Common.h" #include "Common.h"
#include "TypeInfo.h" #include "TypeInfo.h"
#include "TypeLayout.hpp"
#include "Atomic.h" #include "Atomic.h"
#include "PointerBits.h" #include "PointerBits.h"
#include "Utils.hpp" #include "Utils.hpp"
@@ -125,6 +127,7 @@ struct ObjHeader {
static MetaObjHeader* createMetaObject(ObjHeader* object); static MetaObjHeader* createMetaObject(ObjHeader* object);
static void destroyMetaObject(ObjHeader* object); static void destroyMetaObject(ObjHeader* object);
}; };
static_assert(alignof(ObjHeader) <= kotlin::kObjectAlignment);
// Header of value type array objects. Keep layout in sync with that of object header. // Header of value type array objects. Keep layout in sync with that of object header.
struct ArrayHeader { struct ArrayHeader {
@@ -140,6 +143,61 @@ struct ArrayHeader {
// Elements count. Element size is stored in instanceSize_ field of TypeInfo, negated. // Elements count. Element size is stored in instanceSize_ field of TypeInfo, negated.
uint32_t count_; uint32_t count_;
}; };
static_assert(alignof(ArrayHeader) <= kotlin::kObjectAlignment);
#ifndef KONAN_WASM
namespace kotlin {
struct ObjectBody;
struct ArrayBody;
template <>
struct type_layout::descriptor<ObjectBody> {
class type {
public:
using value_type = ObjectBody;
explicit type(const TypeInfo* typeInfo) noexcept : size_(typeInfo->instanceSize_ - sizeof(ObjHeader)) {}
static constexpr size_t alignment() noexcept { return kObjectAlignment; }
uint64_t size() const noexcept { return size_; }
value_type* construct(uint8_t* ptr) noexcept {
RuntimeAssert(isZeroed(std_support::span<uint8_t>(ptr, size_)), "ObjectBodyDescriptor::construct@%p memory is not zeroed", ptr);
return reinterpret_cast<value_type*>(ptr);
}
private:
uint64_t size_;
};
};
template <>
struct type_layout::descriptor<ArrayBody> {
class type {
public:
using value_type = ArrayBody;
explicit type(const TypeInfo* typeInfo, uint32_t count) noexcept :
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
size_(static_cast<uint64_t>(-typeInfo->instanceSize_) * count) {}
static constexpr size_t alignment() noexcept { return kObjectAlignment; }
uint64_t size() const noexcept { return size_; }
value_type* construct(uint8_t* ptr) noexcept {
RuntimeAssert(isZeroed(std_support::span<uint8_t>(ptr, size_)), "ArrayBodyDescriptor::construct@%p memory is not zeroed", ptr);
return reinterpret_cast<ArrayBody*>(ptr);
}
private:
uint64_t size_;
};
};
} // namespace kotlin
#endif
ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj); ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj);
ALWAYS_INLINE bool isShareable(const ObjHeader* obj); ALWAYS_INLINE bool isShareable(const ObjHeader* obj);
kotlin-native/runtime/src/main/cpp/Utils.cpp
+10 -1
View File
@@ -7,6 +7,9 @@
#include <climits> #include <climits>
#include <cstdint> #include <cstdint>
#include <cstring>
using namespace kotlin;
namespace { namespace {
@@ -68,4 +71,10 @@ struct HashCompineImpl<64> {
size_t kotlin::CombineHash(size_t seed, size_t value) { size_t kotlin::CombineHash(size_t seed, size_t value) {
return HashCompineImpl<sizeof(std::size_t) * CHAR_BIT>::fn(seed, value); return HashCompineImpl<sizeof(std::size_t) * CHAR_BIT>::fn(seed, value);
} }
bool kotlin::isZeroed(std_support::span<uint8_t> span) noexcept {
if (span.size() == 0) return true;
if (span[0] != 0) return false;
return memcmp(span.data(), span.data() + 1, span.size() - 1) == 0;
}
kotlin-native/runtime/src/main/cpp/Utils.hpp
+5
View File
@@ -10,6 +10,8 @@
#include <type_traits> #include <type_traits>
#include "std_support/Span.hpp"
namespace kotlin { namespace kotlin {
// A helper for implementing classes with disabled copy constructor and copy assignment. // A helper for implementing classes with disabled copy constructor and copy assignment.
@@ -98,6 +100,9 @@ size_t CombineHash(size_t seed, size_t value);
#define ownerOf(type, field, ref) *reinterpret_cast<type*>(reinterpret_cast<char*>(&ref) - offsetof(type, field)) #define ownerOf(type, field, ref) *reinterpret_cast<type*>(reinterpret_cast<char*>(&ref) - offsetof(type, field))
// Returns `true` if the entire `span` is zeroed.
bool isZeroed(std_support::span<uint8_t> span) noexcept;
} // namespace kotlin } // namespace kotlin
#endif // RUNTIME_UTILS_H #endif // RUNTIME_UTILS_H
kotlin-native/runtime/src/main/cpp/UtilsTest.cpp
+24
View File
@@ -5,6 +5,7 @@
#include "Utils.hpp" #include "Utils.hpp"
#include <array>
#include <type_traits> #include <type_traits>
#include "gmock/gmock.h" #include "gmock/gmock.h"
@@ -73,3 +74,26 @@ TEST(UtilsTest, OwnerOf) {
EXPECT_THAT(&c, &Container::fromX(c.x())); EXPECT_THAT(&c, &Container::fromX(c.x()));
EXPECT_THAT(&c, &Container::fromY(c.y())); EXPECT_THAT(&c, &Container::fromY(c.y()));
} }
TEST(UtilsTest, IsZeroed) {
std::array<uint8_t, 0> empty;
EXPECT_TRUE(isZeroed(empty));
std::array<uint8_t, 1> zeroed1 = {0};
EXPECT_TRUE(isZeroed(zeroed1));
std::array<uint8_t, 1> notZeroed1 = {1};
EXPECT_FALSE(isZeroed(notZeroed1));
std::array<uint8_t, 3> zeroed3 = {0, 0, 0};
EXPECT_TRUE(isZeroed(zeroed3));
std::array<uint8_t, 3> notZeroed3_0 = {1, 0, 0};
EXPECT_FALSE(isZeroed(notZeroed3_0));
std::array<uint8_t, 3> notZeroed3_1 = {0, 1, 0};
EXPECT_FALSE(isZeroed(notZeroed3_1));
std::array<uint8_t, 3> notZeroed3_2 = {0, 0, 1};
EXPECT_FALSE(isZeroed(notZeroed3_2));
}
kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
-1
View File
@@ -6,7 +6,6 @@
#ifndef RUNTIME_MM_GLOBAL_DATA_H #ifndef RUNTIME_MM_GLOBAL_DATA_H
#define RUNTIME_MM_GLOBAL_DATA_H #define RUNTIME_MM_GLOBAL_DATA_H
#include "ObjectFactory.hpp"
#include "GlobalsRegistry.hpp" #include "GlobalsRegistry.hpp"
#include "GC.hpp" #include "GC.hpp"
#include "GCScheduler.hpp" #include "GCScheduler.hpp"
kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp
+82 -72
View File
@@ -17,6 +17,7 @@
#include "Memory.h" #include "Memory.h"
#include "Mutex.hpp" #include "Mutex.hpp"
#include "Porting.h" #include "Porting.h"
#include "TypeLayout.hpp"
#include "Types.h" #include "Types.h"
#include "Utils.hpp" #include "Utils.hpp"
@@ -69,9 +70,9 @@ public:
} }
// Note: This can only be trivially destructible data, as nobody can invoke its destructor. // Note: This can only be trivially destructible data, as nobody can invoke its destructor.
void* Data() noexcept { uint8_t* Data() noexcept {
constexpr size_t kDataOffset = DataOffset(); constexpr size_t kDataOffset = DataOffset();
void* ptr = reinterpret_cast<uint8_t*>(this) + kDataOffset; auto* ptr = reinterpret_cast<uint8_t*>(this) + kDataOffset;
RuntimeAssert(IsAligned(ptr, DataAlignment), "Data=%p is not aligned to %zu", ptr, DataAlignment); RuntimeAssert(IsAligned(ptr, DataAlignment), "Data=%p is not aligned to %zu", ptr, DataAlignment);
return ptr; return ptr;
} }
@@ -79,7 +80,7 @@ public:
// It's a caller responsibility to know if the underlying data is `T`. // It's a caller responsibility to know if the underlying data is `T`.
template <typename T> template <typename T>
T& Data() noexcept { T& Data() noexcept {
return *static_cast<T*>(Data()); return *reinterpret_cast<T*>(Data());
} }
size_t GetAllocatedSize() noexcept { size_t GetAllocatedSize() noexcept {
@@ -438,46 +439,85 @@ private:
template <typename Traits> template <typename Traits>
class ObjectFactory : private Pinned { class ObjectFactory : private Pinned {
using ObjectData = typename Traits::ObjectData;
using Allocator = typename Traits::Allocator; using Allocator = typename Traits::Allocator;
using ObjectData = typename Traits::ObjectData;
struct HeapObjHeader { struct HeapObjHeader {
[[no_unique_address]] // to account for GCs with empty ObjectData using descriptor = type_layout::Composite<HeapObjHeader, ObjectData, ObjHeader>;
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object; static HeapObjHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); }
static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().template fromField<1>(object); }
ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; }
ObjHeader* object() noexcept { return descriptor().template field<1>(this).second; }
private:
HeapObjHeader() = delete;
~HeapObjHeader() = delete;
};
struct HeapObject {
using descriptor = type_layout::Composite<HeapObject, HeapObjHeader, ObjectBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept {
return descriptor{{}, type_layout::descriptor_t<ObjectBody>{typeInfo}};
}
HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; }
private:
HeapObject() = delete;
~HeapObject() = delete;
}; };
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible // Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other. // with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader { struct HeapArrayHeader {
[[no_unique_address]] using descriptor = type_layout::Composite<HeapArrayHeader, ObjectData, ArrayHeader>;
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array; static HeapArrayHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); }
static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().template fromField<1>(array); }
ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; }
ArrayHeader* array() noexcept { return descriptor().template field<1>(this).second; }
private:
HeapArrayHeader() = delete;
~HeapArrayHeader() = delete;
}; };
static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { struct HeapArray {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); using descriptor = type_layout::Composite<HeapArray, HeapArrayHeader, ArrayBody>;
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
}
static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept {
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped return descriptor{{}, type_layout::descriptor_t<ArrayBody>{typeInfo, size}};
// at about half of uint64_t max. }
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
// Note: array body is aligned, but for size computation it is enough to align the sum. HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; }
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
private:
HeapArray() = delete;
~HeapArray() = delete;
};
// Only used for already allocated objects. Cannot overflow size_t.
static size_t GetDataSizeForAllocated(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return HeapArray::make_descriptor(typeInfo, object->array()->count_).size();
} else {
return HeapObject::make_descriptor(typeInfo).size();
}
} }
struct DataSizeProvider { struct DataSizeProvider {
static size_t GetDataSize(void* data) noexcept { static size_t GetDataSize(uint8_t* data) noexcept {
ObjHeader* object = &static_cast<HeapObjHeader*>(data)->object; return GetDataSizeForAllocated(reinterpret_cast<HeapObjHeader*>(data)->object());
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return ArrayAllocatedDataSize(typeInfo, object->array()->count_);
} else {
return ObjectAllocatedDataSize(typeInfo);
}
} }
}; };
@@ -490,32 +530,23 @@ public:
static NodeRef From(ObjHeader* object) noexcept { static NodeRef From(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Must be a heap object"); RuntimeAssert(object->heap(), "Must be a heap object");
auto& heapObject = ownerOf(HeapObjHeader, object, *object); return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapObjHeader::from(object))));
return NodeRef(Storage::Node::FromData(&heapObject));
} }
static NodeRef From(ArrayHeader* array) noexcept { static NodeRef From(ArrayHeader* array) noexcept {
RuntimeAssert(array->obj()->heap(), "Must be a heap object"); RuntimeAssert(array->obj()->heap(), "Must be a heap object");
auto& heapArray = ownerOf(HeapArrayHeader, array, *array); return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapArrayHeader::from(array))));
return NodeRef(Storage::Node::FromData(&heapArray));
} }
static NodeRef From(ObjectData& objectData) noexcept { static NodeRef From(ObjectData& objectData) noexcept {
auto& heapObject = ownerOf(HeapObjHeader, gcData, objectData); return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapObjHeader::from(objectData))));
return NodeRef(Storage::Node::FromData(&heapObject));
} }
NodeRef* operator->() noexcept { return this; } NodeRef* operator->() noexcept { return this; }
ObjectData& ObjectData() noexcept { ObjectData& ObjectData() noexcept { return reinterpret_cast<HeapObjHeader*>(node_.Data())->objectData(); }
// `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can
// be always casted to the other.
return static_cast<HeapObjHeader*>(node_.Data())->gcData;
}
ObjHeader* GetObjHeader() noexcept { ObjHeader* GetObjHeader() noexcept { return reinterpret_cast<HeapObjHeader*>(node_.Data())->object(); }
return &static_cast<HeapObjHeader*>(node_.Data())->object;
}
bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; } bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; }
@@ -550,37 +581,21 @@ public:
ThreadQueue(ObjectFactory& owner, Allocator allocator) noexcept : producer_(owner.storage_, std::move(allocator)) {} ThreadQueue(ObjectFactory& owner, Allocator allocator) noexcept : producer_(owner.storage_, std::move(allocator)) {}
static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
// Only used for already allocated objects. Cannot overflow size_t
auto allocSize = ObjectAllocatedDataSize(typeInfo);
return Storage::Node::GetSizeForDataSize(allocSize);
}
ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept { ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
size_t allocSize = ObjectAllocatedDataSize(typeInfo); auto descriptor = HeapObject::make_descriptor(typeInfo);
auto& node = producer_.Insert(allocSize); auto& heapObject = *descriptor.construct(producer_.Insert(descriptor.size()).Data());
auto* heapObject = new (node.Data()) HeapObjHeader(); ObjHeader* object = heapObject.header(descriptor).object();
auto* object = &heapObject->object;
object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo); object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
// TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
return object; return object;
} }
static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
RuntimeAssert(typeInfo->IsArray(), "Must be an array");
// Only used for already allocated arrays. Cannot overflow size_t
auto allocSize = ArrayAllocatedDataSize(typeInfo, count);
return Storage::Node::GetSizeForDataSize(allocSize);
}
ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
RuntimeAssert(typeInfo->IsArray(), "Must be an array"); RuntimeAssert(typeInfo->IsArray(), "Must be an array");
auto allocSize = ArrayAllocatedDataSize(typeInfo, count); auto descriptor = HeapArray::make_descriptor(typeInfo, count);
auto& node = producer_.Insert(allocSize); auto& heapArray = *descriptor.construct(producer_.Insert(descriptor.size()).Data());
auto* heapArray = new (node.Data()) HeapArrayHeader(); ArrayHeader* array = heapArray.header(descriptor).array();
auto* array = &heapArray->array;
array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo); array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
array->count_ = count; array->count_ = count;
// TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation. // TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
@@ -713,13 +728,8 @@ public:
void ClearForTests() { storage_.ClearForTests(); } void ClearForTests() { storage_.ClearForTests(); }
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept { static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object"); auto dataSize = GetDataSizeForAllocated(object);
const auto* typeInfo = object->type_info(); return Storage::Node::GetSizeForDataSize(dataSize);
if (typeInfo->IsArray()) {
return ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_);
} else {
return ThreadQueue::ObjectAllocatedSize(typeInfo);
}
} }
private: private:
kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp
+3 -3
View File
@@ -62,7 +62,7 @@ template <typename T, size_t DataAlignment>
std_support::vector<T> Collect(ObjectFactoryStorage<DataAlignment>& storage) { std_support::vector<T> Collect(ObjectFactoryStorage<DataAlignment>& storage) {
std_support::vector<T> result; std_support::vector<T> result;
for (auto& node : storage.LockForIter()) { for (auto& node : storage.LockForIter()) {
result.push_back(*static_cast<T*>(node.Data())); result.push_back(*reinterpret_cast<T*>(node.Data()));
} }
return result; return result;
} }
@@ -71,7 +71,7 @@ template <typename T, size_t DataAlignment>
std_support::vector<T> Collect(Consumer<ObjectFactoryStorage<DataAlignment>>& consumer) { std_support::vector<T> Collect(Consumer<ObjectFactoryStorage<DataAlignment>>& consumer) {
std_support::vector<T> result; std_support::vector<T> result;
for (auto& node : consumer) { for (auto& node : consumer) {
result.push_back(*static_cast<T*>(node.Data())); result.push_back(*reinterpret_cast<T*>(node.Data()));
} }
return result; return result;
} }
@@ -710,7 +710,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) {
} }
for (auto& node : iter) { for (auto& node : iter) {
int element = *static_cast<int*>(node.Data()); int element = *reinterpret_cast<int*>(node.Data());
actualBefore.push_back(element); actualBefore.push_back(element);
} }
} }
kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
-1
View File
@@ -11,7 +11,6 @@
#include "GlobalData.hpp" #include "GlobalData.hpp"
#include "GlobalsRegistry.hpp" #include "GlobalsRegistry.hpp"
#include "GC.hpp" #include "GC.hpp"
#include "ObjectFactory.hpp"
#include "ShadowStack.hpp" #include "ShadowStack.hpp"
#include "SpecialRefRegistry.hpp" #include "SpecialRefRegistry.hpp"
#include "ThreadLocalStorage.hpp" #include "ThreadLocalStorage.hpp"
kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp
+1
View File
@@ -7,6 +7,7 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "Porting.h"
#include "ScopedThread.hpp" #include "ScopedThread.hpp"
#include "ThreadData.hpp" #include "ThreadData.hpp"
kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp
+1
View File
@@ -12,6 +12,7 @@
#include "CallsChecker.hpp" #include "CallsChecker.hpp"
#include "Logging.hpp" #include "Logging.hpp"
#include "Porting.h"
#include "SafePoint.hpp" #include "SafePoint.hpp"
#include "StackTrace.hpp" #include "StackTrace.hpp"