azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Decouple ObjectFactory from concrete ObjectData ^KT-60928

Browse Source 
* Use type_layout to declaratively express heap object headers in both
  custom allocator and ObjectFactory.
* Invoke constructor (w/o invoking Kotin constructors) for created
  objects and arrays from both custom allocator and ObjectFactory.
  Previously:
  - custom allocator only checked body for nullability (now this is
    performed in body constructor)
  - ObjectFactory only constructed ObjectData
* In each GC have a AllocatorImpl.hpp and ObjectData.hpp headers
  the first encapsulating allocator-specific types, the second
  containing specific ObjectData implementation.
* In each GC have a separate ObjectFactoryTraits that does not
  actually depend on the specific GC anymore.
* Each GC now expose ObjectData (as undefined type) and its descriptor,
  the latter being used by the custom allocator and ObjectFactory.
* Descriptors for ObjectBody and ArrayBody now live in Memory.h and the
  code calculating size is now shared. Their constructors check that the
  memory is zeroed (Kotlin constructors will expect this).
This commit is contained in:
Alexander Shabalin
2023-07-28 15:08:15 +02:00
committed by Space Team
parent d956a5504d
commit 83a70ddf8b
37 changed files with 901 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/custom_alloc/cpp/CustomAllocator.cpp
+11 -27
View File
@@ -28,19 +28,6 @@
namespace kotlin::alloc {
size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(heapObjectHeaderSize + membersSize, kObjectAlignment);
}
uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
// Note: array body is aligned, but for size computation it is enough to align the sum.
return AlignUp<uint64_t>(heapArrayHeaderSize + membersSize, kObjectAlignment);
}
CustomAllocator::CustomAllocator(Heap& heap) noexcept : heap_(heap), nextFitPage_(nullptr), extraObjectPage_(nullptr) {
CustomAllocInfo("CustomAllocator::CustomAllocator(heap)");
memset(fixedBlockPages_, 0, sizeof(fixedBlockPages_));
@@ -52,9 +39,9 @@ CustomAllocator::~CustomAllocator() {
ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
size_t allocSize = ObjectAllocatedDataSize(typeInfo);
uint8_t* heapObject = Allocate(allocSize);
auto* object = reinterpret_cast<ObjHeader*>(heapObject + gcDataSize);
auto descriptor = HeapObject::make_descriptor(typeInfo);
auto& heapObject = *descriptor.construct(Allocate(descriptor.size()));
ObjHeader* object = heapObject.header(descriptor).object();
if (typeInfo->flags_ & TF_HAS_FINALIZER) {
auto* extraObject = CreateExtraObject();
object->typeInfoOrMeta_ = reinterpret_cast<TypeInfo*>(new (extraObject) mm::ExtraObjectData(object, typeInfo));
@@ -69,9 +56,9 @@ ObjHeader* CustomAllocator::CreateObject(const TypeInfo* typeInfo) noexcept {
ArrayHeader* CustomAllocator::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
CustomAllocDebug("CustomAllocator@%p::CreateArray(%d)", this ,count);
RuntimeAssert(typeInfo->IsArray(), "Must be an array");
auto allocSize = ArrayAllocatedDataSize(typeInfo, count);
uint8_t* heapArray = Allocate(allocSize);
auto* array = reinterpret_cast<ArrayHeader*>(heapArray + gcDataSize);
auto descriptor = HeapArray::make_descriptor(typeInfo, count);
auto& heapArray = *descriptor.construct(Allocate(descriptor.size()));
ArrayHeader* array = heapArray.header(descriptor).array();
array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
array->count_ = count;
return array;
@@ -121,9 +108,9 @@ size_t CustomAllocator::GetAllocatedHeapSize(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return ArrayAllocatedDataSize(typeInfo, object->array()->count_);
return HeapArray::make_descriptor(typeInfo, object->array()->count_).size();
} else {
return ObjectAllocatedDataSize(typeInfo);
return HeapObject::make_descriptor(typeInfo).size();
}
}
@@ -131,16 +118,13 @@ uint8_t* CustomAllocator::Allocate(uint64_t size) noexcept {
RuntimeAssert(size, "CustomAllocator::Allocate cannot allocate 0 bytes");
CustomAllocDebug("CustomAllocator::Allocate(%" PRIu64 ")", size);
uint64_t cellCount = (size + sizeof(Cell) - 1) / sizeof(Cell);
uint8_t* ptr;
if (cellCount <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE) {
ptr = AllocateInFixedBlockPage(cellCount);
return AllocateInFixedBlockPage(cellCount);
} else if (cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE) {
ptr = AllocateInSingleObjectPage(cellCount);
return AllocateInSingleObjectPage(cellCount);
} else {
ptr = AllocateInNextFitPage(cellCount);
return AllocateInNextFitPage(cellCount);
}
RuntimeAssert(ptr[0] == 0 && memcmp(ptr, ptr + 1, size - 1) == 0, "CustomAllocator::Allocate: memory not zero!");
return ptr;
}
uint8_t* CustomAllocator::AllocateInSingleObjectPage(uint64_t cellCount) noexcept {
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+4 -5
View File
@@ -23,7 +23,6 @@
#include "GCStatistics.hpp"
#include "KAssert.h"
#include "Memory.h"
#include "ObjectFactory.hpp"
namespace {
@@ -33,14 +32,14 @@ std::atomic<size_t> allocatedBytesCounter;
namespace kotlin::alloc {
bool SweepObject(uint8_t* heapObjHeader, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
if (gc::GC::SweepObject(heapObjHeader)) {
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& gcHandle) noexcept {
auto* heapObjHeader = reinterpret_cast<HeapObjHeader*>(object);
if (gc::tryResetMark(heapObjHeader->objectData())) {
CustomAllocDebug("SweepObject(%p): still alive", heapObjHeader);
gcHandle.addKeptObject();
return true;
}
auto* objHeader = reinterpret_cast<ObjHeader*>(heapObjHeader + gcDataSize);
auto* extraObject = mm::ExtraObjectData::Get(objHeader);
auto* extraObject = mm::ExtraObjectData::Get(heapObjHeader->object());
if (extraObject) {
if (!extraObject->getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE)) {
CustomAllocDebug("SweepObject(%p): needs to be finalized, extraObject at %p", heapObjHeader, extraObject);
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp
+71 -4
View File
@@ -15,15 +15,74 @@
#include "AtomicStack.hpp"
#include "ExtraObjectData.hpp"
#include "ExtraObjectPage.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "Memory.h"
#include "GC.hpp"
#include "TypeLayout.hpp"
namespace kotlin::alloc {
const size_t gcDataSize = AlignUp(gc::GC::objectDataSize, kObjectAlignment);
const size_t heapObjectHeaderSize = AlignUp(gcDataSize + sizeof(ObjHeader), kObjectAlignment);
const size_t heapArrayHeaderSize = AlignUp(gcDataSize + sizeof(ArrayHeader), kObjectAlignment);
struct HeapObjHeader {
using descriptor = type_layout::Composite<HeapObjHeader, gc::GC::ObjectData, ObjHeader>;
static HeapObjHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); }
static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().fromField<1>(object); }
gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; }
ObjHeader* object() noexcept { return descriptor().field<1>(this).second; }
private:
HeapObjHeader() = delete;
~HeapObjHeader() = delete;
};
struct HeapObject {
using descriptor = type_layout::Composite<HeapObject, HeapObjHeader, ObjectBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept {
return descriptor{{}, type_layout::descriptor_t<ObjectBody>{typeInfo}};
}
HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; }
private:
HeapObject() = delete;
~HeapObject() = delete;
};
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader {
using descriptor = type_layout::Composite<HeapArrayHeader, gc::GC::ObjectData, ArrayHeader>;
static HeapArrayHeader& from(gc::GC::ObjectData& objectData) noexcept { return *descriptor().fromField<0>(&objectData); }
static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().fromField<1>(array); }
gc::GC::ObjectData& objectData() noexcept { return *descriptor().field<0>(this).second; }
ArrayHeader* array() noexcept { return descriptor().field<1>(this).second; }
private:
HeapArrayHeader() = delete;
~HeapArrayHeader() = delete;
};
struct HeapArray {
using descriptor = type_layout::Composite<HeapArray, HeapArrayHeader, ArrayBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept {
return descriptor{{}, type_layout::descriptor_t<ArrayBody>{typeInfo, size}};
}
HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.field<0>(this).second; }
private:
HeapArray() = delete;
~HeapArray() = delete;
};
// Returns `true` if the `object` must be kept alive still.
bool SweepObject(uint8_t* object, FinalizerQueue& finalizerQueue, gc::GCHandle::GCSweepScope& sweepScope) noexcept;
@@ -37,6 +96,14 @@ void Free(void* ptr, size_t size) noexcept;
size_t GetAllocatedBytes() noexcept;
inline gc::GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return HeapObjHeader::from(object).objectData();
}
inline ObjHeader* objectForObjectData(gc::GC::ObjectData& objectData) noexcept {
return HeapObjHeader::from(objectData).object();
}
} // namespace kotlin::alloc
#endif
kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp
+3 -3
View File
@@ -93,18 +93,18 @@ std_support::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
for (auto* page : fixedBlockPages_[blockSize].GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
}
}
}
for (auto* page : nextFitPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
}
}
for (auto* page : singleObjectPages_.GetPages()) {
for (auto* block : page->GetAllocatedBlocks()) {
allocated.push_back(reinterpret_cast<ObjHeader*>(block + gcDataSize));
allocated.push_back(reinterpret_cast<HeapObjHeader*>(block)->object());
}
}
std_support::vector<ObjHeader*> unfinalized;
kotlin-native/runtime/src/gc/cms/cpp/AllocatorImpl.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
namespace kotlin::gc {
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return kotlin::alloc::objectDataForObject(object);
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return kotlin::alloc::objectForObjectData(objectData);
}
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
} // namespace kotlin::gc
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
void OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return ObjectFactory::NodeRef::From(object).ObjectData();
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
}
using FinalizerQueue = ObjectFactory::FinalizerQueue;
using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp
+11 -44
View File
@@ -20,40 +20,12 @@
#include "GCState.hpp"
#include "GCStatistics.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "Heap.hpp"
#endif
using namespace kotlin;
namespace {
[[clang::no_destroy]] std::mutex gcMutex;
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
return objectData.tryResetMark();
}
};
struct ProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(obj).ObjectData();
return objectData.marked();
}
};
template<typename Body>
ScopedThread createGCThread(const char* name, Body&& body) {
return ScopedThread(ScopedThread::attributes().name(name), [name, body] {
@@ -63,8 +35,9 @@ ScopedThread createGCThread(const char* name, Body&& body) {
});
}
#ifndef CUSTOM_ALLOCATOR
// TODO move to common
[[maybe_unused]] inline void checkMarkCorrectness(mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::Iterable& heap) {
[[maybe_unused]] inline void checkMarkCorrectness(gc::ObjectFactory::Iterable& heap) {
if (compiler::runtimeAssertsMode() == compiler::RuntimeAssertsMode::kIgnore) return;
for (auto objRef: heap) {
auto obj = objRef.GetObjHeader();
@@ -72,23 +45,17 @@ ScopedThread createGCThread(const char* name, Body&& body) {
if (objData.marked()) {
traverseReferredObjects(obj, [obj](ObjHeader* field) {
if (field->heap()) {
auto& fieldObjData =
mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(field).ObjectData();
auto& fieldObjData = gc::ObjectFactory::NodeRef::From(field).ObjectData();
RuntimeAssert(fieldObjData.marked(), "Field %p of an alive obj %p must be alive", field, obj);
}
});
}
}
}
#endif
} // namespace
void gc::ConcurrentMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept {
CallsCheckerIgnoreGuard guard;
@@ -133,10 +100,11 @@ mm::ThreadData& gc::ConcurrentMarkAndSweep::ThreadData::commonThreadData() const
#ifndef CUSTOM_ALLOCATOR
gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory,
ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept :
bool mutatorsCooperate,
std::size_t auxGCThreads) noexcept :
objectFactory_(objectFactory),
extraObjectDataFactory_(extraObjectDataFactory),
#else
@@ -150,8 +118,7 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep(
state_.finalized(epoch);
}),
markDispatcher_(mutatorsCooperate),
mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); }))
{
mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) {
for (std::size_t i = 0; i < auxGCThreads; ++i) {
auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); }));
}
@@ -250,7 +217,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gcHandle.threadsAreResumed();
}
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
if (compiler::concurrentWeakSweep()) {
// Expected to happen outside STW.
@@ -261,9 +228,9 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
}
#ifndef CUSTOM_ALLOCATOR
gc::SweepExtraObjects<SweepTraits>(gcHandle, *extraObjectFactoryIterable);
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
extraObjectFactoryIterable = std::nullopt;
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, *objectFactoryIterable);
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread();
#else
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp
+8 -36
View File
@@ -8,29 +8,21 @@
#include <atomic>
#include <cstddef>
#include "Allocator.hpp"
#include "AllocatorImpl.hpp"
#include "Barriers.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerProcessor.hpp"
#include "GCScheduler.hpp"
#include "GCState.hpp"
#include "GCStatistics.hpp"
#include "IntrusiveList.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ObjectFactory.hpp"
#include "ObjectData.hpp"
#include "ParallelMark.hpp"
#include "ScopedThread.hpp"
#include "ThreadData.hpp"
#include "Types.h"
#include "Utils.hpp"
#include "std_support/Memory.hpp"
#include "MarkStack.hpp"
#include "ParallelMark.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "Heap.hpp"
#endif
namespace kotlin {
namespace gc {
@@ -39,25 +31,15 @@ namespace gc {
// TODO: Also make marking run concurrently with Kotlin threads.
class ConcurrentMarkAndSweep : private Pinned {
public:
class ThreadData : private Pinned {
public:
using ObjectData = mark::ObjectData;
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept
: gc_(gc), threadData_(threadData) {}
explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {}
~ThreadData() = default;
void OnOOM(size_t size) noexcept;
void OnSuspendForGC() noexcept;
void safePoint() noexcept { barriers_.onCheckpoint(); }
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
BarriersThreadData& barriers() noexcept { return barriers_; }
bool tryLockRootSet();
@@ -80,26 +62,16 @@ public:
std::atomic<bool> cooperative_ = false;
};
using ObjectData = ThreadData::ObjectData;
using Allocator = ThreadData::Allocator;
#ifndef CUSTOM_ALLOCATOR
using FinalizerQueue = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueue;
using FinalizerQueueTraits = mm::ObjectFactory<ConcurrentMarkAndSweep>::FinalizerQueueTraits;
#else
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
#endif
#ifdef CUSTOM_ALLOCATOR
explicit ConcurrentMarkAndSweep(gcScheduler::GCScheduler& scheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept;
#else
ConcurrentMarkAndSweep(
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory,
ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& scheduler,
bool mutatorsCooperate, std::size_t auxGCThreads) noexcept;
bool mutatorsCooperate,
std::size_t auxGCThreads) noexcept;
#endif
~ConcurrentMarkAndSweep();
@@ -121,7 +93,7 @@ private:
void PerformFullGC(int64_t epoch) noexcept;
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<ConcurrentMarkAndSweep>& objectFactory_;
ObjectFactory& objectFactory_;
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
#else
alloc::Heap heap_;
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+104 -109
View File
@@ -193,11 +193,6 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#endif
}
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().marked();
}
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
@@ -263,12 +258,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -276,12 +271,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(gc::isMarked(stack3.header()), false);
});
}
@@ -309,12 +304,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -322,12 +317,12 @@ TEST_P(ConcurrentMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(gc::isMarked(stack3.header()), false);
});
}
@@ -337,8 +332,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -352,8 +347,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -372,8 +367,8 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header()));
@@ -390,14 +385,14 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(gc::isMarked(weak1.header()), false);
EXPECT_THAT(weak1.get(), nullptr);
});
}
@@ -420,12 +415,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -433,12 +428,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -467,14 +462,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -482,12 +477,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -516,14 +511,14 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -533,12 +528,12 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -555,18 +550,18 @@ TEST_P(ConcurrentMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
});
}
@@ -595,14 +590,14 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -611,12 +606,12 @@ TEST_P(ConcurrentMarkAndSweepTest, RunGCTwice) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -633,12 +628,12 @@ TEST_P(ConcurrentMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
});
}
@@ -652,14 +647,14 @@ TEST_P(ConcurrentMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(object.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(object.header()), false);
});
}
@@ -1165,9 +1160,9 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(object1_local.header()), false);
ASSERT_THAT(gc::isMarked(weak.load()->header()), false);
ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr;
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp
+20 -11
View File
@@ -9,9 +9,8 @@
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp"
#include "ThreadSuspension.hpp"
#include "MarkStack.hpp"
#include "std_support/Memory.hpp"
using namespace kotlin;
@@ -86,7 +85,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifdef CUSTOM_ALLOCATOR
return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#else
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
return ObjectFactory::GetAllocatedHeapSize(object);
#endif
}
@@ -145,20 +144,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
}
bool gc::isMarked(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.marked();
return objectDataForObject(object).marked();
}
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_RESULT_OF(gc::WeakRefRead, object);
}
// static
const size_t gc::GC::objectDataSize = sizeof(ConcurrentMarkAndSweep::ObjectData);
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<ConcurrentMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
return objectData.tryResetMark();
}
// static
@@ -172,3 +166,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif
}
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return sizeof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return alignof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return new (ptr) gc::GC::ObjectData();
}
kotlin-native/runtime/src/gc/cms/cpp/GCImpl.hpp
+11 -17
View File
@@ -7,19 +7,12 @@
#include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "ConcurrentMarkAndSweep.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin {
namespace gc {
using GCImpl = ConcurrentMarkAndSweep;
class GC::Impl : private Pinned {
public:
#ifdef CUSTOM_ALLOCATOR
@@ -29,17 +22,17 @@ public:
#endif
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif
GCImpl& gc() noexcept { return gc_; }
ConcurrentMarkAndSweep& gc() noexcept { return gc_; }
private:
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_;
ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif
GCImpl gc_;
ConcurrentMarkAndSweep gc_;
};
class GC::ThreadData::Impl : private Pinned {
@@ -47,7 +40,7 @@ public:
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
gc_(gc.impl_->gc(), threadData),
#ifndef CUSTOM_ALLOCATOR
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
}
#else
@@ -55,20 +48,21 @@ public:
}
#endif
GCImpl::ThreadData& gc() noexcept { return gc_; }
ConcurrentMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#else
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#endif
private:
GCImpl::ThreadData gc_;
ConcurrentMarkAndSweep::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_;
#else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif
};
kotlin-native/runtime/src/gc/cms/cpp/MarkStack.hpp → kotlin-native/runtime/src/gc/cms/cpp/ObjectData.hpp
+14 -23
View File
@@ -1,28 +1,23 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <atomic>
#include <cstddef>
#include <type_traits>
#include "Allocator.hpp"
#include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "IntrusiveList.hpp"
#include "ObjectFactory.hpp"
#include "Types.h"
#include "Utils.hpp"
#include "std_support/Memory.hpp"
#include "KAssert.h"
namespace kotlin::gc::mark {
namespace kotlin::gc {
class ObjectData {
struct ObjectFactoryTraits {
using ObjectData = ObjectData;
class Allocator;
};
class GC::ObjectData {
public:
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
bool tryMark() noexcept { return trySetNext(reinterpret_cast<ObjectData*>(1)); }
bool marked() const noexcept { return next() != nullptr; }
@@ -32,10 +27,6 @@ public:
return true;
}
ObjHeader* objHeader() noexcept { // FIXME const
return ObjectFactory::NodeRef::From(*this).GetObjHeader();
}
private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
@@ -52,6 +43,6 @@ private:
std::atomic<ObjectData*> next_ = nullptr;
};
static_assert(std::is_trivially_destructible_v<GC::ObjectData>);
} // namespace kotlin::gc::mark
} // namespace kotlin::gc
kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp
-1
View File
@@ -1,6 +1,5 @@
#include "ParallelMark.hpp"
#include "MarkStack.hpp"
#include "MarkAndSweepUtils.hpp"
#include "GCStatistics.hpp"
#include "Utils.hpp"
kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp
+8 -10
View File
@@ -4,12 +4,12 @@
#include <condition_variable>
#include "GCStatistics.hpp"
#include "MarkStack.hpp"
#include "std_support/Vector.hpp"
#include "ManuallyScoped.hpp"
#include "ObjectData.hpp"
#include "ParallelProcessor.hpp"
#include "ThreadRegistry.hpp"
#include "Utils.hpp"
#include "ParallelProcessor.hpp"
#include "ManuallyScoped.hpp"
#include "std_support/Vector.hpp"
namespace kotlin::gc::mark {
@@ -70,14 +70,13 @@ private:
* Mark workers are able to balance work between each other through sharing/stealing.
*/
class ParallelMark : private Pinned {
using MarkStackImpl = intrusive_forward_list<ObjectData>;
using MarkStackImpl = intrusive_forward_list<GC::ObjectData>;
// work balancing parameters were chosen pretty arbitrary
using ParallelProcessor = ParallelProcessor<MarkStackImpl, 512, 4096>;
public:
class MarkTraits {
public:
using MarkQueue = ParallelProcessor::Worker;
using ObjectFactory = ObjectData::ObjectFactory;
static void clear(MarkQueue& queue) noexcept {
RuntimeAssert(queue.localEmpty(), "Mark queue must be empty");
@@ -86,19 +85,18 @@ public:
static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop();
if (obj) {
auto node = ObjectFactory::NodeRef::From(*obj);
return node->GetObjHeader();
return objectForObjectData(*obj);
}
return nullptr;
}
static ALWAYS_INLINE bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData();
auto& objectData = objectDataForObject(object);
return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData);
}
static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept {
auto& objectData = ObjectFactory::NodeRef::From(object).ObjectData();
auto& objectData = objectDataForObject(object);
return objectData.tryMark();
}
kotlin-native/runtime/src/gc/common/cpp/FinalizerProcessor.hpp
-1
View File
@@ -13,7 +13,6 @@
#include "KAssert.h"
#include "Memory.h"
#include "ObjectFactory.hpp"
#include "Runtime.h"
#include "ScopedThread.hpp"
#include "Utils.hpp"
kotlin-native/runtime/src/gc/common/cpp/GC.hpp
+25 -3
View File
@@ -51,6 +51,14 @@ public:
std_support::unique_ptr<Impl> impl_;
};
// Header to be placed before each heap object. GC will use this to keep its data if needed.
// This is used via `type_layout::descriptor_t`, which is specialized below.
// If GC doesn't need any data, it can make `size()` return 0 and `alignment()`
// return 1.
// Note: GC does not deinitialize `ObjectData`, so the implementations must ensure that
// the destructor is a trivial one.
class ObjectData;
explicit GC(gcScheduler::GCScheduler& gcScheduler) noexcept;
~GC();
@@ -75,9 +83,6 @@ public:
void WaitFinished(int64_t epoch) noexcept;
void WaitFinalizers(int64_t epoch) noexcept;
static const size_t objectDataSize;
static bool SweepObject(void* objectData) noexcept;
static void DestroyExtraObjectData(mm::ExtraObjectData& extraObject) noexcept;
private:
@@ -87,7 +92,24 @@ private:
bool isMarked(ObjHeader* object) noexcept;
OBJ_GETTER(tryRef, std::atomic<ObjHeader*>& object) noexcept;
// This will drop the mark bit if it was set and return `true`.
// If the mark bit was unset, this will return `false`.
bool tryResetMark(GC::ObjectData& objectData) noexcept;
inline constexpr bool kSupportsMultipleMutators = true;
} // namespace gc
template <>
struct type_layout::descriptor<gc::GC::ObjectData> {
struct type {
using value_type = gc::GC::ObjectData;
static uint64_t size() noexcept;
static size_t alignment() noexcept;
static value_type* construct(uint8_t* ptr) noexcept;
};
};
} // namespace kotlin
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+19
View File
@@ -7,6 +7,7 @@
#define RUNTIME_GC_COMMON_MARK_AND_SWEEP_UTILS_H
#include "ExtraObjectData.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "FinalizerHooks.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
@@ -164,6 +165,20 @@ typename Traits::ObjectFactory::FinalizerQueue Sweep(GCHandle handle, typename T
return Sweep<Traits>(handle, iter);
}
template <typename T>
struct DefaultSweepTraits {
using ObjectFactory = T;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData& object) noexcept {
auto* baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
return gc::isMarked(baseObject);
}
static bool TryResetMark(typename ObjectFactory::NodeRef node) noexcept { return gc::tryResetMark(node.ObjectData()); }
};
template <typename Traits>
void collectRootSetForThread(GCHandle gcHandle, typename Traits::MarkQueue& markQueue, mm::ThreadData& thread) {
auto handle = gcHandle.collectThreadRoots(thread);
@@ -235,6 +250,10 @@ void processWeaks(GCHandle gcHandle, mm::SpecialRefRegistry& registry) noexcept
}
}
struct DefaultProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept { return gc::isMarked(obj); }
};
} // namespace gc
} // namespace kotlin
kotlin-native/runtime/src/gc/noop/cpp/AllocatorImpl.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "Heap.hpp"
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = Allocator;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(); }
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp
+21 -8
View File
@@ -7,11 +7,12 @@
#include "Common.h"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "NoOpGC.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp"
#include "ThreadData.hpp"
#include "std_support/Memory.hpp"
#include "GCStatistics.hpp"
#include "ObjectOps.hpp"
using namespace kotlin;
@@ -79,7 +80,7 @@ gc::GC::~GC() = default;
// static
size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifndef CUSTOM_ALLOCATOR
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
return ObjectFactory::GetAllocatedHeapSize(object);
#else
return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#endif
@@ -133,11 +134,8 @@ ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed));
}
// static
const size_t gc::GC::objectDataSize = 0; // sizeof(NoOpGC::ObjectData) with [[no_unique_address]]
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
RuntimeAssert(false, "Should not reach here");
return true;
}
@@ -152,3 +150,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif
}
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return 0;
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return 1;
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return reinterpret_cast<gc::GC::ObjectData*>(ptr);
}
kotlin-native/runtime/src/gc/noop/cpp/GCImpl.hpp
+11 -18
View File
@@ -7,36 +7,28 @@
#include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "NoOpGC.hpp"
#include "ObjectFactory.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin {
namespace gc {
using GCImpl = NoOpGC;
class GC::Impl : private Pinned {
public:
Impl() noexcept = default;
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif
GCImpl& gc() noexcept { return gc_; }
NoOpGC& gc() noexcept { return gc_; }
private:
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_;
ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif
GCImpl gc_;
NoOpGC gc_;
};
class GC::ThreadData::Impl : private Pinned {
@@ -45,24 +37,25 @@ public:
Impl(GC& gc, mm::ThreadData& threadData) noexcept : alloc_(gc.impl_->gc().heap()) {}
#else
Impl(GC& gc, mm::ThreadData& threadData) noexcept :
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {}
#endif
GCImpl::ThreadData& gc() noexcept { return gc_; }
NoOpGC::ThreadData& gc() noexcept { return gc_; }
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#else
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#endif
private:
GCImpl::ThreadData gc_;
NoOpGC::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_;
#else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif
};
kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp
+2 -15
View File
@@ -8,14 +8,11 @@
#include <cstddef>
#include "Allocator.hpp"
#include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "Logging.hpp"
#include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "Heap.hpp"
#endif
namespace kotlin {
namespace mm {
@@ -28,21 +25,11 @@ namespace gc {
// TODO: It can be made more efficient.
class NoOpGC : private Pinned {
public:
class ObjectData {};
using Allocator = gc::Allocator;
class ThreadData : private Pinned {
public:
using ObjectData = NoOpGC::ObjectData;
ThreadData() noexcept {}
~ThreadData() = default;
void OnOOM(size_t size) noexcept {}
Allocator CreateAllocator() noexcept { return Allocator(); }
private:
};
kotlin-native/runtime/src/gc/stms/cpp/AllocatorImpl.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
namespace kotlin::gc {
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return kotlin::alloc::objectDataForObject(object);
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return kotlin::alloc::objectForObjectData(objectData);
}
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
} // namespace kotlin::gc
#else
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "GC.hpp"
#include "GlobalData.hpp"
#include "Logging.hpp"
#include "ObjectFactory.hpp"
namespace kotlin::gc {
struct ObjectFactoryTraits {
using Allocator = AllocatorWithGC<Allocator, ObjectFactoryTraits>;
using ObjectData = gc::GC::ObjectData;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
void OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
};
using ObjectFactory = mm::ObjectFactory<ObjectFactoryTraits>;
inline GC::ObjectData& objectDataForObject(ObjHeader* object) noexcept {
return ObjectFactory::NodeRef::From(object).ObjectData();
}
inline ObjHeader* objectForObjectData(GC::ObjectData& objectData) noexcept {
return ObjectFactory::NodeRef::From(objectData)->GetObjHeader();
}
using FinalizerQueue = ObjectFactory::FinalizerQueue;
using FinalizerQueueTraits = ObjectFactory::FinalizerQueueTraits;
} // namespace kotlin::gc
#endif
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp
+23 -12
View File
@@ -6,12 +6,13 @@
#include "GCImpl.hpp"
#include "GC.hpp"
#include "GCStatistics.hpp"
#include "GlobalData.hpp"
#include "MarkAndSweepUtils.hpp"
#include "ObjectAlloc.hpp"
#include "ObjectOps.hpp"
#include "SameThreadMarkAndSweep.hpp"
#include "std_support/Memory.hpp"
#include "GlobalData.hpp"
#include "GCStatistics.hpp"
#include "ObjectOps.hpp"
using namespace kotlin;
@@ -81,7 +82,7 @@ size_t gc::GC::GetAllocatedHeapSize(ObjHeader* object) noexcept {
#ifdef CUSTOM_ALLOCATOR
return alloc::CustomAllocator::GetAllocatedHeapSize(object);
#else
return mm::ObjectFactory<GCImpl>::GetAllocatedHeapSize(object);
return ObjectFactory::GetAllocatedHeapSize(object);
#endif
}
@@ -140,20 +141,15 @@ void gc::GC::WaitFinalizers(int64_t epoch) noexcept {
}
bool gc::isMarked(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.marked();
return objectDataForObject(object).marked();
}
ALWAYS_INLINE OBJ_GETTER(gc::tryRef, std::atomic<ObjHeader*>& object) noexcept {
RETURN_OBJ(object.load(std::memory_order_relaxed));
}
// static
const size_t gc::GC::objectDataSize = sizeof(SameThreadMarkAndSweep::ObjectData);
// static
ALWAYS_INLINE bool gc::GC::SweepObject(void *objectData) noexcept {
return reinterpret_cast<SameThreadMarkAndSweep::ObjectData*>(objectData)->tryResetMark();
ALWAYS_INLINE bool gc::tryResetMark(GC::ObjectData& objectData) noexcept {
return objectData.tryResetMark();
}
// static
@@ -167,3 +163,18 @@ ALWAYS_INLINE void gc::GC::DestroyExtraObjectData(mm::ExtraObjectData& extraObje
extraObject.setFlag(mm::ExtraObjectData::FLAGS_FINALIZED);
#endif
}
// static
ALWAYS_INLINE uint64_t type_layout::descriptor<gc::GC::ObjectData>::type::size() noexcept {
return sizeof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE size_t type_layout::descriptor<gc::GC::ObjectData>::type::alignment() noexcept {
return alignof(gc::GC::ObjectData);
}
// static
ALWAYS_INLINE gc::GC::ObjectData* type_layout::descriptor<gc::GC::ObjectData>::type::construct(uint8_t* ptr) noexcept {
return new (ptr) gc::GC::ObjectData();
}
kotlin-native/runtime/src/gc/stms/cpp/GCImpl.hpp
+11 -17
View File
@@ -7,19 +7,12 @@
#include "GC.hpp"
#include "AllocatorImpl.hpp"
#include "SameThreadMarkAndSweep.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#else
#include "ExtraObjectDataFactory.hpp"
#endif
namespace kotlin {
namespace gc {
using GCImpl = SameThreadMarkAndSweep;
class GC::Impl : private Pinned {
public:
#ifdef CUSTOM_ALLOCATOR
@@ -27,17 +20,17 @@ public:
#else
explicit Impl(gcScheduler::GCScheduler& gcScheduler) noexcept : gc_(objectFactory_, extraObjectDataFactory_, gcScheduler) {}
mm::ObjectFactory<gc::GCImpl>& objectFactory() noexcept { return objectFactory_; }
ObjectFactory& objectFactory() noexcept { return objectFactory_; }
mm::ExtraObjectDataFactory& extraObjectDataFactory() noexcept { return extraObjectDataFactory_; }
#endif
GCImpl& gc() noexcept { return gc_; }
SameThreadMarkAndSweep& gc() noexcept { return gc_; }
private:
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<gc::GCImpl> objectFactory_;
ObjectFactory objectFactory_;
mm::ExtraObjectDataFactory extraObjectDataFactory_;
#endif
GCImpl gc_;
SameThreadMarkAndSweep gc_;
};
class GC::ThreadData::Impl : private Pinned {
@@ -48,25 +41,26 @@ public:
alloc_(gc.impl_->gc().heap()) {
}
#else
objectFactoryThreadQueue_(gc.impl_->objectFactory(), gc_.CreateAllocator()),
objectFactoryThreadQueue_(gc.impl_->objectFactory(), objectFactoryTraits_.CreateAllocator()),
extraObjectDataFactoryThreadQueue_(gc.impl_->extraObjectDataFactory()) {
}
#endif
GCImpl::ThreadData& gc() noexcept { return gc_; }
SameThreadMarkAndSweep::ThreadData& gc() noexcept { return gc_; }
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<GCImpl>::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; }
mm::ExtraObjectDataFactory::ThreadQueue& extraObjectDataFactoryThreadQueue() noexcept { return extraObjectDataFactoryThreadQueue_; }
#else
alloc::CustomAllocator& alloc() noexcept { return alloc_; }
#endif
private:
GCImpl::ThreadData gc_;
SameThreadMarkAndSweep::ThreadData gc_;
#ifdef CUSTOM_ALLOCATOR
alloc::CustomAllocator alloc_;
#else
mm::ObjectFactory<GCImpl>::ThreadQueue objectFactoryThreadQueue_;
[[no_unique_address]] ObjectFactoryTraits objectFactoryTraits_;
ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
mm::ExtraObjectDataFactory::ThreadQueue extraObjectDataFactoryThreadQueue_;
#endif
};
kotlin-native/runtime/src/gc/stms/cpp/ObjectData.hpp
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include <type_traits>
#include "AllocatorImpl.hpp"
#include "GC.hpp"
#include "IntrusiveList.hpp"
#include "KAssert.h"
namespace kotlin::gc {
class GC::ObjectData {
public:
bool tryMark() noexcept { return trySetNext(reinterpret_cast<ObjectData*>(1)); }
bool marked() const noexcept { return next_ != nullptr; }
bool tryResetMark() noexcept {
if (next_ == nullptr) return false;
next_ = nullptr;
return true;
}
private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_; }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_ != nullptr) {
return false;
}
next_ = next;
return true;
}
ObjectData* next_ = nullptr;
};
static_assert(std::is_trivially_destructible_v<GC::ObjectData>);
} // namespace kotlin::gc
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp
+5 -48
View File
@@ -14,64 +14,21 @@
#include "Logging.hpp"
#include "MarkAndSweepUtils.hpp"
#include "Memory.h"
#include "ObjectAlloc.hpp"
#include "RootSet.hpp"
#include "Runtime.h"
#include "ThreadData.hpp"
#include "ThreadRegistry.hpp"
#include "ThreadSuspension.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomFinalizerProcessor.hpp"
#endif
using namespace kotlin;
namespace {
struct SweepTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
static bool IsMarkedByExtraObject(mm::ExtraObjectData &object) noexcept {
auto *baseObject = object.GetBaseObject();
if (!baseObject->heap()) return true;
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(baseObject).ObjectData();
return objectData.marked();
}
static bool TryResetMark(ObjectFactory::NodeRef node) noexcept {
auto& objectData = node.ObjectData();
return objectData.tryResetMark();
}
};
struct FinalizeTraits {
using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
};
struct ProcessWeaksTraits {
static bool IsMarked(ObjHeader* obj) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(obj).ObjectData();
return objectData.marked();
}
};
} // namespace
void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept {
RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size);
// TODO: This will print the log for "manual" scheduling. Fix this.
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinished();
}
#ifdef CUSTOM_ALLOCATOR
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
gcScheduler::GCScheduler& gcScheduler) noexcept :
#else
gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler) noexcept :
ObjectFactory& objectFactory, mm::ExtraObjectDataFactory& extraObjectDataFactory, gcScheduler::GCScheduler& gcScheduler) noexcept :
objectFactory_(objectFactory),
extraObjectDataFactory_(extraObjectDataFactory),
@@ -139,7 +96,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
gc::Mark<internal::MarkTraits>(gcHandle, markQueue_);
gc::processWeaks<ProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
#ifndef CUSTOM_ALLOCATOR
// Taking the locks before the pause is completed. So that any destroying thread
@@ -147,9 +104,9 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
std::optional objectFactoryIterable = objectFactory_.LockForIter();
gc::SweepExtraObjects<SweepTraits>(gcHandle, *extraObjectFactoryIterable);
gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
extraObjectFactoryIterable = std::nullopt;
auto finalizerQueue = gc::Sweep<SweepTraits>(gcHandle, *objectFactoryIterable);
auto finalizerQueue = gc::Sweep<DefaultSweepTraits<ObjectFactory>>(gcHandle, *objectFactoryIterable);
objectFactoryIterable = std::nullopt;
kotlin::compactObjectPoolInMainThread();
#else
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp
+10 -74
View File
@@ -8,24 +8,17 @@
#include <cstddef>
#include "Allocator.hpp"
#include "ExtraObjectDataFactory.hpp"
#include "AllocatorImpl.hpp"
#include "FinalizerProcessor.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp"
#include "GCState.hpp"
#include "GlobalData.hpp"
#include "IntrusiveList.hpp"
#include "ObjectFactory.hpp"
#include "ObjectData.hpp"
#include "Types.h"
#include "Utils.hpp"
#ifdef CUSTOM_ALLOCATOR
#include "CustomAllocator.hpp"
#include "CustomFinalizerProcessor.hpp"
#include "ExtraObjectPage.hpp"
#include "GCApi.hpp"
#include "Heap.hpp"
#endif
namespace kotlin {
namespace mm {
@@ -38,70 +31,20 @@ namespace gc {
// TODO: Rename to StopTheWorldMarkAndSweep.
class SameThreadMarkAndSweep : private Pinned {
public:
class ObjectData {
public:
bool tryMark() noexcept {
return trySetNext(reinterpret_cast<ObjectData*>(1));
}
bool marked() const noexcept { return next_ != nullptr; }
bool tryResetMark() noexcept {
if (next_ == nullptr) return false;
next_ = nullptr;
return true;
}
private:
friend struct DefaultIntrusiveForwardListTraits<ObjectData>;
ObjectData* next() const noexcept { return next_; }
void setNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
next_ = next;
}
bool trySetNext(ObjectData* next) noexcept {
RuntimeAssert(next, "next cannot be nullptr");
if (next_ != nullptr) {
return false;
}
next_ = next;
return true;
}
ObjectData* next_ = nullptr;
};
using MarkQueue = intrusive_forward_list<ObjectData>;
using MarkQueue = intrusive_forward_list<GC::ObjectData>;
class ThreadData : private Pinned {
public:
using ObjectData = SameThreadMarkAndSweep::ObjectData;
using Allocator = AllocatorWithGC<Allocator, ThreadData>;
ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept {}
~ThreadData() = default;
void OnOOM(size_t size) noexcept;
Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); }
private:
};
using Allocator = ThreadData::Allocator;
#ifdef CUSTOM_ALLOCATOR
using FinalizerQueue = alloc::FinalizerQueue;
using FinalizerQueueTraits = alloc::FinalizerQueueTraits;
SameThreadMarkAndSweep(gcScheduler::GCScheduler& gcScheduler) noexcept;
#else
using FinalizerQueue = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueue;
using FinalizerQueueTraits = mm::ObjectFactory<SameThreadMarkAndSweep>::FinalizerQueueTraits;
SameThreadMarkAndSweep(
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory,
ObjectFactory& objectFactory,
mm::ExtraObjectDataFactory& extraObjectDataFactory,
gcScheduler::GCScheduler& gcScheduler) noexcept;
#endif
@@ -122,7 +65,7 @@ private:
void PerformFullGC(int64_t epoch) noexcept;
#ifndef CUSTOM_ALLOCATOR
mm::ObjectFactory<SameThreadMarkAndSweep>& objectFactory_;
ObjectFactory& objectFactory_;
mm::ExtraObjectDataFactory& extraObjectDataFactory_;
#else
alloc::Heap heap_;
@@ -145,21 +88,14 @@ struct MarkTraits {
static ObjHeader* tryDequeue(MarkQueue& queue) noexcept {
if (auto* top = queue.try_pop_front()) {
auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(*top);
return node->GetObjHeader();
return objectForObjectData(*top);
}
return nullptr;
}
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return queue.try_push_front(objectData);
}
static bool tryEnqueue(MarkQueue& queue, ObjHeader* object) noexcept { return queue.try_push_front(objectDataForObject(object)); }
static bool tryMark(ObjHeader* object) noexcept {
auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
return objectData.tryMark();
}
static bool tryMark(ObjHeader* object) noexcept { return objectDataForObject(object).tryMark(); }
static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
auto process = object->type_info()->processObjectInMark;
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+104 -109
View File
@@ -194,11 +194,6 @@ std_support::vector<ObjHeader*> Alive(mm::ThreadData& threadData) {
#endif
}
bool IsMarked(ObjHeader* objHeader) {
auto nodeRef = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(objHeader);
return nodeRef.ObjectData().marked();
}
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
@@ -239,12 +234,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -252,12 +247,12 @@ TEST_F(SameThreadMarkAndSweepTest, RootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(gc::isMarked(stack3.header()), false);
});
}
@@ -285,12 +280,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(global2.header()), false);
ASSERT_THAT(IsMarked(global3.header()), false);
ASSERT_THAT(IsMarked(stack1.header()), false);
ASSERT_THAT(IsMarked(stack2.header()), false);
ASSERT_THAT(IsMarked(stack3.header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(global2.header()), false);
ASSERT_THAT(gc::isMarked(global3.header()), false);
ASSERT_THAT(gc::isMarked(stack1.header()), false);
ASSERT_THAT(gc::isMarked(stack2.header()), false);
ASSERT_THAT(gc::isMarked(stack3.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -298,12 +293,12 @@ TEST_F(SameThreadMarkAndSweepTest, InterconnectedRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global1.header(), global2.header(), global3.header(), stack1.header(), stack2.header(), stack3.header()));
EXPECT_THAT(IsMarked(global1.header()), false);
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(IsMarked(global3.header()), false);
EXPECT_THAT(IsMarked(stack1.header()), false);
EXPECT_THAT(IsMarked(stack2.header()), false);
EXPECT_THAT(IsMarked(stack3.header()), false);
EXPECT_THAT(gc::isMarked(global1.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global3.header()), false);
EXPECT_THAT(gc::isMarked(stack1.header()), false);
EXPECT_THAT(gc::isMarked(stack2.header()), false);
EXPECT_THAT(gc::isMarked(stack3.header()), false);
});
}
@@ -313,8 +308,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjects) {
auto& object2 = AllocateObject(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -328,8 +323,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
auto& object2 = AllocateObjectWithFinalizer(threadData);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), object2.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
EXPECT_CALL(finalizerHook(), Call(object1.header()));
EXPECT_CALL(finalizerHook(), Call(object2.header()));
@@ -348,8 +343,8 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
EXPECT_CALL(finalizerHook(), Call(weak1.header()));
@@ -366,14 +361,14 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(weak1.header()), false);
ASSERT_THAT(weak1.get(), object1.header());
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(gc::isMarked(weak1.header()), false);
EXPECT_THAT(weak1.get(), nullptr);
});
}
@@ -396,12 +391,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -409,12 +404,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectReferencedFromRootSet) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -443,14 +438,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -458,12 +453,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCycles) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -492,14 +487,14 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
EXPECT_CALL(finalizerHook(), Call(object5.header()));
EXPECT_CALL(finalizerHook(), Call(object6.header()));
@@ -509,12 +504,12 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesAndFinalizers) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -531,18 +526,18 @@ TEST_F(SameThreadMarkAndSweepTest, ObjectsWithCyclesIntoRootSet) {
object2->field1 = stack.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), stack.header(), object1.header(), object2.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
});
}
@@ -571,14 +566,14 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header(),
object5.header(), object6.header()));
ASSERT_THAT(IsMarked(global.header()), false);
ASSERT_THAT(IsMarked(stack.header()), false);
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(object2.header()), false);
ASSERT_THAT(IsMarked(object3.header()), false);
ASSERT_THAT(IsMarked(object4.header()), false);
ASSERT_THAT(IsMarked(object5.header()), false);
ASSERT_THAT(IsMarked(object6.header()), false);
ASSERT_THAT(gc::isMarked(global.header()), false);
ASSERT_THAT(gc::isMarked(stack.header()), false);
ASSERT_THAT(gc::isMarked(object1.header()), false);
ASSERT_THAT(gc::isMarked(object2.header()), false);
ASSERT_THAT(gc::isMarked(object3.header()), false);
ASSERT_THAT(gc::isMarked(object4.header()), false);
ASSERT_THAT(gc::isMarked(object5.header()), false);
ASSERT_THAT(gc::isMarked(object6.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
@@ -587,12 +582,12 @@ TEST_F(SameThreadMarkAndSweepTest, RunGCTwice) {
Alive(threadData),
testing::UnorderedElementsAre(
global.header(), stack.header(), object1.header(), object2.header(), object3.header(), object4.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(stack.header()), false);
EXPECT_THAT(IsMarked(object1.header()), false);
EXPECT_THAT(IsMarked(object2.header()), false);
EXPECT_THAT(IsMarked(object3.header()), false);
EXPECT_THAT(IsMarked(object4.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(stack.header()), false);
EXPECT_THAT(gc::isMarked(object1.header()), false);
EXPECT_THAT(gc::isMarked(object2.header()), false);
EXPECT_THAT(gc::isMarked(object3.header()), false);
EXPECT_THAT(gc::isMarked(object4.header()), false);
});
}
@@ -609,12 +604,12 @@ TEST_F(SameThreadMarkAndSweepTest, PermanentObjects) {
global2->field1 = global1.header();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global2.header()));
EXPECT_THAT(IsMarked(global2.header()), false);
EXPECT_THAT(gc::isMarked(global2.header()), false);
});
}
@@ -628,14 +623,14 @@ TEST_F(SameThreadMarkAndSweepTest, SameObjectInRootSet) {
ASSERT_THAT(global.header(), stack.header());
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(object.header()), false);
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(global.header(), object.header()));
EXPECT_THAT(IsMarked(global.header()), false);
EXPECT_THAT(IsMarked(object.header()), false);
EXPECT_THAT(gc::isMarked(global.header()), false);
EXPECT_THAT(gc::isMarked(object.header()), false);
});
}
@@ -1141,9 +1136,9 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1_local.header(), weak.load()->header(), global1.header()));
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT(gc::isMarked(global1.header()), false);
ASSERT_THAT(gc::isMarked(object1_local.header()), false);
ASSERT_THAT(gc::isMarked(weak.load()->header()), false);
ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr;
kotlin-native/runtime/src/main/cpp/Memory.h
+58
View File
@@ -19,9 +19,11 @@
#include <utility>
#include "Alignment.hpp"
#include "KAssert.h"
#include "Common.h"
#include "TypeInfo.h"
#include "TypeLayout.hpp"
#include "Atomic.h"
#include "PointerBits.h"
#include "Utils.hpp"
@@ -125,6 +127,7 @@ struct ObjHeader {
static MetaObjHeader* createMetaObject(ObjHeader* object);
static void destroyMetaObject(ObjHeader* object);
};
static_assert(alignof(ObjHeader) <= kotlin::kObjectAlignment);
// Header of value type array objects. Keep layout in sync with that of object header.
struct ArrayHeader {
@@ -140,6 +143,61 @@ struct ArrayHeader {
// Elements count. Element size is stored in instanceSize_ field of TypeInfo, negated.
uint32_t count_;
};
static_assert(alignof(ArrayHeader) <= kotlin::kObjectAlignment);
#ifndef KONAN_WASM
namespace kotlin {
struct ObjectBody;
struct ArrayBody;
template <>
struct type_layout::descriptor<ObjectBody> {
class type {
public:
using value_type = ObjectBody;
explicit type(const TypeInfo* typeInfo) noexcept : size_(typeInfo->instanceSize_ - sizeof(ObjHeader)) {}
static constexpr size_t alignment() noexcept { return kObjectAlignment; }
uint64_t size() const noexcept { return size_; }
value_type* construct(uint8_t* ptr) noexcept {
RuntimeAssert(isZeroed(std_support::span<uint8_t>(ptr, size_)), "ObjectBodyDescriptor::construct@%p memory is not zeroed", ptr);
return reinterpret_cast<value_type*>(ptr);
}
private:
uint64_t size_;
};
};
template <>
struct type_layout::descriptor<ArrayBody> {
class type {
public:
using value_type = ArrayBody;
explicit type(const TypeInfo* typeInfo, uint32_t count) noexcept :
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
size_(static_cast<uint64_t>(-typeInfo->instanceSize_) * count) {}
static constexpr size_t alignment() noexcept { return kObjectAlignment; }
uint64_t size() const noexcept { return size_; }
value_type* construct(uint8_t* ptr) noexcept {
RuntimeAssert(isZeroed(std_support::span<uint8_t>(ptr, size_)), "ArrayBodyDescriptor::construct@%p memory is not zeroed", ptr);
return reinterpret_cast<ArrayBody*>(ptr);
}
private:
uint64_t size_;
};
};
} // namespace kotlin
#endif
ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj);
ALWAYS_INLINE bool isShareable(const ObjHeader* obj);
kotlin-native/runtime/src/main/cpp/Utils.cpp
+10 -1
View File
@@ -7,6 +7,9 @@
#include <climits>
#include <cstdint>
#include <cstring>
using namespace kotlin;
namespace {
@@ -68,4 +71,10 @@ struct HashCompineImpl<64> {
size_t kotlin::CombineHash(size_t seed, size_t value) {
return HashCompineImpl<sizeof(std::size_t) * CHAR_BIT>::fn(seed, value);
}
}
bool kotlin::isZeroed(std_support::span<uint8_t> span) noexcept {
if (span.size() == 0) return true;
if (span[0] != 0) return false;
return memcmp(span.data(), span.data() + 1, span.size() - 1) == 0;
}
kotlin-native/runtime/src/main/cpp/Utils.hpp
+5
View File
@@ -10,6 +10,8 @@
#include <type_traits>
#include "std_support/Span.hpp"
namespace kotlin {
// A helper for implementing classes with disabled copy constructor and copy assignment.
@@ -98,6 +100,9 @@ size_t CombineHash(size_t seed, size_t value);
#define ownerOf(type, field, ref) *reinterpret_cast<type*>(reinterpret_cast<char*>(&ref) - offsetof(type, field))
// Returns `true` if the entire `span` is zeroed.
bool isZeroed(std_support::span<uint8_t> span) noexcept;
} // namespace kotlin
#endif // RUNTIME_UTILS_H
kotlin-native/runtime/src/main/cpp/UtilsTest.cpp
+24
View File
@@ -5,6 +5,7 @@
#include "Utils.hpp"
#include <array>
#include <type_traits>
#include "gmock/gmock.h"
@@ -73,3 +74,26 @@ TEST(UtilsTest, OwnerOf) {
EXPECT_THAT(&c, &Container::fromX(c.x()));
EXPECT_THAT(&c, &Container::fromY(c.y()));
}
TEST(UtilsTest, IsZeroed) {
std::array<uint8_t, 0> empty;
EXPECT_TRUE(isZeroed(empty));
std::array<uint8_t, 1> zeroed1 = {0};
EXPECT_TRUE(isZeroed(zeroed1));
std::array<uint8_t, 1> notZeroed1 = {1};
EXPECT_FALSE(isZeroed(notZeroed1));
std::array<uint8_t, 3> zeroed3 = {0, 0, 0};
EXPECT_TRUE(isZeroed(zeroed3));
std::array<uint8_t, 3> notZeroed3_0 = {1, 0, 0};
EXPECT_FALSE(isZeroed(notZeroed3_0));
std::array<uint8_t, 3> notZeroed3_1 = {0, 1, 0};
EXPECT_FALSE(isZeroed(notZeroed3_1));
std::array<uint8_t, 3> notZeroed3_2 = {0, 0, 1};
EXPECT_FALSE(isZeroed(notZeroed3_2));
}
kotlin-native/runtime/src/mm/cpp/GlobalData.hpp
-1
View File
@@ -6,7 +6,6 @@
#ifndef RUNTIME_MM_GLOBAL_DATA_H
#define RUNTIME_MM_GLOBAL_DATA_H
#include "ObjectFactory.hpp"
#include "GlobalsRegistry.hpp"
#include "GC.hpp"
#include "GCScheduler.hpp"
kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp
+82 -72
View File
@@ -17,6 +17,7 @@
#include "Memory.h"
#include "Mutex.hpp"
#include "Porting.h"
#include "TypeLayout.hpp"
#include "Types.h"
#include "Utils.hpp"
@@ -69,9 +70,9 @@ public:
}
// Note: This can only be trivially destructible data, as nobody can invoke its destructor.
void* Data() noexcept {
uint8_t* Data() noexcept {
constexpr size_t kDataOffset = DataOffset();
void* ptr = reinterpret_cast<uint8_t*>(this) + kDataOffset;
auto* ptr = reinterpret_cast<uint8_t*>(this) + kDataOffset;
RuntimeAssert(IsAligned(ptr, DataAlignment), "Data=%p is not aligned to %zu", ptr, DataAlignment);
return ptr;
}
@@ -79,7 +80,7 @@ public:
// It's a caller responsibility to know if the underlying data is `T`.
template <typename T>
T& Data() noexcept {
return *static_cast<T*>(Data());
return *reinterpret_cast<T*>(Data());
}
size_t GetAllocatedSize() noexcept {
@@ -438,46 +439,85 @@ private:
template <typename Traits>
class ObjectFactory : private Pinned {
using ObjectData = typename Traits::ObjectData;
using Allocator = typename Traits::Allocator;
using ObjectData = typename Traits::ObjectData;
struct HeapObjHeader {
[[no_unique_address]] // to account for GCs with empty ObjectData
ObjectData gcData;
alignas(kObjectAlignment) ObjHeader object;
using descriptor = type_layout::Composite<HeapObjHeader, ObjectData, ObjHeader>;
static HeapObjHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); }
static HeapObjHeader& from(ObjHeader* object) noexcept { return *descriptor().template fromField<1>(object); }
ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; }
ObjHeader* object() noexcept { return descriptor().template field<1>(this).second; }
private:
HeapObjHeader() = delete;
~HeapObjHeader() = delete;
};
struct HeapObject {
using descriptor = type_layout::Composite<HeapObject, HeapObjHeader, ObjectBody>;
static descriptor make_descriptor(const TypeInfo* typeInfo) noexcept {
return descriptor{{}, type_layout::descriptor_t<ObjectBody>{typeInfo}};
}
HeapObjHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; }
private:
HeapObject() = delete;
~HeapObject() = delete;
};
// Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible
// with `ObjHeader`: the former can always be casted to the other.
struct HeapArrayHeader {
[[no_unique_address]]
ObjectData gcData;
alignas(kObjectAlignment) ArrayHeader array;
using descriptor = type_layout::Composite<HeapArrayHeader, ObjectData, ArrayHeader>;
static HeapArrayHeader& from(ObjectData& objectData) noexcept { return *descriptor().template fromField<0>(&objectData); }
static HeapArrayHeader& from(ArrayHeader* array) noexcept { return *descriptor().template fromField<1>(array); }
ObjectData& objectData() noexcept { return *descriptor().template field<0>(this).second; }
ArrayHeader* array() noexcept { return descriptor().template field<1>(this).second; }
private:
HeapArrayHeader() = delete;
~HeapArrayHeader() = delete;
};
static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
}
struct HeapArray {
using descriptor = type_layout::Composite<HeapArray, HeapArrayHeader, ArrayBody>;
static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
// at about half of uint64_t max.
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
// Note: array body is aligned, but for size computation it is enough to align the sum.
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
static descriptor make_descriptor(const TypeInfo* typeInfo, uint32_t size) noexcept {
return descriptor{{}, type_layout::descriptor_t<ArrayBody>{typeInfo, size}};
}
HeapArrayHeader& header(descriptor descriptor) noexcept { return *descriptor.template field<0>(this).second; }
private:
HeapArray() = delete;
~HeapArray() = delete;
};
// Only used for already allocated objects. Cannot overflow size_t.
static size_t GetDataSizeForAllocated(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return HeapArray::make_descriptor(typeInfo, object->array()->count_).size();
} else {
return HeapObject::make_descriptor(typeInfo).size();
}
}
struct DataSizeProvider {
static size_t GetDataSize(void* data) noexcept {
ObjHeader* object = &static_cast<HeapObjHeader*>(data)->object;
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return ArrayAllocatedDataSize(typeInfo, object->array()->count_);
} else {
return ObjectAllocatedDataSize(typeInfo);
}
static size_t GetDataSize(uint8_t* data) noexcept {
return GetDataSizeForAllocated(reinterpret_cast<HeapObjHeader*>(data)->object());
}
};
@@ -490,32 +530,23 @@ public:
static NodeRef From(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Must be a heap object");
auto& heapObject = ownerOf(HeapObjHeader, object, *object);
return NodeRef(Storage::Node::FromData(&heapObject));
return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapObjHeader::from(object))));
}
static NodeRef From(ArrayHeader* array) noexcept {
RuntimeAssert(array->obj()->heap(), "Must be a heap object");
auto& heapArray = ownerOf(HeapArrayHeader, array, *array);
return NodeRef(Storage::Node::FromData(&heapArray));
return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapArrayHeader::from(array))));
}
static NodeRef From(ObjectData& objectData) noexcept {
auto& heapObject = ownerOf(HeapObjHeader, gcData, objectData);
return NodeRef(Storage::Node::FromData(&heapObject));
return NodeRef(Storage::Node::FromData(reinterpret_cast<uint8_t*>(&HeapObjHeader::from(objectData))));
}
NodeRef* operator->() noexcept { return this; }
ObjectData& ObjectData() noexcept {
// `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can
// be always casted to the other.
return static_cast<HeapObjHeader*>(node_.Data())->gcData;
}
ObjectData& ObjectData() noexcept { return reinterpret_cast<HeapObjHeader*>(node_.Data())->objectData(); }
ObjHeader* GetObjHeader() noexcept {
return &static_cast<HeapObjHeader*>(node_.Data())->object;
}
ObjHeader* GetObjHeader() noexcept { return reinterpret_cast<HeapObjHeader*>(node_.Data())->object(); }
bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; }
@@ -550,37 +581,21 @@ public:
ThreadQueue(ObjectFactory& owner, Allocator allocator) noexcept : producer_(owner.storage_, std::move(allocator)) {}
static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
// Only used for already allocated objects. Cannot overflow size_t
auto allocSize = ObjectAllocatedDataSize(typeInfo);
return Storage::Node::GetSizeForDataSize(allocSize);
}
ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept {
RuntimeAssert(!typeInfo->IsArray(), "Must not be an array");
size_t allocSize = ObjectAllocatedDataSize(typeInfo);
auto& node = producer_.Insert(allocSize);
auto* heapObject = new (node.Data()) HeapObjHeader();
auto* object = &heapObject->object;
auto descriptor = HeapObject::make_descriptor(typeInfo);
auto& heapObject = *descriptor.construct(producer_.Insert(descriptor.size()).Data());
ObjHeader* object = heapObject.header(descriptor).object();
object->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
// TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
return object;
}
static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
RuntimeAssert(typeInfo->IsArray(), "Must be an array");
// Only used for already allocated arrays. Cannot overflow size_t
auto allocSize = ArrayAllocatedDataSize(typeInfo, count);
return Storage::Node::GetSizeForDataSize(allocSize);
}
ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept {
RuntimeAssert(typeInfo->IsArray(), "Must be an array");
auto allocSize = ArrayAllocatedDataSize(typeInfo, count);
auto& node = producer_.Insert(allocSize);
auto* heapArray = new (node.Data()) HeapArrayHeader();
auto* array = &heapArray->array;
auto descriptor = HeapArray::make_descriptor(typeInfo, count);
auto& heapArray = *descriptor.construct(producer_.Insert(descriptor.size()).Data());
ArrayHeader* array = heapArray.header(descriptor).array();
array->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
array->count_ = count;
// TODO: Consider supporting TF_IMMUTABLE: mark instance as frozen upon creation.
@@ -713,13 +728,8 @@ public:
void ClearForTests() { storage_.ClearForTests(); }
static size_t GetAllocatedHeapSize(ObjHeader* object) noexcept {
RuntimeAssert(object->heap(), "Object must be a heap object");
const auto* typeInfo = object->type_info();
if (typeInfo->IsArray()) {
return ThreadQueue::ArrayAllocatedSize(typeInfo, object->array()->count_);
} else {
return ThreadQueue::ObjectAllocatedSize(typeInfo);
}
auto dataSize = GetDataSizeForAllocated(object);
return Storage::Node::GetSizeForDataSize(dataSize);
}
private:
kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp
+3 -3
View File
@@ -62,7 +62,7 @@ template <typename T, size_t DataAlignment>
std_support::vector<T> Collect(ObjectFactoryStorage<DataAlignment>& storage) {
std_support::vector<T> result;
for (auto& node : storage.LockForIter()) {
result.push_back(*static_cast<T*>(node.Data()));
result.push_back(*reinterpret_cast<T*>(node.Data()));
}
return result;
}
@@ -71,7 +71,7 @@ template <typename T, size_t DataAlignment>
std_support::vector<T> Collect(Consumer<ObjectFactoryStorage<DataAlignment>>& consumer) {
std_support::vector<T> result;
for (auto& node : consumer) {
result.push_back(*static_cast<T*>(node.Data()));
result.push_back(*reinterpret_cast<T*>(node.Data()));
}
return result;
}
@@ -710,7 +710,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) {
}
for (auto& node : iter) {
int element = *static_cast<int*>(node.Data());
int element = *reinterpret_cast<int*>(node.Data());
actualBefore.push_back(element);
}
}
kotlin-native/runtime/src/mm/cpp/ThreadData.hpp
-1
View File
@@ -11,7 +11,6 @@
#include "GlobalData.hpp"
#include "GlobalsRegistry.hpp"
#include "GC.hpp"
#include "ObjectFactory.hpp"
#include "ShadowStack.hpp"
#include "SpecialRefRegistry.hpp"
#include "ThreadLocalStorage.hpp"
kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp
+1
View File
@@ -7,6 +7,7 @@
#include "gtest/gtest.h"
#include "Porting.h"
#include "ScopedThread.hpp"
#include "ThreadData.hpp"
kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp
+1
View File
@@ -12,6 +12,7 @@
#include "CallsChecker.hpp"
#include "Logging.hpp"
#include "Porting.h"
#include "SafePoint.hpp"
#include "StackTrace.hpp"