From 98251f42f240ed68b53fdc688598528b347688ad Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Tue, 4 Oct 2022 09:51:08 +0000 Subject: [PATCH] [K/N] Specialize marking call for objects ^KT-54163 Merge-request: KT-MR-7232 Merged-by: Alexander Shabalin --- .../kotlin/backend/konan/llvm/ContextUtils.kt | 5 ++ .../backend/konan/llvm/RTTIGenerator.kt | 45 +++++++--- .../cpp/CompilerCInterface.cpp | 5 ++ .../src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp | 36 ++------ .../src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp | 29 +++++++ .../runtime/src/gc/cms/cpp/GCImpl.cpp | 16 ++++ .../runtime/src/gc/common/cpp/GC.hpp | 4 + .../src/gc/common/cpp/MarkAndSweepUtils.hpp | 86 ++++++++++++------- .../common/cpp/MarkAndSweepUtilsMarkTest.cpp | 8 ++ .../runtime/src/gc/noop/cpp/GCImpl.cpp | 9 ++ .../runtime/src/gc/stms/cpp/GCImpl.cpp | 16 ++++ .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 30 +------ .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 28 ++++++ .../runtime/src/legacymm/cpp/Memory.cpp | 16 ++++ kotlin-native/runtime/src/main/cpp/Memory.h | 5 ++ .../runtime/src/main/cpp/ObjCExport.mm | 2 + .../src/main/cpp/ObjectTestSupport.hpp | 23 +++-- kotlin-native/runtime/src/main/cpp/TypeInfo.h | 12 +-- kotlin-native/runtime/src/mm/cpp/Memory.cpp | 17 ++++ 19 files changed, 280 insertions(+), 112 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 903c28b90f8..ac646aaa7ef 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -522,6 +522,11 @@ internal class Llvm(val context: Context, val module: LLVMModuleRef) : RuntimeAw val Kotlin_mm_safePointFunctionPrologue by lazyRtFunction val Kotlin_mm_safePointWhileLoopBody by lazyRtFunction + val Kotlin_processObjectInMark by lazyRtFunction + val Kotlin_processArrayInMark by lazyRtFunction + val Kotlin_processFieldInMark by lazyRtFunction + val Kotlin_processEmptyObjectInMark by lazyRtFunction + val tlsMode by lazy { when (target) { KonanTarget.WASM32, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt index 0368707bddb..6dff6f01e8b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt @@ -98,7 +98,8 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { flags: Int, classId: Int, writableTypeInfo: ConstPointer?, - associatedObjects: ConstPointer?) : + associatedObjects: ConstPointer?, + processObjectInMark: ConstPointer?) : Struct( runtime.typeInfoType, @@ -134,7 +135,9 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { *listOfNotNull(writableTypeInfo).toTypedArray(), - associatedObjects + associatedObjects, + + processObjectInMark, ) private fun kotlinStringLiteral(string: String?): ConstPointer = if (string == null) { @@ -255,7 +258,11 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { flagsFromClass(irClass) or reflectionInfo.reflectionFlags, context.getLayoutBuilder(irClass).classId, llvmDeclarations.writableTypeInfoGlobal?.pointer, - associatedObjects = genAssociatedObjects(irClass) + associatedObjects = genAssociatedObjects(irClass), + processObjectInMark = when { + irClass.symbol == context.ir.symbols.array -> constPointer(context.llvm.Kotlin_processArrayInMark.llvmValue) + else -> genProcessObjectInMark(bodyType) + } ) val typeInfoGlobalValue = if (!irClass.typeInfoHasVtableAttached) { @@ -271,14 +278,17 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { exportTypeInfoIfRequired(irClass, irClass.llvmTypeInfoPtr) } - private fun getObjOffsets(bodyType: LLVMTypeRef): List = + private fun getIndicesOfObjectFields(bodyType: LLVMTypeRef) : List = getStructElements(bodyType).mapIndexedNotNull { index, type -> - if (isObjectType(type)) { - LLVMOffsetOfElement(llvmTargetData, bodyType, index) - } else { - null + index.takeIf { + isObjectType(type) } - }.map { Int32(it.toInt()) } + } + + private fun getObjOffsets(bodyType: LLVMTypeRef): List = + getIndicesOfObjectFields(bodyType).map { index -> + Int32(LLVMOffsetOfElement(llvmTargetData, bodyType, index).toInt()) + } fun vtable(irClass: IrClass): ConstArray { // TODO: compile-time resolution limits binary compatibility. @@ -473,6 +483,20 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { ) } + private fun genProcessObjectInMark(classType: LLVMTypeRef) : ConstPointer { + val indicesOfObjectFields = getIndicesOfObjectFields(classType) + return when { + indicesOfObjectFields.isEmpty() -> { + // TODO: Try to generate it here instead of importing from the runtime. + constPointer(context.llvm.Kotlin_processEmptyObjectInMark.llvmValue) + } + else -> { + // TODO: specialize for "small" objects + constPointer(context.llvm.Kotlin_processObjectInMark.llvmValue) + } + } + } + // TODO: extract more code common with generate(). fun generateSyntheticInterfaceImpl( irClass: IrClass, @@ -545,7 +569,8 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { flags = flagsFromClass(irClass) or (if (immutable) TF_IMMUTABLE else 0), classId = typeHierarchyInfo.classIdLo, writableTypeInfo = writableTypeInfo, - associatedObjects = null + associatedObjects = null, + processObjectInMark = genProcessObjectInMark(bodyType), ), vtable) typeInfoWithVtableGlobal.setInitializer(typeInfoWithVtable) diff --git a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp index bd0cce35a3e..0da6af51b3f 100644 --- a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp +++ b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp @@ -76,6 +76,11 @@ touchFunction(Kotlin_mm_switchThreadStateRunnable) touchFunction(Kotlin_mm_safePointFunctionPrologue) touchFunction(Kotlin_mm_safePointWhileLoopBody) +touchFunction(Kotlin_processObjectInMark) +touchFunction(Kotlin_processArrayInMark) +touchFunction(Kotlin_processFieldInMark) +touchFunction(Kotlin_processEmptyObjectInMark) + #ifdef __cplusplus } // extern "C" #endif diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 53d1cc44dfd..70df78d35f7 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -28,31 +28,6 @@ namespace { [[clang::no_destroy]] std::condition_variable markingCondVar; [[clang::no_destroy]] std::atomic markingRequested = false; -struct MarkTraits { - using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue; - - static bool isEmpty(const MarkQueue& queue) noexcept { - return queue.empty(); - } - - static void clear(MarkQueue& queue) noexcept { - queue.clear(); - } - - static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto& top = queue.front(); - queue.pop_front(); - auto node = mm::ObjectFactory::NodeRef::From(top); - return node->GetObjHeader(); - } - - static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - if (!objectData.atomicSetToBlack()) return; - queue.push_front(objectData); - } -}; - struct SweepTraits { using ObjectFactory = mm::ObjectFactory; using ExtraObjectsFactory = mm::ExtraObjectDataFactory; @@ -106,8 +81,8 @@ NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendFo lock.unlock(); RuntimeLogDebug({kTagGC}, "Parallel marking in thread %d", konan::currentThreadId()); MarkQueue markQueue; - gc::collectRootSetForThread(markQueue, threadData_); - MarkStats stats = gc::Mark(markQueue); + gc::collectRootSetForThread(markQueue, threadData_); + MarkStats stats = gc::Mark(markQueue); gc_.MergeMarkStats(stats); } @@ -185,7 +160,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // Can be unsafe, because we've stopped the world. auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); - auto markStats = gc::Mark(markQueue_); + auto markStats = gc::Mark(markQueue_); MergeMarkStats(markStats); RuntimeLogDebug({kTagGC}, "Waiting for marking in threads"); @@ -272,7 +247,8 @@ void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept { NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking() noexcept { std::unique_lock lock(markingMutex); markingRequested = false; - gc::collectRootSet(markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); }); + gc::collectRootSet( + markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); }); RuntimeLogDebug({kTagGC}, "Requesting marking in threads"); markingCondVar.notify_all(); } @@ -280,4 +256,4 @@ NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartM void gc::ConcurrentMarkAndSweep::MergeMarkStats(gc::MarkStats stats) noexcept { std::unique_lock lock(markingMutex); lastGCMarkStats_.merge(stats); -} +} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 24aaaa577da..0ef4d0b289f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -136,5 +136,34 @@ private: MarkingBehavior markingBehavior_; }; +namespace internal { +struct MarkTraits { + using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue; + + static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); } + + static void clear(MarkQueue& queue) noexcept { queue.clear(); } + + static ObjHeader* dequeue(MarkQueue& queue) noexcept { + auto& top = queue.front(); + queue.pop_front(); + auto node = mm::ObjectFactory::NodeRef::From(top); + return node->GetObjHeader(); + } + + static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); + if (!objectData.atomicSetToBlack()) return; + queue.push_front(objectData); + } + + static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { + auto process = object->type_info()->processObjectInMark; + RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object); + process(static_cast(&markQueue), object); + } +}; +} // namespace internal + } // namespace gc } // namespace kotlin diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index f7b57dbe435..f593272e76d 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -6,6 +6,7 @@ #include "GCImpl.hpp" #include "GC.hpp" +#include "MarkAndSweepUtils.hpp" #include "ThreadSuspension.hpp" #include "std_support/Memory.hpp" @@ -93,3 +94,18 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept { bool gc::GC::FinalizersThreadIsRunning() noexcept { return impl_->gc().FinalizersThreadIsRunning(); } + +// static +ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept { + gc::internal::processObjectInMark(state, object); +} + +// static +ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept { + gc::internal::processArrayInMark(state, array); +} + +// static +ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { + gc::internal::processFieldInMark(state, field); +} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 5631df4a965..23df367fb0f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -66,6 +66,10 @@ public: void StopFinalizerThreadIfRunning() noexcept; bool FinalizersThreadIsRunning() noexcept; + static void processObjectInMark(void* state, ObjHeader* object) noexcept; + static void processArrayInMark(void* state, ArrayHeader* array) noexcept; + static void processFieldInMark(void* state, ObjHeader* field) noexcept; + private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index be69191a189..7eef2a1bbcb 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -22,6 +22,56 @@ namespace kotlin { namespace gc { +namespace internal { + +template +void processFieldInMark(void* state, ObjHeader* field) noexcept { + auto& markQueue = *static_cast(state); + if (field->heap()) { + Traits::enqueue(markQueue, field); + } +} + +template +void processObjectInMark(void* state, ObjHeader* object) noexcept { + auto* typeInfo = object->type_info(); + RuntimeAssert(typeInfo != theArrayTypeInfo, "Must not be an array of objects"); + for (int i = 0; i < typeInfo->objOffsetsCount_; ++i) { + auto* field = *reinterpret_cast(reinterpret_cast(object) + typeInfo->objOffsets_[i]); + if (!field) continue; + processFieldInMark(state, field); + } +} + +template +void processArrayInMark(void* state, ArrayHeader* array) noexcept { + RuntimeAssert(array->type_info() == theArrayTypeInfo, "Must be an array of objects"); + auto* begin = ArrayAddressOfElementAt(array, 0); + auto* end = ArrayAddressOfElementAt(array, array->count_); + for (auto* it = begin; it != end; ++it) { + auto* field = *it; + if (!field) continue; + processFieldInMark(state, field); + } +} + +template +bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexcept { + if (isNullOrMarker(object)) + return false; + + if (object->heap()) { + Traits::enqueue(markQueue, object); + } else { + // Each permanent and stack object has own entry in the root set, so it's okay to only process objects in heap. + Traits::processInMark(markQueue, object); + RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object); + } + return true; +} + +} // namespace internal + struct MarkStats { // How many objects are alive. size_t aliveHeapSet = 0; @@ -51,11 +101,7 @@ MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept { stats.aliveHeapSet++; stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top); - traverseReferredObjects(top, [&](ObjHeader* field) noexcept { - if (field->heap()) { - Traits::enqueue(markQueue, field); - } - }); + Traits::processInMark(markQueue, top); if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) { if (auto weakCounter = extraObjectData->GetWeakReferenceCounter()) { @@ -124,20 +170,9 @@ void collectRootSetForThread(typename Traits::MarkQueue& markQueue, mm::ThreadDa thread.gc().OnStoppedForGC(); size_t stack = 0; size_t tls = 0; + // TODO: Remove useless mm::ThreadRootSet abstraction. for (auto value : mm::ThreadRootSet(thread)) { - auto* object = value.object; - if (!isNullOrMarker(object)) { - if (object->heap()) { - Traits::enqueue(markQueue, object); - } else { - traverseReferredObjects(object, [&](ObjHeader* field) noexcept { - // Each permanent and stack object has own entry in the root set. - if (field->heap()) { - Traits::enqueue(markQueue, field); - } - }); - RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object); - } + if (internal::collectRoot(markQueue, value.object)) { switch (value.source) { case mm::ThreadRootSet::Source::kStack: ++stack; @@ -156,20 +191,9 @@ void collectRootSetGlobals(typename Traits::MarkQueue& markQueue) noexcept { mm::StableRefRegistry::Instance().ProcessDeletions(); size_t global = 0; size_t stableRef = 0; + // TODO: Remove useless mm::GlobalRootSet abstraction. for (auto value : mm::GlobalRootSet()) { - auto* object = value.object; - if (!isNullOrMarker(object)) { - if (object->heap()) { - Traits::enqueue(markQueue, object); - } else { - traverseReferredObjects(object, [&](ObjHeader* field) noexcept { - // Each permanent and stack object has own entry in the root set. - if (field->heap()) { - Traits::enqueue(markQueue, field); - } - }); - RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object); - } + if (internal::collectRoot(markQueue, value.object)) { switch (value.source) { case mm::GlobalRootSet::Source::kGlobal: ++global; diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index d955e6dae9a..ad477a8f509 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -145,6 +145,14 @@ public: } } + static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { + if (object->type_info() == theArrayTypeInfo) { + gc::internal::processArrayInMark(static_cast(&markQueue), object->array()); + } else { + gc::internal::processObjectInMark(static_cast(&markQueue), object); + } + } + private: static ScopedMarkTraits* instance_; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 181ba39354a..bf724db035d 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -76,3 +76,12 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept {} bool gc::GC::FinalizersThreadIsRunning() noexcept { return false; } + +// static +ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept {} + +// static +ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept {} + +// static +ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index c80c1cb1ab9..338534bcae5 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -6,6 +6,7 @@ #include "GCImpl.hpp" #include "GC.hpp" +#include "MarkAndSweepUtils.hpp" #include "std_support/Memory.hpp" using namespace kotlin; @@ -88,3 +89,18 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept {} bool gc::GC::FinalizersThreadIsRunning() noexcept { return false; } + +// static +ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept { + gc::internal::processObjectInMark(state, object); +} + +// static +ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept { + gc::internal::processArrayInMark(state, array); +} + +// static +ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { + gc::internal::processFieldInMark(state, field); +} \ No newline at end of file diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 165efed3a31..6687ed84f4d 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -22,32 +22,6 @@ using namespace kotlin; namespace { -struct MarkTraits { - using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue; - - static bool isEmpty(const MarkQueue& queue) noexcept { - return queue.empty(); - } - - static void clear(MarkQueue& queue) noexcept { - queue.clear(); - } - - static ObjHeader* dequeue(MarkQueue& queue) noexcept { - auto& top = queue.front(); - queue.pop_front(); - auto node = mm::ObjectFactory::NodeRef::From(top); - return node->GetObjHeader(); - } - - static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { - auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); - if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return; - objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); - queue.push_front(objectData); - } -}; - struct SweepTraits { using ObjectFactory = mm::ObjectFactory; using ExtraObjectsFactory = mm::ExtraObjectDataFactory; @@ -154,7 +128,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogInfo( {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); - gc::collectRootSet(markQueue_, [] (mm::ThreadData&) { return true; }); + gc::collectRootSet(markQueue_, [](mm::ThreadData&) { return true; }); auto timeRootSetUs = konan::getTimeMicros(); // Can be unsafe, because we've stopped the world. auto objectsCountBefore = objectFactory_.GetSizeUnsafe(); @@ -162,7 +136,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogInfo( {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(), timeRootSetUs - timeSuspendUs); - auto markStats = gc::Mark(markQueue_); + auto markStats = gc::Mark(markQueue_); auto timeMarkUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs); scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index ceef89f5745..afe98620e6d 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -110,6 +110,34 @@ private: namespace internal { +struct MarkTraits { + using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue; + + static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); } + + static void clear(MarkQueue& queue) noexcept { queue.clear(); } + + static ObjHeader* dequeue(MarkQueue& queue) noexcept { + auto& top = queue.front(); + queue.pop_front(); + auto node = mm::ObjectFactory::NodeRef::From(top); + return node->GetObjHeader(); + } + + static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept { + auto& objectData = mm::ObjectFactory::NodeRef::From(object).ObjectData(); + if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return; + objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack); + queue.push_front(objectData); + } + + static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { + auto process = object->type_info()->processObjectInMark; + RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object); + process(static_cast(&markQueue), object); + } +}; + SameThreadMarkAndSweep::SafepointFlag loadSafepointFlag() noexcept; } // namespace internal diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 9a56b5c1908..ef2f7569d5a 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3796,6 +3796,22 @@ CODEGEN_INLINE_POLICY RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() { // no-op, used by the new MM only. } +RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processObjectInMark(void* state, ObjHeader* object) { + // no-op, used by the new MM only. +} + +RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processArrayInMark(void* state, ObjHeader* object) { + // no-op, used by the new MM only. +} + +RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processFieldInMark(void* state, ObjHeader* field) { + // no-op, used by the new MM only. +} + +RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object) { + // no-op, used by the new MM only. +} + } // extern "C" #if !KONAN_NO_EXCEPTIONS diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 9bb053bfedd..c656695016b 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -561,4 +561,9 @@ bool FinalizersThreadIsRunning() noexcept; } // namespace kotlin +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object); +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* state, ObjHeader* object); +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field); +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object); + #endif // RUNTIME_MEMORY_H diff --git a/kotlin-native/runtime/src/main/cpp/ObjCExport.mm b/kotlin-native/runtime/src/main/cpp/ObjCExport.mm index 8b1dcdce8c4..6cc7884aa80 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjCExport.mm +++ b/kotlin-native/runtime/src/main/cpp/ObjCExport.mm @@ -661,10 +661,12 @@ static const TypeInfo* createTypeInfo( if ((superType->flags_ & TF_IMMUTABLE) != 0) { result->flags_ |= TF_IMMUTABLE; } + result->processObjectInMark = superType->processObjectInMark; } else { result->instanceSize_ = fieldsInfo->instanceSize_; result->objOffsets_ = fieldsInfo->objOffsets_; result->objOffsetsCount_ = fieldsInfo->objOffsetsCount_; + result->processObjectInMark = fieldsInfo->processObjectInMark; } result->classId_ = superType->classId_; diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp index a9d23405e19..d9e7d0ab16b 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp @@ -28,8 +28,10 @@ private: int32_t instanceSize_ = 0; std_support::vector objOffsets_; + int32_t objOffsetsCount_ = 0; int32_t flags_ = 0; const TypeInfo* superType_ = nullptr; + void (*processObjectInMark_)(void*, ObjHeader*) = nullptr; }; public: @@ -57,7 +59,16 @@ public: template class ArrayBuilder : public Builder { public: - ArrayBuilder() noexcept { instanceSize_ = -static_cast(sizeof(Payload)); } + ArrayBuilder() noexcept { + instanceSize_ = -static_cast(sizeof(Payload)); + if constexpr (std::is_same_v) { + // Following RTTIGenerator.kt + objOffsetsCount_ = 1; + processObjectInMark_ = Kotlin_processArrayInMark; + } else { + processObjectInMark_ = Kotlin_processEmptyObjectInMark; + } + } ArrayBuilder&& addFlag(Konan_TypeFlags flag) noexcept { flags_ |= flag; @@ -75,12 +86,8 @@ public: typeInfo_.instanceSize_ = builder.instanceSize_; objOffsets_ = std::move(builder.objOffsets_); typeInfo_.objOffsets_ = objOffsets_.data(); - if (&typeInfo_ == theArrayTypeInfo) { - // Following RTTIGenerator.kt - typeInfo_.objOffsetsCount_ = 1; - } else { - typeInfo_.objOffsetsCount_ = objOffsets_.size(); - } + typeInfo_.objOffsetsCount_ = builder.objOffsetsCount_; + typeInfo_.processObjectInMark = builder.processObjectInMark_; typeInfo_.flags_ = builder.flags_; typeInfo_.superType_ = builder.superType_; } @@ -174,6 +181,8 @@ TypeInfoHolder::ObjectBuilder::ObjectBuilder() noexcept { auto& actualField = payload.*field; objOffsets_.push_back(reinterpret_cast(&actualField) - reinterpret_cast(object.header())); } + objOffsetsCount_ = objOffsets_.size(); + processObjectInMark_ = Kotlin_processObjectInMark; } namespace internal { diff --git a/kotlin-native/runtime/src/main/cpp/TypeInfo.h b/kotlin-native/runtime/src/main/cpp/TypeInfo.h index df973debe7a..fc35e96eebd 100644 --- a/kotlin-native/runtime/src/main/cpp/TypeInfo.h +++ b/kotlin-native/runtime/src/main/cpp/TypeInfo.h @@ -132,16 +132,16 @@ struct TypeInfo { // Null-terminated array. const AssociatedObjectTableRecord* associatedObjects; + // Invoked on an object during mark phase. + // TODO: Consider providing a generic traverse method instead. + void (*processObjectInMark)(void* state, ObjHeader* object); + // vtable starts just after declared contents of the TypeInfo: // void* const vtable_[]; #ifdef __cplusplus - inline VTableElement const* vtable() const { - return reinterpret_cast(this + 1); - } + inline VTableElement const* vtable() const { return reinterpret_cast(this + 1); } - inline VTableElement* vtable() { - return reinterpret_cast(this + 1); - } + inline VTableElement* vtable() { return reinterpret_cast(this + 1); } inline bool IsArray() const { return instanceSize_ < 0; } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 3e52eb3f6a2..d43475b74a5 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -632,3 +632,20 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept { bool kotlin::FinalizersThreadIsRunning() noexcept { return mm::GlobalData::Instance().gc().FinalizersThreadIsRunning(); } + +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object) { + gc::GC::processObjectInMark(state, object); +} + +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* state, ObjHeader* object) { + gc::GC::processArrayInMark(state, object->array()); +} + +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field) { + gc::GC::processFieldInMark(state, field); +} + +RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object) { + // Empty object. Nothing to do. + // TODO: Try to generate it in the code generator. +} \ No newline at end of file