[K/N] Specialize marking call for objects ^KT-54163

Merge-request: KT-MR-7232
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp

@@ -76,6 +76,11 @@ touchFunction(Kotlin_mm_switchThreadStateRunnable)
 touchFunction(Kotlin_mm_safePointFunctionPrologue)
 touchFunction(Kotlin_mm_safePointWhileLoopBody)
 
+touchFunction(Kotlin_processObjectInMark)
+touchFunction(Kotlin_processArrayInMark)
+touchFunction(Kotlin_processFieldInMark)
+touchFunction(Kotlin_processEmptyObjectInMark)
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -28,31 +28,6 @@ namespace {
     [[clang::no_destroy]] std::condition_variable markingCondVar;
     [[clang::no_destroy]] std::atomic<bool> markingRequested = false;
 
-struct MarkTraits {
-    using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue;
-
-    static bool isEmpty(const MarkQueue& queue) noexcept {
-        return queue.empty();
-    }
-
-    static void clear(MarkQueue& queue) noexcept {
-        queue.clear();
-    }
-
-    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
-        auto& top = queue.front();
-        queue.pop_front();
-        auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(top);
-        return node->GetObjHeader();
-    }
-
-    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
-        if (!objectData.atomicSetToBlack()) return;
-        queue.push_front(objectData);
-    }
-};
-
 struct SweepTraits {
     using ObjectFactory = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>;
     using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
@@ -106,8 +81,8 @@ NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendFo
     lock.unlock();
     RuntimeLogDebug({kTagGC}, "Parallel marking in thread %d", konan::currentThreadId());
     MarkQueue markQueue;
-    gc::collectRootSetForThread<MarkTraits>(markQueue, threadData_);
-    MarkStats stats = gc::Mark<MarkTraits>(markQueue);
+    gc::collectRootSetForThread<internal::MarkTraits>(markQueue, threadData_);
+    MarkStats stats = gc::Mark<internal::MarkTraits>(markQueue);
     gc_.MergeMarkStats(stats);
 }
 
@@ -185,7 +160,7 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     // Can be unsafe, because we've stopped the world.
     auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
 
-    auto markStats = gc::Mark<MarkTraits>(markQueue_);
+    auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
     MergeMarkStats(markStats);
 
     RuntimeLogDebug({kTagGC}, "Waiting for marking in threads");
@@ -272,7 +247,8 @@ void gc::ConcurrentMarkAndSweep::WaitForThreadsReadyToMark() noexcept {
 NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartMarking() noexcept {
         std::unique_lock lock(markingMutex);
         markingRequested = false;
-        gc::collectRootSet<MarkTraits>(markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); });
+        gc::collectRootSet<internal::MarkTraits>(
+                markQueue_, [](mm::ThreadData& thread) { return !thread.gc().impl().gc().marking_.load(); });
         RuntimeLogDebug({kTagGC}, "Requesting marking in threads");
         markingCondVar.notify_all();
 }
@@ -280,4 +256,4 @@ NO_EXTERNAL_CALLS_CHECK void gc::ConcurrentMarkAndSweep::CollectRootSetAndStartM
 void gc::ConcurrentMarkAndSweep::MergeMarkStats(gc::MarkStats stats) noexcept {
     std::unique_lock lock(markingMutex);
     lastGCMarkStats_.merge(stats);
-}
+}

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -136,5 +136,34 @@ private:
     MarkingBehavior markingBehavior_;
 };
 
+namespace internal {
+struct MarkTraits {
+    using MarkQueue = gc::ConcurrentMarkAndSweep::MarkQueue;
+
+    static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
+
+    static void clear(MarkQueue& queue) noexcept { queue.clear(); }
+
+    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+        auto& top = queue.front();
+        queue.pop_front();
+        auto node = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(top);
+        return node->GetObjHeader();
+    }
+
+    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+        auto& objectData = mm::ObjectFactory<gc::ConcurrentMarkAndSweep>::NodeRef::From(object).ObjectData();
+        if (!objectData.atomicSetToBlack()) return;
+        queue.push_front(objectData);
+    }
+
+    static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
+        auto process = object->type_info()->processObjectInMark;
+        RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object);
+        process(static_cast<void*>(&markQueue), object);
+    }
+};
+} // namespace internal
+
 } // namespace gc
 } // namespace kotlin

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -6,6 +6,7 @@
 #include "GCImpl.hpp"
 
 #include "GC.hpp"
+#include "MarkAndSweepUtils.hpp"
 #include "ThreadSuspension.hpp"
 #include "std_support/Memory.hpp"
 
@@ -93,3 +94,18 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept {
 bool gc::GC::FinalizersThreadIsRunning() noexcept {
     return impl_->gc().FinalizersThreadIsRunning();
 }
+
+// static
+ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept {
+    gc::internal::processObjectInMark<gc::internal::MarkTraits>(state, object);
+}
+
+// static
+ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept {
+    gc::internal::processArrayInMark<gc::internal::MarkTraits>(state, array);
+}
+
+// static
+ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {
+    gc::internal::processFieldInMark<gc::internal::MarkTraits>(state, field);
+}

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -66,6 +66,10 @@ public:
     void StopFinalizerThreadIfRunning() noexcept;
     bool FinalizersThreadIsRunning() noexcept;
 
+    static void processObjectInMark(void* state, ObjHeader* object) noexcept;
+    static void processArrayInMark(void* state, ArrayHeader* array) noexcept;
+    static void processFieldInMark(void* state, ObjHeader* field) noexcept;
+
 private:
     std_support::unique_ptr<Impl> impl_;
 };

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -22,6 +22,56 @@
 namespace kotlin {
 namespace gc {
 
+namespace internal {
+
+template <typename Traits>
+void processFieldInMark(void* state, ObjHeader* field) noexcept {
+    auto& markQueue = *static_cast<typename Traits::MarkQueue*>(state);
+    if (field->heap()) {
+        Traits::enqueue(markQueue, field);
+    }
+}
+
+template <typename Traits>
+void processObjectInMark(void* state, ObjHeader* object) noexcept {
+    auto* typeInfo = object->type_info();
+    RuntimeAssert(typeInfo != theArrayTypeInfo, "Must not be an array of objects");
+    for (int i = 0; i < typeInfo->objOffsetsCount_; ++i) {
+        auto* field = *reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(object) + typeInfo->objOffsets_[i]);
+        if (!field) continue;
+        processFieldInMark<Traits>(state, field);
+    }
+}
+
+template <typename Traits>
+void processArrayInMark(void* state, ArrayHeader* array) noexcept {
+    RuntimeAssert(array->type_info() == theArrayTypeInfo, "Must be an array of objects");
+    auto* begin = ArrayAddressOfElementAt(array, 0);
+    auto* end = ArrayAddressOfElementAt(array, array->count_);
+    for (auto* it = begin; it != end; ++it) {
+        auto* field = *it;
+        if (!field) continue;
+        processFieldInMark<Traits>(state, field);
+    }
+}
+
+template <typename Traits>
+bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexcept {
+    if (isNullOrMarker(object))
+        return false;
+
+    if (object->heap()) {
+        Traits::enqueue(markQueue, object);
+    } else {
+        // Each permanent and stack object has own entry in the root set, so it's okay to only process objects in heap.
+        Traits::processInMark(markQueue, object);
+        RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object);
+    }
+    return true;
+}
+
+} // namespace internal
+
 struct MarkStats {
     // How many objects are alive.
     size_t aliveHeapSet = 0;
@@ -51,11 +101,7 @@ MarkStats Mark(typename Traits::MarkQueue& markQueue) noexcept {
         stats.aliveHeapSet++;
         stats.aliveHeapSetBytes += mm::GetAllocatedHeapSize(top);
 
-        traverseReferredObjects(top, [&](ObjHeader* field) noexcept {
-            if (field->heap()) {
-                Traits::enqueue(markQueue, field);
-            }
-        });
+        Traits::processInMark(markQueue, top);
 
         if (auto* extraObjectData = mm::ExtraObjectData::Get(top)) {
             if (auto weakCounter = extraObjectData->GetWeakReferenceCounter()) {
@@ -124,20 +170,9 @@ void collectRootSetForThread(typename Traits::MarkQueue& markQueue, mm::ThreadDa
     thread.gc().OnStoppedForGC();
     size_t stack = 0;
     size_t tls = 0;
+    // TODO: Remove useless mm::ThreadRootSet abstraction.
     for (auto value : mm::ThreadRootSet(thread)) {
-        auto* object = value.object;
-        if (!isNullOrMarker(object)) {
-            if (object->heap()) {
-                Traits::enqueue(markQueue, object);
-            } else {
-                traverseReferredObjects(object, [&](ObjHeader* field) noexcept {
-                    // Each permanent and stack object has own entry in the root set.
-                    if (field->heap()) {
-                        Traits::enqueue(markQueue, field);
-                    }
-                });
-                RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object);
-            }
+        if (internal::collectRoot<Traits>(markQueue, value.object)) {
             switch (value.source) {
                 case mm::ThreadRootSet::Source::kStack:
                     ++stack;
@@ -156,20 +191,9 @@ void collectRootSetGlobals(typename Traits::MarkQueue& markQueue) noexcept {
     mm::StableRefRegistry::Instance().ProcessDeletions();
     size_t global = 0;
     size_t stableRef = 0;
+    // TODO: Remove useless mm::GlobalRootSet abstraction.
     for (auto value : mm::GlobalRootSet()) {
-        auto* object = value.object;
-        if (!isNullOrMarker(object)) {
-            if (object->heap()) {
-                Traits::enqueue(markQueue, object);
-            } else {
-                traverseReferredObjects(object, [&](ObjHeader* field) noexcept {
-                    // Each permanent and stack object has own entry in the root set.
-                    if (field->heap()) {
-                        Traits::enqueue(markQueue, field);
-                    }
-                });
-                RuntimeAssert(!object->has_meta_object(), "Non-heap object %p may not have an extra object data", object);
-            }
+        if (internal::collectRoot<Traits>(markQueue, value.object)) {
             switch (value.source) {
                 case mm::GlobalRootSet::Source::kGlobal:
                     ++global;

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -145,6 +145,14 @@ public:
         }
     }
 
+    static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
+        if (object->type_info() == theArrayTypeInfo) {
+            gc::internal::processArrayInMark<ScopedMarkTraits>(static_cast<void*>(&markQueue), object->array());
+        } else {
+            gc::internal::processObjectInMark<ScopedMarkTraits>(static_cast<void*>(&markQueue), object);
+        }
+    }
+
 private:
     static ScopedMarkTraits* instance_;
 

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -76,3 +76,12 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept {}
 bool gc::GC::FinalizersThreadIsRunning() noexcept {
     return false;
 }
+
+// static
+ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept {}
+
+// static
+ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept {}
+
+// static
+ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {}

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -6,6 +6,7 @@
 #include "GCImpl.hpp"
 
 #include "GC.hpp"
+#include "MarkAndSweepUtils.hpp"
 #include "std_support/Memory.hpp"
 
 using namespace kotlin;
@@ -88,3 +89,18 @@ void gc::GC::StopFinalizerThreadIfRunning() noexcept {}
 bool gc::GC::FinalizersThreadIsRunning() noexcept {
     return false;
 }
+
+// static
+ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept {
+    gc::internal::processObjectInMark<gc::internal::MarkTraits>(state, object);
+}
+
+// static
+ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept {
+    gc::internal::processArrayInMark<gc::internal::MarkTraits>(state, array);
+}
+
+// static
+ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {
+    gc::internal::processFieldInMark<gc::internal::MarkTraits>(state, field);
+}

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -22,32 +22,6 @@ using namespace kotlin;
 
 namespace {
 
-struct MarkTraits {
-    using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue;
-
-    static bool isEmpty(const MarkQueue& queue) noexcept {
-        return queue.empty();
-    }
-
-    static void clear(MarkQueue& queue) noexcept {
-        queue.clear();
-    }
-
-    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
-        auto& top = queue.front();
-        queue.pop_front();
-        auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(top);
-        return node->GetObjHeader();
-    }
-
-    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
-        auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
-        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
-        objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
-        queue.push_front(objectData);
-    }
-};
-
 struct SweepTraits {
     using ObjectFactory = mm::ObjectFactory<gc::SameThreadMarkAndSweep>;
     using ExtraObjectsFactory = mm::ExtraObjectDataFactory;
@@ -154,7 +128,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
 
         RuntimeLogInfo(
                 {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_);
-        gc::collectRootSet<MarkTraits>(markQueue_, [] (mm::ThreadData&) { return true; });
+        gc::collectRootSet<internal::MarkTraits>(markQueue_, [](mm::ThreadData&) { return true; });
         auto timeRootSetUs = konan::getTimeMicros();
         // Can be unsafe, because we've stopped the world.
         auto objectsCountBefore = objectFactory_.GetSizeUnsafe();
@@ -162,7 +136,7 @@ bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept {
         RuntimeLogInfo(
                 {kTagGC}, "Collected root set of size %zu in %" PRIu64 " microseconds", markQueue_.size(),
                 timeRootSetUs - timeSuspendUs);
-        auto markStats = gc::Mark<MarkTraits>(markQueue_);
+        auto markStats = gc::Mark<internal::MarkTraits>(markQueue_);
         auto timeMarkUs = konan::getTimeMicros();
         RuntimeLogDebug({kTagGC}, "Marked %zu objects in %" PRIu64 " microseconds", markStats.aliveHeapSet, timeMarkUs - timeRootSetUs);
         scheduler.gcData().UpdateAliveSetBytes(markStats.aliveHeapSetBytes);

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp

@@ -110,6 +110,34 @@ private:
 
 namespace internal {
 
+struct MarkTraits {
+    using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue;
+
+    static bool isEmpty(const MarkQueue& queue) noexcept { return queue.empty(); }
+
+    static void clear(MarkQueue& queue) noexcept { queue.clear(); }
+
+    static ObjHeader* dequeue(MarkQueue& queue) noexcept {
+        auto& top = queue.front();
+        queue.pop_front();
+        auto node = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(top);
+        return node->GetObjHeader();
+    }
+
+    static void enqueue(MarkQueue& queue, ObjHeader* object) noexcept {
+        auto& objectData = mm::ObjectFactory<gc::SameThreadMarkAndSweep>::NodeRef::From(object).ObjectData();
+        if (objectData.color() == gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack) return;
+        objectData.setColor(gc::SameThreadMarkAndSweep::ObjectData::Color::kBlack);
+        queue.push_front(objectData);
+    }
+
+    static void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept {
+        auto process = object->type_info()->processObjectInMark;
+        RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object);
+        process(static_cast<void*>(&markQueue), object);
+    }
+};
+
 SameThreadMarkAndSweep::SafepointFlag loadSafepointFlag() noexcept;
 
 } // namespace internal

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -3796,6 +3796,22 @@ CODEGEN_INLINE_POLICY RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() {
     // no-op, used by the new MM only.
 }
 
+RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processObjectInMark(void* state, ObjHeader* object) {
+    // no-op, used by the new MM only.
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processArrayInMark(void* state, ObjHeader* object) {
+    // no-op, used by the new MM only.
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processFieldInMark(void* state, ObjHeader* field) {
+    // no-op, used by the new MM only.
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object) {
+    // no-op, used by the new MM only.
+}
+
 } // extern "C"
 
 #if !KONAN_NO_EXCEPTIONS

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -561,4 +561,9 @@ bool FinalizersThreadIsRunning() noexcept;
 
 } // namespace kotlin
 
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object);
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* state, ObjHeader* object);
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field);
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object);
+
 #endif // RUNTIME_MEMORY_H

kotlin-native/runtime/src/main/cpp/ObjCExport.mm

@@ -661,10 +661,12 @@ static const TypeInfo* createTypeInfo(
     if ((superType->flags_ & TF_IMMUTABLE) != 0) {
       result->flags_ |= TF_IMMUTABLE;
     }
+    result->processObjectInMark = superType->processObjectInMark;
   } else {
     result->instanceSize_ = fieldsInfo->instanceSize_;
     result->objOffsets_ = fieldsInfo->objOffsets_;
     result->objOffsetsCount_ = fieldsInfo->objOffsetsCount_;
+    result->processObjectInMark = fieldsInfo->processObjectInMark;
   }
 
   result->classId_ = superType->classId_;

kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp

@@ -28,8 +28,10 @@ private:
 
         int32_t instanceSize_ = 0;
         std_support::vector<int32_t> objOffsets_;
+        int32_t objOffsetsCount_ = 0;
         int32_t flags_ = 0;
         const TypeInfo* superType_ = nullptr;
+        void (*processObjectInMark_)(void*, ObjHeader*) = nullptr;
     };
 
 public:
@@ -57,7 +59,16 @@ public:
     template <typename Payload>
     class ArrayBuilder : public Builder {
     public:
-        ArrayBuilder() noexcept { instanceSize_ = -static_cast<int32_t>(sizeof(Payload)); }
+        ArrayBuilder() noexcept {
+            instanceSize_ = -static_cast<int32_t>(sizeof(Payload));
+            if constexpr (std::is_same_v<Payload, ObjHeader*>) {
+                // Following RTTIGenerator.kt
+                objOffsetsCount_ = 1;
+                processObjectInMark_ = Kotlin_processArrayInMark;
+            } else {
+                processObjectInMark_ = Kotlin_processEmptyObjectInMark;
+            }
+        }
 
         ArrayBuilder&& addFlag(Konan_TypeFlags flag) noexcept {
             flags_ |= flag;
@@ -75,12 +86,8 @@ public:
         typeInfo_.instanceSize_ = builder.instanceSize_;
         objOffsets_ = std::move(builder.objOffsets_);
         typeInfo_.objOffsets_ = objOffsets_.data();
-        if (&typeInfo_ == theArrayTypeInfo) {
-            // Following RTTIGenerator.kt
-            typeInfo_.objOffsetsCount_ = 1;
-        } else {
-            typeInfo_.objOffsetsCount_ = objOffsets_.size();
-        }
+        typeInfo_.objOffsetsCount_ = builder.objOffsetsCount_;
+        typeInfo_.processObjectInMark = builder.processObjectInMark_;
         typeInfo_.flags_ = builder.flags_;
         typeInfo_.superType_ = builder.superType_;
     }
@@ -174,6 +181,8 @@ TypeInfoHolder::ObjectBuilder<Payload>::ObjectBuilder() noexcept {
         auto& actualField = payload.*field;
         objOffsets_.push_back(reinterpret_cast<uintptr_t>(&actualField) - reinterpret_cast<uintptr_t>(object.header()));
     }
+    objOffsetsCount_ = objOffsets_.size();
+    processObjectInMark_ = Kotlin_processObjectInMark;
 }
 
 namespace internal {

kotlin-native/runtime/src/main/cpp/TypeInfo.h

@@ -132,16 +132,16 @@ struct TypeInfo {
     // Null-terminated array.
     const AssociatedObjectTableRecord* associatedObjects;
 
+    // Invoked on an object during mark phase.
+    // TODO: Consider providing a generic traverse method instead.
+    void (*processObjectInMark)(void* state, ObjHeader* object);
+
     // vtable starts just after declared contents of the TypeInfo:
     // void* const vtable_[];
 #ifdef __cplusplus
-    inline VTableElement const* vtable() const {
-      return reinterpret_cast<VTableElement const*>(this + 1);
-    }
+    inline VTableElement const* vtable() const { return reinterpret_cast<VTableElement const*>(this + 1); }
 
-    inline VTableElement* vtable() {
-      return reinterpret_cast<VTableElement*>(this + 1);
-    }
+    inline VTableElement* vtable() { return reinterpret_cast<VTableElement*>(this + 1); }
 
     inline bool IsArray() const { return instanceSize_ < 0; }
 

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -632,3 +632,20 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {
 bool kotlin::FinalizersThreadIsRunning() noexcept {
     return mm::GlobalData::Instance().gc().FinalizersThreadIsRunning();
 }
+
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object) {
+    gc::GC::processObjectInMark(state, object);
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* state, ObjHeader* object) {
+    gc::GC::processArrayInMark(state, object->array());
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field) {
+    gc::GC::processFieldInMark(state, field);
+}
+
+RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object) {
+    // Empty object. Nothing to do.
+    // TODO: Try to generate it in the code generator.
+}