[K/N] Prevent unmarked objects from appearing during concurrent weak processing

Merge-request: KT-MR-11614
Merged-by: Alexey Glushko <aleksei.glushko@jetbrains.com>

kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp

@@ -18,6 +18,7 @@ using namespace kotlin;
 namespace {
 
 std::atomic<ObjHeader* (*)(ObjHeader*)> weakRefBarrier = nullptr;
+std::atomic<int64_t> weakProcessingEpoch = 0;
 
 ObjHeader* weakRefBarrierImpl(ObjHeader* weakReferee) noexcept {
     if (!weakReferee) return nullptr;
@@ -36,47 +37,62 @@ NO_INLINE ObjHeader* weakRefReadSlowPath(std::atomic<ObjHeader*>& weakReferee) n
     return barrier ? barrier(weak) : weak;
 }
 
-void waitForThreadsToReachCheckpoint() {
-    // Reset checkpoint on all threads.
-    for (auto& thr : mm::ThreadRegistry::Instance().LockForIter()) {
-        thr.gc().impl().gc().barriers().resetCheckpoint();
-    }
-
-    mm::SafePointActivator safePointActivator;
-
-    // Wait for all threads to either have passed safepoint or to be in the native state.
-    // Either of these mean that none of them are inside a weak reference accessing code.
-    mm::ThreadRegistry::Instance().waitAllThreads([](mm::ThreadData& thread) noexcept {
-        return thread.gc().impl().gc().barriers().visitedCheckpoint() || thread.suspensionData().suspendedOrNative();
-    });
-}
-
 } // namespace
 
-void gc::BarriersThreadData::onCheckpoint() noexcept {
-    visitedCheckpoint_.store(true, std::memory_order_release);
+void gc::BarriersThreadData::onThreadRegistration() noexcept {
+    if (weakRefBarrier.load(std::memory_order_acquire) != nullptr) {
+        startMarkingNewObjects(GCHandle::getByEpoch(weakProcessingEpoch.load(std::memory_order_relaxed)));
+    }
 }
 
-void gc::BarriersThreadData::resetCheckpoint() noexcept {
-    visitedCheckpoint_.store(false, std::memory_order_release);
+ALWAYS_INLINE void gc::BarriersThreadData::onSafePoint() noexcept {}
+
+void gc::BarriersThreadData::startMarkingNewObjects(gc::GCHandle gcHandle) noexcept {
+    RuntimeAssert(weakRefBarrier.load(std::memory_order_relaxed) != nullptr, "New allocations marking may only be requested by weak ref barriers");
+    markHandle_ = gcHandle.mark();
 }
 
-bool gc::BarriersThreadData::visitedCheckpoint() const noexcept {
-    return visitedCheckpoint_.load(std::memory_order_acquire);
+void gc::BarriersThreadData::stopMarkingNewObjects() noexcept {
+    RuntimeAssert(weakRefBarrier.load(std::memory_order_relaxed) == nullptr, "New allocations marking could only been requested by weak ref barriers");
+    markHandle_ = std::nullopt;
 }
 
-void gc::EnableWeakRefBarriers() noexcept {
+bool gc::BarriersThreadData::shouldMarkNewObjects() const noexcept {
+    return markHandle_.has_value();
+}
+
+ALWAYS_INLINE void gc::BarriersThreadData::onAllocation(ObjHeader* allocated) {
+    if (compiler::concurrentWeakSweep()) {
+        bool shouldMark = shouldMarkNewObjects();
+        bool barriersEnabled = weakRefBarrier.load(std::memory_order_relaxed) != nullptr;
+        RuntimeAssert(shouldMark == barriersEnabled, "New allocations marking must happen with and only with weak ref barriers");
+        if (shouldMark) {
+            auto& objectData = objectDataForObject(allocated);
+            bool wasUnmarked = objectData.tryMark();
+            RuntimeAssert(wasUnmarked, "No one else could mark this newly allocated object before");
+            markHandle_->addObject();
+        }
+    }
+}
+
+void gc::EnableWeakRefBarriers(int64_t epoch) noexcept {
+    auto mutators = mm::ThreadRegistry::Instance().LockForIter();
+    weakProcessingEpoch.store(epoch, std::memory_order_relaxed);
     weakRefBarrier.store(weakRefBarrierImpl, std::memory_order_seq_cst);
+    for (auto& mutator: mutators) {
+        mutator.gc().impl().gc().barriers().startMarkingNewObjects(GCHandle::getByEpoch(epoch));
+    }
 }
 
 void gc::DisableWeakRefBarriers() noexcept {
+    auto mutators = mm::ThreadRegistry::Instance().LockForIter();
     weakRefBarrier.store(nullptr, std::memory_order_seq_cst);
-    waitForThreadsToReachCheckpoint();
+    for (auto& mutator: mutators) {
+        mutator.gc().impl().gc().barriers().stopMarkingNewObjects();
+    }
 }
 
-OBJ_GETTER(kotlin::gc::WeakRefRead, std::atomic<ObjHeader*>& weakReferee) noexcept {
-    // TODO: Make this work with GCs that can stop thread at any point.
-
+OBJ_GETTER(gc::WeakRefRead, std::atomic<ObjHeader*>& weakReferee) noexcept {
     if (!compiler::concurrentWeakSweep()) {
         RETURN_OBJ(weakReferee.load(std::memory_order_relaxed));
     }

kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp

@@ -10,23 +10,26 @@
 
 #include "Memory.h"
 #include "Utils.hpp"
+#include "GCStatistics.hpp"
 
 namespace kotlin::gc {
 
 class BarriersThreadData : private Pinned {
 public:
-    void onCheckpoint() noexcept;
-    void resetCheckpoint() noexcept;
-    bool visitedCheckpoint() const noexcept;
+    void onThreadRegistration() noexcept;
+    void onSafePoint() noexcept;
+    
+    void startMarkingNewObjects(GCHandle gcHandle) noexcept;
+    void stopMarkingNewObjects() noexcept;
+    bool shouldMarkNewObjects() const noexcept;
 
+    void onAllocation(ObjHeader* allocated);
 private:
-    std::atomic<bool> visitedCheckpoint_ = false;
+    std::optional<GCHandle::GCMarkScope> markHandle_{};
 };
 
 // Must be called during STW.
-void EnableWeakRefBarriers() noexcept;
-
-// Must be called outside STW.
+void EnableWeakRefBarriers(int64_t epoch) noexcept;
 void DisableWeakRefBarriers() noexcept;
 
 OBJ_GETTER(WeakRefRead, std::atomic<ObjHeader*>& weakReferee) noexcept;

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -175,21 +175,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     RuntimeAssert(didSuspend, "Only GC thread can request suspension");
     gcHandle.suspensionRequested();
 
-    // TODO (WaitForThreadsReadyToMark())
-    RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "GC must run on unregistered thread");
-
     markDispatcher_.waitForThreadsPauseMutation();
     GCLogDebug(epoch, "All threads have paused mutation");
     gcHandle.threadsAreSuspended();
 
-#ifdef CUSTOM_ALLOCATOR
-    // This should really be done by each individual thread while waiting
-    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
-        thread.gc().impl().alloc().PrepareForGC();
-    }
-    heap_.PrepareForGC();
-#endif
-
     auto& scheduler = gcScheduler_;
     scheduler.onGCStart();
 
@@ -201,17 +190,8 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
 
     mm::WaitForThreadsSuspension();
 
-#ifndef CUSTOM_ALLOCATOR
-    // Taking the locks before the pause is completed. So that any destroying thread
-    // would not publish into the global state at an unexpected time.
-    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
-    std::optional objectFactoryIterable = objectFactory_.LockForIter();
-    checkMarkCorrectness(*objectFactoryIterable);
-#endif
-
     if (compiler::concurrentWeakSweep()) {
-        // Expected to happen inside STW.
-        gc::EnableWeakRefBarriers();
+        EnableWeakRefBarriers(epoch);
 
         mm::ResumeThreads();
         gcHandle.threadsAreResumed();
@@ -220,13 +200,42 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 
     if (compiler::concurrentWeakSweep()) {
-        // Expected to happen outside STW.
-        gc::DisableWeakRefBarriers();
-    } else {
-        mm::ResumeThreads();
-        gcHandle.threadsAreResumed();
+        bool didSuspend = mm::RequestThreadsSuspension();
+        RuntimeAssert(didSuspend, "Only GC thread can request suspension");
+        gcHandle.suspensionRequested();
+
+        mm::WaitForThreadsSuspension();
+        GCLogDebug(gcHandle.getEpoch(), "All threads have paused mutation");
+        gcHandle.threadsAreSuspended();
+        DisableWeakRefBarriers();
     }
 
+    // TODO outline as mark_.isolateMarkedHeapAndFinishMark()
+    // By this point all the alive heap must be marked.
+    // All the mutations (incl. allocations) after this method will be subject for the next GC.
+#ifdef CUSTOM_ALLOCATOR
+    // This should really be done by each individual thread while waiting
+    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
+        thread.gc().impl().alloc().PrepareForGC();
+    }
+    auto& heap = mm::GlobalData::Instance().gc().impl().gc().heap();
+    heap.PrepareForGC();
+#else
+    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
+        thread.gc().PublishObjectFactory();
+    }
+
+    // Taking the locks before the pause is completed. So that any destroying thread
+    // would not publish into the global state at an unexpected time.
+    std::optional objectFactoryIterable = objectFactory_.LockForIter();
+    std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();
+
+    checkMarkCorrectness(*objectFactoryIterable);
+#endif
+
+    mm::ResumeThreads();
+    gcHandle.threadsAreResumed();
+
 #ifndef CUSTOM_ALLOCATOR
     gc::SweepExtraObjects<DefaultSweepTraits<ObjectFactory>>(gcHandle, *extraObjectFactoryIterable);
     extraObjectFactoryIterable = std::nullopt;

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp

@@ -38,7 +38,9 @@ public:
 
         void OnSuspendForGC() noexcept;
 
-        void safePoint() noexcept { barriers_.onCheckpoint(); }
+        void safePoint() noexcept { barriers_.onSafePoint(); }
+
+        void onThreadRegistration() noexcept { barriers_.onThreadRegistration(); }
 
         BarriersThreadData& barriers() noexcept { return barriers_; }
 

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp

@@ -21,6 +21,7 @@
 #include "ObjectTestSupport.hpp"
 #include "SafePoint.hpp"
 #include "SingleThreadExecutor.hpp"
+#include "StableRef.hpp"
 #include "TestSupport.hpp"
 #include "ThreadData.hpp"
 #include "WeakRef.hpp"
@@ -1034,6 +1035,51 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) {
     }
 }
 
+TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeakNewObj) {
+    std_support::vector<Mutator> mutators(kDefaultThreadCount);
+
+    // Make sure all mutators are initialized.
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait();
+    }
+
+    std_support::vector<std::future<void>> gcFutures;
+    auto epoch = mm::GlobalData::Instance().gc().Schedule();
+    std::atomic<bool> gcDone = false;
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (auto& mutator : mutators) {
+        gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept {
+            mm::safePoint(threadData);
+
+            auto& object = AllocateObject(threadData);
+            auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& {
+                ObjHolder holder;
+                return InstallWeakReference(threadData, object.header(), holder.slot());
+            })();
+            EXPECT_NE(objectWeak.get(), nullptr);
+
+            auto& extraObj = *mm::ExtraObjectData::Get(object.header());
+            extraObj.ClearRegularWeakReferenceImpl();
+            extraObj.Uninstall();
+            mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj);
+
+            while (!gcDone.load(std::memory_order_relaxed)) {
+                mm::safePoint(threadData);
+            }
+        }));
+    }
+
+    mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
+    gcDone.store(true, std::memory_order_relaxed);
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+}
 
 TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
     std_support::vector<Mutator> mutators(kDefaultThreadCount);
@@ -1104,31 +1150,6 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         future.wait();
     }
 
-#ifndef CUSTOM_ALLOCATOR
-    // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything).
-
-    std_support::vector<ObjHeader*> expectedAlive;
-    for (int i = 0; i < kDefaultThreadCount; ++i) {
-        expectedAlive.push_back(globals[i]);
-        expectedAlive.push_back(locals[i]);
-        expectedAlive.push_back(reachables[i]);
-    }
-
-    for (auto& mutator : mutators) {
-        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
-    }
-
-    for (int i = 0; i < kDefaultThreadCount; ++i) {
-        std_support::vector<ObjHeader*> aliveForThisThread(expectedAlive.begin(), expectedAlive.end());
-        aliveForThisThread.push_back(globals[kDefaultThreadCount + i]);
-        aliveForThisThread.push_back(locals[kDefaultThreadCount + i]);
-        aliveForThisThread.push_back(reachables[kDefaultThreadCount + i]);
-        // Unreachables for new threads were not collected.
-        aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]);
-        EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread));
-    }
-#else
-    // Custom allocator does not have a notion of objects alive only for some thread
     std_support::vector<ObjHeader*> expectedAlive;
     for (int i = 0; i < kDefaultThreadCount; ++i) {
         expectedAlive.push_back(globals[i]);
@@ -1140,9 +1161,27 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         // Unreachables for new threads were not collected.
         expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
     }
-    // All threads see the same alive objects with the custom alloctor, enough to check a single mutator.
+
+#ifndef CUSTOM_ALLOCATOR
+    // Force mutators to publish their internal heaps
+    std_support::vector<std::future<void>> publishFutures;
+    for (auto& mutator: mutators) {
+        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+            threadData.gc().PublishObjectFactory();
+        }));
+    }
+    for (auto& mutator: newMutators) {
+        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+            threadData.gc().PublishObjectFactory();
+        }));
+    }
+    for (auto& future : publishFutures) {
+        future.wait();
+    }
+#endif
+
+    // All threads see the same alive objects, enough to check a single mutator.
     EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));
-#endif // CUSTOM_ALLOCATOR
 }
 
 TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {

kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp

@@ -19,7 +19,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::Publish() noexcept {
+void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
     impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
@@ -36,19 +36,25 @@ void gc::GC::ThreadData::ClearForTests() noexcept {
 }
 
 ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept {
+    ObjHeader* obj;
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
+    obj = impl_->objectFactoryThreadQueue().CreateObject(typeInfo);
 #else
-    return impl_->alloc().CreateObject(typeInfo);
+    obj = impl_->alloc().CreateObject(typeInfo);
 #endif
+    impl().gc().barriers().onAllocation(obj);
+    return obj;
 }
 
 ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    ArrayHeader* arr;
 #ifndef CUSTOM_ALLOCATOR
-    return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
+    arr = impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements);
 #else
-    return impl_->alloc().CreateArray(typeInfo, elements);
+    arr = impl_->alloc().CreateArray(typeInfo, elements);
 #endif
+    impl().gc().barriers().onAllocation(arr->obj());
+    return arr;
 }
 
 ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject(
@@ -76,6 +82,10 @@ void gc::GC::ThreadData::safePoint() noexcept {
     impl_->gc().safePoint();
 }
 
+void gc::GC::ThreadData::onThreadRegistration() noexcept {
+    impl_->gc().onThreadRegistration();
+}
+
 gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique<Impl>(gcScheduler)) {}
 
 gc::GC::~GC() = default;

kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp

@@ -169,7 +169,7 @@ private:
     GCHandle gcHandle_ = GCHandle::invalid();
     MarkPacer pacer_;
     std::optional<mm::ThreadRegistry::Iterable> lockedMutatorsList_;
-    ManuallyScoped<ParallelProcessor> parallelProcessor_;
+    ManuallyScoped<ParallelProcessor> parallelProcessor_{};
 
     std::mutex workerCreationMutex_;
     std::atomic<std::size_t> activeWorkersCount_ = 0;

kotlin-native/runtime/src/gc/common/cpp/GC.hpp

@@ -35,7 +35,7 @@ public:
 
         Impl& impl() noexcept { return *impl_; }
 
-        void Publish() noexcept;
+        void PublishObjectFactory() noexcept;
         void ClearForTests() noexcept;
 
         ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept;
@@ -47,6 +47,8 @@ public:
 
         void safePoint() noexcept;
 
+        void onThreadRegistration() noexcept;
+
     private:
         std_support::unique_ptr<Impl> impl_;
     };

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp

@@ -20,8 +20,12 @@ extern "C" {
 void Kotlin_Internal_GC_GCInfoBuilder_setEpoch(KRef thiz, KLong value);
 void Kotlin_Internal_GC_GCInfoBuilder_setStartTime(KRef thiz, KLong value);
 void Kotlin_Internal_GC_GCInfoBuilder_setEndTime(KRef thiz, KLong value);
-void Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(KRef thiz, KLong value);
-void Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(KRef thiz, KLong value);
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(KRef thiz, KLong value);
 void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong value);
 void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz,
                                                  KLong threadLocalReferences, KLong stackReferences,
@@ -77,8 +81,15 @@ struct GCInfo {
     std::optional<uint64_t> epoch;
     std::optional<KLong> startTime; // time since process start
     std::optional<KLong> endTime;
-    std::optional<KLong> pauseStartTime;
-    std::optional<KLong> pauseEndTime;
+
+    std::optional<KLong> firstPauseRequestTime;
+    std::optional<KLong> firstPauseStartTime;
+    std::optional<KLong> firstPauseEndTime;
+
+    std::optional<KLong> secondPauseRequestTime;
+    std::optional<KLong> secondPauseStartTime;
+    std::optional<KLong> secondPauseEndTime;
+
     std::optional<KLong> finalizersDoneTime;
     std::optional<RootSetStatistics> rootSet;
     std::optional<kotlin::gc::MarkStats> markStats;
@@ -91,8 +102,12 @@ struct GCInfo {
         Kotlin_Internal_GC_GCInfoBuilder_setEpoch(builder, static_cast<KLong>(*epoch));
         if (startTime) Kotlin_Internal_GC_GCInfoBuilder_setStartTime(builder, *startTime);
         if (endTime) Kotlin_Internal_GC_GCInfoBuilder_setEndTime(builder, *endTime);
-        if (pauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(builder, *pauseStartTime);
-        if (pauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(builder, *pauseEndTime);
+        if (firstPauseRequestTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(builder, *firstPauseRequestTime);
+        if (firstPauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(builder, *firstPauseStartTime);
+        if (firstPauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(builder, *firstPauseEndTime);
+        if (secondPauseRequestTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(builder, *secondPauseRequestTime);
+        if (secondPauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(builder, *secondPauseStartTime);
+        if (secondPauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(builder, *secondPauseEndTime);
         if (finalizersDoneTime) Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(builder, *finalizersDoneTime);
         if (rootSet)
             Kotlin_Internal_GC_GCInfoBuilder_setRootSet(
@@ -223,9 +238,21 @@ void GCHandle::finished() {
                     epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->sizeBytes,
                     stat->memoryUsageAfter.heap->sizeBytes);
         }
-        if (stat->pauseStartTime && stat->pauseEndTime) {
-            auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
-            GCLogInfo(epoch_, "Mutators pause time: %" PRIu64 " microseconds.", time);
+        if (stat->firstPauseRequestTime && stat->firstPauseStartTime) {
+            auto time = (*stat->firstPauseStartTime - *stat->firstPauseRequestTime) / 1000;
+            GCLogInfo(epoch_, "Time to pause #1: %" PRIu64 " microseconds.", time);
+        }
+        if (stat->firstPauseRequestTime && stat->firstPauseEndTime) {
+            auto time = (*stat->firstPauseEndTime - *stat->firstPauseRequestTime) / 1000;
+            GCLogInfo(epoch_, "Mutators pause time #1: %" PRIu64 " microseconds.", time);
+        }
+        if (stat->secondPauseRequestTime && stat->secondPauseStartTime) {
+            auto time = (*stat->secondPauseStartTime - *stat->secondPauseRequestTime) / 1000;
+            GCLogInfo(epoch_, "Time to pause #2: %" PRIu64 " microseconds.", time);
+        }
+        if (stat->secondPauseRequestTime && stat->secondPauseEndTime) {
+            auto time = (*stat->secondPauseEndTime - *stat->secondPauseRequestTime) / 1000;
+            GCLogInfo(epoch_, "Mutators pause time #2: %" PRIu64 " microseconds.", time);
         }
         if (stat->startTime) {
             auto time = (*current.endTime - *current.startTime) / 1000;
@@ -242,14 +269,30 @@ void GCHandle::suspensionRequested() {
     std::lock_guard guard(lock);
     GCLogDebug(epoch_, "Requested thread suspension");
     if (auto* stat = statByEpoch(epoch_)) {
-        stat->pauseStartTime = static_cast<KLong>(konan::getTimeNanos());
+        auto requestTime = static_cast<KLong>(konan::getTimeNanos());
+        if (!stat->firstPauseRequestTime) {
+            stat->firstPauseRequestTime = requestTime;
+        } else {
+            RuntimeAssert(!stat->secondPauseRequestTime, "GCStatistics support max two pauses per GC epoch");
+            stat->secondPauseRequestTime = requestTime;
+        }
     }
 }
 void GCHandle::threadsAreSuspended() {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
-        if (stat->pauseStartTime) {
-            auto time = (konan::getTimeNanos() - *stat->pauseStartTime) / 1000;
+        auto startTime = static_cast<KLong>(konan::getTimeNanos());
+        std::optional<KLong> requestTime;
+        if (!stat->firstPauseStartTime) {
+            stat->firstPauseStartTime = startTime;
+            requestTime = stat->firstPauseRequestTime;
+        } else {
+            RuntimeAssert(!stat->secondPauseStartTime, "GCStatistics support max two pauses per GC epoch");
+            stat->secondPauseStartTime = startTime;
+            requestTime = stat->secondPauseRequestTime;
+        }
+        if (requestTime) {
+            auto time = (startTime - *requestTime) / 1000;
             GCLogDebug(epoch_, "Suspended all threads in %" PRIu64 " microseconds", time);
             return;
         }
@@ -267,9 +310,18 @@ void GCHandle::threadsAreSuspended() {
 void GCHandle::threadsAreResumed() {
     std::lock_guard guard(lock);
     if (auto* stat = statByEpoch(epoch_)) {
-        stat->pauseEndTime = static_cast<KLong>(konan::getTimeNanos());
-        if (stat->pauseStartTime) {
-            auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000;
+        auto endTime = static_cast<KLong>(konan::getTimeNanos());
+        std::optional<KLong> startTime;
+        if (!stat->firstPauseEndTime) {
+            stat->firstPauseEndTime = endTime;
+            startTime = stat->firstPauseStartTime;
+        } else {
+            RuntimeAssert(!stat->secondPauseEndTime, "GCStatistics support max two pauses per GC epoch");
+            stat->secondPauseEndTime = endTime;
+            startTime = stat->secondPauseStartTime;
+        }
+        if (startTime) {
+            auto time = (endTime - *startTime) / 1000;
             GCLogDebug(epoch_, "Resume all threads. Total pause time is %" PRId64 " microseconds.", time);
             return;
         }
@@ -358,6 +410,8 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept {
     }
 }
 
+GCHandle::GCMarkScope GCHandle::mark() { return GCMarkScope(*this); }
+
 GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
 
 GCHandle::GCSweepScope::~GCSweepScope() {
@@ -397,11 +451,27 @@ GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){
                threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime());
 }
 
-GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){}
+void GCHandle::GCMarkScope::swap(GCHandle::GCMarkScope& other) noexcept {
+    std::swap(handle_, other.handle_);
+    std::swap(startTime_, other.startTime_);
+}
+
+GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle) {}
+
+GCHandle::GCMarkScope::GCMarkScope(GCHandle::GCMarkScope&& that) noexcept {
+    swap(that);
+}
+
+GCHandle::GCMarkScope& GCHandle::GCMarkScope::operator=(GCHandle::GCMarkScope that) noexcept {
+    swap(that);
+    return *this;
+}
 
 GCHandle::GCMarkScope::~GCMarkScope() {
-    handle_.marked(stats_);
-    GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime());
+    if (handle_.isValid()) {
+        handle_.marked(stats_);
+        GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime());
+    }
 }
 
 gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {}

kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp

@@ -95,16 +95,7 @@ public:
         void addThreadLocalRoot() { threadLocalRoots_++; }
     };
 
-    class GCMarkScope : GCStageScopeUsTimer, Pinned {
-        GCHandle& handle_;
-        MarkStats stats_;
-
-    public:
-        explicit GCMarkScope(GCHandle& handle);
-        ~GCMarkScope();
-
-        void addObject() noexcept { ++stats_.markedCount; }
-    };
+    class GCMarkScope;
 
     class GCProcessWeaksScope : GCStageScopeUsTimer, Pinned {
         GCHandle& handle_;
@@ -151,9 +142,24 @@ public:
     GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); }
     GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); }
     GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); }
-    GCMarkScope mark() { return GCMarkScope(*this); }
+    GCMarkScope mark();
     GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); }
 
     MarkStats getMarked();
 };
+
+class GCHandle::GCMarkScope : GCStageScopeUsTimer {
+    GCHandle handle_ = GCHandle::invalid();
+    MarkStats stats_;
+
+    void swap(GCMarkScope& other) noexcept;
+
+public:
+    explicit GCMarkScope(GCHandle& handle);
+    GCMarkScope(GCMarkScope&& that) noexcept;
+    GCMarkScope& operator=(GCMarkScope that) noexcept;
+    ~GCMarkScope();
+
+    void addObject() noexcept { ++stats_.markedCount; }
+};
 }

kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp

@@ -20,7 +20,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::Publish() noexcept {
+void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
     impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
@@ -73,6 +73,8 @@ void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
 void gc::GC::ThreadData::safePoint() noexcept {}
 
+void gc::GC::ThreadData::onThreadRegistration() noexcept {}
+
 gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique<Impl>()) {}
 
 gc::GC::~GC() = default;

kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp

@@ -20,7 +20,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im
 
 gc::GC::ThreadData::~ThreadData() = default;
 
-void gc::GC::ThreadData::Publish() noexcept {
+void gc::GC::ThreadData::PublishObjectFactory() noexcept {
 #ifndef CUSTOM_ALLOCATOR
     impl_->extraObjectDataFactoryThreadQueue().Publish();
     impl_->objectFactoryThreadQueue().Publish();
@@ -73,6 +73,8 @@ void gc::GC::ThreadData::OnSuspendForGC() noexcept { }
 
 void gc::GC::ThreadData::safePoint() noexcept {}
 
+void gc::GC::ThreadData::onThreadRegistration() noexcept {}
+
 gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique<Impl>(gcScheduler)) {}
 
 gc::GC::~GC() = default;

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -99,6 +99,10 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 
 #ifndef CUSTOM_ALLOCATOR
+    for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
+        thread.gc().PublishObjectFactory();
+    }
+
     // Taking the locks before the pause is completed. So that any destroying thread
     // would not publish into the global state at an unexpected time.
     std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter();

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -1010,6 +1010,52 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) {
     }
 }
 
+TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeakNewObj) {
+    std_support::vector<Mutator> mutators(kDefaultThreadCount);
+
+    // Make sure all mutators are initialized.
+    for (int i = 0; i < kDefaultThreadCount; ++i) {
+        mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait();
+    }
+
+    std_support::vector<std::future<void>> gcFutures;
+    auto epoch = mm::GlobalData::Instance().gc().Schedule();
+    std::atomic<bool> gcDone = false;
+
+    // Spin until thread suspension is requested.
+    while (!mm::IsThreadSuspensionRequested()) {
+    }
+
+    for (auto& mutator : mutators) {
+        gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept {
+            mm::safePoint(threadData);
+
+            auto& object = AllocateObject(threadData);
+            auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& {
+                ObjHolder holder;
+                return InstallWeakReference(threadData, object.header(), holder.slot());
+            })();
+            EXPECT_NE(objectWeak.get(), nullptr);
+
+            auto& extraObj = *mm::ExtraObjectData::Get(object.header());
+            extraObj.ClearRegularWeakReferenceImpl();
+            extraObj.Uninstall();
+            mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj);
+
+            while (!gcDone.load(std::memory_order_relaxed)) {
+                mm::safePoint(threadData);
+            }
+        }));
+    }
+
+    mm::GlobalData::Instance().gc().WaitFinalizers(epoch);
+    gcDone.store(true, std::memory_order_relaxed);
+
+    for (auto& future : gcFutures) {
+        future.wait();
+    }
+}
+
 TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
     std_support::vector<Mutator> mutators(kDefaultThreadCount);
     std_support::vector<ObjHeader*> globals(2 * kDefaultThreadCount);
@@ -1079,31 +1125,6 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         future.wait();
     }
 
-#ifndef CUSTOM_ALLOCATOR
-    // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything).
-
-    std_support::vector<ObjHeader*> expectedAlive;
-    for (int i = 0; i < kDefaultThreadCount; ++i) {
-        expectedAlive.push_back(globals[i]);
-        expectedAlive.push_back(locals[i]);
-        expectedAlive.push_back(reachables[i]);
-    }
-
-    for (auto& mutator : mutators) {
-        EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive));
-    }
-
-    for (int i = 0; i < kDefaultThreadCount; ++i) {
-        std_support::vector<ObjHeader*> aliveForThisThread(expectedAlive.begin(), expectedAlive.end());
-        aliveForThisThread.push_back(globals[kDefaultThreadCount + i]);
-        aliveForThisThread.push_back(locals[kDefaultThreadCount + i]);
-        aliveForThisThread.push_back(reachables[kDefaultThreadCount + i]);
-        // Unreachables for new threads were not collected.
-        aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]);
-        EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread));
-    }
-#else
-    // Custom allocator does not have a notion of objects alive only for some thread
     std_support::vector<ObjHeader*> expectedAlive;
     for (int i = 0; i < kDefaultThreadCount; ++i) {
         expectedAlive.push_back(globals[i]);
@@ -1115,9 +1136,27 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
         // Unreachables for new threads were not collected.
         expectedAlive.push_back(unreachables[kDefaultThreadCount + i]);
     }
-    // All threads see the same alive objects with the custom alloctor, enough to check a single mutator.
+
+#ifndef CUSTOM_ALLOCATOR
+    // Force mutators to publish their internal heaps
+    std_support::vector<std::future<void>> publishFutures;
+    for (auto& mutator: mutators) {
+        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+            threadData.gc().PublishObjectFactory();
+        }));
+    }
+    for (auto& mutator: newMutators) {
+        publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) {
+            threadData.gc().PublishObjectFactory();
+        }));
+    }
+    for (auto& future : publishFutures) {
+        future.wait();
+    }
+#endif
+
+    // All threads see the same alive objects, enough to check a single mutator.
     EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive));
-#endif // CUSTOM_ALLOCATOR
 }
 
 

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt

@@ -53,10 +53,16 @@ public class RootSetStatistics(
  * @property startTimeNs Time, when garbage collector run is started, meausered by [kotlin.system.getTimeNanos].
  * @property endTimeNs Time, when garbage collector run is ended, measured by [kotlin.system.getTimeNanos].
  *                     After this point, most of the memory is reclaimed, and a new garbage collector run can start.
- * @property pauseStartTimeNs Time, when mutator threads are suspended, mesured by [kotlin.system.getTimeNanos].
- * @property pauseEndTimeNs Time, when mutator threads are unsuspended, mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the first time,
+ *                                   mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseStartTimeNs Time, when mutator threads are suspended for the first time, mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseEndTimeNs Time, when mutator threads are unsuspended for the first time, mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the second time,
+ *                                    mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseStartTimeNs Time, when mutator threads are suspended for the second time, mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseEndTimeNs Time, when mutator threads are unsuspended for the second time, mesured by [kotlin.system.getTimeNanos].
  * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos].
- *                                If null, memory reclamation is still in progress.
+ *                               If null, memory reclamation is still in progress.
  * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details.
  * @property memoryUsageAfter Memory usage at the start of garbage collector run, separated by memory pools.
  *                            The set of memory pools depends on the collector implementation.
@@ -74,8 +80,12 @@ public class GCInfo(
         val epoch: Long,
         val startTimeNs: Long,
         val endTimeNs: Long,
-        val pauseStartTimeNs: Long,
-        val pauseEndTimeNs: Long,
+        val firstPauseRequestTimeNs: Long,
+        val firstPauseStartTimeNs: Long,
+        val firstPauseEndTimeNs: Long,
+        val secondPauseRequestTimeNs: Long?,
+        val secondPauseStartTimeNs: Long?,
+        val secondPauseEndTimeNs: Long?,
         val postGcCleanupTimeNs: Long?,
         val rootSet: RootSetStatistics,
         val memoryUsageBefore: Map<String, MemoryUsage>,
@@ -89,8 +99,12 @@ public class GCInfo(
                         info.epoch,
                         info.startTimeNs,
                         info.endTimeNs,
-                        info.pauseStartTimeNs,
-                        info.pauseEndTimeNs,
+                        info.firstPauseRequestTimeNs,
+                        info.firstPauseStartTimeNs,
+                        info.firstPauseEndTimeNs,
+                        info.secondPauseRequestTimeNs,
+                        info.secondPauseStartTimeNs,
+                        info.secondPauseEndTimeNs,
                         info.postGcCleanupTimeNs,
                         info.rootSet.let {
                             RootSetStatistics(

kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt

@@ -71,8 +71,14 @@ public class RootSetStatistics(
  * @property startTimeNs Time, when garbage collector run is started, meausered by [kotlin.system.getTimeNanos].
  * @property endTimeNs Time, when garbage collector run is ended, measured by [kotlin.system.getTimeNanos].
  *                     After this point, most of the memory is reclaimed, and a new garbage collector run can start.
- * @property pauseStartTimeNs Time, when mutator threads are suspended, mesured by [kotlin.system.getTimeNanos].
- * @property pauseEndTimeNs Time, when mutator threads are unsuspended, mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the first time,
+ *                                   mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseStartTimeNs Time, when mutator threads are suspended for the first time, mesured by [kotlin.system.getTimeNanos].
+ * @property firstPauseEndTimeNs Time, when mutator threads are unsuspended for the first time, mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the second time,
+ *                                    mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseStartTimeNs Time, when mutator threads are suspended for the second time, mesured by [kotlin.system.getTimeNanos].
+ * @property secondPauseEndTimeNs Time, when mutator threads are unsuspended for the second time, mesured by [kotlin.system.getTimeNanos].
  * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos].
  *                                If null, memory reclamation is still in progress.
  * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details.
@@ -93,8 +99,12 @@ public class GCInfo(
         val epoch: Long,
         val startTimeNs: Long,
         val endTimeNs: Long,
-        val pauseStartTimeNs: Long,
-        val pauseEndTimeNs: Long,
+        val firstPauseRequestTimeNs: Long,
+        val firstPauseStartTimeNs: Long,
+        val firstPauseEndTimeNs: Long,
+        val secondPauseRequestTimeNs: Long?,
+        val secondPauseStartTimeNs: Long?,
+        val secondPauseEndTimeNs: Long?,
         val postGcCleanupTimeNs: Long?,
         val rootSet: RootSetStatistics,
         val markedCount: Long,
@@ -116,8 +126,12 @@ private class GCInfoBuilder() {
     var epoch: Long? = null
     var startTimeNs: Long? = null
     var endTimeNs: Long? = null
-    var pauseStartTimeNs: Long? = null
-    var pauseEndTimeNs: Long? = null
+    var firstPauseRequestTimeNs: Long? = null
+    var firstPauseStartTimeNs: Long? = null
+    var firstPauseEndTimeNs: Long? = null
+    var secondPauseRequestTimeNs: Long? = null
+    var secondPauseStartTimeNs: Long? = null
+    var secondPauseEndTimeNs: Long? = null
     var postGcCleanupTimeNs: Long? = null
     var rootSet: RootSetStatistics? = null
     var markedCount: Long? = null
@@ -140,14 +154,34 @@ private class GCInfoBuilder() {
         endTimeNs = value
     }
 
-    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime")
-    private fun setPauseStartTime(value: Long) {
-        pauseStartTimeNs = value
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime")
+    private fun setFirstPauseRequestTime(value: Long) {
+        firstPauseRequestTimeNs = value
     }
 
-    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime")
-    private fun setPauseEndTime(value: Long) {
-        pauseEndTimeNs = value
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime")
+    private fun setFirstPauseStartTime(value: Long) {
+        firstPauseStartTimeNs = value
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime")
+    private fun setFirstPauseEndTime(value: Long) {
+        firstPauseEndTimeNs = value
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime")
+    private fun setSecondPauseRequestTime(value: Long) {
+        secondPauseRequestTimeNs = value
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime")
+    private fun setSecondPauseStartTime(value: Long) {
+        secondPauseStartTimeNs = value
+    }
+
+    @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime")
+    private fun setSecondPauseEndTime(value: Long) {
+        secondPauseEndTimeNs = value
     }
 
     @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime")
@@ -191,8 +225,12 @@ private class GCInfoBuilder() {
                 epoch ?: return null,
                 startTimeNs ?: return null,
                 endTimeNs ?: return null,
-                pauseStartTimeNs ?: return null,
-                pauseEndTimeNs ?: return null,
+                firstPauseRequestTimeNs ?: return null,
+                firstPauseStartTimeNs ?: return null,
+                firstPauseEndTimeNs ?: return null,
+                secondPauseRequestTimeNs,
+                secondPauseStartTimeNs,
+                secondPauseEndTimeNs,
                 postGcCleanupTimeNs,
                 rootSet ?: return null,
                 markedCount ?: return null,

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -78,7 +78,7 @@ TEST_F(ExtraObjectDataTest, ConcurrentInstall) {
             }
             auto& extraData = mm::ExtraObjectData::Install(object.header());
             actual[i] = &extraData;
-            mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->Publish();
+            mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->gc().PublishObjectFactory();
         });
     }
 

kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp

@@ -214,7 +214,12 @@ public:
             owner_.all_.splice(owner_.all_.end(), std::move(queue_));
         }
 
-        void clearForTests() noexcept { queue_.clear(); }
+        void clearForTests() noexcept {
+            for (auto& specialRef: queue_) {
+                specialRef.dispose();
+            }
+            queue_.clear();
+        }
 
         [[nodiscard("must be manually disposed")]] StableRef createStableRef(ObjHeader* object) noexcept;
         [[nodiscard("must be manually disposed")]] WeakRef createWeakRef(ObjHeader* object) noexcept;

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -63,7 +63,6 @@ public:
         // TODO: These use separate locks, which is inefficient.
         globalsThreadQueue_.Publish();
         specialRefRegistry_.publish();
-        gc_.Publish();
     }
 
     void ClearForTests() noexcept {

kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp

@@ -18,11 +18,13 @@ mm::ThreadRegistry& mm::ThreadRegistry::Instance() noexcept {
 }
 
 mm::ThreadRegistry::Node* mm::ThreadRegistry::RegisterCurrentThread() noexcept {
+    auto lock = list_.LockForIter();
     auto* threadDataNode = list_.Emplace(konan::currentThreadId());
     AssertThreadState(threadDataNode->Get(), ThreadState::kNative);
     Node*& currentDataNode = currentThreadDataNode_;
     RuntimeAssert(!IsCurrentThreadRegistered(), "This thread already had some data assigned to it.");
     currentDataNode = threadDataNode;
+    threadDataNode->Get()->gc().onThreadRegistration();
     return threadDataNode;
 }
 

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -248,10 +248,22 @@ void Kotlin_Internal_GC_GCInfoBuilder_setStartTime(KRef thiz, KLong value) {
 void Kotlin_Internal_GC_GCInfoBuilder_setEndTime(KRef thiz, KLong value) {
     throw std::runtime_error("Not implemented for tests");
 }
-void Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(KRef thiz, KLong value) {
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(KRef thiz, KLong value) {
     throw std::runtime_error("Not implemented for tests");
 }
-void Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(KRef thiz, KLong value) {
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(KRef thiz, KLong value) {
+    throw std::runtime_error("Not implemented for tests");
+}
+void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(KRef thiz, KLong value) {
+    throw std::runtime_error("Not implemented for tests");
+}
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(KRef thiz, KLong value) {
+    throw std::runtime_error("Not implemented for tests");
+}
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(KRef thiz, KLong value) {
+    throw std::runtime_error("Not implemented for tests");
+}
+void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(KRef thiz, KLong value) {
     throw std::runtime_error("Not implemented for tests");
 }
 void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong value) {