diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp index 676bf00a352..4a913188c88 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp @@ -18,6 +18,7 @@ using namespace kotlin; namespace { std::atomic weakRefBarrier = nullptr; +std::atomic weakProcessingEpoch = 0; ObjHeader* weakRefBarrierImpl(ObjHeader* weakReferee) noexcept { if (!weakReferee) return nullptr; @@ -36,47 +37,62 @@ NO_INLINE ObjHeader* weakRefReadSlowPath(std::atomic& weakReferee) n return barrier ? barrier(weak) : weak; } -void waitForThreadsToReachCheckpoint() { - // Reset checkpoint on all threads. - for (auto& thr : mm::ThreadRegistry::Instance().LockForIter()) { - thr.gc().impl().gc().barriers().resetCheckpoint(); - } - - mm::SafePointActivator safePointActivator; - - // Wait for all threads to either have passed safepoint or to be in the native state. - // Either of these mean that none of them are inside a weak reference accessing code. - mm::ThreadRegistry::Instance().waitAllThreads([](mm::ThreadData& thread) noexcept { - return thread.gc().impl().gc().barriers().visitedCheckpoint() || thread.suspensionData().suspendedOrNative(); - }); -} - } // namespace -void gc::BarriersThreadData::onCheckpoint() noexcept { - visitedCheckpoint_.store(true, std::memory_order_release); +void gc::BarriersThreadData::onThreadRegistration() noexcept { + if (weakRefBarrier.load(std::memory_order_acquire) != nullptr) { + startMarkingNewObjects(GCHandle::getByEpoch(weakProcessingEpoch.load(std::memory_order_relaxed))); + } } -void gc::BarriersThreadData::resetCheckpoint() noexcept { - visitedCheckpoint_.store(false, std::memory_order_release); +ALWAYS_INLINE void gc::BarriersThreadData::onSafePoint() noexcept {} + +void gc::BarriersThreadData::startMarkingNewObjects(gc::GCHandle gcHandle) noexcept { + RuntimeAssert(weakRefBarrier.load(std::memory_order_relaxed) != nullptr, "New allocations marking may only be requested by weak ref barriers"); + markHandle_ = gcHandle.mark(); } -bool gc::BarriersThreadData::visitedCheckpoint() const noexcept { - return visitedCheckpoint_.load(std::memory_order_acquire); +void gc::BarriersThreadData::stopMarkingNewObjects() noexcept { + RuntimeAssert(weakRefBarrier.load(std::memory_order_relaxed) == nullptr, "New allocations marking could only been requested by weak ref barriers"); + markHandle_ = std::nullopt; } -void gc::EnableWeakRefBarriers() noexcept { +bool gc::BarriersThreadData::shouldMarkNewObjects() const noexcept { + return markHandle_.has_value(); +} + +ALWAYS_INLINE void gc::BarriersThreadData::onAllocation(ObjHeader* allocated) { + if (compiler::concurrentWeakSweep()) { + bool shouldMark = shouldMarkNewObjects(); + bool barriersEnabled = weakRefBarrier.load(std::memory_order_relaxed) != nullptr; + RuntimeAssert(shouldMark == barriersEnabled, "New allocations marking must happen with and only with weak ref barriers"); + if (shouldMark) { + auto& objectData = objectDataForObject(allocated); + bool wasUnmarked = objectData.tryMark(); + RuntimeAssert(wasUnmarked, "No one else could mark this newly allocated object before"); + markHandle_->addObject(); + } + } +} + +void gc::EnableWeakRefBarriers(int64_t epoch) noexcept { + auto mutators = mm::ThreadRegistry::Instance().LockForIter(); + weakProcessingEpoch.store(epoch, std::memory_order_relaxed); weakRefBarrier.store(weakRefBarrierImpl, std::memory_order_seq_cst); + for (auto& mutator: mutators) { + mutator.gc().impl().gc().barriers().startMarkingNewObjects(GCHandle::getByEpoch(epoch)); + } } void gc::DisableWeakRefBarriers() noexcept { + auto mutators = mm::ThreadRegistry::Instance().LockForIter(); weakRefBarrier.store(nullptr, std::memory_order_seq_cst); - waitForThreadsToReachCheckpoint(); + for (auto& mutator: mutators) { + mutator.gc().impl().gc().barriers().stopMarkingNewObjects(); + } } -OBJ_GETTER(kotlin::gc::WeakRefRead, std::atomic& weakReferee) noexcept { - // TODO: Make this work with GCs that can stop thread at any point. - +OBJ_GETTER(gc::WeakRefRead, std::atomic& weakReferee) noexcept { if (!compiler::concurrentWeakSweep()) { RETURN_OBJ(weakReferee.load(std::memory_order_relaxed)); } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp index eef4d874fb9..95673756bf0 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp @@ -10,23 +10,26 @@ #include "Memory.h" #include "Utils.hpp" +#include "GCStatistics.hpp" namespace kotlin::gc { class BarriersThreadData : private Pinned { public: - void onCheckpoint() noexcept; - void resetCheckpoint() noexcept; - bool visitedCheckpoint() const noexcept; + void onThreadRegistration() noexcept; + void onSafePoint() noexcept; + + void startMarkingNewObjects(GCHandle gcHandle) noexcept; + void stopMarkingNewObjects() noexcept; + bool shouldMarkNewObjects() const noexcept; + void onAllocation(ObjHeader* allocated); private: - std::atomic visitedCheckpoint_ = false; + std::optional markHandle_{}; }; // Must be called during STW. -void EnableWeakRefBarriers() noexcept; - -// Must be called outside STW. +void EnableWeakRefBarriers(int64_t epoch) noexcept; void DisableWeakRefBarriers() noexcept; OBJ_GETTER(WeakRefRead, std::atomic& weakReferee) noexcept; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index 4207e004731..41da794a038 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -175,21 +175,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { RuntimeAssert(didSuspend, "Only GC thread can request suspension"); gcHandle.suspensionRequested(); - // TODO (WaitForThreadsReadyToMark()) - RuntimeAssert(!kotlin::mm::IsCurrentThreadRegistered(), "GC must run on unregistered thread"); - markDispatcher_.waitForThreadsPauseMutation(); GCLogDebug(epoch, "All threads have paused mutation"); gcHandle.threadsAreSuspended(); -#ifdef CUSTOM_ALLOCATOR - // This should really be done by each individual thread while waiting - for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { - thread.gc().impl().alloc().PrepareForGC(); - } - heap_.PrepareForGC(); -#endif - auto& scheduler = gcScheduler_; scheduler.onGCStart(); @@ -201,17 +190,8 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { mm::WaitForThreadsSuspension(); -#ifndef CUSTOM_ALLOCATOR - // Taking the locks before the pause is completed. So that any destroying thread - // would not publish into the global state at an unexpected time. - std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); - std::optional objectFactoryIterable = objectFactory_.LockForIter(); - checkMarkCorrectness(*objectFactoryIterable); -#endif - if (compiler::concurrentWeakSweep()) { - // Expected to happen inside STW. - gc::EnableWeakRefBarriers(); + EnableWeakRefBarriers(epoch); mm::ResumeThreads(); gcHandle.threadsAreResumed(); @@ -220,13 +200,42 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); if (compiler::concurrentWeakSweep()) { - // Expected to happen outside STW. - gc::DisableWeakRefBarriers(); - } else { - mm::ResumeThreads(); - gcHandle.threadsAreResumed(); + bool didSuspend = mm::RequestThreadsSuspension(); + RuntimeAssert(didSuspend, "Only GC thread can request suspension"); + gcHandle.suspensionRequested(); + + mm::WaitForThreadsSuspension(); + GCLogDebug(gcHandle.getEpoch(), "All threads have paused mutation"); + gcHandle.threadsAreSuspended(); + DisableWeakRefBarriers(); } + // TODO outline as mark_.isolateMarkedHeapAndFinishMark() + // By this point all the alive heap must be marked. + // All the mutations (incl. allocations) after this method will be subject for the next GC. +#ifdef CUSTOM_ALLOCATOR + // This should really be done by each individual thread while waiting + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + thread.gc().impl().alloc().PrepareForGC(); + } + auto& heap = mm::GlobalData::Instance().gc().impl().gc().heap(); + heap.PrepareForGC(); +#else + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + thread.gc().PublishObjectFactory(); + } + + // Taking the locks before the pause is completed. So that any destroying thread + // would not publish into the global state at an unexpected time. + std::optional objectFactoryIterable = objectFactory_.LockForIter(); + std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); + + checkMarkCorrectness(*objectFactoryIterable); +#endif + + mm::ResumeThreads(); + gcHandle.threadsAreResumed(); + #ifndef CUSTOM_ALLOCATOR gc::SweepExtraObjects>(gcHandle, *extraObjectFactoryIterable); extraObjectFactoryIterable = std::nullopt; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 6913da5cac3..6028c41c0d7 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -38,7 +38,9 @@ public: void OnSuspendForGC() noexcept; - void safePoint() noexcept { barriers_.onCheckpoint(); } + void safePoint() noexcept { barriers_.onSafePoint(); } + + void onThreadRegistration() noexcept { barriers_.onThreadRegistration(); } BarriersThreadData& barriers() noexcept { return barriers_; } diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 312ac4a7fa4..93163c89d65 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -21,6 +21,7 @@ #include "ObjectTestSupport.hpp" #include "SafePoint.hpp" #include "SingleThreadExecutor.hpp" +#include "StableRef.hpp" #include "TestSupport.hpp" #include "ThreadData.hpp" #include "WeakRef.hpp" @@ -1034,6 +1035,51 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { } } +TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeakNewObj) { + std_support::vector mutators(kDefaultThreadCount); + + // Make sure all mutators are initialized. + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + std_support::vector> gcFutures; + auto epoch = mm::GlobalData::Instance().gc().Schedule(); + std::atomic gcDone = false; + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (auto& mutator : mutators) { + gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { + mm::safePoint(threadData); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { + ObjHolder holder; + return InstallWeakReference(threadData, object.header(), holder.slot()); + })(); + EXPECT_NE(objectWeak.get(), nullptr); + + auto& extraObj = *mm::ExtraObjectData::Get(object.header()); + extraObj.ClearRegularWeakReferenceImpl(); + extraObj.Uninstall(); + mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj); + + while (!gcDone.load(std::memory_order_relaxed)) { + mm::safePoint(threadData); + } + })); + } + + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); + gcDone.store(true, std::memory_order_relaxed); + + for (auto& future : gcFutures) { + future.wait(); + } +} TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { std_support::vector mutators(kDefaultThreadCount); @@ -1104,31 +1150,6 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { future.wait(); } -#ifndef CUSTOM_ALLOCATOR - // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything). - - std_support::vector expectedAlive; - for (int i = 0; i < kDefaultThreadCount; ++i) { - expectedAlive.push_back(globals[i]); - expectedAlive.push_back(locals[i]); - expectedAlive.push_back(reachables[i]); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - std_support::vector aliveForThisThread(expectedAlive.begin(), expectedAlive.end()); - aliveForThisThread.push_back(globals[kDefaultThreadCount + i]); - aliveForThisThread.push_back(locals[kDefaultThreadCount + i]); - aliveForThisThread.push_back(reachables[kDefaultThreadCount + i]); - // Unreachables for new threads were not collected. - aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]); - EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread)); - } -#else - // Custom allocator does not have a notion of objects alive only for some thread std_support::vector expectedAlive; for (int i = 0; i < kDefaultThreadCount; ++i) { expectedAlive.push_back(globals[i]); @@ -1140,9 +1161,27 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { // Unreachables for new threads were not collected. expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); } - // All threads see the same alive objects with the custom alloctor, enough to check a single mutator. + +#ifndef CUSTOM_ALLOCATOR + // Force mutators to publish their internal heaps + std_support::vector> publishFutures; + for (auto& mutator: mutators) { + publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PublishObjectFactory(); + })); + } + for (auto& mutator: newMutators) { + publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PublishObjectFactory(); + })); + } + for (auto& future : publishFutures) { + future.wait(); + } +#endif + + // All threads see the same alive objects, enough to check a single mutator. EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); -#endif // CUSTOM_ALLOCATOR } TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 2a84ca89cd0..4308caa424e 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -19,7 +19,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Publish() noexcept { +void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); @@ -36,19 +36,25 @@ void gc::GC::ThreadData::ClearForTests() noexcept { } ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeInfo) noexcept { + ObjHeader* obj; #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); + obj = impl_->objectFactoryThreadQueue().CreateObject(typeInfo); #else - return impl_->alloc().CreateObject(typeInfo); + obj = impl_->alloc().CreateObject(typeInfo); #endif + impl().gc().barriers().onAllocation(obj); + return obj; } ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { + ArrayHeader* arr; #ifndef CUSTOM_ALLOCATOR - return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); + arr = impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); #else - return impl_->alloc().CreateArray(typeInfo, elements); + arr = impl_->alloc().CreateArray(typeInfo, elements); #endif + impl().gc().barriers().onAllocation(arr->obj()); + return arr; } ALWAYS_INLINE mm::ExtraObjectData& gc::GC::ThreadData::CreateExtraObjectDataForObject( @@ -76,6 +82,10 @@ void gc::GC::ThreadData::safePoint() noexcept { impl_->gc().safePoint(); } +void gc::GC::ThreadData::onThreadRegistration() noexcept { + impl_->gc().onThreadRegistration(); +} + gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} gc::GC::~GC() = default; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp index 621d3d4f79e..82be3d73836 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp @@ -169,7 +169,7 @@ private: GCHandle gcHandle_ = GCHandle::invalid(); MarkPacer pacer_; std::optional lockedMutatorsList_; - ManuallyScoped parallelProcessor_; + ManuallyScoped parallelProcessor_{}; std::mutex workerCreationMutex_; std::atomic activeWorkersCount_ = 0; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index af39b14c44f..385ac6b3c5f 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -35,7 +35,7 @@ public: Impl& impl() noexcept { return *impl_; } - void Publish() noexcept; + void PublishObjectFactory() noexcept; void ClearForTests() noexcept; ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; @@ -47,6 +47,8 @@ public: void safePoint() noexcept; + void onThreadRegistration() noexcept; + private: std_support::unique_ptr impl_; }; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp index cad3841a186..fcc76cbc296 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.cpp @@ -20,8 +20,12 @@ extern "C" { void Kotlin_Internal_GC_GCInfoBuilder_setEpoch(KRef thiz, KLong value); void Kotlin_Internal_GC_GCInfoBuilder_setStartTime(KRef thiz, KLong value); void Kotlin_Internal_GC_GCInfoBuilder_setEndTime(KRef thiz, KLong value); -void Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(KRef thiz, KLong value); -void Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(KRef thiz, KLong value); +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(KRef thiz, KLong value); void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong value); void Kotlin_Internal_GC_GCInfoBuilder_setRootSet(KRef thiz, KLong threadLocalReferences, KLong stackReferences, @@ -77,8 +81,15 @@ struct GCInfo { std::optional epoch; std::optional startTime; // time since process start std::optional endTime; - std::optional pauseStartTime; - std::optional pauseEndTime; + + std::optional firstPauseRequestTime; + std::optional firstPauseStartTime; + std::optional firstPauseEndTime; + + std::optional secondPauseRequestTime; + std::optional secondPauseStartTime; + std::optional secondPauseEndTime; + std::optional finalizersDoneTime; std::optional rootSet; std::optional markStats; @@ -91,8 +102,12 @@ struct GCInfo { Kotlin_Internal_GC_GCInfoBuilder_setEpoch(builder, static_cast(*epoch)); if (startTime) Kotlin_Internal_GC_GCInfoBuilder_setStartTime(builder, *startTime); if (endTime) Kotlin_Internal_GC_GCInfoBuilder_setEndTime(builder, *endTime); - if (pauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(builder, *pauseStartTime); - if (pauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(builder, *pauseEndTime); + if (firstPauseRequestTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(builder, *firstPauseRequestTime); + if (firstPauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(builder, *firstPauseStartTime); + if (firstPauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(builder, *firstPauseEndTime); + if (secondPauseRequestTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(builder, *secondPauseRequestTime); + if (secondPauseStartTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(builder, *secondPauseStartTime); + if (secondPauseEndTime) Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(builder, *secondPauseEndTime); if (finalizersDoneTime) Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(builder, *finalizersDoneTime); if (rootSet) Kotlin_Internal_GC_GCInfoBuilder_setRootSet( @@ -223,9 +238,21 @@ void GCHandle::finished() { epoch_, "Heap memory usage: before %" PRIu64 " bytes, after %" PRIu64 " bytes", stat->memoryUsageBefore.heap->sizeBytes, stat->memoryUsageAfter.heap->sizeBytes); } - if (stat->pauseStartTime && stat->pauseEndTime) { - auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000; - GCLogInfo(epoch_, "Mutators pause time: %" PRIu64 " microseconds.", time); + if (stat->firstPauseRequestTime && stat->firstPauseStartTime) { + auto time = (*stat->firstPauseStartTime - *stat->firstPauseRequestTime) / 1000; + GCLogInfo(epoch_, "Time to pause #1: %" PRIu64 " microseconds.", time); + } + if (stat->firstPauseRequestTime && stat->firstPauseEndTime) { + auto time = (*stat->firstPauseEndTime - *stat->firstPauseRequestTime) / 1000; + GCLogInfo(epoch_, "Mutators pause time #1: %" PRIu64 " microseconds.", time); + } + if (stat->secondPauseRequestTime && stat->secondPauseStartTime) { + auto time = (*stat->secondPauseStartTime - *stat->secondPauseRequestTime) / 1000; + GCLogInfo(epoch_, "Time to pause #2: %" PRIu64 " microseconds.", time); + } + if (stat->secondPauseRequestTime && stat->secondPauseEndTime) { + auto time = (*stat->secondPauseEndTime - *stat->secondPauseRequestTime) / 1000; + GCLogInfo(epoch_, "Mutators pause time #2: %" PRIu64 " microseconds.", time); } if (stat->startTime) { auto time = (*current.endTime - *current.startTime) / 1000; @@ -242,14 +269,30 @@ void GCHandle::suspensionRequested() { std::lock_guard guard(lock); GCLogDebug(epoch_, "Requested thread suspension"); if (auto* stat = statByEpoch(epoch_)) { - stat->pauseStartTime = static_cast(konan::getTimeNanos()); + auto requestTime = static_cast(konan::getTimeNanos()); + if (!stat->firstPauseRequestTime) { + stat->firstPauseRequestTime = requestTime; + } else { + RuntimeAssert(!stat->secondPauseRequestTime, "GCStatistics support max two pauses per GC epoch"); + stat->secondPauseRequestTime = requestTime; + } } } void GCHandle::threadsAreSuspended() { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { - if (stat->pauseStartTime) { - auto time = (konan::getTimeNanos() - *stat->pauseStartTime) / 1000; + auto startTime = static_cast(konan::getTimeNanos()); + std::optional requestTime; + if (!stat->firstPauseStartTime) { + stat->firstPauseStartTime = startTime; + requestTime = stat->firstPauseRequestTime; + } else { + RuntimeAssert(!stat->secondPauseStartTime, "GCStatistics support max two pauses per GC epoch"); + stat->secondPauseStartTime = startTime; + requestTime = stat->secondPauseRequestTime; + } + if (requestTime) { + auto time = (startTime - *requestTime) / 1000; GCLogDebug(epoch_, "Suspended all threads in %" PRIu64 " microseconds", time); return; } @@ -267,9 +310,18 @@ void GCHandle::threadsAreSuspended() { void GCHandle::threadsAreResumed() { std::lock_guard guard(lock); if (auto* stat = statByEpoch(epoch_)) { - stat->pauseEndTime = static_cast(konan::getTimeNanos()); - if (stat->pauseStartTime) { - auto time = (*stat->pauseEndTime - *stat->pauseStartTime) / 1000; + auto endTime = static_cast(konan::getTimeNanos()); + std::optional startTime; + if (!stat->firstPauseEndTime) { + stat->firstPauseEndTime = endTime; + startTime = stat->firstPauseStartTime; + } else { + RuntimeAssert(!stat->secondPauseEndTime, "GCStatistics support max two pauses per GC epoch"); + stat->secondPauseEndTime = endTime; + startTime = stat->secondPauseStartTime; + } + if (startTime) { + auto time = (endTime - *startTime) / 1000; GCLogDebug(epoch_, "Resume all threads. Total pause time is %" PRId64 " microseconds.", time); return; } @@ -358,6 +410,8 @@ void GCHandle::sweptExtraObjects(gc::SweepStats stats) noexcept { } } +GCHandle::GCMarkScope GCHandle::mark() { return GCMarkScope(*this); } + GCHandle::GCSweepScope::GCSweepScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} GCHandle::GCSweepScope::~GCSweepScope() { @@ -397,11 +451,27 @@ GCHandle::GCThreadRootSetScope::~GCThreadRootSetScope(){ threadData_.threadId(), stackRoots_, threadLocalRoots_, getStageTime()); } -GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle){} +void GCHandle::GCMarkScope::swap(GCHandle::GCMarkScope& other) noexcept { + std::swap(handle_, other.handle_); + std::swap(startTime_, other.startTime_); +} + +GCHandle::GCMarkScope::GCMarkScope(kotlin::gc::GCHandle& handle) : handle_(handle) {} + +GCHandle::GCMarkScope::GCMarkScope(GCHandle::GCMarkScope&& that) noexcept { + swap(that); +} + +GCHandle::GCMarkScope& GCHandle::GCMarkScope::operator=(GCHandle::GCMarkScope that) noexcept { + swap(that); + return *this; +} GCHandle::GCMarkScope::~GCMarkScope() { - handle_.marked(stats_); - GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime()); + if (handle_.isValid()) { + handle_.marked(stats_); + GCLogDebug(handle_.getEpoch(), "Marked %" PRIu64 " objects in %" PRIu64 " microseconds.", stats_.markedCount, getStageTime()); + } } gc::GCHandle::GCProcessWeaksScope::GCProcessWeaksScope(gc::GCHandle& handle) noexcept : handle_(handle) {} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp index e4447086b6c..82157d1d970 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCStatistics.hpp @@ -95,16 +95,7 @@ public: void addThreadLocalRoot() { threadLocalRoots_++; } }; - class GCMarkScope : GCStageScopeUsTimer, Pinned { - GCHandle& handle_; - MarkStats stats_; - - public: - explicit GCMarkScope(GCHandle& handle); - ~GCMarkScope(); - - void addObject() noexcept { ++stats_.markedCount; } - }; + class GCMarkScope; class GCProcessWeaksScope : GCStageScopeUsTimer, Pinned { GCHandle& handle_; @@ -151,9 +142,24 @@ public: GCSweepExtraObjectsScope sweepExtraObjects() { return GCSweepExtraObjectsScope(*this); } GCGlobalRootSetScope collectGlobalRoots() { return GCGlobalRootSetScope(*this); } GCThreadRootSetScope collectThreadRoots(mm::ThreadData& threadData) { return GCThreadRootSetScope(*this, threadData); } - GCMarkScope mark() { return GCMarkScope(*this); } + GCMarkScope mark(); GCProcessWeaksScope processWeaks() noexcept { return GCProcessWeaksScope(*this); } MarkStats getMarked(); }; + +class GCHandle::GCMarkScope : GCStageScopeUsTimer { + GCHandle handle_ = GCHandle::invalid(); + MarkStats stats_; + + void swap(GCMarkScope& other) noexcept; + +public: + explicit GCMarkScope(GCHandle& handle); + GCMarkScope(GCMarkScope&& that) noexcept; + GCMarkScope& operator=(GCMarkScope that) noexcept; + ~GCMarkScope(); + + void addObject() noexcept { ++stats_.markedCount; } +}; } diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 081b49adc41..ae33fdd2d96 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -20,7 +20,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Publish() noexcept { +void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); @@ -73,6 +73,8 @@ void gc::GC::ThreadData::OnSuspendForGC() noexcept { } void gc::GC::ThreadData::safePoint() noexcept {} +void gc::GC::ThreadData::onThreadRegistration() noexcept {} + gc::GC::GC(gcScheduler::GCScheduler&) noexcept : impl_(std_support::make_unique()) {} gc::GC::~GC() = default; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 3b54c9bc996..8f7deb2d586 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -20,7 +20,7 @@ gc::GC::ThreadData::ThreadData(GC& gc, mm::ThreadData& threadData) noexcept : im gc::GC::ThreadData::~ThreadData() = default; -void gc::GC::ThreadData::Publish() noexcept { +void gc::GC::ThreadData::PublishObjectFactory() noexcept { #ifndef CUSTOM_ALLOCATOR impl_->extraObjectDataFactoryThreadQueue().Publish(); impl_->objectFactoryThreadQueue().Publish(); @@ -73,6 +73,8 @@ void gc::GC::ThreadData::OnSuspendForGC() noexcept { } void gc::GC::ThreadData::safePoint() noexcept {} +void gc::GC::ThreadData::onThreadRegistration() noexcept {} + gc::GC::GC(gcScheduler::GCScheduler& gcScheduler) noexcept : impl_(std_support::make_unique(gcScheduler)) {} gc::GC::~GC() = default; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 31855aceec2..5041890376c 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -99,6 +99,10 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); #ifndef CUSTOM_ALLOCATOR + for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) { + thread.gc().PublishObjectFactory(); + } + // Taking the locks before the pause is completed. So that any destroying thread // would not publish into the global state at an unexpected time. std::optional extraObjectFactoryIterable = extraObjectDataFactory_.LockForIter(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 5d38e22df44..990e8390c3a 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -1010,6 +1010,52 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { } } +TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeakNewObj) { + std_support::vector mutators(kDefaultThreadCount); + + // Make sure all mutators are initialized. + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); + } + + std_support::vector> gcFutures; + auto epoch = mm::GlobalData::Instance().gc().Schedule(); + std::atomic gcDone = false; + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) { + } + + for (auto& mutator : mutators) { + gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { + mm::safePoint(threadData); + + auto& object = AllocateObject(threadData); + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { + ObjHolder holder; + return InstallWeakReference(threadData, object.header(), holder.slot()); + })(); + EXPECT_NE(objectWeak.get(), nullptr); + + auto& extraObj = *mm::ExtraObjectData::Get(object.header()); + extraObj.ClearRegularWeakReferenceImpl(); + extraObj.Uninstall(); + mm::GlobalData::Instance().gc().DestroyExtraObjectData(extraObj); + + while (!gcDone.load(std::memory_order_relaxed)) { + mm::safePoint(threadData); + } + })); + } + + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); + gcDone.store(true, std::memory_order_relaxed); + + for (auto& future : gcFutures) { + future.wait(); + } +} + TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { std_support::vector mutators(kDefaultThreadCount); std_support::vector globals(2 * kDefaultThreadCount); @@ -1079,31 +1125,6 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { future.wait(); } -#ifndef CUSTOM_ALLOCATOR - // Old mutators don't even see alive objects from the new threads yet (as the latter ones have not published anything). - - std_support::vector expectedAlive; - for (int i = 0; i < kDefaultThreadCount; ++i) { - expectedAlive.push_back(globals[i]); - expectedAlive.push_back(locals[i]); - expectedAlive.push_back(reachables[i]); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAreArray(expectedAlive)); - } - - for (int i = 0; i < kDefaultThreadCount; ++i) { - std_support::vector aliveForThisThread(expectedAlive.begin(), expectedAlive.end()); - aliveForThisThread.push_back(globals[kDefaultThreadCount + i]); - aliveForThisThread.push_back(locals[kDefaultThreadCount + i]); - aliveForThisThread.push_back(reachables[kDefaultThreadCount + i]); - // Unreachables for new threads were not collected. - aliveForThisThread.push_back(unreachables[kDefaultThreadCount + i]); - EXPECT_THAT(newMutators[i].Alive(), testing::UnorderedElementsAreArray(aliveForThisThread)); - } -#else - // Custom allocator does not have a notion of objects alive only for some thread std_support::vector expectedAlive; for (int i = 0; i < kDefaultThreadCount; ++i) { expectedAlive.push_back(globals[i]); @@ -1115,9 +1136,27 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { // Unreachables for new threads were not collected. expectedAlive.push_back(unreachables[kDefaultThreadCount + i]); } - // All threads see the same alive objects with the custom alloctor, enough to check a single mutator. + +#ifndef CUSTOM_ALLOCATOR + // Force mutators to publish their internal heaps + std_support::vector> publishFutures; + for (auto& mutator: mutators) { + publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PublishObjectFactory(); + })); + } + for (auto& mutator: newMutators) { + publishFutures.emplace_back(mutator.Execute([](mm::ThreadData& threadData, Mutator& mutator) { + threadData.gc().PublishObjectFactory(); + })); + } + for (auto& future : publishFutures) { + future.wait(); + } +#endif + + // All threads see the same alive objects, enough to check a single mutator. EXPECT_THAT(mutators[0].Alive(), testing::UnorderedElementsAreArray(expectedAlive)); -#endif // CUSTOM_ALLOCATOR } diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt index 5c435d69742..e2032fc7793 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/gc/GCInfo.kt @@ -53,10 +53,16 @@ public class RootSetStatistics( * @property startTimeNs Time, when garbage collector run is started, meausered by [kotlin.system.getTimeNanos]. * @property endTimeNs Time, when garbage collector run is ended, measured by [kotlin.system.getTimeNanos]. * After this point, most of the memory is reclaimed, and a new garbage collector run can start. - * @property pauseStartTimeNs Time, when mutator threads are suspended, mesured by [kotlin.system.getTimeNanos]. - * @property pauseEndTimeNs Time, when mutator threads are unsuspended, mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the first time, + * mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseStartTimeNs Time, when mutator threads are suspended for the first time, mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseEndTimeNs Time, when mutator threads are unsuspended for the first time, mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the second time, + * mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseStartTimeNs Time, when mutator threads are suspended for the second time, mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseEndTimeNs Time, when mutator threads are unsuspended for the second time, mesured by [kotlin.system.getTimeNanos]. * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos]. - * If null, memory reclamation is still in progress. + * If null, memory reclamation is still in progress. * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details. * @property memoryUsageAfter Memory usage at the start of garbage collector run, separated by memory pools. * The set of memory pools depends on the collector implementation. @@ -74,8 +80,12 @@ public class GCInfo( val epoch: Long, val startTimeNs: Long, val endTimeNs: Long, - val pauseStartTimeNs: Long, - val pauseEndTimeNs: Long, + val firstPauseRequestTimeNs: Long, + val firstPauseStartTimeNs: Long, + val firstPauseEndTimeNs: Long, + val secondPauseRequestTimeNs: Long?, + val secondPauseStartTimeNs: Long?, + val secondPauseEndTimeNs: Long?, val postGcCleanupTimeNs: Long?, val rootSet: RootSetStatistics, val memoryUsageBefore: Map, @@ -89,8 +99,12 @@ public class GCInfo( info.epoch, info.startTimeNs, info.endTimeNs, - info.pauseStartTimeNs, - info.pauseEndTimeNs, + info.firstPauseRequestTimeNs, + info.firstPauseStartTimeNs, + info.firstPauseEndTimeNs, + info.secondPauseRequestTimeNs, + info.secondPauseStartTimeNs, + info.secondPauseEndTimeNs, info.postGcCleanupTimeNs, info.rootSet.let { RootSetStatistics( diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt index 92d22b72c32..5cbb371c945 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/runtime/GCInfo.kt @@ -71,8 +71,14 @@ public class RootSetStatistics( * @property startTimeNs Time, when garbage collector run is started, meausered by [kotlin.system.getTimeNanos]. * @property endTimeNs Time, when garbage collector run is ended, measured by [kotlin.system.getTimeNanos]. * After this point, most of the memory is reclaimed, and a new garbage collector run can start. - * @property pauseStartTimeNs Time, when mutator threads are suspended, mesured by [kotlin.system.getTimeNanos]. - * @property pauseEndTimeNs Time, when mutator threads are unsuspended, mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the first time, + * mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseStartTimeNs Time, when mutator threads are suspended for the first time, mesured by [kotlin.system.getTimeNanos]. + * @property firstPauseEndTimeNs Time, when mutator threads are unsuspended for the first time, mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseRequestTimeNs Time, when the garbage collector thread requested suspension of mutator threads for the second time, + * mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseStartTimeNs Time, when mutator threads are suspended for the second time, mesured by [kotlin.system.getTimeNanos]. + * @property secondPauseEndTimeNs Time, when mutator threads are unsuspended for the second time, mesured by [kotlin.system.getTimeNanos]. * @property postGcCleanupTimeNs Time, when all memory is reclaimed, measured by [kotlin.system.getTimeNanos]. * If null, memory reclamation is still in progress. * @property rootSet The number of objects in each root set pool. Check [RootSetStatistics] doc for details. @@ -93,8 +99,12 @@ public class GCInfo( val epoch: Long, val startTimeNs: Long, val endTimeNs: Long, - val pauseStartTimeNs: Long, - val pauseEndTimeNs: Long, + val firstPauseRequestTimeNs: Long, + val firstPauseStartTimeNs: Long, + val firstPauseEndTimeNs: Long, + val secondPauseRequestTimeNs: Long?, + val secondPauseStartTimeNs: Long?, + val secondPauseEndTimeNs: Long?, val postGcCleanupTimeNs: Long?, val rootSet: RootSetStatistics, val markedCount: Long, @@ -116,8 +126,12 @@ private class GCInfoBuilder() { var epoch: Long? = null var startTimeNs: Long? = null var endTimeNs: Long? = null - var pauseStartTimeNs: Long? = null - var pauseEndTimeNs: Long? = null + var firstPauseRequestTimeNs: Long? = null + var firstPauseStartTimeNs: Long? = null + var firstPauseEndTimeNs: Long? = null + var secondPauseRequestTimeNs: Long? = null + var secondPauseStartTimeNs: Long? = null + var secondPauseEndTimeNs: Long? = null var postGcCleanupTimeNs: Long? = null var rootSet: RootSetStatistics? = null var markedCount: Long? = null @@ -140,14 +154,34 @@ private class GCInfoBuilder() { endTimeNs = value } - @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime") - private fun setPauseStartTime(value: Long) { - pauseStartTimeNs = value + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime") + private fun setFirstPauseRequestTime(value: Long) { + firstPauseRequestTimeNs = value } - @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime") - private fun setPauseEndTime(value: Long) { - pauseEndTimeNs = value + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime") + private fun setFirstPauseStartTime(value: Long) { + firstPauseStartTimeNs = value + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime") + private fun setFirstPauseEndTime(value: Long) { + firstPauseEndTimeNs = value + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime") + private fun setSecondPauseRequestTime(value: Long) { + secondPauseRequestTimeNs = value + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime") + private fun setSecondPauseStartTime(value: Long) { + secondPauseStartTimeNs = value + } + + @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime") + private fun setSecondPauseEndTime(value: Long) { + secondPauseEndTimeNs = value } @ExportForCppRuntime("Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime") @@ -191,8 +225,12 @@ private class GCInfoBuilder() { epoch ?: return null, startTimeNs ?: return null, endTimeNs ?: return null, - pauseStartTimeNs ?: return null, - pauseEndTimeNs ?: return null, + firstPauseRequestTimeNs ?: return null, + firstPauseStartTimeNs ?: return null, + firstPauseEndTimeNs ?: return null, + secondPauseRequestTimeNs, + secondPauseStartTimeNs, + secondPauseEndTimeNs, postGcCleanupTimeNs, rootSet ?: return null, markedCount ?: return null, diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index 25b2c42aaf1..9f4ac4c5822 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -78,7 +78,7 @@ TEST_F(ExtraObjectDataTest, ConcurrentInstall) { } auto& extraData = mm::ExtraObjectData::Install(object.header()); actual[i] = &extraData; - mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->Publish(); + mm::GlobalData::Instance().threadRegistry().CurrentThreadData()->gc().PublishObjectFactory(); }); } diff --git a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp index bffc34fb383..cd53b34f110 100644 --- a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp +++ b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp @@ -214,7 +214,12 @@ public: owner_.all_.splice(owner_.all_.end(), std::move(queue_)); } - void clearForTests() noexcept { queue_.clear(); } + void clearForTests() noexcept { + for (auto& specialRef: queue_) { + specialRef.dispose(); + } + queue_.clear(); + } [[nodiscard("must be manually disposed")]] StableRef createStableRef(ObjHeader* object) noexcept; [[nodiscard("must be manually disposed")]] WeakRef createWeakRef(ObjHeader* object) noexcept; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 77cee8eb21e..62ed319537d 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -63,7 +63,6 @@ public: // TODO: These use separate locks, which is inefficient. globalsThreadQueue_.Publish(); specialRefRegistry_.publish(); - gc_.Publish(); } void ClearForTests() noexcept { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp index 7d7c9f2ce71..91a56cd5637 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp @@ -18,11 +18,13 @@ mm::ThreadRegistry& mm::ThreadRegistry::Instance() noexcept { } mm::ThreadRegistry::Node* mm::ThreadRegistry::RegisterCurrentThread() noexcept { + auto lock = list_.LockForIter(); auto* threadDataNode = list_.Emplace(konan::currentThreadId()); AssertThreadState(threadDataNode->Get(), ThreadState::kNative); Node*& currentDataNode = currentThreadDataNode_; RuntimeAssert(!IsCurrentThreadRegistered(), "This thread already had some data assigned to it."); currentDataNode = threadDataNode; + threadDataNode->Get()->gc().onThreadRegistration(); return threadDataNode; } diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index 7c34503b1b1..5a6c01bf5f5 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -248,10 +248,22 @@ void Kotlin_Internal_GC_GCInfoBuilder_setStartTime(KRef thiz, KLong value) { void Kotlin_Internal_GC_GCInfoBuilder_setEndTime(KRef thiz, KLong value) { throw std::runtime_error("Not implemented for tests"); } -void Kotlin_Internal_GC_GCInfoBuilder_setPauseStartTime(KRef thiz, KLong value) { +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseRequestTime(KRef thiz, KLong value) { throw std::runtime_error("Not implemented for tests"); } -void Kotlin_Internal_GC_GCInfoBuilder_setPauseEndTime(KRef thiz, KLong value) { +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseStartTime(KRef thiz, KLong value) { + throw std::runtime_error("Not implemented for tests"); +} +void Kotlin_Internal_GC_GCInfoBuilder_setFirstPauseEndTime(KRef thiz, KLong value) { + throw std::runtime_error("Not implemented for tests"); +} +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseRequestTime(KRef thiz, KLong value) { + throw std::runtime_error("Not implemented for tests"); +} +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseStartTime(KRef thiz, KLong value) { + throw std::runtime_error("Not implemented for tests"); +} +void Kotlin_Internal_GC_GCInfoBuilder_setSecondPauseEndTime(KRef thiz, KLong value) { throw std::runtime_error("Not implemented for tests"); } void Kotlin_Internal_GC_GCInfoBuilder_setPostGcCleanupTime(KRef thiz, KLong value) {