diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index d4449b7ae1f..bf0f408e862 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -186,7 +186,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } ?: arg } - private val defaultGcMarkSingleThreaded get() = target.family == Family.MINGW + private val defaultGcMarkSingleThreaded get() = target.family == Family.MINGW && gc == GC.PARALLEL_MARK_CONCURRENT_SWEEP val gcMarkSingleThreaded: Boolean by lazy { configuration.get(BinaryOptions.gcMarkSingleThreaded) ?: defaultGcMarkSingleThreaded @@ -203,6 +203,12 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration "Mutators cooperation is not supported during single threaded mark") } false + } else if (gc == GC.CONCURRENT_MARK_AND_SWEEP) { + if (mutatorsCooperate == true) { + configuration.report(CompilerMessageSeverity.STRONG_WARNING, + "Mutators cooperation is not yet supported in CMS GC") + } + false } else { mutatorsCooperate ?: true } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLogging.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLogging.kt index e07486815f7..9f7b8d70de4 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLogging.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/RuntimeLogging.kt @@ -30,6 +30,7 @@ enum class LoggingTag(val ord: Int) { Alloc(6), Balancing(7), Barriers(8), + GCMark(9), ; companion object { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 841d2be1241..ef44f283336 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -481,7 +481,6 @@ internal class CodegenLlvmHelpers(private val generationState: NativeGenerationS val Kotlin_processObjectInMark by lazyRtFunction val Kotlin_processArrayInMark by lazyRtFunction - val Kotlin_processFieldInMark by lazyRtFunction val Kotlin_processEmptyObjectInMark by lazyRtFunction val UpdateVolatileHeapRef by lazyRtFunction diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/EscapeAnalysis.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/EscapeAnalysis.kt index 7223f12e34d..0a2ecb218c2 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/EscapeAnalysis.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/EscapeAnalysis.kt @@ -1828,6 +1828,10 @@ internal object EscapeAnalysis { IntraproceduralAnalysis(context, moduleDFG, externalModulesDFG, callGraph).analyze() InterproceduralAnalysis(context, generationState, callGraph, intraproceduralAnalysisResult, externalModulesDFG, lifetimes, propagateExiledToHeapObjects = context.config.memoryModel != MemoryModel.EXPERIMENTAL + // The GC must be careful not to scan exiled objects, that have already became dead, + // as they may reference other already destroyed stack-allocated objects. + // TODO somehow tag these object, so that GC could handle them properly. + || context.config.gc == GC.CONCURRENT_MARK_AND_SWEEP ).analyze() } catch (t: Throwable) { val extraUserInfo = diff --git a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp index 706d1257705..25e7e853ab3 100644 --- a/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp +++ b/kotlin-native/runtime/src/compiler_interface/cpp/CompilerCInterface.cpp @@ -81,7 +81,6 @@ touchFunction(Kotlin_mm_safePointWhileLoopBody) touchFunction(Kotlin_processObjectInMark) touchFunction(Kotlin_processArrayInMark) -touchFunction(Kotlin_processFieldInMark) touchFunction(Kotlin_processEmptyObjectInMark) touchFunction(Kotlin_arrayGetElementAddress) diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp index dbd14300d64..461c48a94e1 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ @@ -33,12 +33,14 @@ void gc::barriers::BarriersThreadData::onThreadRegistration() noexcept { ALWAYS_INLINE void gc::barriers::BarriersThreadData::onSafePoint() noexcept {} void gc::barriers::BarriersThreadData::startMarkingNewObjects(gc::GCHandle gcHandle) noexcept { - RuntimeAssert(markBarriersEnabled.load(std::memory_order_relaxed), "New allocations marking may only be requested by mark barriers"); + RuntimeAssert(markBarriersEnabled.load(std::memory_order_relaxed), + "New allocations marking may only be requested by mark barriers"); markHandle_ = gcHandle.mark(); } void gc::barriers::BarriersThreadData::stopMarkingNewObjects() noexcept { - RuntimeAssert(!markBarriersEnabled.load(std::memory_order_relaxed), "New allocations marking could only been requested by mark barriers"); + RuntimeAssert(!markBarriersEnabled.load(std::memory_order_relaxed), + "New allocations marking could only been requested by mark barriers"); markHandle_ = std::nullopt; } @@ -48,7 +50,8 @@ bool gc::barriers::BarriersThreadData::shouldMarkNewObjects() const noexcept { ALWAYS_INLINE void gc::barriers::BarriersThreadData::onAllocation(ObjHeader* allocated) { bool shouldMark = shouldMarkNewObjects(); - RuntimeAssert(shouldMark == markBarriersEnabled.load(std::memory_order_relaxed), "New allocations marking must happen with and only with mark barriers"); + RuntimeAssert(shouldMark == markBarriersEnabled.load(std::memory_order_relaxed), + "New allocations marking must happen with and only with mark barriers"); BarriersLogDebug(shouldMark, "Allocation %p", allocated); if (shouldMark) { auto& objectData = alloc::objectDataForObject(allocated); @@ -87,7 +90,7 @@ NO_INLINE void beforeHeapRefUpdateSlowPath(mm::DirectRefAccessor ref, ObjHeader* // TODO perhaps it would be better to path the thread data from outside auto& threadData = *mm::ThreadRegistry::Instance().CurrentThreadData(); auto& markQueue = *threadData.gc().impl().gc().mark().markQueue(); - gc::mark::ParallelMark::MarkTraits::tryEnqueue(markQueue, prev); + gc::mark::ConcurrentMark::MarkTraits::tryEnqueue(markQueue, prev); // No need to add the marked object in statistics here. // Objects will be counted on dequeue. } @@ -102,3 +105,30 @@ ALWAYS_INLINE void gc::barriers::beforeHeapRefUpdate(mm::DirectRefAccessor ref, BarriersLogDebug(false, "Write *%p <- %p (%p overwritten)", ref.location(), value, ref.load()); } } + +namespace { + +// TODO decide whether it's really beneficial to NO_INLINE the slow path +NO_INLINE void weakRefReadInMarkSlowPath(ObjHeader* weakReferee) noexcept { + auto& threadData = *mm::ThreadRegistry::Instance().CurrentThreadData(); + auto& markQueue = *threadData.gc().impl().gc().mark().markQueue(); + gc::mark::ConcurrentMark::MarkTraits::tryEnqueue(markQueue, weakReferee); +} + +} // namespace + + +ALWAYS_INLINE OBJ_GETTER(gc::barriers::weakRefReadBarrier, std::atomic& weakReferee) noexcept { + // TODO be careful with atomics when conucrrent weak sweep is supported + auto weak = weakReferee.load(std::memory_order_relaxed); + if (!weak) return nullptr; + bool mark = markBarriersEnabled.load(std::memory_order_acquire); + if (__builtin_expect(mark, false)) { + BarriersLogDebug(true, "[mark] Weak read %p", weak); + weakRefReadInMarkSlowPath(weak); + } else { + // TODO reintroduce after-mark barriers that check mark bit like in PMCS, when concurrent weak sweep is supported + BarriersLogDebug(false, "Weak read %p", weak); + } + return weak; +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp index ffb3504580d..238f50c105b 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/Barriers.hpp @@ -1,5 +1,5 @@ /* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ @@ -12,13 +12,14 @@ #include "GCStatistics.hpp" #include "ReferenceOps.hpp" +/** See. `ConcurrentMark` */ namespace kotlin::gc::barriers { class BarriersThreadData : private Pinned { public: void onThreadRegistration() noexcept; void onSafePoint() noexcept; - + void startMarkingNewObjects(GCHandle gcHandle) noexcept; void stopMarkingNewObjects() noexcept; bool shouldMarkNewObjects() const noexcept; @@ -34,6 +35,6 @@ void disableMarkBarriers() noexcept; void beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* value) noexcept; -// TODO re-introduce weak barriers again +OBJ_GETTER(weakRefReadBarrier, std::atomic& weakReferee) noexcept; } // namespace kotlin::gc diff --git a/kotlin-native/runtime/src/gc/cms/cpp/BarriersTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/BarriersTest.cpp index 6f4c31d213e..0d6e21d3125 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/BarriersTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/BarriersTest.cpp @@ -7,9 +7,8 @@ #include "gtest/gtest.h" #include "Barriers.hpp" -#include "ParallelMark.hpp" +#include "ConcurrentMark.hpp" #include "GCImpl.hpp" -#include "ManuallyScoped.hpp" #include "ObjectTestSupport.hpp" #include "ObjectOps.hpp" #include "ReferenceOps.hpp" @@ -57,7 +56,7 @@ public: } private: - gc::mark::ParallelMark::ParallelProcessor parProc_; + gc::mark::ConcurrentMark::ParallelProcessor parProc_; }; } // namespace diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.cpp new file mode 100644 index 00000000000..e4e31db2e85 --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.cpp @@ -0,0 +1,112 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ConcurrentMark.hpp" + +#include "MarkAndSweepUtils.hpp" +#include "GCStatistics.hpp" +#include "Utils.hpp" +#include "GCImpl.hpp" + +using namespace kotlin; + +void gc::mark::ConcurrentMark::beginMarkingEpoch(gc::GCHandle gcHandle) { + gcHandle_ = gcHandle; + + lockedMutatorsList_ = mm::ThreadRegistry::Instance().LockForIter(); + + parallelProcessor_.construct(); +} + +void gc::mark::ConcurrentMark::endMarkingEpoch() { + parallelProcessor_.destroy(); + resetMutatorFlags(); + lockedMutatorsList_ = std::nullopt; +} + +void gc::mark::ConcurrentMark::runMainInSTW() { + ParallelProcessor::Worker mainWorker(*parallelProcessor_); + GCLogDebug(gcHandle().getEpoch(), "Creating main (#0) mark worker"); + + // create mutator mark queues + for (auto& thread: *lockedMutatorsList_) { + thread.gc().impl().gc().mark().markQueue().construct(*parallelProcessor_); + } + + completeMutatorsRootSet(mainWorker); + + // global root set must be collected after all the mutator's global data have been published + collectRootSetGlobals (gcHandle(), mainWorker); + + barriers::enableMarkBarriers(gcHandle().getEpoch()); + + resumeTheWorld(gcHandle()); + + // build mark closure + parallelMark(mainWorker); + + // TODO resume the world much later when the mark closure is completed + stopTheWorld(gcHandle(), "GC stop the world #2: complete mark closure"); + + barriers::disableMarkBarriers(); + + bool refsRemainInMutatorQueues = false; + do { + for (auto& mutator: *lockedMutatorsList_) { + const bool markQueueNowEmpty = mutator.gc().impl().gc().mark().markQueue()->forceFlush(); + if (!markQueueNowEmpty) { + refsRemainInMutatorQueues = true; + } + } + + parallelProcessor_->resetForNewWork(); + // complete mark closure form newly found objects + parallelMark(mainWorker); + } while (refsRemainInMutatorQueues); + + for (auto& thread: *lockedMutatorsList_) { + auto& markQueue = thread.gc().impl().gc().mark().markQueue(); + RuntimeAssert(markQueue->retainsNoWork(), ""); // TODO move into queue's destuctor? + markQueue.destroy(); + } +} + +void gc::mark::ConcurrentMark::runOnMutator(mm::ThreadData&) { + // no-op +} + + +gc::GCHandle& gc::mark::ConcurrentMark::gcHandle() { + RuntimeAssert(gcHandle_.isValid(), "GCHandle must be initialized"); + return gcHandle_; +} + + +void gc::mark::ConcurrentMark::completeMutatorsRootSet(MarkTraits::MarkQueue& markQueue) { + // workers compete for mutators to collect their root set + for (auto& thread: *lockedMutatorsList_) { + tryCollectRootSet(thread, markQueue); + } +} + +void gc::mark::ConcurrentMark::tryCollectRootSet(mm::ThreadData& thread, MarkTraits::MarkQueue& markQueue) { + auto& gcData = thread.gc().impl().gc(); + if (!gcData.tryLockRootSet()) return; + + GCLogDebug(gcHandle().getEpoch(), "Root set collection on thread %d for thread %d", konan::currentThreadId(), thread.threadId()); + gcData.publish(); + collectRootSetForThread (gcHandle(), markQueue, thread); +} + +void gc::mark::ConcurrentMark::parallelMark(ParallelProcessor::Worker& worker) { + GCLogDebug(gcHandle().getEpoch(), "Mark loop has begun"); + Mark (gcHandle(), worker); +} + +void gc::mark::ConcurrentMark::resetMutatorFlags() { + for (auto& mut: *lockedMutatorsList_) { + mut.gc().impl().gc().clearMarkFlags(); + } +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.hpp new file mode 100644 index 00000000000..61a43e99e2e --- /dev/null +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMark.hpp @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include + +#include "GCStatistics.hpp" +#include "ManuallyScoped.hpp" +#include "ObjectData.hpp" +#include "ParallelProcessor.hpp" +#include "SafePoint.hpp" +#include "ThreadRegistry.hpp" +#include "Utils.hpp" + +namespace kotlin::gc::mark { + +/** + * Implementation of a Mark GC phase, that runs concurrent with mutator threads, using a "Snapshot at the Beginning" approach. + * Steps: + * 1. Pause all mutators, collect their root sets. + * 2. Mutators resume, maintaining a weak tri-collor invariant with the help of: + * - Deletion write barrier (Dijkstra et al): + * Remembers overwritten values in a thread-local mark queue. + * Barrier+write combination doesn't need to be atomic, + * as only references from mark phase beginning matter for the SatB approach. + * - Read barrier for weak refs: + * Remembers each object read via a weak reference in a thread-local mark queue. + * Prevents the possibility of inserting a strong reference to a ewakly-reachable object behind the mark front. + * 3. Concurrently, the marker thread builds a mark closure. Unmarked objects hidden in a mutator mark queues may still exist. + * 4. Pause mutators once more, drain local mark queues, and complete the mark closure, this time non-concurrently. + * // TODO build closure fully concurrent + * 5. Process and clean weak references. // TODO process weak refs concurrently + * 6. Prepare mared heap for sweeping and resume mutation. // TODO prepare heap without a pause + */ +class ConcurrentMark : private Pinned { + using MarkStackImpl = intrusive_forward_list; + +public: + // work balancing parameters were chosen pretty arbitrary + using ParallelProcessor = ParallelProcessor; + using MutatorQueue = ParallelProcessor::WorkSource; + + class MarkTraits { + public: + using MarkQueue = ParallelProcessor::Worker; + using AnyQueue = ParallelProcessor::WorkSource; + + static constexpr auto kAllowHeapToStackRefs = false; + + static void clear(AnyQueue& queue) noexcept { + RuntimeAssert(queue.localEmpty(), "Mark queue must be empty"); + } + + static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept { + auto* obj = queue.tryPop(); + if (obj) { + auto object = alloc::objectForObjectData(*obj); + RuntimeLogDebug({logging::Tag::kGCMark}, "Dequeued %p", object); + return object; + } + return nullptr; + } + + static ALWAYS_INLINE bool tryEnqueue(AnyQueue& queue, ObjHeader* object) noexcept { + auto& objectData = alloc::objectDataForObject(object); + bool pushed = queue.tryPush(objectData); + if (pushed) { + RuntimeLogDebug({logging::Tag::kGCMark}, "Enqueued %p", object); + } + return pushed; + } + + static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept { + auto& objectData = alloc::objectDataForObject(object); + bool pushed = objectData.tryMark(); + if (pushed) { + RuntimeLogDebug({logging::Tag::kGCMark}, "Marked %p", object); + } + return pushed; + } + + static ALWAYS_INLINE void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { + auto process = object->type_info()->processObjectInMark; + RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object); + process(static_cast(&markQueue), object); + } + }; + + class ThreadData : private Pinned { + public: + auto& markQueue() noexcept { return markQueue_; } + private: + ManuallyScoped markQueue_{}; + }; + + void beginMarkingEpoch(GCHandle gcHandle); + void endMarkingEpoch(); + + /** To be run by a single "main" GC thread during STW. */ + void runMainInSTW(); + + /** + * To be run by mutator threads that would like to participate in mark. + * Will wait for STW detection by a "main" routine. + */ + void runOnMutator(mm::ThreadData& mutatorThread); + +private: + GCHandle& gcHandle(); + + void completeMutatorsRootSet(MarkTraits::MarkQueue& markQueue); + void tryCollectRootSet(mm::ThreadData& thread, ParallelProcessor::Worker& markQueue); + void parallelMark(ParallelProcessor::Worker& worker); + + void resetMutatorFlags(); + + GCHandle gcHandle_ = GCHandle::invalid(); + std::optional lockedMutatorsList_; + ManuallyScoped parallelProcessor_{}; +}; +} // namespace kotlin::gc::mark diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp index c6ec04689ea..bf4dcc9b02f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp @@ -55,8 +55,6 @@ ScopedThread createGCThread(const char* name, Body&& body) { } // namespace void gc::ConcurrentMarkAndSweep::ThreadData::OnSuspendForGC() noexcept { - CallsCheckerIgnoreGuard guard; - gc_.markDispatcher_.runOnMutator(commonThreadData()); } @@ -91,12 +89,10 @@ gc::ConcurrentMarkAndSweep::ConcurrentMarkAndSweep( GCHandle::getByEpoch(epoch).finalizersDone(); state_.finalized(epoch); }), - markDispatcher_(mutatorsCooperate), mainThread_(createGCThread("Main GC thread", [this] { mainGCThreadBody(); })) { - for (std::size_t i = 0; i < auxGCThreads; ++i) { - auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); })); - } - RuntimeLogInfo({kTagGC}, "Parallel Mark & Concurrent Sweep GC initialized"); + RuntimeAssert(!mutatorsCooperate, "Cooperative mutators aren't supported yet"); + RuntimeAssert(auxGCThreads == 0, "Auxiliary GC threads aren't supported yet"); + RuntimeLogInfo({kTagGC}, "Concurrent Mark & Sweep GC initialized"); } gc::ConcurrentMarkAndSweep::~ConcurrentMarkAndSweep() { @@ -119,7 +115,7 @@ bool gc::ConcurrentMarkAndSweep::FinalizersThreadIsRunning() noexcept { } void gc::ConcurrentMarkAndSweep::mainGCThreadBody() { - RuntimeLogWarning({kTagGC}, "Initializing Concurrent Mark and Sweep GC. Concurrent mark is not implemented yet."); + RuntimeLogWarning({kTagGC}, "Initializing Concurrent Mark and Sweep GC."); while (true) { auto epoch = state_.waitScheduled(); if (epoch.has_value()) { @@ -128,14 +124,6 @@ void gc::ConcurrentMarkAndSweep::mainGCThreadBody() { break; } } - markDispatcher_.requestShutdown(); -} - -void gc::ConcurrentMarkAndSweep::auxiliaryGCThreadBody() { - RuntimeAssert(!compiler::gcMarkSingleThreaded(), "Should not reach here during single threaded mark"); - while (!markDispatcher_.shutdownRequested()) { - markDispatcher_.runAuxiliary(); - } } void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { @@ -145,7 +133,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { markDispatcher_.beginMarkingEpoch(gcHandle); GCLogDebug(epoch, "Main GC requested marking in mutators"); - stopTheWorld(gcHandle); + stopTheWorld(gcHandle, "GC stop the world #1: collect root set"); auto& scheduler = gcScheduler_; scheduler.onGCStart(); @@ -207,15 +195,3 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { // TODO: Consider having an always on sleeping finalizer thread. finalizerProcessor_.ScheduleTasks(std::move(finalizerQueue), epoch); } - -void gc::ConcurrentMarkAndSweep::reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, std::size_t auxGCThreads) noexcept { - if (compiler::gcMarkSingleThreaded()) { - RuntimeCheck(auxGCThreads == 0, "Auxiliary GC threads must not be created with gcMarkSingleThread"); - return; - } - std::unique_lock mainGCLock(gcMutex); - markDispatcher_.reset(maxParallelism, mutatorsCooperate, [this] { auxThreads_.clear(); }); - for (std::size_t i = 0; i < auxGCThreads; ++i) { - auxThreads_.emplace_back(createGCThread("Auxiliary GC thread", [this] { auxiliaryGCThreadBody(); })); - } -} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index fdb463491f4..43422f405c8 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -19,7 +19,7 @@ #include "IntrusiveList.hpp" #include "MarkAndSweepUtils.hpp" #include "ObjectData.hpp" -#include "ParallelMark.hpp" +#include "ConcurrentMark.hpp" #include "ScopedThread.hpp" #include "ThreadData.hpp" #include "Types.h" @@ -57,7 +57,7 @@ public: ConcurrentMarkAndSweep& gc_; mm::ThreadData& threadData_; barriers::BarriersThreadData barriers_; - mark::ParallelMark::ThreadData mark_; + mark::ConcurrentMark::ThreadData mark_; std::atomic rootSetLocked_ = false; std::atomic published_ = false; @@ -71,13 +71,10 @@ public: void StopFinalizerThreadIfRunning() noexcept; bool FinalizersThreadIsRunning() noexcept; - void reconfigure(std::size_t maxParallelism, bool mutatorsCooperate, size_t auxGCThreads) noexcept; - GCStateHolder& state() noexcept { return state_; } private: void mainGCThreadBody(); - void auxiliaryGCThreadBody(); void PerformFullGC(int64_t epoch) noexcept; alloc::Allocator& allocator_; @@ -86,7 +83,7 @@ private: GCStateHolder state_; FinalizerProcessor finalizerProcessor_; - mark::ParallelMark markDispatcher_; + mark::ConcurrentMark markDispatcher_; ScopedThread mainThread_; std::vector auxThreads_; }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index f4bc1b3cf89..64ed1cadcb9 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -11,7 +11,6 @@ #include "GCImpl.hpp" #include "GlobalData.hpp" -#include "SafePoint.hpp" #include "StableRef.hpp" #include "TestSupport.hpp" #include "TracingGCTest.hpp" @@ -34,65 +33,5 @@ public: } // namespace -TYPED_TEST_P(TracingGCTest, CMSMultipleMutatorsWeak) { - std::vector mutators(kDefaultThreadCount); - ObjHeader* globalRoot = nullptr; - test_support::RegularWeakReferenceImpl* weak = nullptr; - - mutators[0] - .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { - auto& global = mutator.AddGlobalRoot(); - - auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { - ObjHolder holder; - return test_support::InstallWeakReference(threadData, object.header(), holder.slot()); - })(); - global->field1 = objectWeak.header(); - weak = &objectWeak; - globalRoot = global.header(); - }) - .wait(); - - // Make sure all mutators are initialized. - for (int i = 1; i < kDefaultThreadCount; ++i) { - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) {}).wait(); - } - - std::vector> gcFutures; - auto epoch = mm::GlobalData::Instance().gc().Schedule(); - std::atomic gcDone = false; - - // Spin until thread suspension is requested. - while (!mm::IsThreadSuspensionRequested()) {} - - for (auto& mutator : mutators) { - gcFutures.emplace_back(mutator.Execute([&](mm::ThreadData& threadData, Mutator& mutator) noexcept { - bool dead = false; - while (!gcDone.load(std::memory_order_relaxed)) { - mm::safePoint(threadData); - auto weakReferee = weak->get(); - if (dead) { - EXPECT_THAT(weakReferee, nullptr); - } else if (weakReferee == nullptr) { - dead = true; - } - } - EXPECT_THAT(weak->get(), nullptr); - })); - } - - mm::GlobalData::Instance().gc().WaitFinalizers(epoch); - gcDone.store(true, std::memory_order_relaxed); - - for (auto& future : gcFutures) { - future.wait(); - } - - for (auto& mutator : mutators) { - EXPECT_THAT(mutator.Alive(), testing::UnorderedElementsAre(globalRoot, weak->header())); - } -} - -REGISTER_TYPED_TEST_SUITE_WITH_LISTS(TracingGCTest, TRACING_GC_TEST_LIST, CMSMultipleMutatorsWeak); +REGISTER_TYPED_TEST_SUITE_WITH_LISTS(TracingGCTest, TRACING_GC_TEST_LIST); INSTANTIATE_TYPED_TEST_SUITE_P(CMS, TracingGCTest, ConcurrentMarkAndSweepTest); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 42c3c428ad7..21991220dc9 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -59,17 +59,12 @@ bool gc::GC::FinalizersThreadIsRunning() noexcept { // static ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) noexcept { - gc::internal::processObjectInMark(state, object); + gc::internal::processObjectInMark(state, object); } // static ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept { - gc::internal::processArrayInMark(state, array); -} - -// static -ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { - gc::internal::processFieldInMark(state, field); + gc::internal::processArrayInMark(state, array); } int64_t gc::GC::Schedule() noexcept { @@ -89,7 +84,7 @@ ALWAYS_INLINE void gc::beforeHeapRefUpdate(mm::DirectRefAccessor ref, ObjHeader* } ALWAYS_INLINE OBJ_GETTER(gc::weakRefReadBarrier, std::atomic& weakReferee) noexcept { - RETURN_OBJ(weakReferee.load(std::memory_order_relaxed)); + RETURN_RESULT_OF(gc::barriers::weakRefReadBarrier, weakReferee); } bool gc::isMarked(ObjHeader* object) noexcept { diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp deleted file mode 100644 index 6820d972418..00000000000 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.cpp +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "ParallelMark.hpp" - -#include "MarkAndSweepUtils.hpp" -#include "GCStatistics.hpp" -#include "Utils.hpp" - -// required to access gc thread data -#include "GCImpl.hpp" - -using namespace kotlin; - -namespace { - -template -void spinWait(Cond&& until) { - while (!until()) { - std::this_thread::yield(); - } -} - -} // namespace - -bool gc::mark::MarkPacer::is(gc::mark::MarkPacer::Phase phase) const { - std::unique_lock lock(mutex_); - return phase_ == phase; -} - -void gc::mark::MarkPacer::begin(gc::mark::MarkPacer::Phase phase) { - { - std::unique_lock lock(mutex_); - phase_ = phase; - } - cond_.notify_all(); -} - -void gc::mark::MarkPacer::wait(gc::mark::MarkPacer::Phase phase) { - std::unique_lock lock(mutex_); - cond_.wait(lock, [=]() { return phase_ >= phase; }); -} - -void gc::mark::MarkPacer::beginEpoch(uint64_t epoch) { - epoch_ = epoch; - begin(Phase::kReady); - GCLogDebug(epoch_.load(), "Mark is ready to recruit workers in a new epoch."); -} - -void gc::mark::MarkPacer::waitNewEpochReadyOrShutdown() const { - std::unique_lock lock(mutex_); - cond_.wait(lock, [this]() { return phase_ >= Phase::kReady; }); -} - -void gc::mark::MarkPacer::waitEpochFinished(uint64_t currentEpoch) const { - std::unique_lock lock(mutex_); - cond_.wait(lock, [this, currentEpoch]() { - return phase_ == Phase::kIdle || phase_ == Phase::kShutdown || epoch_.load(std::memory_order_relaxed) > currentEpoch; - }); -} - -bool gc::mark::MarkPacer::acceptingNewWorkers() const { - std::unique_lock lock(mutex_); - return Phase::kReady <= phase_ && phase_ <= Phase::kParallelMark; -} - - -gc::mark::ParallelMark::ParallelMark(bool mutatorsCooperate) { - std::size_t maxParallelism = std::thread::hardware_concurrency(); - if (maxParallelism == 0) { - maxParallelism = std::numeric_limits::max(); - } - setParallelismLevel(maxParallelism, mutatorsCooperate); -} - -void gc::mark::ParallelMark::beginMarkingEpoch(gc::GCHandle gcHandle) { - gcHandle_ = gcHandle; - - lockedMutatorsList_ = mm::ThreadRegistry::Instance().LockForIter(); - - parallelProcessor_.construct(); - - if (!compiler::gcMarkSingleThreaded()) { - std::unique_lock guard(workerCreationMutex_); - pacer_.beginEpoch(gcHandle.getEpoch()); - // main worker is always accounted, so others would not be able to exhaust all the parallelism before main is instantiated - activeWorkersCount_ = 1; - } -} - -void gc::mark::ParallelMark::endMarkingEpoch() { - if (!compiler::gcMarkSingleThreaded()) { - // We must now wait for every worker to finish the Mark procedure: - // wake up from possible waiting, publish statistics, etc. - // Only then it's safe to destroy the parallelProcessor and proceed to other GC tasks such as sweep. - spinWait([=]() { return activeWorkersCount_.load(std::memory_order_relaxed) == 0; }); - - std::unique_lock guard(workerCreationMutex_); - RuntimeAssert(activeWorkersCount_ == 0, "All the workers must already finish"); - pacer_.begin(MarkPacer::Phase::kIdle); - } - parallelProcessor_.destroy(); - resetMutatorFlags(); - lockedMutatorsList_ = std::nullopt; -} - -void gc::mark::ParallelMark::runMainInSTW() { - if (compiler::gcMarkSingleThreaded()) { - ParallelProcessor::Worker worker(*parallelProcessor_); - gc::collectRootSet(gcHandle(), worker, [] (mm::ThreadData&) { return true; }); - gc::Mark(gcHandle(), worker); - } else { - RuntimeAssert(activeWorkersCount_ > 0, "Main worker must always be accounted"); - ParallelProcessor::Worker mainWorker(*parallelProcessor_); - GCLogDebug(gcHandle().getEpoch(), "Creating main (#0) mark worker"); - - pacer_.begin(MarkPacer::Phase::kRootSet); - completeMutatorsRootSet(mainWorker); - spinWait([this] { - return allMutators([](mm::ThreadData& mut) { return mut.gc().impl().gc().published(); }); - }); - // global root set must be collected after all the mutator's global data have been published - collectRootSetGlobals(gcHandle(), mainWorker); - - pacer_.begin(MarkPacer::Phase::kParallelMark); - parallelMark(mainWorker); - } -} - -void gc::mark::ParallelMark::runOnMutator(mm::ThreadData& mutatorThread) { - if (compiler::gcMarkSingleThreaded() || !mutatorsCooperate_) return; - - auto parallelWorker = createWorker(); - if (parallelWorker) { - GCLogDebug(gcHandle().getEpoch(), "Mutator thread cooperates in marking"); - - tryCollectRootSet(mutatorThread, *parallelWorker); - - completeRootSetAndMark(*parallelWorker); - } -} - -void gc::mark::ParallelMark::runAuxiliary() { - RuntimeAssert(!compiler::gcMarkSingleThreaded(), "Should not reach here during single threaded mark"); - - pacer_.waitNewEpochReadyOrShutdown(); - if (pacer_.is(MarkPacer::Phase::kShutdown)) return; - - auto curEpoch = gcHandle().getEpoch(); - auto parallelWorker = createWorker(); - if (parallelWorker) { - completeRootSetAndMark(*parallelWorker); - } - - pacer_.waitEpochFinished(curEpoch); -} - -void gc::mark::ParallelMark::requestShutdown() { - pacer_.begin(MarkPacer::Phase::kShutdown); -} - -bool gc::mark::ParallelMark::shutdownRequested() const { - return pacer_.is(MarkPacer::Phase::kShutdown); -} - - -gc::GCHandle& gc::mark::ParallelMark::gcHandle() { - RuntimeAssert(gcHandle_.isValid(), "GCHandle must be initialized"); - return gcHandle_; -} - -void gc::mark::ParallelMark::setParallelismLevel(size_t maxParallelism, bool mutatorsCooperate) { - RuntimeCheck(maxParallelism > 0, "Parallelism level can't be 0"); - maxParallelism_ = maxParallelism; - mutatorsCooperate_ = mutatorsCooperate; - RuntimeLogInfo({kTagGC}, - "Set up parallel mark with maxParallelism = %zu and %s" "cooperative mutators", - maxParallelism_, (mutatorsCooperate_ ? "" : "non-")); -} - -void gc::mark::ParallelMark::completeRootSetAndMark(ParallelProcessor::Worker& parallelWorker) { - pacer_.wait(MarkPacer::Phase::kRootSet); - completeMutatorsRootSet(parallelWorker); - pacer_.wait(MarkPacer::Phase::kParallelMark); - parallelMark(parallelWorker); -} - -void gc::mark::ParallelMark::completeMutatorsRootSet(MarkTraits::MarkQueue& markQueue) { - // workers compete for mutators to collect their root set - for (auto& thread: *lockedMutatorsList_) { - tryCollectRootSet(thread, markQueue); - } -} - -void gc::mark::ParallelMark::tryCollectRootSet(mm::ThreadData& thread, MarkTraits::MarkQueue& markQueue) { - auto& gcData = thread.gc().impl().gc(); - if (!gcData.tryLockRootSet()) return; - - GCLogDebug(gcHandle().getEpoch(), "Root set collection on thread %d for thread %d", - konan::currentThreadId(), thread.threadId()); - gcData.publish(); - collectRootSetForThread(gcHandle(), markQueue, thread); -} - -void gc::mark::ParallelMark::parallelMark(ParallelProcessor::Worker& worker) { - GCLogDebug(gcHandle().getEpoch(), "Mark loop has begun"); - Mark(gcHandle(), worker); - - std::unique_lock guard(workerCreationMutex_); - activeWorkersCount_.fetch_sub(1, std::memory_order_relaxed); -} - -std::optional gc::mark::ParallelMark::createWorker() { - std::unique_lock guard(workerCreationMutex_); - if (!pacer_.acceptingNewWorkers() || - activeWorkersCount_.load(std::memory_order_relaxed) >= maxParallelism_ || - activeWorkersCount_.load(std::memory_order_relaxed) == 0) return std::nullopt; - - auto num = activeWorkersCount_.fetch_add(1, std::memory_order_relaxed); - GCLogDebug(gcHandle().getEpoch(), "Creating mark worker #%zu", num); - return std::make_optional(*parallelProcessor_); -} - -void gc::mark::ParallelMark::resetMutatorFlags() { - for (auto& mut: *lockedMutatorsList_) { - auto& gcData = mut.gc().impl().gc(); - if (!compiler::gcMarkSingleThreaded()) { - // single threaded mark do not use this flag - RuntimeAssert(gcData.published(), "Must have been published during mark"); - } - gcData.clearMarkFlags(); - } -} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp deleted file mode 100644 index 957679fcbe7..00000000000 --- a/kotlin-native/runtime/src/gc/cms/cpp/ParallelMark.hpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#pragma once - -#include -#include - -#include "GCStatistics.hpp" -#include "ManuallyScoped.hpp" -#include "ObjectData.hpp" -#include "ParallelProcessor.hpp" -#include "ThreadRegistry.hpp" -#include "Utils.hpp" - -namespace kotlin::gc::mark { - -class MarkPacer : private Pinned { -public: - enum class Phase { - /** Mark is not in progress. */ - kIdle, - /** - * MarkDispatcher is ready to recruit new workers. - * - * In case of cooperative mark mutator threads are welcome to mark their own root sets. - * Each thread is free to start as soon as it reaches a safe point. - * No need to wait for others. - */ - kReady, - /** - * All mutator threads must be in a safe state at this point: - * 1) Suspended on a safe point; - * 2) In the native code; - * 3) Registered as cooperative markers during previous phase. - * - * Now all the GC workers are summoned to participate in a root set collection. - */ - kRootSet, - /** - * Root set is collected. No more workers can be instantiated, time to begin parallel mark. - * Parallel mark can't stop before all the created workers begin the marking. - */ - kParallelMark, - /** A shutdown was requested. There is nothing more to wait for. */ - kShutdown, - }; - - bool is(Phase phase) const; - void begin(Phase phase); - void wait(Phase phase); - - void beginEpoch(uint64_t epoch); - void waitNewEpochReadyOrShutdown() const; - void waitEpochFinished(uint64_t epoch) const; - - bool acceptingNewWorkers() const; - -private: - std::atomic epoch_ = 0; - Phase phase_ = Phase::kIdle; - mutable std::mutex mutex_; - mutable std::condition_variable cond_; -}; - -/** - * Parallel mark dispatcher. - * Mark can be performed on one or more threads. - * Each threads wanting to participate have to execute an appropriate run- routine when ready to mark. - * There must be exactly one executor of a `runMainInSTW()`. - * - * Mark workers are able to balance work between each other through sharing/stealing. - */ -class ParallelMark : private Pinned { - using MarkStackImpl = intrusive_forward_list; - // work balancing parameters were chosen pretty arbitrary - -public: - using ParallelProcessor = ParallelProcessor; - using MutatorQueue = ParallelProcessor::WorkSource; - - class MarkTraits { - public: - using MarkQueue = ParallelProcessor::Worker; - using AnyQueue = ParallelProcessor::WorkSource; - - static void clear(AnyQueue& queue) noexcept { - RuntimeAssert(queue.localEmpty(), "Mark queue must be empty"); - } - - static ALWAYS_INLINE ObjHeader* tryDequeue(MarkQueue& queue) noexcept { - auto* obj = compiler::gcMarkSingleThreaded() ? queue.tryPopLocal() : queue.tryPop(); - if (obj) { - return alloc::objectForObjectData(*obj); - } - return nullptr; - } - - static ALWAYS_INLINE bool tryEnqueue(AnyQueue& queue, ObjHeader* object) noexcept { - auto& objectData = alloc::objectDataForObject(object); - return compiler::gcMarkSingleThreaded() ? queue.tryPushLocal(objectData) : queue.tryPush(objectData); - } - - static ALWAYS_INLINE bool tryMark(ObjHeader* object) noexcept { - auto& objectData = alloc::objectDataForObject(object); - return objectData.tryMark(); - } - - static ALWAYS_INLINE void processInMark(MarkQueue& markQueue, ObjHeader* object) noexcept { - auto process = object->type_info()->processObjectInMark; - RuntimeAssert(process != nullptr, "Got null processObjectInMark for object %p", object); - process(static_cast(&markQueue), object); - } - }; - - class ThreadData : private Pinned { - public: - auto& markQueue() noexcept { return markQueue_; } - private: - ManuallyScoped markQueue_{}; - }; - - explicit ParallelMark(bool mutatorsCooperate); - - void beginMarkingEpoch(gc::GCHandle gcHandle); - void endMarkingEpoch(); - - /** To be run by a single "main" GC thread during STW. */ - void runMainInSTW(); - - /** - * To be run by mutator threads that would like to participate in mark. - * Will wait for STW detection by a "main" routine. - */ - void runOnMutator(mm::ThreadData& mutatorThread); - - /** - * To be run by auxiliary GC threads. - * Will wait for STW detection by a "main" routine. - */ - void runAuxiliary(); - - void requestShutdown(); - bool shutdownRequested() const; - - template - void reset(std::size_t maxParallelism, bool mutatorsCooperate, Pred waitForWorkersToFinish) { - pacer_.begin(MarkPacer::Phase::kShutdown); - waitForWorkersToFinish(); - pacer_.begin(MarkPacer::Phase::kIdle); - setParallelismLevel(maxParallelism, mutatorsCooperate); - } - -private: - GCHandle& gcHandle(); - - void setParallelismLevel(size_t maxParallelism, bool mutatorsCooperate); - - template - bool allMutators(Pred predicate) noexcept { - for (auto& thread : *lockedMutatorsList_) { - if (!predicate(thread)) { - return false; - } - } - return true; - } - - void completeRootSetAndMark(ParallelProcessor::Worker& parallelWorker); - void completeMutatorsRootSet(MarkTraits::MarkQueue& markQueue); - void tryCollectRootSet(mm::ThreadData& thread, ParallelProcessor::Worker& markQueue); - void parallelMark(ParallelProcessor::Worker& worker); - - std::optional createWorker(); - - void resetMutatorFlags(); - - std::size_t maxParallelism_ = 1; - bool mutatorsCooperate_ = false; - - GCHandle gcHandle_ = GCHandle::invalid(); - MarkPacer pacer_; - std::optional lockedMutatorsList_; - ManuallyScoped parallelProcessor_{}; - - std::mutex workerCreationMutex_; - std::atomic activeWorkersCount_ = 0; -}; - -} // namespace kotlin::gc::mark - diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 4ef88a3ce2b..cbedd069f74 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -72,7 +72,6 @@ public: static void processObjectInMark(void* state, ObjHeader* object) noexcept; static void processArrayInMark(void* state, ArrayHeader* array) noexcept; - static void processFieldInMark(void* state, ObjHeader* field) noexcept; // TODO: These should exist only in the scheduler. int64_t Schedule() noexcept; diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp index 14a88907558..f5c9d943440 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.cpp @@ -5,8 +5,8 @@ #include "MarkAndSweepUtils.hpp" -void kotlin::gc::stopTheWorld(kotlin::gc::GCHandle gcHandle) noexcept { - bool didSuspend = mm::RequestThreadsSuspension(); +void kotlin::gc::stopTheWorld(GCHandle gcHandle, const char* reason) noexcept { + bool didSuspend = mm::RequestThreadsSuspension(reason); RuntimeAssert(didSuspend, "Only GC thread can request suspension"); gcHandle.suspensionRequested(); diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 4538a841f55..d1b06577d63 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -26,27 +26,32 @@ namespace gc { namespace internal { template -void processFieldInMark(void* state, ObjHeader* field) noexcept { +void processFieldInMark(void* state, ObjHeader* object, ObjHeader* field) noexcept { auto& markQueue = *static_cast(state); if (field->heap()) { Traits::tryEnqueue(markQueue, field); } + if constexpr (!Traits::kAllowHeapToStackRefs) { + if (object->heap()) { + RuntimeAssert(!field->local(), "Heap object %p references stack object %p[typeInfo=%p]", object, field, field->type_info()); + } + } } template void processObjectInMark(void* state, ObjHeader* object) noexcept { - traverseClassObjectFields(object, [state] (auto fieldAccessor) noexcept { + traverseClassObjectFields(object, [=] (auto fieldAccessor) noexcept { if (ObjHeader* field = fieldAccessor.direct()) { - processFieldInMark(state, field); + processFieldInMark(state, object, field); } }); } template void processArrayInMark(void* state, ArrayHeader* array) noexcept { - traverseArrayOfObjectsElements(array, [state] (auto elemAccessor) noexcept { + traverseArrayOfObjectsElements(array, [=] (auto elemAccessor) noexcept { if (ObjHeader* elem = elemAccessor.direct()) { - processFieldInMark(state, elem); + processFieldInMark(state, array->obj(), elem); } }); } @@ -180,11 +185,11 @@ struct DefaultProcessWeaksTraits { static bool IsMarked(ObjHeader* obj) noexcept { return gc::isMarked(obj); } }; -void stopTheWorld(GCHandle gcHandle) noexcept; +void stopTheWorld(GCHandle gcHandle, const char* reason) noexcept; void resumeTheWorld(GCHandle gcHandle) noexcept; [[nodiscard]] inline auto stopTheWorldInScope(GCHandle gcHandle) noexcept { - return ScopeGuard([=]() { stopTheWorld(gcHandle); }, [=]() { resumeTheWorld(gcHandle); }); + return ScopeGuard([=]() { stopTheWorld(gcHandle, "GC stop the world"); }, [=]() { resumeTheWorld(gcHandle); }); } } // namespace gc diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index d922c418d5b..329c7f351d9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -75,6 +75,8 @@ class ScopedMarkTraits : private Pinned { public: using MarkQueue = std::vector; + static constexpr auto kAllowHeapToStackRefs = true; + ScopedMarkTraits() { RuntimeAssert(instance_ == nullptr, "Only one ScopedMarkTraits is allowed"); instance_ = this; diff --git a/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp index 435c8402c3b..94ddab71e3a 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp @@ -1073,6 +1073,180 @@ TYPED_TEST_P(TracingGCTest, FreeObjectWithFreeWeakReversedOrder) { f1.wait(); } +TYPED_TEST_P(TracingGCTest, MutateBetweenSafePoints) { + constexpr auto kQuant = 5; + constexpr auto kGcNumber = 5; + constexpr auto kMaxItersPerGC = 10; + + std::atomic stopMutation = false; + std::atomic startMutation = false; + std::atomic initializedMutators = 0; + std::atomic gcEpoch = 0; + + Mutator scheduler; + std::future schedulerFuture = scheduler.Execute([&](mm::ThreadData& threadData, Mutator& mutator) { + while (initializedMutators < kDefaultThreadCount) { /* wait */ } + startMutation = true; + for (int i = 0; i < kGcNumber; ++i) { + gcEpoch = i; + mm::GlobalData::Instance().gcScheduler().scheduleAndWaitFinalized(); + } + stopMutation = true; + }); + + std::vector mutators(kDefaultThreadCount); + std::vector> mutatorFutures; + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutatorFutures.emplace_back(mutators[i].Execute([&](mm::ThreadData& threadData, Mutator& mutator) { + StackObjectHolder head{threadData}; + ObjHeader* tail = head.header(); + + auto append = [&](int count) { + for (int i = 0; i < count; ++i) { + auto& next = AllocateObject(threadData); + auto& tailObj = test_support::Object::FromObjHeader(tail); + tailObj->field1 = next.header(); + tail = next.header(); + } + }; + + auto cut = [&](ObjHeader* first, int count) { + ObjHeader* last = first; + for (int i = 0; i < count; ++i) { + auto& lastObj = test_support::Object::FromObjHeader(last); + last = lastObj->field1.accessor().load(); + } + auto& firstObj = test_support::Object::FromObjHeader(first); + auto& lastObj = test_support::Object::FromObjHeader(last); + firstObj->field1.accessor() = lastObj->field1.accessor().load(); + }; + + // Initialize + append(kQuant * 2); + ++initializedMutators; + while (!startMutation.load()) { /* wait */ }; + + int iter = 0; + while (!stopMutation.load()) { + // do not let mutator outpace gc to much + while (iter > (gcEpoch.load() * kMaxItersPerGC) && !stopMutation.load()) { + mm::safePoint(threadData); + } + + auto oldTail = tail; + append(kQuant); + // [head]->(a)->...->(b)->...->(oldTail)->(c)->...->(d) + mm::safePoint(threadData); + + append(kQuant); + // [head]->(a)->...->(b)->...->(oldTail)->(c)->...->(d)->...->(tail) + mm::safePoint(threadData); + + cut(head.header(), kQuant); + // (a)->...-+ + // v + // [head]->(b)->...->(oldTail)->(c)->...->(d)->...->(tail) + mm::safePoint(threadData); + + cut(oldTail, kQuant); + // (a)->...-+ (c)->...-+ + // v v + // [head]->(b)->...->(oldTail)->(d)->...->(tail) + mm::safePoint(threadData); + ++iter; + } + + std::size_t length = 0; + auto* node = &test_support::Object::FromObjHeader(head.header()); + while ((*node)->field1.accessor().load() != nullptr) { + node = &test_support::Object::FromObjHeader((*node)->field1.accessor().load()); + ++length; + } + + EXPECT_THAT(length, kQuant * 2); + })); + } + + schedulerFuture.wait(); + for (auto& future : mutatorFutures) { + future.wait(); + } +} + +TYPED_TEST_P(TracingGCTest, WeakResuractionInMark) { + constexpr auto kObjectsPerThread = 100; + std::vector mutators(kDefaultThreadCount); + std::vector weaks(kDefaultThreadCount); + std::vector*> roots(kDefaultThreadCount); + + // initialize threads + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([&, i](mm::ThreadData& threadData, Mutator& mutator) { + // make sure GC will have somthing to do whil we play with weak references + for (int j = 0; j < kObjectsPerThread; ++j) { + auto& object = AllocateObject(threadData); + object->field1 = roots[i]->header(); + roots[i] = &object; + } + mutator.AddGlobalRoot(roots[i]->header()); + + auto& weakReferee = AllocateObject(threadData); + auto& weakRef = [&threadData, &weakReferee]() -> test_support::RegularWeakReferenceImpl& { + ObjHolder holder; + return test_support::InstallWeakReference(threadData, weakReferee.header(), holder.slot()); + }(); + EXPECT_NE(weakRef.get(), nullptr); + weaks[i] = &weakRef; + mutator.AddGlobalRoot(weakRef.header()); + }).wait(); + } + + auto epoch = mm::GlobalData::Instance().gc().Schedule(); + std::atomic gcDone = false; + + // Spin until thread suspension is requested. + while (!mm::IsThreadSuspensionRequested()) {} + + std::vector> mutatorFutures; + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutatorFutures.emplace_back(mutators[i].Execute([&, i](mm::ThreadData& threadData, Mutator& mutator) noexcept { + safePoint(threadData); + + auto weakReferee = weaks[i]->get(); + (*roots[i])->field2 = weakReferee; + bool resurected = weakReferee != nullptr; + + while (!gcDone.load(std::memory_order_relaxed)) { + safePoint(threadData); + } + if (resurected) { + EXPECT_NE(weaks[i]->get(), nullptr); + } else { + EXPECT_EQ(weaks[i]->get(), nullptr); + } + })); + } + + mm::GlobalData::Instance().gc().WaitFinalizers(epoch); + gcDone = true; + + for (auto& future : mutatorFutures) { + future.wait(); + } + + for (int i = 0; i < kDefaultThreadCount; ++i) { + mutators[i].Execute([&, i](mm::ThreadData&, Mutator&) noexcept { + if (auto weakReferee = weaks[i]->get()) { + auto& extraObj = *mm::ExtraObjectData::Get(weakReferee); + extraObj.ClearRegularWeakReferenceImpl(); + extraObj.Uninstall(); + alloc::destroyExtraObjectData(extraObj); + } + }).wait(); + } +} + #define TRACING_GC_TEST_LIST \ RootSet, \ InterconnectedRootSet, \ @@ -1092,7 +1266,9 @@ TYPED_TEST_P(TracingGCTest, FreeObjectWithFreeWeakReversedOrder) { MultipleMutatorsAddToRootSetAfterCollectionRequested, \ CrossThreadReference, \ NewThreadsWhileRequestingCollection, \ - FreeObjectWithFreeWeakReversedOrder + FreeObjectWithFreeWeakReversedOrder, \ + MutateBetweenSafePoints, \ + WeakResuractionInMark template class STWMarkGCTest : public TracingGCTest {}; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index bcedacaa980..7e4b72cfbd0 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -48,9 +48,6 @@ ALWAYS_INLINE void gc::GC::processObjectInMark(void* state, ObjHeader* object) n // static ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) noexcept {} -// static -ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept {} - int64_t gc::GC::Schedule() noexcept { return 0; } diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp index 06b37d3a11f..5a5025fb391 100644 --- a/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/pmcs/cpp/GCImpl.cpp @@ -67,11 +67,6 @@ ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) n gc::internal::processArrayInMark(state, array); } -// static -ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { - gc::internal::processFieldInMark(state, field); -} - int64_t gc::GC::Schedule() noexcept { return impl_->gc().state().schedule(); } diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMark.hpp b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMark.hpp index 6d41d096411..7b6173e87f4 100644 --- a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMark.hpp +++ b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMark.hpp @@ -82,6 +82,8 @@ public: public: using MarkQueue = ParallelProcessor::Worker; + static constexpr auto kAllowHeapToStackRefs = true; + static void clear(MarkQueue& queue) noexcept { RuntimeAssert(queue.localEmpty(), "Mark queue must be empty"); } diff --git a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp index cda9871e130..f24c07aa9c0 100644 --- a/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp +++ b/kotlin-native/runtime/src/gc/pmcs/cpp/ParallelMarkConcurrentSweep.cpp @@ -148,7 +148,7 @@ void gc::ParallelMarkConcurrentSweep::PerformFullGC(int64_t epoch) noexcept { markDispatcher_.beginMarkingEpoch(gcHandle); GCLogDebug(epoch, "Main GC requested marking in mutators"); - stopTheWorld(gcHandle); + stopTheWorld(gcHandle, "GC stop the world #1: mark"); auto& scheduler = gcScheduler_; scheduler.onGCStart(); @@ -168,7 +168,7 @@ void gc::ParallelMarkConcurrentSweep::PerformFullGC(int64_t epoch) noexcept { gc::processWeaks(gcHandle, mm::SpecialRefRegistry::instance()); if (compiler::concurrentWeakSweep()) { - stopTheWorld(gcHandle); + stopTheWorld(gcHandle, "GC stop the world #2: prepare heap for sweep"); gc::DisableWeakRefBarriers(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index 35fc1fe38f7..e3e103ab449 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -60,11 +60,6 @@ ALWAYS_INLINE void gc::GC::processArrayInMark(void* state, ArrayHeader* array) n gc::internal::processArrayInMark(state, array); } -// static -ALWAYS_INLINE void gc::GC::processFieldInMark(void* state, ObjHeader* field) noexcept { - gc::internal::processFieldInMark(state, field); -} - int64_t gc::GC::Schedule() noexcept { return impl_->gc().state().schedule(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 03f722d02b1..91c2a6ed5a4 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -57,7 +57,7 @@ bool gc::SameThreadMarkAndSweep::FinalizersThreadIsRunning() noexcept { void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept { auto gcHandle = GCHandle::create(epoch); - stopTheWorld(gcHandle); + stopTheWorld(gcHandle, "GC stop the world"); auto& scheduler = gcScheduler_; scheduler.onGCStart(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index cd2f86eb7d6..602f426a269 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -68,6 +68,8 @@ namespace internal { struct MarkTraits { using MarkQueue = gc::SameThreadMarkAndSweep::MarkQueue; + static constexpr auto kAllowHeapToStackRefs = true; + static void clear(MarkQueue& queue) noexcept { queue.clear(); } static ObjHeader* tryDequeue(MarkQueue& queue) noexcept { diff --git a/kotlin-native/runtime/src/main/cpp/Logging.hpp b/kotlin-native/runtime/src/main/cpp/Logging.hpp index 82a7edb237e..d7427b8f974 100644 --- a/kotlin-native/runtime/src/main/cpp/Logging.hpp +++ b/kotlin-native/runtime/src/main/cpp/Logging.hpp @@ -38,8 +38,9 @@ enum class Tag : int32_t { kAlloc = 6, kBalancing = 7, kBarriers = 8, + kGCMark = 9, - kEnumSize = 9 + kEnumSize = 10 }; namespace internal { @@ -65,6 +66,7 @@ inline const char* name(Tag tag) { case Tag::kAlloc: return "alloc"; case Tag::kBalancing: return "balancing"; case Tag::kBarriers: return "barriers"; + case Tag::kGCMark: return "gcMark"; case Tag::kEnumSize: break; } diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 32d5c8e45bd..9ff035f680e 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -77,7 +77,9 @@ struct ObjHeader { * Hardware guaranties on many supported platforms doesn't allow this to happen. */ const TypeInfo* type_info() const { - return atomicGetRelaxed(&clearPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK)->typeInfo_); + const TypeInfo* typeInfo = atomicGetRelaxed(&clearPointerBits(typeInfoOrMetaRelaxed(), OBJECT_TAG_MASK)->typeInfo_); + RuntimeAssert(typeInfo != nullptr, "TypeInfo ptr in object %p in null", this); + return typeInfo; } bool has_meta_object() const { @@ -599,7 +601,6 @@ void compactObjectPoolInCurrentThread() noexcept; RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processObjectInMark(void* state, ObjHeader* object); RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* state, ObjHeader* object); -RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field); RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object); #endif // RUNTIME_MEMORY_H diff --git a/kotlin-native/runtime/src/main/cpp/ParallelProcessor.hpp b/kotlin-native/runtime/src/main/cpp/ParallelProcessor.hpp index 74cec8eadf4..e8bab4a6536 100644 --- a/kotlin-native/runtime/src/main/cpp/ParallelProcessor.hpp +++ b/kotlin-native/runtime/src/main/cpp/ParallelProcessor.hpp @@ -95,7 +95,7 @@ public: explicit WorkSource(ParallelProcessor& dispatcher) : dispatcher_(dispatcher) {} ALWAYS_INLINE bool retainsNoWork() const noexcept { - return batch_.empty() && this->localEmpty(); + return batch_.empty() && overflowList().empty(); } ALWAYS_INLINE bool tryPush(typename ListImpl::reference value) noexcept { @@ -119,7 +119,7 @@ public: if (!batch_.empty()) { bool released = dispatcher_.releaseBatch(std::move(batch_)); if (released) { - RuntimeLogDebug({ kTagBalancing }, "Work butch flushed"); + RuntimeLogDebug({ kTagBalancing }, "Work batch flushed"); batch_ = Batch{}; } else { RuntimeLogDebug({ kTagBalancing }, "Failed to force flush work queue"); @@ -130,7 +130,7 @@ public: if (overflowList().empty()) { return true; } else { - RuntimeLogDebug({ kTagBalancing }, "Refiling batch from overflow list"); + RuntimeLogDebug({ kTagBalancing }, "Refilling batch from overflow list"); batch_.fillFrom(overflowList()); } } @@ -141,6 +141,10 @@ public: return this->localQueue_; } + const ListImpl& overflowList() const noexcept { + return this->localQueue_; + } + ParallelProcessor& dispatcher_; Batch batch_; }; @@ -222,6 +226,13 @@ public: return registeredWorkers_.load(std::memory_order_relaxed); } + /** Prepare for a new round of work processing. Must be called only after a previous round is fully finished. */ + void resetForNewWork() noexcept { + RuntimeAssert(allDone_, "A work processing iteration must be finished"); + RuntimeAssert(waitingWorkers_ == 0, "There must be no workers sleeping inside processing loop"); + allDone_ = false; + } + private: bool releaseBatch(Batch&& batch) { RuntimeAssert(!batch.empty(), "A batch to release into shared pool must be non-empty"); diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index d08b33d89c3..cc1b4196f40 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -575,10 +575,6 @@ RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processArrayInMark(void* st gc::GC::processArrayInMark(state, object->array()); } -RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processFieldInMark(void* state, ObjHeader* field) { - gc::GC::processFieldInMark(state, field); -} - RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(void* state, ObjHeader* object) { // Empty object. Nothing to do. // TODO: Try to generate it in the code generator. diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index 3e8d9bd2d9b..1e9da672b6e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -23,7 +23,7 @@ namespace { } // namespace -std::atomic kotlin::mm::internal::gSuspensionRequested = false; +std::atomic mm::internal::gSuspensionReauestReason = nullptr; ALWAYS_INLINE mm::ThreadSuspensionData::MutatorPauseHandle::MutatorPauseHandle(const char* reason, mm::ThreadData& threadData) noexcept : reason_(reason), threadData_(threadData), pauseStartTimeMicros_(konan::getTimeMicros()) @@ -65,7 +65,7 @@ kotlin::ThreadState kotlin::mm::ThreadSuspensionData::setState(kotlin::ThreadSta NO_EXTERNAL_CALLS_CHECK void kotlin::mm::ThreadSuspensionData::suspendIfRequested() noexcept { if (IsThreadSuspensionRequested()) { - auto pauseHandle = pauseMutationInScope("stop the world"); + auto pauseHandle = pauseMutationInScope(internal::gSuspensionReauestReason.load(std::memory_order_relaxed)); threadData_.gc().OnSuspendForGC(); std::unique_lock lock(gSuspensionMutex); @@ -80,18 +80,18 @@ ALWAYS_INLINE mm::ThreadSuspensionData::MutatorPauseHandle mm::ThreadSuspensionD return MutatorPauseHandle(reason, threadData_); } -bool kotlin::mm::RequestThreadsSuspension() noexcept { +bool kotlin::mm::RequestThreadsSuspension(internal::SuspensionReason reason) noexcept { CallsCheckerIgnoreGuard guard; RuntimeAssert(gSafePointActivator == std::nullopt, "Current thread already suspended threads."); { std::unique_lock lock(gSuspensionMutex); // Someone else has already suspended threads. - if (internal::gSuspensionRequested.load(std::memory_order_relaxed)) { + if (internal::gSuspensionReauestReason.load(std::memory_order_relaxed) != nullptr) { return false; } gSafePointActivator = mm::SafePointActivator(); - internal::gSuspensionRequested.store(true); + internal::gSuspensionReauestReason.store(reason); } return true; @@ -116,7 +116,7 @@ void kotlin::mm::ResumeThreads() noexcept { // https://en.cppreference.com/w/cpp/thread/condition_variable { std::unique_lock lock(gSuspensionMutex); - internal::gSuspensionRequested.store(false); + internal::gSuspensionReauestReason.store(nullptr); } gSuspensionCondVar.notify_all(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp index d94ef8431eb..f6383b4bb1a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp @@ -9,6 +9,7 @@ #include #include "Memory.h" +#include "Utils.hpp" namespace kotlin { namespace mm { @@ -17,14 +18,15 @@ class ThreadData; namespace internal { -extern std::atomic gSuspensionRequested; +using SuspensionReason = const char*; +extern std::atomic gSuspensionReauestReason; } // namespace internal inline bool IsThreadSuspensionRequested() noexcept { // Must use seq_cst ordering for synchronization with GC // in native->runnable transition. - return internal::gSuspensionRequested.load(); + return internal::gSuspensionReauestReason.load(); } class ThreadSuspensionData : private Pinned { @@ -71,7 +73,7 @@ private: mm::ThreadData& threadData_; }; -bool RequestThreadsSuspension() noexcept; +bool RequestThreadsSuspension(const char* reason) noexcept; void WaitForThreadsSuspension() noexcept; /** @@ -80,8 +82,8 @@ void WaitForThreadsSuspension() noexcept; * of this call will be suspended on exit from the Native state. * Returns false if some other thread has suspended the threads. */ -inline bool SuspendThreads() noexcept { - if (!RequestThreadsSuspension()) { +inline bool SuspendThreads(internal::SuspensionReason reason) noexcept { + if (!RequestThreadsSuspension(reason)) { return false; } WaitForThreadsSuspension(); diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp index 3bdd90ae458..051f7a3f353 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp @@ -128,7 +128,7 @@ TEST_F(ThreadSuspensionTest, SimpleStartStop) { reportProgress(i, kIterations); canStart = true; - mm::SuspendThreads(); + mm::SuspendThreads("test"); auto suspended = collectSuspended(); EXPECT_THAT(suspended, testing::Each(true)); EXPECT_EQ(mm::IsThreadSuspensionRequested(), true); @@ -171,7 +171,7 @@ TEST_F(ThreadSuspensionTest, SwitchStateToNative) { reportProgress(i, kIterations); canStart = true; - mm::SuspendThreads(); + mm::SuspendThreads("test"); EXPECT_EQ(mm::IsThreadSuspensionRequested(), true); mm::ResumeThreads(); @@ -196,7 +196,7 @@ TEST_F(ThreadSuspensionTest, ConcurrentSuspend) { ready[i] = true; waitUntilThreadsAreReady(); - bool success = mm::SuspendThreads(); + bool success = mm::SuspendThreads("test"); if (success) { successCount++; auto allThreadData = collectThreadData(); @@ -248,7 +248,7 @@ TEST_F(ThreadSuspensionTest, FileInitializationWithSuspend) { waitUntilThreadsAreReady(); auto gcThread = std::async(std::launch::async, [] { - mm::RequestThreadsSuspension(); + mm::RequestThreadsSuspension("test"); mm::WaitForThreadsSuspension(); mm::ResumeThreads(); });