From 84e7b53926d8aeda1dedfd5f4b128caa74f92f10 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Tue, 14 Sep 2021 13:03:28 +0300 Subject: [PATCH] [K/N] Rework GCScheduler to schedule GC on its own. --- .../backend/konan/CheckExternalCalls.kt | 4 +- .../runtime/src/gc/common/cpp/GCScheduler.cpp | 139 +++++++--- .../runtime/src/gc/common/cpp/GCScheduler.hpp | 169 +++++++++---- .../gc/stms/cpp/SameThreadMarkAndSweep.cpp | 238 ++++++++++-------- .../gc/stms/cpp/SameThreadMarkAndSweep.hpp | 12 +- .../stms/cpp/SameThreadMarkAndSweepTest.cpp | 7 + .../runtime/src/main/cpp/SingleLockList.hpp | 8 + kotlin-native/runtime/src/mm/cpp/Memory.cpp | 6 +- .../runtime/src/mm/cpp/ThreadData.hpp | 4 +- .../runtime/src/mm/cpp/ThreadSuspension.cpp | 9 +- .../runtime/src/mm/cpp/ThreadSuspension.hpp | 11 +- 11 files changed, 401 insertions(+), 206 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt index 07adae68c77..dedc2c64d55 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt @@ -53,7 +53,7 @@ private class CallsChecker(val context: Context, goodFunctions: List) { private fun LLVMValueRef.getPossiblyExternalCalledFunction(): ExternalCallInfo? { fun isIndirectCallArgument(value: LLVMValueRef) = LLVMIsALoadInst(value) != null || LLVMIsAArgument(value) != null || - LLVMIsAPHINode(value) != null || LLVMIsASelectInst(value) != null || LLVMIsACallInst(value) != null + LLVMIsAPHINode(value) != null || LLVMIsASelectInst(value) != null || LLVMIsACallInst(value) != null || LLVMIsAExtractElementInst(value) != null fun cleanCalledFunction(value: LLVMValueRef): ExternalCallInfo? { return when { @@ -202,4 +202,4 @@ internal fun addFunctionsListSymbolForChecker(context: Context) { ?.setInitializer(Int32(functions.size)) ?: throw IllegalStateException("$functionListSizeGlobal global not found") context.verifyBitCode() -} \ No newline at end of file +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp index a801e7fca69..2213e00c0ee 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp @@ -6,50 +6,119 @@ #include "GCScheduler.hpp" #include "CompilerConstants.hpp" +#include "GlobalData.hpp" +#include "KAssert.h" #include "Porting.h" +#include "RepeatedTimer.hpp" +#include "ThreadRegistry.hpp" +#include "ThreadData.hpp" using namespace kotlin; -bool gc::GCScheduler::ThreadData::OnSafePointSlowPath() noexcept { - const auto result = onSafePoint_(allocatedBytes_, safePointsCounter_); - ClearCountersAndUpdateThresholds(); - return result; +namespace { + +class GCEmptySchedulerData : public gc::GCSchedulerData { + void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {} + void OnPerformFullGC() noexcept override {} +}; + +class GCSchedulerDataWithTimer : public gc::GCSchedulerData { +public: + explicit GCSchedulerDataWithTimer(gc::GCSchedulerConfig& config, std::function scheduleGC) noexcept : + config_(config), scheduleGC_(std::move(scheduleGC)), timer_(std::chrono::microseconds(config_.regularGcIntervalUs), [this]() { + OnTimer(); + return std::chrono::microseconds(config_.regularGcIntervalUs); + }) {} + + void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override { + size_t allocatedBytes = threadData.allocatedBytes(); + if (allocatedBytes > config_.allocationThresholdBytes) { + RuntimeAssert(static_cast(scheduleGC_), "scheduleGC_ cannot be empty"); + scheduleGC_(); + } + } + + void OnPerformFullGC() noexcept override {} + +private: + void OnTimer() noexcept { + auto allThreadsAreNative = []() { + auto threadRegistryIter = mm::GlobalData::Instance().threadRegistry().LockForIter(); + return std::all_of(threadRegistryIter.begin(), threadRegistryIter.end(), [](mm::ThreadData& thread) { + return thread.state() == ThreadState::kNative; + }); + }(); + // Don't run, if kotlin code is not being executed. + if (allThreadsAreNative) return; + + // TODO: Probably makes sense to check memory usage of the process. + scheduleGC_(); + } + + gc::GCSchedulerConfig& config_; + + std::function scheduleGC_; + RepeatedTimer timer_; +}; + +class GCSchedulerDataWithoutTimer : public gc::GCSchedulerData { +public: + using CurrentTimeCallback = std::function; + + GCSchedulerDataWithoutTimer( + gc::GCSchedulerConfig& config, std::function scheduleGC, CurrentTimeCallback currentTimeCallbackNs) noexcept : + config_(config), + currentTimeCallbackNs_(std::move(currentTimeCallbackNs)), + timeOfLastGcNs_(currentTimeCallbackNs_()), + scheduleGC_(std::move(scheduleGC)) {} + + void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override { + size_t allocatedBytes = threadData.allocatedBytes(); + if (allocatedBytes > config_.allocationThresholdBytes || + currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs) { + RuntimeAssert(static_cast(scheduleGC_), "scheduleGC_ cannot be empty"); + scheduleGC_(); + } + } + + void OnPerformFullGC() noexcept override { timeOfLastGcNs_ = currentTimeCallbackNs_(); } + +private: + gc::GCSchedulerConfig& config_; + CurrentTimeCallback currentTimeCallbackNs_; + + std::atomic timeOfLastGcNs_; + std::function scheduleGC_; +}; + +} // namespace + +KStdUniquePtr gc::internal::MakeEmptyGCSchedulerData() noexcept { + return ::make_unique(); } -void gc::GCScheduler::ThreadData::ClearCountersAndUpdateThresholds() noexcept { - allocatedBytes_ = 0; - safePointsCounter_ = 0; - - allocatedBytesThreshold_ = config_.allocationThresholdBytes; - safePointsCounterThreshold_ = config_.threshold; +KStdUniquePtr gc::internal::MakeGCSchedulerDataWithTimer( + GCSchedulerConfig& config, std::function scheduleGC) noexcept { + return ::make_unique(config, std::move(scheduleGC)); } -gc::GCSchedulerConfig::GCSchedulerConfig() noexcept { - if (compiler::gcAggressive()) { - // TODO: Make it even more aggressive and run on a subset of backend.native tests. - threshold = 1000; - allocationThresholdBytes = 10000; - cooldownThresholdNs = 0; +KStdUniquePtr gc::internal::MakeGCSchedulerDataWithoutTimer( + GCSchedulerConfig& config, std::function scheduleGC, std::function currentTimeCallbackNs) noexcept { + return ::make_unique(config, std::move(scheduleGC), std::move(currentTimeCallbackNs)); +} + +KStdUniquePtr gc::internal::MakeGCSchedulerData(GCSchedulerConfig& config, std::function scheduleGC) noexcept { + if (internal::useGCTimer()) { + return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC)); + } else { + return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); }); } } -gc::GCScheduler::GCData::GCData(gc::GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept : - config_(config), currentTimeCallbackNs_(std::move(currentTimeCallbackNs)), timeOfLastGcNs_(currentTimeCallbackNs_()) {} - -bool gc::GCScheduler::GCData::OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept { - if (allocatedBytes > config_.allocationThresholdBytes) return true; - - return currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs; -} - -void gc::GCScheduler::GCData::OnPerformFullGC() noexcept { - timeOfLastGcNs_ = currentTimeCallbackNs_(); -} - -gc::GCScheduler::GCScheduler() noexcept : gcData_(config_, []() { return konan::getTimeNanos(); }) {} - -gc::GCScheduler::ThreadData gc::GCScheduler::NewThreadData() noexcept { - return ThreadData(config_, [this](size_t allocatedBytes, size_t safePointsCounter) { - return gcData().OnSafePoint(allocatedBytes, safePointsCounter); - }); +void gc::GCScheduler::SetScheduleGC(std::function scheduleGC) noexcept { + RuntimeAssert(static_cast(scheduleGC), "scheduleGC cannot be empty"); + RuntimeAssert(!static_cast(scheduleGC_), "scheduleGC must not have been set"); + scheduleGC_ = std::move(scheduleGC); + RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call"); + gcData_ = internal::MakeGCSchedulerData(config_, scheduleGC_); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp index 5ac9360426c..c168dfaa119 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp @@ -10,99 +10,162 @@ #include #include #include +#include +#include "CompilerConstants.hpp" +#include "Types.h" #include "Utils.hpp" namespace kotlin { namespace gc { +namespace internal { + +inline bool useGCTimer() noexcept { +#if KONAN_NO_THREADS + return false; +#else + // With aggressive mode we use safepoint counting to drive GC. + return !compiler::gcAggressive(); +#endif +} + +} // namespace internal + struct GCSchedulerConfig { std::atomic threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine). std::atomic allocationThresholdBytes = 10 * 1024 * 1024; // 10MiB by default. std::atomic cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default. std::atomic autoTune = false; + std::atomic regularGcIntervalUs = 200 * 1000; // 200 milliseconds by default. - GCSchedulerConfig() noexcept; + GCSchedulerConfig() noexcept { + if (compiler::gcAggressive()) { + // TODO: Make it even more aggressive and run on a subset of backend.native tests. + threshold = 1000; + allocationThresholdBytes = 10000; + cooldownThresholdNs = 0; + } + } }; -// TODO: Consider calling GC from the scheduler itself. -class GCScheduler : private Pinned { +class GCSchedulerThreadData; + +class GCSchedulerData { public: - class ThreadData { - public: - using OnSafePointCallback = std::function; + virtual ~GCSchedulerData() = default; - static constexpr size_t kFunctionEpilogueWeight = 1; - static constexpr size_t kLoopBodyWeight = 1; - static constexpr size_t kExceptionUnwindWeight = 1; + // Called by different mutator threads. + virtual void OnSafePoint(GCSchedulerThreadData& threadData) noexcept = 0; - explicit ThreadData(GCSchedulerConfig& config, OnSafePointCallback onSafePoint) noexcept : - config_(config), onSafePoint_(std::move(onSafePoint)) { - ClearCountersAndUpdateThresholds(); - } + // Always called by the GC thread. + virtual void OnPerformFullGC() noexcept = 0; +}; - // Should be called on encountering a safepoint. - bool OnSafePointRegular(size_t weight) noexcept { +class GCSchedulerThreadData { +public: + static constexpr size_t kFunctionEpilogueWeight = 1; + static constexpr size_t kLoopBodyWeight = 1; + static constexpr size_t kExceptionUnwindWeight = 1; + + explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function onSafePoint) noexcept : + config_(config), onSafePoint_(std::move(onSafePoint)) { + ClearCountersAndUpdateThresholds(); + } + + // Should be called on encountering a safepoint. + void OnSafePointRegular(size_t weight) noexcept { + if (!internal::useGCTimer()) { safePointsCounter_ += weight; if (safePointsCounter_ < safePointsCounterThreshold_) { - return false; + return; } - return OnSafePointSlowPath(); + OnSafePointSlowPath(); } + } - // Should be called on encountering a safepoint placed by the allocator. - // TODO: Should this even be a safepoint (i.e. a place, where we suspend)? - bool OnSafePointAllocation(size_t size) noexcept { - allocatedBytes_ += size; - if (allocatedBytes_ < allocatedBytesThreshold_) { - return false; - } - return OnSafePointSlowPath(); + // Should be called on encountering a safepoint placed by the allocator. + // TODO: Should this even be a safepoint (i.e. a place, where we suspend)? + void OnSafePointAllocation(size_t size) noexcept { + allocatedBytes_ += size; + if (allocatedBytes_ < allocatedBytesThreshold_) { + return; } + OnSafePointSlowPath(); + } - void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); } + void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); } - private: - bool OnSafePointSlowPath() noexcept; - void ClearCountersAndUpdateThresholds() noexcept; + size_t allocatedBytes() const noexcept { return allocatedBytes_; } - GCSchedulerConfig& config_; - OnSafePointCallback onSafePoint_; + size_t safePointsCounter() const noexcept { return safePointsCounter_; } - size_t allocatedBytes_ = 0; - size_t allocatedBytesThreshold_ = 0; - size_t safePointsCounter_ = 0; - size_t safePointsCounterThreshold_ = 0; - }; +private: + void OnSafePointSlowPath() noexcept { + onSafePoint_(*this); + ClearCountersAndUpdateThresholds(); + } - class GCData { - public: - using CurrentTimeCallback = std::function; + void ClearCountersAndUpdateThresholds() noexcept { + allocatedBytes_ = 0; + safePointsCounter_ = 0; - GCData(GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept; + allocatedBytesThreshold_ = config_.allocationThresholdBytes; + safePointsCounterThreshold_ = config_.threshold; + } - // May be called by different threads via `ThreadData`. - bool OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept; + GCSchedulerConfig& config_; + std::function onSafePoint_; - // Always called by the GC thread. - void OnPerformFullGC() noexcept; + size_t allocatedBytes_ = 0; + size_t allocatedBytesThreshold_ = 0; + size_t safePointsCounter_ = 0; + size_t safePointsCounterThreshold_ = 0; +}; - private: - GCSchedulerConfig& config_; - CurrentTimeCallback currentTimeCallbackNs_; +namespace internal { - std::atomic timeOfLastGcNs_; - }; +KStdUniquePtr MakeEmptyGCSchedulerData() noexcept; +KStdUniquePtr MakeGCSchedulerDataWithTimer(GCSchedulerConfig& config, std::function scheduleGC) noexcept; +KStdUniquePtr MakeGCSchedulerDataWithoutTimer( + GCSchedulerConfig& config, std::function scheduleGC, std::function currentTimeCallbackNs) noexcept; +KStdUniquePtr MakeGCSchedulerData(GCSchedulerConfig& config, std::function scheduleGC) noexcept; - GCScheduler() noexcept; +} // namespace internal + +class GCScheduler : private Pinned { +public: + GCScheduler() noexcept = default; GCSchedulerConfig& config() noexcept { return config_; } - GCData& gcData() noexcept { return gcData_; } - ThreadData NewThreadData() noexcept; + // Only valid after `SetScheduleGC` is called. + GCSchedulerData& gcData() noexcept { + RuntimeAssert(gcData_ != nullptr, "Cannot be called before SetScheduleGC"); + return *gcData_; + } + + // Can only be called once. + void SetScheduleGC(std::function scheduleGC) noexcept; + + GCSchedulerThreadData NewThreadData() noexcept { + return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->OnSafePoint(threadData); }); + } + + template + KStdUniquePtr ReplaceGCSchedulerDataForTests(F&& factory) noexcept { + RuntimeAssert(static_cast(scheduleGC_), "Can only be called after SetScheduleGC"); + + auto other = std::forward(factory)(config_, scheduleGC_); + RuntimeAssert(other != nullptr, "factory cannot return a null"); + using std::swap; + swap(gcData_, other); + return other; + } private: GCSchedulerConfig config_; - GCData gcData_; + KStdUniquePtr gcData_; + std::function scheduleGC_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 3a18d6ed2fa..e5bda47ad4f 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -51,52 +51,38 @@ struct FinalizeTraits { using ObjectFactory = mm::ObjectFactory; }; +// Global, because it's accessed on a hot path: avoid memory load from `this`. +std::atomic gSafepointFlag = gc::SameThreadMarkAndSweep::SafepointFlag::kNone; + } // namespace -void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept { - SafePointRegular(GCScheduler::ThreadData::kFunctionEpilogueWeight); +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept { + SafePointRegular(GCSchedulerThreadData::kFunctionEpilogueWeight); } -void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(GCScheduler::ThreadData::kLoopBodyWeight); +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { + SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight); } -void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept { - SafePointRegular(GCScheduler::ThreadData::kExceptionUnwindWeight); +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept { + SafePointRegular(GCSchedulerThreadData::kExceptionUnwindWeight); } void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - threadData_.suspensionData().suspendIfRequested(); - auto& scheduler = threadData_.gcScheduler(); - if (scheduler.OnSafePointAllocation(size)) { - RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointAllocation size=%zu", size); - PerformFullGC(); + threadData_.gcScheduler().OnSafePointAllocation(size); + SafepointFlag flag = gSafepointFlag.load(); + if (flag != SafepointFlag::kNone) { + SafePointSlowPath(flag); } } void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept { - mm::ObjectFactory::FinalizerQueue finalizerQueue; - { - // Switch state to native to simulate this thread being a GC thread. - // As a bonus, if we failed to suspend threads (which means some other thread asked for a GC), - // we will automatically suspend at the scope exit. - // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`. - ThreadStateGuard guard(ThreadState::kNative); - finalizerQueue = gc_.PerformFullGC(); + auto didGC = gc_.PerformFullGC(); + + if (!didGC) { + // If we failed to suspend threads, someone else might be asking to suspend them. + threadData_.suspensionData().suspendIfRequested(); } - - // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would - // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing - // kotlin code. - - // TODO: These will actually need to be run on a separate thread. - // TODO: Cannot use `threadData_` here, because there's no way to transform `mm::ThreadData` into `MemoryState*`. - AssertThreadState(ThreadState::kRunnable); - RuntimeLogDebug({kTagGC}, "Starting to run finalizers"); - auto timeBeforeUs = konan::getTimeMicros(); - finalizerQueue.Finalize(); - auto timeAfterUs = konan::getTimeMicros(); - RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs); } void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { @@ -104,102 +90,144 @@ void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { PerformFullGC(); } -void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { + threadData_.gcScheduler().OnSafePointRegular(weight); + SafepointFlag flag = gSafepointFlag.load(); + if (flag != SafepointFlag::kNone) { + SafePointSlowPath(flag); + } +} + +NO_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointSlowPath(SafepointFlag flag) noexcept { + RuntimeAssert(flag != SafepointFlag::kNone, "Must've been handled by the caller"); + // No need to check for kNeedsSuspend, because `suspendIfRequested` checks for its own atomic. threadData_.suspensionData().suspendIfRequested(); - auto& scheduler = threadData_.gcScheduler(); - if (scheduler.OnSafePointRegular(weight)) { - RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointRegular weight=%zu", weight); + if (flag == SafepointFlag::kNeedsGC) { + RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePoint"); PerformFullGC(); } } -mm::ObjectFactory::FinalizerQueue gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { +gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept { + mm::GlobalData::Instance().gcScheduler().SetScheduleGC([]() { + RuntimeLogDebug({kTagGC}, "Scheduling GC by thread %d", konan::currentThreadId()); + gSafepointFlag = SafepointFlag::kNeedsGC; + }); +} + +bool gc::SameThreadMarkAndSweep::PerformFullGC() noexcept { RuntimeLogDebug({kTagGC}, "Attempt to suspend threads by thread %d", konan::currentThreadId()); auto timeStartUs = konan::getTimeMicros(); - bool didSuspend = mm::SuspendThreads(); - auto timeSuspendUs = konan::getTimeMicros(); + bool didSuspend = mm::RequestThreadsSuspension(); if (!didSuspend) { - RuntimeLogDebug({kTagGC}, "Failed to suspend threads"); + RuntimeLogDebug({kTagGC}, "Failed to suspend threads by thread %d", konan::currentThreadId()); // Somebody else suspended the threads, and so ran a GC. // TODO: This breaks if suspension is used by something apart from GC. - return {}; + return false; } - RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); + RuntimeLogDebug({kTagGC}, "Requested thread suspension by thread %d", konan::currentThreadId()); + gSafepointFlag = SafepointFlag::kNeedsSuspend; - auto& scheduler = mm::GlobalData::Instance().gcScheduler(); - scheduler.gcData().OnPerformFullGC(); + mm::ObjectFactory::FinalizerQueue finalizerQueue; + { + // Switch state to native to simulate this thread being a GC thread. + ThreadStateGuard guard(ThreadState::kNative); - RuntimeLogInfo({kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); - KStdVector graySet; - for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { - // TODO: Maybe it's more efficient to do by the suspending thread? - thread.Publish(); - thread.gcScheduler().OnStoppedForGC(); - size_t stack = 0; - size_t tls = 0; - for (auto value : mm::ThreadRootSet(thread)) { + mm::WaitForThreadsSuspension(); + auto timeSuspendUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); + + auto& scheduler = mm::GlobalData::Instance().gcScheduler(); + scheduler.gcData().OnPerformFullGC(); + + RuntimeLogInfo( + {kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); + KStdVector graySet; + for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { + // TODO: Maybe it's more efficient to do by the suspending thread? + thread.Publish(); + thread.gcScheduler().OnStoppedForGC(); + size_t stack = 0; + size_t tls = 0; + for (auto value : mm::ThreadRootSet(thread)) { + if (!isNullOrMarker(value.object)) { + graySet.push_back(value.object); + switch (value.source) { + case mm::ThreadRootSet::Source::kStack: + ++stack; + break; + case mm::ThreadRootSet::Source::kTLS: + ++tls; + break; + } + } + } + RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls); + } + mm::StableRefRegistry::Instance().ProcessDeletions(); + size_t global = 0; + size_t stableRef = 0; + for (auto value : mm::GlobalRootSet()) { if (!isNullOrMarker(value.object)) { graySet.push_back(value.object); switch (value.source) { - case mm::ThreadRootSet::Source::kStack: - ++stack; + case mm::GlobalRootSet::Source::kGlobal: + ++global; break; - case mm::ThreadRootSet::Source::kTLS: - ++tls; + case mm::GlobalRootSet::Source::kStableRef: + ++stableRef; break; } } } - RuntimeLogDebug({kTagGC}, "Collected root set for thread stack=%zu tls=%zu", stack, tls); + auto timeRootSetUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef); + + // Can be unsafe, because we've stopped the world. + auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + + RuntimeLogInfo( + {kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(), + stableRef, timeRootSetUs - timeSuspendUs); + gc::Mark(std::move(graySet)); + auto timeMarkUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs); + finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); + auto timeSweepUs = konan::getTimeMicros(); + RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs); + + // Can be unsafe, because we've stopped the world. + auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + + gSafepointFlag = SafepointFlag::kNone; + mm::ResumeThreads(); + auto timeResumeUs = konan::getTimeMicros(); + + RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs); + + auto finalizersCount = finalizerQueue.size(); + auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount; + + RuntimeLogInfo( + {kTagGC}, + "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64 + " microseconds", + epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs); + ++epoch_; + lastGCTimestampUs_ = timeResumeUs; } - mm::StableRefRegistry::Instance().ProcessDeletions(); - size_t global = 0; - size_t stableRef = 0; - for (auto value : mm::GlobalRootSet()) { - if (!isNullOrMarker(value.object)) { - graySet.push_back(value.object); - switch (value.source) { - case mm::GlobalRootSet::Source::kGlobal: - ++global; - break; - case mm::GlobalRootSet::Source::kStableRef: - ++stableRef; - break; - } - } - } - auto timeRootSetUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Collected global root set global=%zu stableRef=%zu", global, stableRef); - // Can be unsafe, because we've stopped the world. - auto objectsCountBefore = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); + // Finalizers are run after threads are resumed, because finalizers may request GC themselves, which would + // try to suspend threads again. Also, we run finalizers in the runnable state, because they may be executing + // kotlin code. - RuntimeLogInfo({kTagGC}, "Collected root set of size %zu of which %zu are stable refs in %" PRIu64 " microseconds", graySet.size(), stableRef, timeRootSetUs - timeSuspendUs); - gc::Mark(std::move(graySet)); - auto timeMarkUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Marked in %" PRIu64 " microseconds", timeMarkUs - timeRootSetUs); - auto finalizerQueue = gc::Sweep(mm::GlobalData::Instance().objectFactory()); - auto timeSweepUs = konan::getTimeMicros(); - RuntimeLogDebug({kTagGC}, "Sweeped in %" PRIu64 " microseconds", timeSweepUs - timeMarkUs); + // TODO: These will actually need to be run on a separate thread. + AssertThreadState(ThreadState::kRunnable); + RuntimeLogDebug({kTagGC}, "Starting to run finalizers"); + auto timeBeforeUs = konan::getTimeMicros(); + finalizerQueue.Finalize(); + auto timeAfterUs = konan::getTimeMicros(); + RuntimeLogInfo({kTagGC}, "Finished running finalizers in %" PRIu64 " microseconds", timeAfterUs - timeBeforeUs); - // Can be unsafe, because we've stopped the world. - auto objectsCountAfter = mm::GlobalData::Instance().objectFactory().GetSizeUnsafe(); - - mm::ResumeThreads(); - auto timeResumeUs = konan::getTimeMicros(); - - RuntimeLogDebug({kTagGC}, "Resumed threads in %" PRIu64 " microseconds.", timeResumeUs - timeSweepUs); - - auto finalizersCount = finalizerQueue.size(); - auto collectedCount = objectsCountBefore - objectsCountAfter - finalizersCount; - - RuntimeLogInfo( - {kTagGC}, - "Finished GC epoch %zu. Collected %zu objects, to be finalized %zu objects, %zu objects remain. Total pause time %" PRIu64 - " microseconds", - epoch_, collectedCount, finalizersCount, objectsCountAfter, timeResumeUs - timeStartUs); - ++epoch_; - lastGCTimestampUs_ = timeResumeUs; - - return finalizerQueue; + return true; } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index f5f0afa571c..7346faeb941 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -24,6 +24,12 @@ namespace gc { // Stop-the-world Mark-and-Sweep that runs on mutator threads. Can support targets that do not have threads. class SameThreadMarkAndSweep : private Pinned { public: + enum class SafepointFlag { + kNone, + kNeedsSuspend, + kNeedsGC, + }; + class ObjectData { public: enum class Color { @@ -57,16 +63,18 @@ public: private: void SafePointRegular(size_t weight) noexcept; + void SafePointSlowPath(SafepointFlag flag) noexcept; SameThreadMarkAndSweep& gc_; mm::ThreadData& threadData_; }; - SameThreadMarkAndSweep() noexcept = default; + SameThreadMarkAndSweep() noexcept; ~SameThreadMarkAndSweep() = default; private: - mm::ObjectFactory::FinalizerQueue PerformFullGC() noexcept; + // Returns `true` if GC has happened, and `false` if not (because someone else has suspended the threads). + bool PerformFullGC() noexcept; size_t epoch_ = 0; uint64_t lastGCTimestampUs_ = 0; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 8b4fa379054..258763de888 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -215,9 +215,16 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader class SameThreadMarkAndSweepTest : public testing::Test { public: + SameThreadMarkAndSweepTest() { + mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests( + [](auto& config, auto scheduleGC) { return gc::internal::MakeEmptyGCSchedulerData(); }); + } + ~SameThreadMarkAndSweepTest() { mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().objectFactory().ClearForTests(); + mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests( + [](auto& config, auto scheduleGC) { return gc::internal::MakeGCSchedulerData(config, std::move(scheduleGC)); }); } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } diff --git a/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp b/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp index 65b376fd7a5..9352e2f0c13 100644 --- a/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp +++ b/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp @@ -7,8 +7,10 @@ #define RUNTIME_SINGLE_LOCK_LIST_H #include +#include #include #include +#include #include "Alloc.h" #include "Mutex.hpp" @@ -52,6 +54,12 @@ public: class Iterator { public: + using difference_type = void; + using value_type = Value; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::forward_iterator_tag; + explicit Iterator(Node* node) noexcept : node_(node) {} Value& operator*() noexcept { return node_->value_; } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index ba872a978ad..d519284ef84 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -511,19 +511,19 @@ extern "C" void CheckGlobalsAccessible() { // Always accessible } -extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() { +extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointFunctionEpilogue() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); threadData->gc().SafePointFunctionEpilogue(); } -extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() { +extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointWhileLoopBody() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); threadData->gc().SafePointLoopBody(); } -extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() { +extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointExceptionUnwind() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); threadData->gc().SafePointExceptionUnwind(); diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 239d9d9c5ba..c81c19343a4 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -58,7 +58,7 @@ public: KStdVector>& initializingSingletons() noexcept { return initializingSingletons_; } - gc::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; } + gc::GCSchedulerThreadData& gcScheduler() noexcept { return gcScheduler_; } gc::GC::ThreadData& gc() noexcept { return gc_; } @@ -83,7 +83,7 @@ private: ThreadLocalStorage tls_; StableRefRegistry::ThreadQueue stableRefThreadQueue_; ShadowStack shadowStack_; - gc::GCScheduler::ThreadData gcScheduler_; + gc::GCSchedulerThreadData gcScheduler_; gc::GC::ThreadData gc_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; KStdVector> initializingSingletons_; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index 738a637a934..3d202c546ca 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -11,6 +11,7 @@ #include #include "Logging.hpp" +#include "StackTrace.hpp" namespace { @@ -60,7 +61,7 @@ NO_EXTERNAL_CALLS_CHECK void kotlin::mm::ThreadSuspensionData::suspendIfRequeste } } -bool kotlin::mm::SuspendThreads() noexcept { +NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::RequestThreadsSuspension() noexcept { RuntimeAssert(gSuspensionRequestedByCurrentThread == false, "Current thread already suspended threads."); { std::unique_lock lock(gSuspensionMutex); @@ -72,12 +73,14 @@ bool kotlin::mm::SuspendThreads() noexcept { } gSuspensionRequestedByCurrentThread = true; + return true; +} + +void kotlin::mm::WaitForThreadsSuspension() noexcept { // Spin wating for threads to suspend. Ignore Native threads. while(!allThreads(isSuspendedOrNative)) { yield(); } - - return true; } void kotlin::mm::ResumeThreads() noexcept { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp index c3b3f953735..e453a5cb4c2 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp @@ -54,13 +54,22 @@ private: void suspendIfRequestedSlowPath() noexcept; }; +bool RequestThreadsSuspension() noexcept; +void WaitForThreadsSuspension() noexcept; + /** * Suspends all threads registered in ThreadRegistry except threads that are in the Native state. * Blocks until all such threads are suspended. Threads that are in the Native state on the moment * of this call will be suspended on exit from the Native state. * Returns false if some other thread has suspended the threads. */ -bool SuspendThreads() noexcept; +inline bool SuspendThreads() noexcept { + if (!RequestThreadsSuspension()) { + return false; + } + WaitForThreadsSuspension(); + return true; +} /** * Resumes all threads registered in ThreadRegistry that were suspended by the SuspendThreads call.