diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt index e3790cd3cf1..07adae68c77 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CheckExternalCalls.kt @@ -124,7 +124,11 @@ private class CallsChecker(val context: Context, goodFunctions: List) { else -> { callSiteDescription = functionName calledName = calleeInfo.name - calledPtrLlvm = LLVMBuildBitCast(builder, calleeInfo.calledPtr, int8TypePtr, "") + calledPtrLlvm = when (val typeKind = LLVMGetTypeKind(calleeInfo.calledPtr.type)) { + LLVMTypeKind.LLVMPointerTypeKind -> LLVMBuildBitCast(builder, calleeInfo.calledPtr, int8TypePtr, "") + LLVMTypeKind.LLVMIntegerTypeKind -> LLVMBuildIntToPtr(builder, calleeInfo.calledPtr, int8TypePtr, "") + else -> TODO("Unsupported typeKind=${typeKind} of calledPtr=${llvm2string(calleeInfo.calledPtr)}") + } } } val callSiteDescriptionLlvm = context.llvm.staticData.cStringLiteral(callSiteDescription).llvm diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp new file mode 100644 index 00000000000..a801e7fca69 --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp @@ -0,0 +1,55 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GCScheduler.hpp" + +#include "CompilerConstants.hpp" +#include "Porting.h" + +using namespace kotlin; + +bool gc::GCScheduler::ThreadData::OnSafePointSlowPath() noexcept { + const auto result = onSafePoint_(allocatedBytes_, safePointsCounter_); + ClearCountersAndUpdateThresholds(); + return result; +} + +void gc::GCScheduler::ThreadData::ClearCountersAndUpdateThresholds() noexcept { + allocatedBytes_ = 0; + safePointsCounter_ = 0; + + allocatedBytesThreshold_ = config_.allocationThresholdBytes; + safePointsCounterThreshold_ = config_.threshold; +} + +gc::GCSchedulerConfig::GCSchedulerConfig() noexcept { + if (compiler::gcAggressive()) { + // TODO: Make it even more aggressive and run on a subset of backend.native tests. + threshold = 1000; + allocationThresholdBytes = 10000; + cooldownThresholdNs = 0; + } +} + +gc::GCScheduler::GCData::GCData(gc::GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept : + config_(config), currentTimeCallbackNs_(std::move(currentTimeCallbackNs)), timeOfLastGcNs_(currentTimeCallbackNs_()) {} + +bool gc::GCScheduler::GCData::OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept { + if (allocatedBytes > config_.allocationThresholdBytes) return true; + + return currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs; +} + +void gc::GCScheduler::GCData::OnPerformFullGC() noexcept { + timeOfLastGcNs_ = currentTimeCallbackNs_(); +} + +gc::GCScheduler::GCScheduler() noexcept : gcData_(config_, []() { return konan::getTimeNanos(); }) {} + +gc::GCScheduler::ThreadData gc::GCScheduler::NewThreadData() noexcept { + return ThreadData(config_, [this](size_t allocatedBytes, size_t safePointsCounter) { + return gcData().OnSafePoint(allocatedBytes, safePointsCounter); + }); +} diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp new file mode 100644 index 00000000000..5ac9360426c --- /dev/null +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp @@ -0,0 +1,111 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_GC_COMMON_GC_SCHEDULER_H +#define RUNTIME_GC_COMMON_GC_SCHEDULER_H + +#include +#include +#include +#include + +#include "Utils.hpp" + +namespace kotlin { +namespace gc { + +struct GCSchedulerConfig { + std::atomic threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine). + std::atomic allocationThresholdBytes = 10 * 1024 * 1024; // 10MiB by default. + std::atomic cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default. + std::atomic autoTune = false; + + GCSchedulerConfig() noexcept; +}; + +// TODO: Consider calling GC from the scheduler itself. +class GCScheduler : private Pinned { +public: + class ThreadData { + public: + using OnSafePointCallback = std::function; + + static constexpr size_t kFunctionEpilogueWeight = 1; + static constexpr size_t kLoopBodyWeight = 1; + static constexpr size_t kExceptionUnwindWeight = 1; + + explicit ThreadData(GCSchedulerConfig& config, OnSafePointCallback onSafePoint) noexcept : + config_(config), onSafePoint_(std::move(onSafePoint)) { + ClearCountersAndUpdateThresholds(); + } + + // Should be called on encountering a safepoint. + bool OnSafePointRegular(size_t weight) noexcept { + safePointsCounter_ += weight; + if (safePointsCounter_ < safePointsCounterThreshold_) { + return false; + } + return OnSafePointSlowPath(); + } + + // Should be called on encountering a safepoint placed by the allocator. + // TODO: Should this even be a safepoint (i.e. a place, where we suspend)? + bool OnSafePointAllocation(size_t size) noexcept { + allocatedBytes_ += size; + if (allocatedBytes_ < allocatedBytesThreshold_) { + return false; + } + return OnSafePointSlowPath(); + } + + void OnStoppedForGC() noexcept { ClearCountersAndUpdateThresholds(); } + + private: + bool OnSafePointSlowPath() noexcept; + void ClearCountersAndUpdateThresholds() noexcept; + + GCSchedulerConfig& config_; + OnSafePointCallback onSafePoint_; + + size_t allocatedBytes_ = 0; + size_t allocatedBytesThreshold_ = 0; + size_t safePointsCounter_ = 0; + size_t safePointsCounterThreshold_ = 0; + }; + + class GCData { + public: + using CurrentTimeCallback = std::function; + + GCData(GCSchedulerConfig& config, CurrentTimeCallback currentTimeCallbackNs) noexcept; + + // May be called by different threads via `ThreadData`. + bool OnSafePoint(size_t allocatedBytes, size_t safePointsCounter) noexcept; + + // Always called by the GC thread. + void OnPerformFullGC() noexcept; + + private: + GCSchedulerConfig& config_; + CurrentTimeCallback currentTimeCallbackNs_; + + std::atomic timeOfLastGcNs_; + }; + + GCScheduler() noexcept; + + GCSchedulerConfig& config() noexcept { return config_; } + GCData& gcData() noexcept { return gcData_; } + ThreadData NewThreadData() noexcept; + +private: + GCSchedulerConfig config_; + GCData gcData_; +}; + +} // namespace gc +} // namespace kotlin + +#endif // RUNTIME_GC_COMMON_GC_SCHEDULER_H diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 4e4cfef2cfe..b90592e1c17 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -8,6 +8,7 @@ #include +#include "GCScheduler.hpp" #include "Utils.hpp" #include "Types.h" @@ -47,23 +48,10 @@ public: NoOpGC() noexcept {} ~NoOpGC() = default; - void SetThreshold(size_t value) noexcept { threshold_ = value; } - size_t GetThreshold() noexcept { return threshold_; } - - void SetAllocationThresholdBytes(size_t value) noexcept { allocationThresholdBytes_ = value; } - size_t GetAllocationThresholdBytes() noexcept { return allocationThresholdBytes_; } - - void SetCooldownThresholdUs(uint64_t value) noexcept { cooldownThresholdUs_ = value; } - uint64_t GetCooldownThresholdUs() noexcept { return cooldownThresholdUs_; } - - void SetAutoTune(bool value) noexcept { autoTune_ = value; } - bool GetAutoTune() noexcept { return autoTune_; } + GCScheduler& scheduler() noexcept { return scheduler_; } private: - size_t threshold_ = 0; - size_t allocationThresholdBytes_ = 0; - uint64_t cooldownThresholdUs_ = 0; - bool autoTune_ = false; + GCScheduler scheduler_; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 0ca3a8cd5b6..3a18d6ed2fa 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -54,28 +54,24 @@ struct FinalizeTraits { } // namespace void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept { - SafePointRegular(1); + SafePointRegular(GCScheduler::ThreadData::kFunctionEpilogueWeight); } void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { - SafePointRegular(1); + SafePointRegular(GCScheduler::ThreadData::kLoopBodyWeight); } void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept { - SafePointRegular(1); + SafePointRegular(GCScheduler::ThreadData::kExceptionUnwindWeight); } void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { - size_t allocationOverhead = - gc_.GetAllocationThresholdBytes() == 0 ? allocatedBytes_ : allocatedBytes_ % gc_.GetAllocationThresholdBytes(); - if (threadData_.suspensionData().suspendIfRequested()) { - allocatedBytes_ = 0; - } else if (allocationOverhead + size >= gc_.GetAllocationThresholdBytes()) { + threadData_.suspensionData().suspendIfRequested(); + auto& scheduler = threadData_.gcScheduler(); + if (scheduler.OnSafePointAllocation(size)) { RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointAllocation size=%zu", size); - allocatedBytes_ = 0; PerformFullGC(); } - allocatedBytes_ += size; } void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept { @@ -104,36 +100,16 @@ void gc::SameThreadMarkAndSweep::ThreadData::PerformFullGC() noexcept { } void gc::SameThreadMarkAndSweep::ThreadData::OnOOM(size_t size) noexcept { - RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM"); + RuntimeLogDebug({kTagGC}, "Attempt to GC on OOM at size=%zu", size); PerformFullGC(); } -void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegularSlowPath() noexcept { - safePointsCounter_ = 0; - if (konan::getTimeMicros() - timeOfLastGcUs_ >= gc_.GetCooldownThresholdUs()) { - RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointRegular"); - timeOfLastGcUs_ = konan::getTimeMicros(); - PerformFullGC(); - } -} - void gc::SameThreadMarkAndSweep::ThreadData::SafePointRegular(size_t weight) noexcept { - if (threadData_.suspensionData().suspendIfRequested()) { - safePointsCounter_ = 0; - } else { - safePointsCounter_ += weight; - if (safePointsCounter_ >= gc_.GetThreshold()) { - SafePointRegularSlowPath(); - } - } -} - -gc::SameThreadMarkAndSweep::SameThreadMarkAndSweep() noexcept { - if (compiler::gcAggressive()) { - // TODO: Make it even more aggressive and run on a subset of backend.native tests. - threshold_ = 1000; - allocationThresholdBytes_ = 10000; - cooldownThresholdUs_ = 0; + threadData_.suspensionData().suspendIfRequested(); + auto& scheduler = threadData_.gcScheduler(); + if (scheduler.OnSafePointRegular(weight)) { + RuntimeLogDebug({kTagGC}, "Attempt to GC at SafePointRegular weight=%zu", weight); + PerformFullGC(); } } @@ -150,11 +126,15 @@ mm::ObjectFactory::FinalizerQueue gc::SameThreadMark } RuntimeLogDebug({kTagGC}, "Suspended all threads in %" PRIu64 " microseconds", timeSuspendUs - timeStartUs); + auto& scheduler = mm::GlobalData::Instance().gcScheduler(); + scheduler.gcData().OnPerformFullGC(); + RuntimeLogInfo({kTagGC}, "Started GC epoch %zu. Time since last GC %" PRIu64 " microseconds", epoch_, timeStartUs - lastGCTimestampUs_); KStdVector graySet; for (auto& thread : mm::GlobalData::Instance().threadRegistry().LockForIter()) { // TODO: Maybe it's more efficient to do by the suspending thread? thread.Publish(); + thread.gcScheduler().OnStoppedForGC(); size_t stack = 0; size_t tls = 0; for (auto value : mm::ThreadRootSet(thread)) { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index f02f8c5d891..f5f0afa571c 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -8,6 +8,7 @@ #include +#include "GCScheduler.hpp" #include "ObjectFactory.hpp" #include "Types.h" #include "Utils.hpp" @@ -56,40 +57,19 @@ public: private: void SafePointRegular(size_t weight) noexcept; - void SafePointRegularSlowPath() noexcept; SameThreadMarkAndSweep& gc_; mm::ThreadData& threadData_; - size_t allocatedBytes_ = 0; - size_t safePointsCounter_ = 0; - uint64_t timeOfLastGcUs_ = konan::getTimeMicros(); }; - SameThreadMarkAndSweep() noexcept; + SameThreadMarkAndSweep() noexcept = default; ~SameThreadMarkAndSweep() = default; - void SetThreshold(size_t value) noexcept { threshold_ = value; } - size_t GetThreshold() noexcept { return threshold_; } - - void SetAllocationThresholdBytes(size_t value) noexcept { allocationThresholdBytes_ = value; } - size_t GetAllocationThresholdBytes() noexcept { return allocationThresholdBytes_; } - - void SetCooldownThresholdUs(uint64_t value) noexcept { cooldownThresholdUs_ = value; } - uint64_t GetCooldownThresholdUs() noexcept { return cooldownThresholdUs_; } - - void SetAutoTune(bool value) noexcept { autoTune_ = value; } - bool GetAutoTune() noexcept { return autoTune_; } - private: mm::ObjectFactory::FinalizerQueue PerformFullGC() noexcept; size_t epoch_ = 0; uint64_t lastGCTimestampUs_ = 0; - - size_t threshold_ = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine). - size_t allocationThresholdBytes_ = 10 * 1024 * 1024; // 10MiB by default. - uint64_t cooldownThresholdUs_ = 200 * 1000; // 200 milliseconds by default. - bool autoTune_ = false; }; } // namespace gc diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 4b870f3a9f9..8b4fa379054 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -215,24 +215,7 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader class SameThreadMarkAndSweepTest : public testing::Test { public: - SameThreadMarkAndSweepTest() { - // These tests rely on GC being called only when asked. Setting the thresholds unreachably high for this. - // TODO: Alternatively need to mark "safe" zones, where GC cannot happen. - auto& gc = mm::GlobalData::Instance().gc(); - threshold_ = gc.GetThreshold(); - gc.SetThreshold(std::numeric_limits::max()); - allocationThresholdBytes_ = gc.GetAllocationThresholdBytes(); - gc.SetAllocationThresholdBytes(std::numeric_limits::max()); - cooldownThresholdUs_ = gc.GetCooldownThresholdUs(); - gc.SetCooldownThresholdUs(std::numeric_limits::max()); - } - ~SameThreadMarkAndSweepTest() { - auto& gc = mm::GlobalData::Instance().gc(); - gc.SetThreshold(threshold_); - gc.SetAllocationThresholdBytes(allocationThresholdBytes_); - gc.SetCooldownThresholdUs(cooldownThresholdUs_); - mm::GlobalsRegistry::Instance().ClearForTests(); mm::GlobalData::Instance().objectFactory().ClearForTests(); } @@ -241,9 +224,6 @@ public: private: FinalizerHooksTestSupport finalizerHooks_; - size_t threshold_; - size_t allocationThresholdBytes_; - uint64_t cooldownThresholdUs_; }; } // namespace diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index 5427d32f3c4..c6c096febbb 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -9,6 +9,7 @@ #include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" +#include "GCScheduler.hpp" #include "StableRefRegistry.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" @@ -25,6 +26,7 @@ public: GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; } ObjectFactory& objectFactory() noexcept { return objectFactory_; } + gc::GCScheduler& gcScheduler() noexcept { return gcScheduler_; } gc::GC& gc() noexcept { return gc_; } private: @@ -38,6 +40,7 @@ private: GlobalsRegistry globalsRegistry_; StableRefRegistry stableRefRegistry_; ObjectFactory objectFactory_; + gc::GCScheduler gcScheduler_; gc::GC gc_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index cf8d5c0fc1e..1273a74f127 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -298,11 +298,11 @@ extern "C" void Kotlin_native_internal_GC_setThreshold(ObjHeader*, int32_t value if (value < 0) { ThrowIllegalArgumentException(); } - mm::GlobalData::Instance().gc().SetThreshold(static_cast(value)); + mm::GlobalData::Instance().gcScheduler().config().threshold = static_cast(value); } extern "C" int32_t Kotlin_native_internal_GC_getThreshold(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gc().GetThreshold(); + auto threshold = mm::GlobalData::Instance().gcScheduler().config().threshold.load(); auto maxValue = std::numeric_limits::max(); if (threshold > static_cast(maxValue)) { return maxValue; @@ -324,11 +324,11 @@ extern "C" void Kotlin_native_internal_GC_setThresholdAllocations(ObjHeader*, in if (value < 0) { ThrowIllegalArgumentException(); } - mm::GlobalData::Instance().gc().SetAllocationThresholdBytes(static_cast(value)); + mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes = static_cast(value); } extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) { - auto threshold = mm::GlobalData::Instance().gc().GetAllocationThresholdBytes(); + auto threshold = mm::GlobalData::Instance().gcScheduler().config().allocationThresholdBytes.load(); auto maxValue = std::numeric_limits::max(); if (threshold > static_cast(maxValue)) { return maxValue; @@ -337,11 +337,11 @@ extern "C" int64_t Kotlin_native_internal_GC_getThresholdAllocations(ObjHeader*) } extern "C" void Kotlin_native_internal_GC_setTuneThreshold(ObjHeader*, KBoolean value) { - mm::GlobalData::Instance().gc().SetAutoTune(value); + mm::GlobalData::Instance().gcScheduler().config().autoTune = value; } extern "C" KBoolean Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) { - return mm::GlobalData::Instance().gc().GetAutoTune(); + return mm::GlobalData::Instance().gcScheduler().config().autoTune.load(); } extern "C" OBJ_GETTER(Kotlin_native_internal_GC_detectCycles, ObjHeader*) { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index dc8e57e35f0..239d9d9c5ba 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -11,6 +11,7 @@ #include "GlobalData.hpp" #include "GlobalsRegistry.hpp" #include "GC.hpp" +#include "GCScheduler.hpp" #include "ObjectFactory.hpp" #include "ShadowStack.hpp" #include "StableRefRegistry.hpp" @@ -32,6 +33,7 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()), + gcScheduler_(GlobalData::Instance().gcScheduler().NewThreadData()), gc_(GlobalData::Instance().gc(), *this), objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_), suspensionData_(ThreadState::kNative) {} @@ -56,6 +58,8 @@ public: KStdVector>& initializingSingletons() noexcept { return initializingSingletons_; } + gc::GCScheduler::ThreadData& gcScheduler() noexcept { return gcScheduler_; } + gc::GC::ThreadData& gc() noexcept { return gc_; } ThreadSuspensionData& suspensionData() { return suspensionData_; } @@ -79,6 +83,7 @@ private: ThreadLocalStorage tls_; StableRefRegistry::ThreadQueue stableRefThreadQueue_; ShadowStack shadowStack_; + gc::GCScheduler::ThreadData gcScheduler_; gc::GC::ThreadData gc_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; KStdVector> initializingSingletons_; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp index a893ef579ff..701e9bd495f 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.cpp @@ -39,21 +39,15 @@ void yield() noexcept { std::this_thread::yield(); } -std::atomic gSuspensionRequested = false; THREAD_LOCAL_VARIABLE bool gSuspensionRequestedByCurrentThread = false; std::mutex gSuspensionMutex; std::condition_variable gSuspendsionCondVar; } // namespace -NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::ThreadSuspensionData::suspendIfRequested() noexcept { - if (IsThreadSuspensionRequested()) { - return suspendIfRequestedSlowPath(); - } - return false; -} +std::atomic kotlin::mm::internal::gSuspensionRequested = false; -NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::ThreadSuspensionData::suspendIfRequestedSlowPath() noexcept { +NO_EXTERNAL_CALLS_CHECK void kotlin::mm::ThreadSuspensionData::suspendIfRequestedSlowPath() noexcept { std::unique_lock lock(gSuspensionMutex); if (IsThreadSuspensionRequested()) { auto threadId = konan::currentThreadId(); @@ -61,14 +55,7 @@ NO_EXTERNAL_CALLS_CHECK bool kotlin::mm::ThreadSuspensionData::suspendIfRequeste AutoReset scopedAssign(&suspended_, true); gSuspendsionCondVar.wait(lock, []() { return !IsThreadSuspensionRequested(); }); RuntimeLogDebug({kTagGC, kTagMM}, "Resuming thread %d", threadId); - return true; } - return false; -} - -bool kotlin::mm::IsThreadSuspensionRequested() noexcept { - // TODO: Consider using a more relaxed memory order. - return gSuspensionRequested.load(); } bool kotlin::mm::SuspendThreads() noexcept { @@ -76,7 +63,7 @@ bool kotlin::mm::SuspendThreads() noexcept { { std::unique_lock lock(gSuspensionMutex); bool actual = false; - gSuspensionRequested.compare_exchange_strong(actual, true); + internal::gSuspensionRequested.compare_exchange_strong(actual, true); if (actual) { return false; } @@ -98,7 +85,7 @@ void kotlin::mm::ResumeThreads() noexcept { // https://en.cppreference.com/w/cpp/thread/condition_variable { std::unique_lock lock(gSuspensionMutex); - gSuspensionRequested = false; + internal::gSuspensionRequested = false; } gSuspensionRequestedByCurrentThread = false; gSuspendsionCondVar.notify_all(); diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp index f2c52e6832c..c3b3f953735 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadSuspension.hpp @@ -13,6 +13,17 @@ namespace kotlin { namespace mm { +namespace internal { + +extern std::atomic gSuspensionRequested; + +} // namespace internal + +inline bool IsThreadSuspensionRequested() noexcept { + // TODO: Consider using a more relaxed memory order. + return internal::gSuspensionRequested.load(); +} + class ThreadSuspensionData : private Pinned { public: explicit ThreadSuspensionData(ThreadState initialState) noexcept : state_(initialState), suspended_(false) {} @@ -31,16 +42,18 @@ public: bool suspended() noexcept { return suspended_; } - bool suspendIfRequested() noexcept; + NO_EXTERNAL_CALLS_CHECK void suspendIfRequested() noexcept { + if (IsThreadSuspensionRequested()) { + suspendIfRequestedSlowPath(); + } + } private: std::atomic state_; std::atomic suspended_; - bool suspendIfRequestedSlowPath() noexcept; + void suspendIfRequestedSlowPath() noexcept; }; -bool IsThreadSuspensionRequested() noexcept; - /** * Suspends all threads registered in ThreadRegistry except threads that are in the Native state. * Blocks until all such threads are suspended. Threads that are in the Native state on the moment