diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index e513daac4f0..1a5215c96e3 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -434,7 +434,7 @@ internal abstract class FunctionGenerationContext( val codegen: CodeGenerator, private val startLocation: LocationInfo?, private val endLocation: LocationInfo?, - private val switchToRunnable: Boolean, + switchToRunnable: Boolean, internal val irFunction: IrFunction? = null ) : ContextUtils { @@ -480,10 +480,13 @@ internal abstract class FunctionGenerationContext( protected open val needCleanupLandingpadAndLeaveFrame: Boolean get() = irFunction?.annotations?.hasAnnotation(RuntimeNames.exportForCppRuntime) == true || // Exported to foreign code (!stackLocalsManager.isEmpty() && context.memoryModel != MemoryModel.EXPERIMENTAL) || - (context.memoryModel == MemoryModel.EXPERIMENTAL && switchToRunnable) + switchToRunnable private var setCurrentFrameIsCalled: Boolean = false + private val switchToRunnable: Boolean = + context.memoryModel == MemoryModel.EXPERIMENTAL && switchToRunnable + val stackLocalsManager = StackLocalsManagerImpl(this, stackLocalsInitBb) data class FunctionInvokeInformation( @@ -910,11 +913,6 @@ internal abstract class FunctionGenerationContext( } call(context.llvm.setCurrentFrameFunction, listOf(slotsPhi!!)) setCurrentFrameIsCalled = true - if (context.memoryModel == MemoryModel.EXPERIMENTAL) { - if (!forbidRuntime) { - call(context.llvm.Kotlin_mm_safePointExceptionUnwind, emptyList()) - } - } return landingpad } @@ -1502,7 +1500,7 @@ internal abstract class FunctionGenerationContext( } releaseVars() - handleEpilogueForExperimentalMM(context.llvm.Kotlin_mm_safePointExceptionUnwind) + handleEpilogueExperimentalMM() LLVMBuildResume(builder, landingpad) } } @@ -1514,14 +1512,11 @@ internal abstract class FunctionGenerationContext( * places with inconsistent stack layout. So we setup debug info only for this part of bb. */ startLocation?.let { debugLocation(it, it) } - val needStateSwitch = context.memoryModel == MemoryModel.EXPERIMENTAL && switchToRunnable - if (needsRuntimeInit || needStateSwitch) { + if (needsRuntimeInit || switchToRunnable) { check(!forbidRuntime) { "Attempt to init runtime where runtime usage is forbidden" } call(context.llvm.initRuntimeIfNeeded, emptyList()) } - if (needStateSwitch) { - switchThreadState(Runnable) - } + handlePrologueExperimentalMM() if (needSlots || needCleanupLandingpadAndLeaveFrame) { call(context.llvm.enterFrameFunction, listOf(slotsPhi!!, Int32(vars.skipSlots).llvm, Int32(slotCount).llvm)) } else { @@ -1577,18 +1572,22 @@ internal abstract class FunctionGenerationContext( protected fun onReturn() { releaseVars() - handleEpilogueForExperimentalMM(context.llvm.Kotlin_mm_safePointFunctionEpilogue) + handleEpilogueExperimentalMM() } - private fun handleEpilogueForExperimentalMM(safePointFunction: LlvmCallable) { - if (context.memoryModel == MemoryModel.EXPERIMENTAL) { - if (!forbidRuntime) { - call(safePointFunction, emptyList()) - } - if (switchToRunnable) { - check(!forbidRuntime) { "Generating a bridge when runtime is forbidden" } - switchThreadState(Native) - } + private fun handlePrologueExperimentalMM() { + if (switchToRunnable) { + switchThreadState(Runnable) + } + if (context.memoryModel == MemoryModel.EXPERIMENTAL && !forbidRuntime) { + call(context.llvm.Kotlin_mm_safePointFunctionPrologue, emptyList()) + } + } + + private fun handleEpilogueExperimentalMM() { + if (switchToRunnable) { + check(!forbidRuntime) { "Generating a bridge when runtime is forbidden" } + switchThreadState(Native) } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 10afeecab48..c2f1e7bd215 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -507,9 +507,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) : Runti private val Kotlin_ObjCExport_NSIntegerTypeProvider by lazyRtFunction private val Kotlin_longTypeProvider by lazyRtFunction - val Kotlin_mm_safePointFunctionEpilogue by lazyRtFunction + val Kotlin_mm_safePointFunctionPrologue by lazyRtFunction val Kotlin_mm_safePointWhileLoopBody by lazyRtFunction - val Kotlin_mm_safePointExceptionUnwind by lazyRtFunction val tlsMode by lazy { when (target) { diff --git a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp index 9ea8e845371..3e1a8dc6999 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp @@ -67,9 +67,8 @@ public: class GCSchedulerThreadData { public: - static constexpr size_t kFunctionEpilogueWeight = 1; + static constexpr size_t kFunctionPrologueWeight = 1; static constexpr size_t kLoopBodyWeight = 1; - static constexpr size_t kExceptionUnwindWeight = 1; explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function onSafePoint) noexcept : config_(config), onSafePoint_(std::move(onSafePoint)) { diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index b90592e1c17..bfaa4860bce 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -33,9 +33,8 @@ public: explicit ThreadData(NoOpGC& gc, mm::ThreadData& threadData) noexcept {} ~ThreadData() = default; - void SafePointFunctionEpilogue() noexcept {} + void SafePointFunctionPrologue() noexcept {} void SafePointLoopBody() noexcept {} - void SafePointExceptionUnwind() noexcept {} void SafePointAllocation(size_t size) noexcept {} void PerformFullGC() noexcept {} diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp index 8b9fb350698..c7fc5174e70 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp @@ -64,18 +64,14 @@ std::atomic gSafepointFlag = gc::Same } // namespace -ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionEpilogue() noexcept { - SafePointRegular(GCSchedulerThreadData::kFunctionEpilogueWeight); +ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointFunctionPrologue() noexcept { + SafePointRegular(GCSchedulerThreadData::kFunctionPrologueWeight); } ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointLoopBody() noexcept { SafePointRegular(GCSchedulerThreadData::kLoopBodyWeight); } -ALWAYS_INLINE void gc::SameThreadMarkAndSweep::ThreadData::SafePointExceptionUnwind() noexcept { - SafePointRegular(GCSchedulerThreadData::kExceptionUnwindWeight); -} - void gc::SameThreadMarkAndSweep::ThreadData::SafePointAllocation(size_t size) noexcept { threadData_.gcScheduler().OnSafePointAllocation(size); SafepointFlag flag = gSafepointFlag.load(); diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index 7346faeb941..43231c4e6eb 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -52,7 +52,7 @@ public: explicit ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData) noexcept : gc_(gc), threadData_(threadData) {} ~ThreadData() = default; - void SafePointFunctionEpilogue() noexcept; + void SafePointFunctionPrologue() noexcept; void SafePointLoopBody() noexcept; void SafePointExceptionUnwind() noexcept; void SafePointAllocation(size_t size) noexcept; diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index dcda5ff79cf..3004cb6ea3b 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -785,7 +785,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsCollect) { for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); }); + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); } for (auto& future : gcFutures) { @@ -906,7 +906,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsAddToRootSetAfterCollectionRe for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = mutators[i].Execute([i, expandRootSet](mm::ThreadData& threadData, Mutator& mutator) { expandRootSet(threadData, mutator, i); - threadData.gc().SafePointFunctionEpilogue(); + threadData.gc().SafePointFunctionPrologue(); }); } @@ -970,7 +970,7 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); }); + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); } for (auto& future : gcFutures) { @@ -1034,7 +1034,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { - threadData.gc().SafePointFunctionEpilogue(); + threadData.gc().SafePointFunctionPrologue(); EXPECT_THAT((*weak)->referred, nullptr); }); } @@ -1092,7 +1092,7 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { // All the other threads are stopping at safe points. for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = - mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionEpilogue(); }); + mutators[i].Execute([](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); }); } // GC will be completed first diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 7fddc89a5be..a49cefedd3a 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3733,7 +3733,7 @@ ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_switchThreadStateRunnable() { // no-op, used by the new MM only. } -ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() { +ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionPrologue() { // no-op, used by the new MM only. } @@ -3741,10 +3741,6 @@ ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() { // no-op, used by the new MM only. } -ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() { - // no-op, used by the new MM only. -} - } // extern "C" #if !KONAN_NO_EXCEPTIONS diff --git a/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp b/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp index 83ab6969123..fc359cc3dc9 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerExport.cpp @@ -46,7 +46,6 @@ void EnsureDeclarationsEmitted() { ensureUsed(CheckGlobalsAccessible); ensureUsed(Kotlin_mm_switchThreadStateNative); ensureUsed(Kotlin_mm_switchThreadStateRunnable); - ensureUsed(Kotlin_mm_safePointFunctionEpilogue); + ensureUsed(Kotlin_mm_safePointFunctionPrologue); ensureUsed(Kotlin_mm_safePointWhileLoopBody); - ensureUsed(Kotlin_mm_safePointExceptionUnwind); } diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 665127a0e3e..f9ab6e1b16e 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -332,9 +332,8 @@ ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_switchThreadStateNative(); ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_switchThreadStateRunnable(); // Safe point callbacks from Kotlin code generator. -void Kotlin_mm_safePointFunctionEpilogue() RUNTIME_NOTHROW; +void Kotlin_mm_safePointFunctionPrologue() RUNTIME_NOTHROW; void Kotlin_mm_safePointWhileLoopBody() RUNTIME_NOTHROW; -void Kotlin_mm_safePointExceptionUnwind() RUNTIME_NOTHROW; #ifdef __cplusplus } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 42656d051ae..488c29afba6 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -533,10 +533,10 @@ extern "C" void CheckGlobalsAccessible() { // Always accessible } -extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointFunctionEpilogue() { +extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointFunctionPrologue() { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); AssertThreadState(threadData, ThreadState::kRunnable); - threadData->gc().SafePointFunctionEpilogue(); + threadData->gc().SafePointFunctionPrologue(); } extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointWhileLoopBody() { @@ -545,12 +545,6 @@ extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointWhileLoopBody() threadData->gc().SafePointLoopBody(); } -extern "C" RUNTIME_NOTHROW ALWAYS_INLINE void Kotlin_mm_safePointExceptionUnwind() { - auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - AssertThreadState(threadData, ThreadState::kRunnable); - threadData->gc().SafePointExceptionUnwind(); -} - extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void Kotlin_mm_switchThreadStateNative() { SwitchThreadState(mm::ThreadRegistry::Instance().CurrentThreadData(), ThreadState::kNative); }