[K/N] Rework scheduler type choosing

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt

@@ -25,6 +25,8 @@ object BinaryOptions : BinaryOptionRegistry() {
     val androidProgramType by option<AndroidProgramType>()
 
     val unitSuspendFunctionObjCExport by option<UnitSuspendFunctionObjCExport>()
+
+    val gcSchedulerType by option<GCSchedulerType>()
 }
 
 open class BinaryOption<T : Any>(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.backend.konan
+
+enum class GCSchedulerType(val value: Int) {
+    DISABLED(0),
+    WITH_TIMER(1),
+    ON_SAFE_POINTS(2)
+}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -104,6 +104,13 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
                 ?: SourceInfoType.CORESYMBOLICATION.takeIf { debug && target.supportsCoreSymbolication() }
                 ?: SourceInfoType.NOOP
 
+    val gcSchedulerType: GCSchedulerType by lazy {
+        configuration.get(BinaryOptions.gcSchedulerType) ?: when {
+            !target.supportsThreads() -> GCSchedulerType.ON_SAFE_POINTS
+            gcAggressive -> GCSchedulerType.ON_SAFE_POINTS
+            else -> GCSchedulerType.WITH_TIMER
+        }
+    }
 
     val needVerifyIr: Boolean
         get() = configuration.get(KonanConfigKeys.VERIFY_IR) == true

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2738,6 +2738,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         overrideRuntimeGlobal("Kotlin_gcAggressive", Int32(if (context.config.gcAggressive) 1 else 0))
         overrideRuntimeGlobal("Kotlin_workerExceptionHandling", Int32(context.config.workerExceptionHandling.value))
         overrideRuntimeGlobal("Kotlin_freezingEnabled", Int32(if (context.config.freezing.enableFreezeAtRuntime) 1 else 0))
+        overrideRuntimeGlobal("Kotlin_gcSchedulerType", Int32(context.config.gcSchedulerType.value))
         val getSourceInfoFunctionName = when (context.config.sourceInfoType) {
             SourceInfoType.NOOP -> null
             SourceInfoType.LIBBACKTRACE -> "Kotlin_getSourceInfo_libbacktrace"

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp

@@ -96,34 +96,39 @@ private:
     std::function<void()> scheduleGC_;
 };
 
-} // namespace
 
-KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeEmptyGCSchedulerData() noexcept {
+KStdUniquePtr<gc::GCSchedulerData> MakeEmptyGCSchedulerData() noexcept {
     return ::make_unique<GCEmptySchedulerData>();
 }
 
-KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerDataWithTimer(
+KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerDataWithTimer(
-        GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
+        gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
     return ::make_unique<GCSchedulerDataWithTimer>(config, std::move(scheduleGC));
 }
 
-KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerDataWithoutTimer(
+KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerDataWithoutTimer(
-        GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept {
+        gc::GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept {
     return ::make_unique<GCSchedulerDataWithoutTimer>(config, std::move(scheduleGC), std::move(currentTimeCallbackNs));
 }
 
-KStdUniquePtr<gc::GCSchedulerData> gc::internal::MakeGCSchedulerData(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
+} // namespace
-    if (internal::useGCTimer()) {
+
-        return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC));
+KStdUniquePtr<gc::GCSchedulerData> kotlin::gc::MakeGCSchedulerData(SchedulerType type, gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
-    } else {
+    switch (type) {
-        return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); });
+        case SchedulerType::kDisabled:
+            return MakeEmptyGCSchedulerData();
+        case SchedulerType::kWithTimer:
+            return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC));
+        case SchedulerType::kOnSafepoints:
+            return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); });
     }
 }
 
+
 void gc::GCScheduler::SetScheduleGC(std::function<void()> scheduleGC) noexcept {
     RuntimeAssert(static_cast<bool>(scheduleGC), "scheduleGC cannot be empty");
     RuntimeAssert(!static_cast<bool>(scheduleGC_), "scheduleGC must not have been set");
     scheduleGC_ = std::move(scheduleGC);
     RuntimeAssert(gcData_ == nullptr, "gcData_ must not be set prior to scheduleGC call");
-    gcData_ = internal::MakeGCSchedulerData(config_, scheduleGC_);
+    gcData_ = MakeGCSchedulerData(compiler::getGCSchedulerType(), config_, scheduleGC_);
 }

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp

@@ -20,18 +20,8 @@
 namespace kotlin {
 namespace gc {
 
-namespace internal {
+using SchedulerType = compiler::GCSchedulerType;
 
-inline bool useGCTimer() noexcept {
-#if KONAN_NO_THREADS
-    return false;
-#else
-    // With aggressive mode we use safepoint counting to drive GC.
-    return !compiler::gcAggressive();
-#endif
-}
-
-} // namespace internal
 
 struct GCSchedulerConfig {
     std::atomic<size_t> threshold = 100000; // Roughly 1 safepoint per 10ms (on a subset of examples on one particular machine).
@@ -80,7 +70,7 @@ public:
 
     // Should be called on encountering a safepoint.
     void OnSafePointRegular(size_t weight) noexcept {
-        if (!internal::useGCTimer()) {
+        if (compiler::getGCSchedulerType() == compiler::GCSchedulerType::kOnSafepoints) {
             safePointsCounter_ += weight;
             if (safePointsCounter_ < safePointsCounterThreshold_) {
                 return;
@@ -128,15 +118,6 @@ private:
     size_t safePointsCounterThreshold_ = 0;
 };
 
-namespace internal {
-
-KStdUniquePtr<GCSchedulerData> MakeEmptyGCSchedulerData() noexcept;
-KStdUniquePtr<GCSchedulerData> MakeGCSchedulerDataWithTimer(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept;
-KStdUniquePtr<GCSchedulerData> MakeGCSchedulerDataWithoutTimer(
-        GCSchedulerConfig& config, std::function<void()> scheduleGC, std::function<uint64_t()> currentTimeCallbackNs) noexcept;
-KStdUniquePtr<GCSchedulerData> MakeGCSchedulerData(GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept;
-
-} // namespace internal
 
 class GCScheduler : private Pinned {
 public:
@@ -156,23 +137,17 @@ public:
         return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->OnSafePoint(threadData); });
     }
 
-    template <typename F>
-    KStdUniquePtr<GCSchedulerData> ReplaceGCSchedulerDataForTests(F&& factory) noexcept {
-        RuntimeAssert(static_cast<bool>(scheduleGC_), "Can only be called after SetScheduleGC");
-
-        auto other = std::forward<F>(factory)(config_, scheduleGC_);
-        RuntimeAssert(other != nullptr, "factory cannot return a null");
-        using std::swap;
-        swap(gcData_, other);
-        return other;
-    }
-
 private:
     GCSchedulerConfig config_;
     KStdUniquePtr<GCSchedulerData> gcData_;
     std::function<void()> scheduleGC_;
 };
 
+KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerData(
+        SchedulerType type,
+        GCSchedulerConfig& config,
+        std::function<void()> scheduleGC) noexcept;
+
 } // namespace gc
 } // namespace kotlin
 

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp

@@ -216,17 +216,11 @@ WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader
 
 class SameThreadMarkAndSweepTest : public testing::Test {
 public:
-    SameThreadMarkAndSweepTest() {
-        mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests(
-                [](auto& config, auto scheduleGC) { return gc::internal::MakeEmptyGCSchedulerData(); });
-    }
 
     ~SameThreadMarkAndSweepTest() {
         mm::GlobalsRegistry::Instance().ClearForTests();
         mm::GlobalData::Instance().extraObjectDataFactory().ClearForTests();
         mm::GlobalData::Instance().objectFactory().ClearForTests();
-        mm::GlobalData::Instance().gcScheduler().ReplaceGCSchedulerDataForTests(
-                [](auto& config, auto scheduleGC) { return gc::internal::MakeGCSchedulerData(config, std::move(scheduleGC)); });
     }
 
     testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }

kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp

@@ -13,6 +13,7 @@ using namespace kotlin;
 // These are defined by overrideRuntimeGlobals in IrToBitcode.kt
 RUNTIME_WEAK int32_t Kotlin_destroyRuntimeMode = 1;
 RUNTIME_WEAK int32_t Kotlin_gcAggressive = 0;
+RUNTIME_WEAK int32_t Kotlin_gcSchedulerType = 2;
 RUNTIME_WEAK int32_t Kotlin_workerExceptionHandling = 0;
 RUNTIME_WEAK int32_t Kotlin_freezingEnabled = 1;
 RUNTIME_WEAK const Kotlin_getSourceInfo_FunctionType Kotlin_getSourceInfo_Function = nullptr;
@@ -36,6 +37,10 @@ ALWAYS_INLINE bool compiler::freezingEnabled() noexcept {
     return Kotlin_freezingEnabled != 0;
 }
 
+ALWAYS_INLINE compiler::GCSchedulerType compiler::getGCSchedulerType() noexcept {
+    return static_cast<compiler::GCSchedulerType>(Kotlin_gcSchedulerType);
+}
+
 #ifdef KONAN_ANDROID
 ALWAYS_INLINE bool compiler::printToAndroidLogcat() noexcept {
     return Kotlin_printToAndroidLogcat != 0;

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -54,6 +54,13 @@ enum class WorkerExceptionHandling : int32_t {
     kUseHook = 1,
 };
 
+// Must match GCSchedulerType in GCSchedulerType.kt
+enum class GCSchedulerType {
+    kDisabled = 0,
+    kWithTimer = 1,
+    kOnSafepoints = 2
+};
+
 DestroyRuntimeMode destroyRuntimeMode() noexcept;
 
 bool gcAggressive() noexcept;
@@ -82,6 +89,8 @@ ALWAYS_INLINE inline int getSourceInfo(void* addr, SourceInfo *result, int resul
     }
 }
 
+compiler::GCSchedulerType getGCSchedulerType() noexcept;
+
 #ifdef KONAN_ANDROID
 bool printToAndroidLogcat() noexcept;
 #endif

kotlin-native/runtime/src/main/cpp/TestSupport.hpp

@@ -3,6 +3,8 @@
  * that can be found in the LICENSE file.
  */
 
+#pragma once
+
 #include <functional>
 #include <thread>
 
@@ -31,9 +33,11 @@ public:
         kotlin::SwitchThreadState(memoryState(), ThreadState::kRunnable);
     }
     ~ScopedMemoryInit() {
+        // ClearForTests must not be done concurrently with GC
+        SwitchThreadState(memoryState(), ThreadState::kRunnable, /* reentrant = */ true);
         ClearMemoryForTests(memoryState());
         // Ensure that memory deinit is performed in the native state.
-        SwitchThreadState(memoryState(), ThreadState::kNative, /* reentrant = */ true);
+        SwitchThreadState(memoryState(), ThreadState::kNative);
         DeinitMemoryForTests(memoryState());
     }
 

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -66,6 +66,7 @@ extern "C" {
 extern const int32_t KonanNeedDebugInfo = 1;
 extern const int32_t Kotlin_runtimeAssertsMode = static_cast<int32_t>(kotlin::compiler::RuntimeAssertsMode::kPanic);
 extern const char* const Kotlin_runtimeLogs = nullptr;
+int32_t Kotlin_gcSchedulerType = 0;  // kDisabled
 
 extern const TypeInfo* theAnyTypeInfo = theAnyTypeInfoHolder.typeInfo();
 extern const TypeInfo* theArrayTypeInfo = theArrayTypeInfoHolder.typeInfo();