[K/N] Add a separate aggressive GC scheduler

^KT-48537

Merge-request: KT-MR-5253
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -320,10 +320,6 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                         null
                     }
                 })
-                if (memoryModel != MemoryModel.EXPERIMENTAL && arguments.gcAggressive) {
-                    configuration.report(ERROR, "-Xgc-aggressive is only supported for -memory-model experimental")
-                }
-                put(GARBAGE_COLLECTOR_AGRESSIVE, arguments.gcAggressive)
                 put(PROPERTY_LAZY_INITIALIZATION, when (arguments.propertyLazyInitialization) {
                     null -> {
                         when (memoryModel) {

kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt

@@ -330,9 +330,6 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
     @Argument(value="-Xgc", valueDescription = "<gc>", description = "GC to use, 'noop' and 'stms' are currently supported. Works only with -memory-model experimental")
     var gc: String? = null
 
-    @Argument(value="-Xgc-aggressive", description = "Make GC agressive. Works only with -memory-model experimental")
-    var gcAggressive: Boolean = false
-
     @Argument(value = "-Xir-property-lazy-initialization", valueDescription = "{disable|enable}", description = "Initialize top level properties lazily per file")
     var propertyLazyInitialization: String? = null
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/GCSchedulerType.kt

@@ -5,8 +5,10 @@
 
 package org.jetbrains.kotlin.backend.konan
 
+// Must match `GCSchedulerType` in CompilerConstants.hpp
 enum class GCSchedulerType(val value: Int) {
     DISABLED(0),
     WITH_TIMER(1),
-    ON_SAFE_POINTS(2)
+    ON_SAFE_POINTS(2),
-}
+    AGGRESSIVE(3),
+}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -94,7 +94,6 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         }
         realGc
     }
-    val gcAggressive: Boolean get() = configuration.get(KonanConfigKeys.GARBAGE_COLLECTOR_AGRESSIVE)!!
     val runtimeAssertsMode: RuntimeAssertsMode get() = configuration.get(BinaryOptions.runtimeAssertionsMode) ?: RuntimeAssertsMode.IGNORE
     val workerExceptionHandling: WorkerExceptionHandling get() = configuration.get(KonanConfigKeys.WORKER_EXCEPTION_HANDLING)!!
     val runtimeLogs: String? get() = configuration.get(KonanConfigKeys.RUNTIME_LOGS)
@@ -119,7 +118,6 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
     val gcSchedulerType: GCSchedulerType by lazy {
         configuration.get(BinaryOptions.gcSchedulerType) ?: when {
             !target.supportsThreads() -> GCSchedulerType.ON_SAFE_POINTS
-            gcAggressive -> GCSchedulerType.ON_SAFE_POINTS
             else -> GCSchedulerType.WITH_TIMER
         }
     }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt

@@ -161,7 +161,6 @@ class KonanConfigKeys {
         val DESTROY_RUNTIME_MODE: CompilerConfigurationKey<DestroyRuntimeMode>
                 = CompilerConfigurationKey.create("when to destroy runtime")
         val GARBAGE_COLLECTOR: CompilerConfigurationKey<GC> = CompilerConfigurationKey.create("gc")
-        val GARBAGE_COLLECTOR_AGRESSIVE: CompilerConfigurationKey<Boolean> = CompilerConfigurationKey.create("turn on agressive GC mode")
         val CHECK_LLD_COMPATIBILITY: CompilerConfigurationKey<Boolean> = CompilerConfigurationKey.create("check compatibility with LLD")
         val PROPERTY_LAZY_INITIALIZATION: CompilerConfigurationKey<Boolean>
                 = CompilerConfigurationKey.create("lazy top level properties initialization")

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2735,7 +2735,6 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             return
 
         overrideRuntimeGlobal("Kotlin_destroyRuntimeMode", Int32(context.config.destroyRuntimeMode.value))
-        overrideRuntimeGlobal("Kotlin_gcAggressive", Int32(if (context.config.gcAggressive) 1 else 0))
         overrideRuntimeGlobal("Kotlin_workerExceptionHandling", Int32(context.config.workerExceptionHandling.value))
         overrideRuntimeGlobal("Kotlin_freezingEnabled", Int32(if (context.config.freezing.enableFreezeAtRuntime) 1 else 0))
         overrideRuntimeGlobal("Kotlin_gcSchedulerType", Int32(context.config.gcSchedulerType.value))

kotlin-native/backend.native/tests/build.gradle

@@ -113,7 +113,7 @@ ext.isNoopGC = project.globalTestArgs.contains("-Xgc=noop")
 if (ext.isExperimentalMM) {
     // TODO: Make it more aggressive and only turn it on for a subset of tests.
     tasks.withType(KonanCompileNativeBinary.class).configureEach {
-        extraOpts "-Xgc-aggressive"
+        extraOpts "-Xbinary=gcSchedulerType=aggressive"
     }
 }
 

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.cpp

@@ -18,7 +18,7 @@ using namespace kotlin;
 namespace {
 
 class GCEmptySchedulerData : public gc::GCSchedulerData {
-    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {}
+    void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {}
     void OnPerformFullGC() noexcept override {}
     void UpdateAliveSetBytes(size_t bytes) noexcept override {}
 };
@@ -31,7 +31,7 @@ public:
             return std::chrono::microseconds(config_.regularGcIntervalUs);
         }) {}
 
-    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {
+    void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {
         size_t allocatedBytes = threadData.allocatedBytes();
         if (allocatedBytes > config_.allocationThresholdBytes) {
             RuntimeAssert(static_cast<bool>(scheduleGC_), "scheduleGC_ cannot be empty");
@@ -75,7 +75,7 @@ public:
         timeOfLastGcNs_(currentTimeCallbackNs_()),
         scheduleGC_(std::move(scheduleGC)) {}
 
-    void OnSafePoint(gc::GCSchedulerThreadData& threadData) noexcept override {
+    void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override {
         size_t allocatedBytes = threadData.allocatedBytes();
         if (allocatedBytes > config_.allocationThresholdBytes ||
             currentTimeCallbackNs_() - timeOfLastGcNs_ >= config_.cooldownThresholdNs) {
@@ -96,6 +96,24 @@ private:
     std::function<void()> scheduleGC_;
 };
 
+class GCSchedulerDataAggressive : public gc::GCSchedulerData {
+public:
+    GCSchedulerDataAggressive(gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept :
+        scheduleGC_(std::move(scheduleGC)) {
+        RuntimeLogInfo({kTagGC}, "Initialize GC scheduler config in the aggressive mode");
+        // TODO: Make it even more aggressive and run on a subset of backend.native tests.
+        config.threshold = 1000;
+        config.allocationThresholdBytes = 10000;
+    }
+
+    void UpdateFromThreadData(gc::GCSchedulerThreadData& threadData) noexcept override { scheduleGC_(); }
+
+    void OnPerformFullGC() noexcept override {}
+    void UpdateAliveSetBytes(size_t bytes) noexcept override {}
+
+private:
+    std::function<void()> scheduleGC_;
+};
 
 KStdUniquePtr<gc::GCSchedulerData> MakeEmptyGCSchedulerData() noexcept {
     return ::make_unique<GCEmptySchedulerData>();
@@ -111,6 +129,10 @@ KStdUniquePtr<gc::GCSchedulerData> MakeGCSchedulerDataWithoutTimer(
     return ::make_unique<GCSchedulerDataWithoutTimer>(config, std::move(scheduleGC), std::move(currentTimeCallbackNs));
 }
 
+KStdUniquePtr<gc::GCSchedulerData> MakeGCShedulerDataAggressive(gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
+    return ::make_unique<GCSchedulerDataAggressive>(config, std::move(scheduleGC));
+}
+
 } // namespace
 
 KStdUniquePtr<gc::GCSchedulerData> kotlin::gc::MakeGCSchedulerData(SchedulerType type, gc::GCSchedulerConfig& config, std::function<void()> scheduleGC) noexcept {
@@ -121,6 +143,8 @@ KStdUniquePtr<gc::GCSchedulerData> kotlin::gc::MakeGCSchedulerData(SchedulerType
             return MakeGCSchedulerDataWithTimer(config, std::move(scheduleGC));
         case SchedulerType::kOnSafepoints:
             return MakeGCSchedulerDataWithoutTimer(config, std::move(scheduleGC), []() { return konan::getTimeNanos(); });
+        case SchedulerType::kAggressive:
+            return MakeGCShedulerDataAggressive(config, std::move(scheduleGC));
     }
 }
 

kotlin-native/runtime/src/gc/common/cpp/GCScheduler.hpp

@@ -29,17 +29,6 @@ struct GCSchedulerConfig {
     std::atomic<uint64_t> cooldownThresholdNs = 200 * 1000 * 1000; // 200 milliseconds by default.
     std::atomic<bool> autoTune = false;
     std::atomic<uint64_t> regularGcIntervalUs = 200 * 1000; // 200 milliseconds by default.
-
-    GCSchedulerConfig() noexcept {
-        if (compiler::gcAggressive()) {
-            // TODO: Make a separate GCSchedulerData for the aggressive mode and move this log there.
-            RuntimeLogInfo({kTagGC}, "Initialize GC scheduler config in the aggressive mode");
-            // TODO: Make it even more aggressive and run on a subset of backend.native tests.
-            threshold = 1000;
-            allocationThresholdBytes = 10000;
-            cooldownThresholdNs = 0;
-        }
-    }
 };
 
 class GCSchedulerThreadData;
@@ -49,7 +38,7 @@ public:
     virtual ~GCSchedulerData() = default;
 
     // Called by different mutator threads.
-    virtual void OnSafePoint(GCSchedulerThreadData& threadData) noexcept = 0;
+    virtual void UpdateFromThreadData(GCSchedulerThreadData& threadData) noexcept = 0;
 
     // Always called by the GC thread.
     virtual void OnPerformFullGC() noexcept = 0;
@@ -63,19 +52,25 @@ public:
     static constexpr size_t kFunctionPrologueWeight = 1;
     static constexpr size_t kLoopBodyWeight = 1;
 
-    explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> onSafePoint) noexcept :
+    explicit GCSchedulerThreadData(GCSchedulerConfig& config, std::function<void(GCSchedulerThreadData&)> slowPath) noexcept :
-        config_(config), onSafePoint_(std::move(onSafePoint)) {
+        config_(config), slowPath_(std::move(slowPath)) {
         ClearCountersAndUpdateThresholds();
     }
 
     // Should be called on encountering a safepoint.
     void OnSafePointRegular(size_t weight) noexcept {
-        if (compiler::getGCSchedulerType() == compiler::GCSchedulerType::kOnSafepoints) {
+        // TODO: This is a weird design. Consider replacing switch+virtual functions with pimpl+separate compilation.
-            safePointsCounter_ += weight;
+        switch (compiler::getGCSchedulerType()) {
-            if (safePointsCounter_ < safePointsCounterThreshold_) {
+            case compiler::GCSchedulerType::kOnSafepoints:
+            case compiler::GCSchedulerType::kAggressive:
+                safePointsCounter_ += weight;
+                if (safePointsCounter_ < safePointsCounterThreshold_) {
+                    return;
+                }
+                OnSafePointSlowPath();
+                return;
+            default:
                 return;
-            }
-            OnSafePointSlowPath();
         }
     }
 
@@ -97,7 +92,7 @@ public:
 
 private:
     void OnSafePointSlowPath() noexcept {
-        onSafePoint_(*this);
+        slowPath_(*this);
         ClearCountersAndUpdateThresholds();
     }
 
@@ -110,7 +105,7 @@ private:
     }
 
     GCSchedulerConfig& config_;
-    std::function<void(GCSchedulerThreadData&)> onSafePoint_;
+    std::function<void(GCSchedulerThreadData&)> slowPath_;
 
     size_t allocatedBytes_ = 0;
     size_t allocatedBytesThreshold_ = 0;
@@ -134,7 +129,7 @@ public:
     void SetScheduleGC(std::function<void()> scheduleGC) noexcept;
 
     GCSchedulerThreadData NewThreadData() noexcept {
-        return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->OnSafePoint(threadData); });
+        return GCSchedulerThreadData(config_, [this](auto& threadData) { gcData_->UpdateFromThreadData(threadData); });
     }
 
 private:

kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp

@@ -12,7 +12,6 @@ using namespace kotlin;
 
 // These are defined by overrideRuntimeGlobals in IrToBitcode.kt
 RUNTIME_WEAK int32_t Kotlin_destroyRuntimeMode = 1;
-RUNTIME_WEAK int32_t Kotlin_gcAggressive = 0;
 RUNTIME_WEAK int32_t Kotlin_gcSchedulerType = 2;
 RUNTIME_WEAK int32_t Kotlin_workerExceptionHandling = 0;
 RUNTIME_WEAK int32_t Kotlin_freezingEnabled = 1;
@@ -25,10 +24,6 @@ ALWAYS_INLINE compiler::DestroyRuntimeMode compiler::destroyRuntimeMode() noexce
     return static_cast<compiler::DestroyRuntimeMode>(Kotlin_destroyRuntimeMode);
 }
 
-ALWAYS_INLINE bool compiler::gcAggressive() noexcept {
-    return Kotlin_gcAggressive != 0;
-}
-
 ALWAYS_INLINE compiler::WorkerExceptionHandling compiler::workerExceptionHandling() noexcept {
     return static_cast<compiler::WorkerExceptionHandling>(Kotlin_workerExceptionHandling);
 }

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -58,13 +58,12 @@ enum class WorkerExceptionHandling : int32_t {
 enum class GCSchedulerType {
     kDisabled = 0,
     kWithTimer = 1,
-    kOnSafepoints = 2
+    kOnSafepoints = 2,
+    kAggressive = 3,
 };
 
 DestroyRuntimeMode destroyRuntimeMode() noexcept;
 
-bool gcAggressive() noexcept;
-
 ALWAYS_INLINE inline bool shouldContainDebugInfo() noexcept {
     return KonanNeedDebugInfo != 0;
 }