[K/N] Manually compact mimalloc heap ^KT-53182

Merge-request: KT-MR-7111
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt

@@ -44,6 +44,8 @@ object BinaryOptions : BinaryOptionRegistry() {
     val sanitizer by option<SanitizerKind>()
 
     val mimallocUseDefaultOptions by booleanOption()
+
+    val mimallocUseCompaction by booleanOption()
 }
 
 open class BinaryOption<T : Any>(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -188,6 +188,11 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         configuration.get(BinaryOptions.mimallocUseDefaultOptions) ?: false
     }
 
+    val mimallocUseCompaction by lazy {
+        // Turned off by default, because it slows down allocation.
+        configuration.get(BinaryOptions.mimallocUseCompaction) ?: false
+    }
+
     init {
         if (!platformManager.isEnabled(target)) {
             error("Target ${target.visibleName} is not available on the ${HostManager.hostName} host")

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -2794,6 +2794,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         }
         overrideRuntimeGlobal("Kotlin_appStateTracking", Int32(context.config.appStateTracking.value))
         overrideRuntimeGlobal("Kotlin_mimallocUseDefaultOptions", Int32(if (context.config.mimallocUseDefaultOptions) 1 else 0))
+        overrideRuntimeGlobal("Kotlin_mimallocUseCompaction", Int32(if (context.config.mimallocUseCompaction) 1 else 0))
     }
 
     //-------------------------------------------------------------------------//

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -204,14 +204,16 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     mm::ResumeThreads();
     auto timeResumeUs = konan::getTimeMicros();
 
-    RuntimeLogInfo({kTagGC},
-                    "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %"  PRIu64" microseconds",
-                    timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
+    RuntimeLogInfo(
+            {kTagGC}, "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %" PRIu64 " microseconds",
+            timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
 
     auto finalizerQueue = gc::Sweep<SweepTraits>(objectFactoryIterable);
     auto timeSweepUs = konan::getTimeMicros();
     RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs);
 
+    kotlin::compactObjectPoolInMainThread();
+
     // Can be unsafe, because we have a lock in objectFactoryIterable
     auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
     auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();

kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp

@@ -29,6 +29,7 @@ RUNTIME_WEAK int32_t Kotlin_printToAndroidLogcat = 1;
 // Keep it 0 even when the compiler defaults to 1: if the overriding mechanism breaks, keeping it disabled is safer.
 RUNTIME_WEAK int32_t Kotlin_appStateTracking = 0;
 RUNTIME_WEAK int32_t Kotlin_mimallocUseDefaultOptions = 1;
+RUNTIME_WEAK int32_t Kotlin_mimallocUseCompaction = 0;
 
 ALWAYS_INLINE compiler::DestroyRuntimeMode compiler::destroyRuntimeMode() noexcept {
     return static_cast<compiler::DestroyRuntimeMode>(Kotlin_destroyRuntimeMode);
@@ -68,3 +69,7 @@ ALWAYS_INLINE int compiler::getSourceInfo(void* addr, SourceInfo *result, int re
 ALWAYS_INLINE bool compiler::mimallocUseDefaultOptions() noexcept {
     return Kotlin_mimallocUseDefaultOptions != 0;
 }
+
+ALWAYS_INLINE bool compiler::mimallocUseCompaction() noexcept {
+    return Kotlin_mimallocUseCompaction != 0;
+}

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -110,6 +110,7 @@ bool suspendFunctionsFromAnyThreadFromObjCEnabled() noexcept;
 AppStateTracking appStateTracking() noexcept;
 int getSourceInfo(void* addr, SourceInfo *result, int result_size) noexcept;
 bool mimallocUseDefaultOptions() noexcept;
+bool mimallocUseCompaction() noexcept;
 
 #ifdef KONAN_ANDROID
 bool printToAndroidLogcat() noexcept;

kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp

@@ -13,6 +13,11 @@ namespace kotlin {
 void initObjectPool() noexcept;
 void* allocateInObjectPool(size_t size) noexcept;
 void freeInObjectPool(void* ptr) noexcept;
+// Instruct the allocator to free unused resources.
+void compactObjectPoolInCurrentThread() noexcept;
+// Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread.
+// May do nothing if the main thread is not an event loop.
+void compactObjectPoolInMainThread() noexcept;
 
 template <typename T>
 struct ObjectPoolAllocator {

kotlin-native/runtime/src/main/cpp/Worker.cpp

@@ -32,6 +32,7 @@
 #include "Memory.h"
 #include "Natives.h"
 #include "ObjCMMAPI.h"
+#include "ObjectAlloc.hpp"
 #include "Runtime.h"
 #include "Types.h"
 #include "Worker.h"
@@ -239,6 +240,7 @@ KNativePtr transfer(ObjHolder* holder, KInt mode) {
 }
 
 void waitInNativeState(pthread_cond_t* cond, pthread_mutex_t* mutex) {
+    kotlin::compactObjectPoolInCurrentThread();
     CallWithThreadState<ThreadState::kNative>(pthread_cond_wait, cond, mutex);
 }
 
@@ -246,6 +248,7 @@ void waitInNativeState(pthread_cond_t* cond,
           pthread_mutex_t* mutex,
           uint64_t timeoutNanoseconds,
           uint64_t* microsecondsPassed = nullptr) {
+    kotlin::compactObjectPoolInCurrentThread();
     CallWithThreadState<ThreadState::kNative>(WaitOnCondVar, cond, mutex, timeoutNanoseconds, microsecondsPassed);
 }
 

kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp

@@ -10,6 +10,11 @@
 #include "../../mimalloc/c/include/mimalloc.h"
 #include "Alignment.hpp"
 #include "CompilerConstants.hpp"
+#include "Memory.h"
+
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+#include <dispatch/dispatch.h>
+#endif
 
 using namespace kotlin;
 
@@ -17,11 +22,20 @@ namespace {
 
 std::once_flag initOptions;
 
-}
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT;
+#endif
+
+} // namespace
 
 void kotlin::initObjectPool() noexcept {
     if (!compiler::mimallocUseDefaultOptions()) {
-        std::call_once(initOptions, [] { mi_option_enable(mi_option_reset_decommits); });
+        std::call_once(initOptions, [] {
+            mi_option_enable(mi_option_reset_decommits);
+            if (compiler::mimallocUseCompaction()) {
+                mi_option_set(mi_option_reset_delay, 0);
+            }
+        });
     }
     mi_thread_init();
 }
@@ -33,3 +47,24 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
 void kotlin::freeInObjectPool(void* ptr) noexcept {
     mi_free(ptr);
 }
+
+void kotlin::compactObjectPoolInCurrentThread() noexcept {
+    if (!compiler::mimallocUseCompaction()) return;
+    mi_collect(true);
+}
+
+void kotlin::compactObjectPoolInMainThread() noexcept {
+    if (!compiler::mimallocUseCompaction()) return;
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+    if (scheduledCompactOnMainThread.test_and_set()) {
+        // If it's already scheduled, do nothing.
+        return;
+    }
+    dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) {
+        if (mm::IsCurrentThreadRegistered()) {
+            kotlin::compactObjectPoolInCurrentThread();
+        }
+        scheduledCompactOnMainThread.clear();
+    });
+#endif
+}

kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp

@@ -30,3 +30,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
 void kotlin::freeInObjectPool(void* ptr) noexcept {
     freeImpl(ptr);
 }
+
+void kotlin::compactObjectPoolInCurrentThread() noexcept {}
+
+void kotlin::compactObjectPoolInMainThread() noexcept {}

kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt

@@ -56,6 +56,7 @@ sealed class ClangArgs(
                     "REPORT_BACKTRACE_TO_IOS_CRASH_LOG".takeIf { target.supportsIosCrashLog() },
                     "NEED_SMALL_BINARY".takeIf { target.needSmallBinary() },
                     "TARGET_HAS_ADDRESS_DEPENDENCY".takeIf { target.hasAddressDependencyInMemoryModel() },
+                    "SUPPORTS_GRAND_CENTRAL_DISPATCH".takeIf { target.supportsGrandCentralDispatch },
             ).map { "KONAN_$it=1" }
             val otherOptions = listOfNotNull(
                     "USE_ELF_SYMBOLS=1".takeIf { target.binaryFormat() == BinaryFormat.ELF },

kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt

@@ -172,6 +172,11 @@ fun KonanTarget.hasAddressDependencyInMemoryModel(): Boolean =
          Architecture.MIPS32, Architecture.MIPSEL32, Architecture.WASM32 -> false
      }
 
+val KonanTarget.supportsGrandCentralDispatch
+    get() = when(family) {
+        Family.WATCHOS, Family.IOS, Family.TVOS, Family.OSX -> true
+        else -> false
+    }
 
 // TODO: this is bad function. It should be replaced by capabilities functions like above
 // but two affected targets are too strange, so we postpone it