[K/N] Manually compact mimalloc heap ^KT-53182

Merge-request: KT-MR-7111
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -204,14 +204,16 @@ bool gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     mm::ResumeThreads();
     auto timeResumeUs = konan::getTimeMicros();
 
-    RuntimeLogInfo({kTagGC},
-                    "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %"  PRIu64" microseconds",
-                    timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
+    RuntimeLogInfo(
+            {kTagGC}, "Resumed threads in %" PRIu64 " microseconds. Total pause for most threads is %" PRIu64 " microseconds",
+            timeResumeUs - timeSweepExtraObjectsUs, timeResumeUs - timeStartUs);
 
     auto finalizerQueue = gc::Sweep<SweepTraits>(objectFactoryIterable);
     auto timeSweepUs = konan::getTimeMicros();
     RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs);
 
+    kotlin::compactObjectPoolInMainThread();
+
     // Can be unsafe, because we have a lock in objectFactoryIterable
     auto objectsCountAfter = objectFactory_.GetSizeUnsafe();
     auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe();

kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp

@@ -29,6 +29,7 @@ RUNTIME_WEAK int32_t Kotlin_printToAndroidLogcat = 1;
 // Keep it 0 even when the compiler defaults to 1: if the overriding mechanism breaks, keeping it disabled is safer.
 RUNTIME_WEAK int32_t Kotlin_appStateTracking = 0;
 RUNTIME_WEAK int32_t Kotlin_mimallocUseDefaultOptions = 1;
+RUNTIME_WEAK int32_t Kotlin_mimallocUseCompaction = 0;
 
 ALWAYS_INLINE compiler::DestroyRuntimeMode compiler::destroyRuntimeMode() noexcept {
     return static_cast<compiler::DestroyRuntimeMode>(Kotlin_destroyRuntimeMode);
@@ -68,3 +69,7 @@ ALWAYS_INLINE int compiler::getSourceInfo(void* addr, SourceInfo *result, int re
 ALWAYS_INLINE bool compiler::mimallocUseDefaultOptions() noexcept {
     return Kotlin_mimallocUseDefaultOptions != 0;
 }
+
+ALWAYS_INLINE bool compiler::mimallocUseCompaction() noexcept {
+    return Kotlin_mimallocUseCompaction != 0;
+}

kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp

@@ -110,6 +110,7 @@ bool suspendFunctionsFromAnyThreadFromObjCEnabled() noexcept;
 AppStateTracking appStateTracking() noexcept;
 int getSourceInfo(void* addr, SourceInfo *result, int result_size) noexcept;
 bool mimallocUseDefaultOptions() noexcept;
+bool mimallocUseCompaction() noexcept;
 
 #ifdef KONAN_ANDROID
 bool printToAndroidLogcat() noexcept;

kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp

@@ -13,6 +13,11 @@ namespace kotlin {
 void initObjectPool() noexcept;
 void* allocateInObjectPool(size_t size) noexcept;
 void freeInObjectPool(void* ptr) noexcept;
+// Instruct the allocator to free unused resources.
+void compactObjectPoolInCurrentThread() noexcept;
+// Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread.
+// May do nothing if the main thread is not an event loop.
+void compactObjectPoolInMainThread() noexcept;
 
 template <typename T>
 struct ObjectPoolAllocator {

kotlin-native/runtime/src/main/cpp/Worker.cpp

@@ -32,6 +32,7 @@
 #include "Memory.h"
 #include "Natives.h"
 #include "ObjCMMAPI.h"
+#include "ObjectAlloc.hpp"
 #include "Runtime.h"
 #include "Types.h"
 #include "Worker.h"
@@ -239,6 +240,7 @@ KNativePtr transfer(ObjHolder* holder, KInt mode) {
 }
 
 void waitInNativeState(pthread_cond_t* cond, pthread_mutex_t* mutex) {
+    kotlin::compactObjectPoolInCurrentThread();
     CallWithThreadState<ThreadState::kNative>(pthread_cond_wait, cond, mutex);
 }
 
@@ -246,6 +248,7 @@ void waitInNativeState(pthread_cond_t* cond,
           pthread_mutex_t* mutex,
           uint64_t timeoutNanoseconds,
           uint64_t* microsecondsPassed = nullptr) {
+    kotlin::compactObjectPoolInCurrentThread();
     CallWithThreadState<ThreadState::kNative>(WaitOnCondVar, cond, mutex, timeoutNanoseconds, microsecondsPassed);
 }
 

kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp

@@ -10,6 +10,11 @@
 #include "../../mimalloc/c/include/mimalloc.h"
 #include "Alignment.hpp"
 #include "CompilerConstants.hpp"
+#include "Memory.h"
+
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+#include <dispatch/dispatch.h>
+#endif
 
 using namespace kotlin;
 
@@ -17,11 +22,20 @@ namespace {
 
 std::once_flag initOptions;
 
-}
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT;
+#endif
+
+} // namespace
 
 void kotlin::initObjectPool() noexcept {
     if (!compiler::mimallocUseDefaultOptions()) {
-        std::call_once(initOptions, [] { mi_option_enable(mi_option_reset_decommits); });
+        std::call_once(initOptions, [] {
+            mi_option_enable(mi_option_reset_decommits);
+            if (compiler::mimallocUseCompaction()) {
+                mi_option_set(mi_option_reset_delay, 0);
+            }
+        });
     }
     mi_thread_init();
 }
@@ -33,3 +47,24 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
 void kotlin::freeInObjectPool(void* ptr) noexcept {
     mi_free(ptr);
 }
+
+void kotlin::compactObjectPoolInCurrentThread() noexcept {
+    if (!compiler::mimallocUseCompaction()) return;
+    mi_collect(true);
+}
+
+void kotlin::compactObjectPoolInMainThread() noexcept {
+    if (!compiler::mimallocUseCompaction()) return;
+#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH
+    if (scheduledCompactOnMainThread.test_and_set()) {
+        // If it's already scheduled, do nothing.
+        return;
+    }
+    dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) {
+        if (mm::IsCurrentThreadRegistered()) {
+            kotlin::compactObjectPoolInCurrentThread();
+        }
+        scheduledCompactOnMainThread.clear();
+    });
+#endif
+}

kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp

@@ -30,3 +30,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
 void kotlin::freeInObjectPool(void* ptr) noexcept {
     freeImpl(ptr);
 }
+
+void kotlin::compactObjectPoolInCurrentThread() noexcept {}
+
+void kotlin::compactObjectPoolInMainThread() noexcept {}