diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt index 6acd1644003..e1b5e106f92 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BinaryOptions.kt @@ -44,6 +44,8 @@ object BinaryOptions : BinaryOptionRegistry() { val sanitizer by option() val mimallocUseDefaultOptions by booleanOption() + + val mimallocUseCompaction by booleanOption() } open class BinaryOption( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index eb4a841b80f..45a02f064e0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -188,6 +188,11 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration configuration.get(BinaryOptions.mimallocUseDefaultOptions) ?: false } + val mimallocUseCompaction by lazy { + // Turned off by default, because it slows down allocation. + configuration.get(BinaryOptions.mimallocUseCompaction) ?: false + } + init { if (!platformManager.isEnabled(target)) { error("Target ${target.visibleName} is not available on the ${HostManager.hostName} host") diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 04d7eb54437..0a700e0500b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -2794,6 +2794,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map(objectFactoryIterable); auto timeSweepUs = konan::getTimeMicros(); RuntimeLogDebug({kTagGC}, "Swept in %" PRIu64 " microseconds", timeSweepUs - timeResumeUs); + kotlin::compactObjectPoolInMainThread(); + // Can be unsafe, because we have a lock in objectFactoryIterable auto objectsCountAfter = objectFactory_.GetSizeUnsafe(); auto extraObjectsCountAfter = mm::GlobalData::Instance().extraObjectDataFactory().GetSizeUnsafe(); diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp index 2b054310169..5546560782d 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.cpp @@ -29,6 +29,7 @@ RUNTIME_WEAK int32_t Kotlin_printToAndroidLogcat = 1; // Keep it 0 even when the compiler defaults to 1: if the overriding mechanism breaks, keeping it disabled is safer. RUNTIME_WEAK int32_t Kotlin_appStateTracking = 0; RUNTIME_WEAK int32_t Kotlin_mimallocUseDefaultOptions = 1; +RUNTIME_WEAK int32_t Kotlin_mimallocUseCompaction = 0; ALWAYS_INLINE compiler::DestroyRuntimeMode compiler::destroyRuntimeMode() noexcept { return static_cast(Kotlin_destroyRuntimeMode); @@ -68,3 +69,7 @@ ALWAYS_INLINE int compiler::getSourceInfo(void* addr, SourceInfo *result, int re ALWAYS_INLINE bool compiler::mimallocUseDefaultOptions() noexcept { return Kotlin_mimallocUseDefaultOptions != 0; } + +ALWAYS_INLINE bool compiler::mimallocUseCompaction() noexcept { + return Kotlin_mimallocUseCompaction != 0; +} diff --git a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp index 2ee07217d9a..88c795bed89 100644 --- a/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp +++ b/kotlin-native/runtime/src/main/cpp/CompilerConstants.hpp @@ -110,6 +110,7 @@ bool suspendFunctionsFromAnyThreadFromObjCEnabled() noexcept; AppStateTracking appStateTracking() noexcept; int getSourceInfo(void* addr, SourceInfo *result, int result_size) noexcept; bool mimallocUseDefaultOptions() noexcept; +bool mimallocUseCompaction() noexcept; #ifdef KONAN_ANDROID bool printToAndroidLogcat() noexcept; diff --git a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp index 0ae08ef40db..279deb484aa 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp @@ -13,6 +13,11 @@ namespace kotlin { void initObjectPool() noexcept; void* allocateInObjectPool(size_t size) noexcept; void freeInObjectPool(void* ptr) noexcept; +// Instruct the allocator to free unused resources. +void compactObjectPoolInCurrentThread() noexcept; +// Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread. +// May do nothing if the main thread is not an event loop. +void compactObjectPoolInMainThread() noexcept; template struct ObjectPoolAllocator { diff --git a/kotlin-native/runtime/src/main/cpp/Worker.cpp b/kotlin-native/runtime/src/main/cpp/Worker.cpp index 21071fdc05f..cc0ce802ee3 100644 --- a/kotlin-native/runtime/src/main/cpp/Worker.cpp +++ b/kotlin-native/runtime/src/main/cpp/Worker.cpp @@ -32,6 +32,7 @@ #include "Memory.h" #include "Natives.h" #include "ObjCMMAPI.h" +#include "ObjectAlloc.hpp" #include "Runtime.h" #include "Types.h" #include "Worker.h" @@ -239,6 +240,7 @@ KNativePtr transfer(ObjHolder* holder, KInt mode) { } void waitInNativeState(pthread_cond_t* cond, pthread_mutex_t* mutex) { + kotlin::compactObjectPoolInCurrentThread(); CallWithThreadState(pthread_cond_wait, cond, mutex); } @@ -246,6 +248,7 @@ void waitInNativeState(pthread_cond_t* cond, pthread_mutex_t* mutex, uint64_t timeoutNanoseconds, uint64_t* microsecondsPassed = nullptr) { + kotlin::compactObjectPoolInCurrentThread(); CallWithThreadState(WaitOnCondVar, cond, mutex, timeoutNanoseconds, microsecondsPassed); } diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp index c27cbb2f584..17293ec726f 100644 --- a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp @@ -10,6 +10,11 @@ #include "../../mimalloc/c/include/mimalloc.h" #include "Alignment.hpp" #include "CompilerConstants.hpp" +#include "Memory.h" + +#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH +#include +#endif using namespace kotlin; @@ -17,11 +22,20 @@ namespace { std::once_flag initOptions; -} +#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH +std::atomic_flag scheduledCompactOnMainThread = ATOMIC_FLAG_INIT; +#endif + +} // namespace void kotlin::initObjectPool() noexcept { if (!compiler::mimallocUseDefaultOptions()) { - std::call_once(initOptions, [] { mi_option_enable(mi_option_reset_decommits); }); + std::call_once(initOptions, [] { + mi_option_enable(mi_option_reset_decommits); + if (compiler::mimallocUseCompaction()) { + mi_option_set(mi_option_reset_delay, 0); + } + }); } mi_thread_init(); } @@ -33,3 +47,24 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept { void kotlin::freeInObjectPool(void* ptr) noexcept { mi_free(ptr); } + +void kotlin::compactObjectPoolInCurrentThread() noexcept { + if (!compiler::mimallocUseCompaction()) return; + mi_collect(true); +} + +void kotlin::compactObjectPoolInMainThread() noexcept { + if (!compiler::mimallocUseCompaction()) return; +#if KONAN_SUPPORTS_GRAND_CENTRAL_DISPATCH + if (scheduledCompactOnMainThread.test_and_set()) { + // If it's already scheduled, do nothing. + return; + } + dispatch_async_f(dispatch_get_main_queue(), nullptr, [](void*) { + if (mm::IsCurrentThreadRegistered()) { + kotlin::compactObjectPoolInCurrentThread(); + } + scheduledCompactOnMainThread.clear(); + }); +#endif +} \ No newline at end of file diff --git a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp index 3f64d87a853..65e431a7a70 100644 --- a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp @@ -30,3 +30,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept { void kotlin::freeInObjectPool(void* ptr) noexcept { freeImpl(ptr); } + +void kotlin::compactObjectPoolInCurrentThread() noexcept {} + +void kotlin::compactObjectPoolInMainThread() noexcept {} \ No newline at end of file diff --git a/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt b/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt index 02fa8b699f0..9e21f3d6fd8 100644 --- a/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt +++ b/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt @@ -56,6 +56,7 @@ sealed class ClangArgs( "REPORT_BACKTRACE_TO_IOS_CRASH_LOG".takeIf { target.supportsIosCrashLog() }, "NEED_SMALL_BINARY".takeIf { target.needSmallBinary() }, "TARGET_HAS_ADDRESS_DEPENDENCY".takeIf { target.hasAddressDependencyInMemoryModel() }, + "SUPPORTS_GRAND_CENTRAL_DISPATCH".takeIf { target.supportsGrandCentralDispatch }, ).map { "KONAN_$it=1" } val otherOptions = listOfNotNull( "USE_ELF_SYMBOLS=1".takeIf { target.binaryFormat() == BinaryFormat.ELF }, diff --git a/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt b/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt index 40e310f59c9..6a31013d607 100644 --- a/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt +++ b/kotlin-native/shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt @@ -172,6 +172,11 @@ fun KonanTarget.hasAddressDependencyInMemoryModel(): Boolean = Architecture.MIPS32, Architecture.MIPSEL32, Architecture.WASM32 -> false } +val KonanTarget.supportsGrandCentralDispatch + get() = when(family) { + Family.WATCHOS, Family.IOS, Family.TVOS, Family.OSX -> true + else -> false + } // TODO: this is bad function. It should be replaced by capabilities functions like above // but two affected targets are too strange, so we postpone it