diff --git a/kotlin-native/runtime/src/mimalloc/c/alloc-aligned.c b/kotlin-native/runtime/src/mimalloc/c/alloc-aligned.c index 4cbe58a5c05..43563f92a44 100644 --- a/kotlin-native/runtime/src/mimalloc/c/alloc-aligned.c +++ b/kotlin-native/runtime/src/mimalloc/c/alloc-aligned.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memset, memcpy +#include // memset // ------------------------------------------------------ // Aligned Allocation @@ -21,7 +21,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see ) if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see ) const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - + // try if there is a small block available with just the right alignment const size_t padsize = size + MI_PADDING_SIZE; if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { @@ -46,7 +46,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); return p; } - + // otherwise over-allocate void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero); if (p == NULL) return NULL; @@ -55,7 +55,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask); mi_assert_internal(adjust <= alignment); void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust)); - if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true); + if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true); mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) ); return aligned_p; @@ -137,7 +137,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne memset((uint8_t*)newp + start, 0, newsize - start); } } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; diff --git a/kotlin-native/runtime/src/mimalloc/c/alloc-override-osx.c b/kotlin-native/runtime/src/mimalloc/c/alloc-override-osx.c index 25a6cd4c301..640842b755e 100644 --- a/kotlin-native/runtime/src/mimalloc/c/alloc-override-osx.c +++ b/kotlin-native/runtime/src/mimalloc/c/alloc-override-osx.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -37,7 +37,6 @@ terms of the MIT license. A copy of the license can be found in the file extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); #endif - /* ------------------------------------------------------ malloc zone members ------------------------------------------------------ */ @@ -67,7 +66,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { static void zone_free(malloc_zone_t* zone, void* p) { UNUSED(zone); - return mi_free(p); + mi_free(p); } static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { @@ -192,63 +191,85 @@ static malloc_zone_t* mi_get_default_zone() } } -static void __attribute__((constructor)) _mi_macos_override_malloc() -{ - static malloc_introspection_t intro; - memset(&intro, 0, sizeof(intro)); +static malloc_introspection_t mi_introspect = { + .enumerator = &intro_enumerator, + .good_size = &intro_good_size, + .check = &intro_check, + .print = &intro_print, + .log = &intro_log, + .force_lock = &intro_force_lock, + .force_unlock = &intro_force_unlock, +#if defined(MAC_OS_X_VERSION_10_6) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 + .zone_locked = &intro_zone_locked, + .statistics = &intro_statistics, +#endif +}; - intro.enumerator = &intro_enumerator; - intro.good_size = &intro_good_size; - intro.check = &intro_check; - intro.print = &intro_print; - intro.log = &intro_log; - intro.force_lock = &intro_force_lock; - intro.force_unlock = &intro_force_unlock; +static malloc_zone_t mi_malloc_zone = { + .size = &zone_size, + .zone_name = "mimalloc", + .introspect = &mi_introspect, + .malloc = &zone_malloc, + .calloc = &zone_calloc, + .valloc = &zone_valloc, + .free = &zone_free, + .realloc = &zone_realloc, + .destroy = &zone_destroy, + .batch_malloc = &zone_batch_malloc, + .batch_free = &zone_batch_free, +#if defined(MAC_OS_X_VERSION_10_6) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 + // switch to version 9 on OSX 10.6 to support memalign. + .version = 9, + .memalign = &zone_memalign, + .free_definite_size = &zone_free_definite_size, + .pressure_relief = &zone_pressure_relief, +#else + .version = 4, +#endif +}; - static malloc_zone_t zone; - memset(&zone, 0, sizeof(zone)); - zone.version = 4; - zone.zone_name = "mimalloc"; - zone.size = &zone_size; - zone.introspect = &intro; - zone.malloc = &zone_malloc; - zone.calloc = &zone_calloc; - zone.valloc = &zone_valloc; - zone.free = &zone_free; - zone.realloc = &zone_realloc; - zone.destroy = &zone_destroy; - zone.batch_malloc = &zone_batch_malloc; - zone.batch_free = &zone_batch_free; +#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) +static malloc_zone_t *mi_malloc_default_zone(void) { + return &mi_malloc_zone; +} +// TODO: should use the macros in alloc-override but they aren't available here. +__attribute__((used)) static struct { + const void *replacement; + const void *target; +} replace_malloc_default_zone[] __attribute__((section("__DATA, __interpose"))) = { + { (const void*)mi_malloc_default_zone, (const void*)malloc_default_zone }, +}; +#endif + +static void __attribute__((constructor(0))) _mi_macos_override_malloc() { malloc_zone_t* purgeable_zone = NULL; #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - // switch to version 9 on OSX 10.6 to support memalign. - zone.version = 9; - zone.memalign = &zone_memalign; - zone.free_definite_size = &zone_free_definite_size; - zone.pressure_relief = &zone_pressure_relief; - intro.zone_locked = &intro_zone_locked; - intro.statistics = &intro_statistics; - // force the purgeable zone to exist to avoid strange bugs if (malloc_default_purgeable_zone) { purgeable_zone = malloc_default_purgeable_zone(); } #endif - // Register our zone - malloc_zone_register(&zone); - + // Register our zone. + // thomcc: I think this is still needed to put us in the zone list. + malloc_zone_register(&mi_malloc_zone); // Unregister the default zone, this makes our zone the new default // as that was the last registered. malloc_zone_t *default_zone = mi_get_default_zone(); - malloc_zone_unregister(default_zone); + // thomcc: Unsure if the next test is *always* false or just false in the + // cases I've tried. I'm also unsure if the code inside is needed. at all + if (default_zone != &mi_malloc_zone) { + malloc_zone_unregister(default_zone); - // Reregister the default zone so free and realloc in that zone keep working. - malloc_zone_register(default_zone); + // Reregister the default zone so free and realloc in that zone keep working. + malloc_zone_register(default_zone); + } // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs // earlier than the default zone. @@ -256,7 +277,8 @@ static void __attribute__((constructor)) _mi_macos_override_malloc() malloc_zone_unregister(purgeable_zone); malloc_zone_register(purgeable_zone); } + } #endif // MI_MALLOC_OVERRIDE -#endif \ No newline at end of file +#endif // KONAN_MI_MALLOC \ No newline at end of file diff --git a/kotlin-native/runtime/src/mimalloc/c/alloc-override.c b/kotlin-native/runtime/src/mimalloc/c/alloc-override.c index 59f06c0efd0..6f1c968c668 100644 --- a/kotlin-native/runtime/src/mimalloc/c/alloc-override.c +++ b/kotlin-native/runtime/src/mimalloc/c/alloc-override.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -15,18 +15,18 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__MACH__) && !defined(MI_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_INTERPOSE))) // ------------------------------------------------------ // Override system malloc // ------------------------------------------------------ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))) + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); #endif #define MI_FORWARD1(fun,x) MI_FORWARD(fun) #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) @@ -62,6 +62,13 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(posix_memalign), MI_INTERPOSE_MI(reallocf), MI_INTERPOSE_MI(valloc), + #ifndef MI_OSX_ZONE + // some code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + #else + // We interpose malloc_default_zone in alloc-override-osx.c + MI_INTERPOSE_MI(free), + #endif // some code allocates from a zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us }; @@ -70,13 +77,13 @@ terms of the MIT license. A copy of the license can be found in the file // we just override new/delete which does work in a static library. #else // On all other systems forward to our API - void* malloc(size_t size) MI_FORWARD1(mi_malloc, size); - void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n); - void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize); - void free(void* p) MI_FORWARD0(mi_free, p); + void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) + void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) + void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) + void free(void* p) MI_FORWARD0(mi_free, p) #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) #pragma GCC visibility push(default) #endif @@ -91,18 +98,18 @@ terms of the MIT license. A copy of the license can be found in the file // see // ------------------------------------------------------ #include - void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); - void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) - void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); - void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) #endif #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) @@ -123,30 +130,30 @@ terms of the MIT license. A copy of the license can be found in the file // used by GCC and CLang). // See // ------------------------------------------------------ - void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete - void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] - void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); - void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } - typedef struct mi_nothrow_s { } mi_nothrow_t; + typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) - void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit - void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit - void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); - void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) - void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit - void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit - void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); - void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } @@ -165,13 +172,13 @@ extern "C" { // Posix & Unix functions definitions // ------------------------------------------------------ -void cfree(void* p) MI_FORWARD0(mi_free, p); -void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); -size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p); +void cfree(void* p) MI_FORWARD0(mi_free, p) +void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) +size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) #if !defined(__ANDROID__) -size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); +size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else -size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p); +size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) #endif // no forwarding here due to aliasing/name mangling issues @@ -182,22 +189,23 @@ void* memalign(size_t alignment, size_t size) { return mi_memali int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -// on some glibc `aligned_alloc` is declared `static inline` so we cannot override it (e.g. Conda). This happens -// when _GLIBCXX_HAVE_ALIGNED_ALLOC is not defined. However, in those cases it will use `memalign`, `posix_memalign`, -// or `_aligned_malloc` and we can avoid overriding it ourselves. -// We should always override if using C compilation. (issue #276) -#if _GLIBCXX_HAVE_ALIGNED_ALLOC || !defined(__cplusplus) +// `aligned_alloc` is only available when __USE_ISOC11 is defined. +// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot +// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. +// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it +// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. +#if __USE_ISOC11 void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #endif #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) - void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size); - void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size); - void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size); - void __libc_free(void* p) MI_FORWARD0(mi_free,p); - void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) void* __libc_valloc(size_t size) { return mi_valloc(size); } void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } @@ -209,7 +217,7 @@ void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(a } #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) #pragma GCC visibility pop #endif diff --git a/kotlin-native/runtime/src/mimalloc/c/alloc-posix.c b/kotlin-native/runtime/src/mimalloc/c/alloc-posix.c index 193fe1dfc1a..f8aa7d6fb20 100644 --- a/kotlin-native/runtime/src/mimalloc/c/alloc-posix.c +++ b/kotlin-native/runtime/src/mimalloc/c/alloc-posix.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -17,7 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file // ------------------------------------------------------ #include -#include // memcpy +#include // memset #include // getenv #ifdef _MSC_VER @@ -77,7 +77,7 @@ mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept { } mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { - if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL; + if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL; if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see void* p = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment)); mi_assert_internal(((uintptr_t)p % alignment) == 0); @@ -103,7 +103,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex size_t size = (len+1)*sizeof(unsigned short); unsigned short* p = (unsigned short*)mi_malloc(size); if (p != NULL) { - memcpy(p,s,size); + _mi_memcpy(p,s,size); } return p; } diff --git a/kotlin-native/runtime/src/mimalloc/c/alloc.c b/kotlin-native/runtime/src/mimalloc/c/alloc.c index 3508fb30fdd..66ba8955b26 100644 --- a/kotlin-native/runtime/src/mimalloc/c/alloc.c +++ b/kotlin-native/runtime/src/mimalloc/c/alloc.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include // memset, memcpy, strlen +#include // memset, strlen #include // malloc, exit #define MI_IN_ALLOC_C @@ -23,27 +23,34 @@ terms of the MIT license. A copy of the license can be found in the file // Fall back to generic allocation only if the list is empty. extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept { mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); - mi_block_t* block = page->free; + mi_block_t* const block = page->free; if (mi_unlikely(block == NULL)) { return _mi_malloc_generic(heap, size); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); // pop from the free list - page->free = mi_block_next(page, block); page->used++; + page->free = mi_block_next(page, block); mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); + #if (MI_DEBUG>0) if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); } #elif (MI_SECURE!=0) block->next = 0; // don't leak internal data #endif -#if (MI_STAT>1) + +#if (MI_STAT>0) const size_t bsize = mi_page_usable_block_size(page); if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_increase(heap, normal, bsize); + mi_heap_stat_counter_increase(heap, normal_count, 1); +#if (MI_STAT>1) const size_t bin = _mi_bin(bsize); - mi_heap_stat_increase(heap, normal[bin], 1); + mi_heap_stat_increase(heap, normal_bins[bin], 1); +#endif } #endif + #if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST) mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); @@ -54,6 +61,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } #endif + return block; } @@ -282,6 +290,49 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co } #endif +// only maintain stats for smaller objects if requested +#if (MI_STAT>0) +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { +#if (MI_STAT < 2) + UNUSED(block); +#endif + mi_heap_t* const heap = mi_heap_get_default(); + const size_t bsize = mi_page_usable_block_size(page); +#if (MI_STAT>1) + const size_t usize = mi_page_usable_size_of(page, block); + mi_heap_stat_decrease(heap, malloc, usize); +#endif + if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, normal, bsize); +#if (MI_STAT > 1) + mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); +#endif + } +} +#else +static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { + UNUSED(page); UNUSED(block); +} +#endif + +#if (MI_STAT>0) +// maintain stats for huge objects +static void mi_stat_huge_free(const mi_page_t* page) { + mi_heap_t* const heap = mi_heap_get_default(); + const size_t bsize = mi_page_block_size(page); // to match stats in `page.c:mi_page_huge_alloc` + if (bsize <= MI_HUGE_OBJ_SIZE_MAX) { + mi_heap_stat_decrease(heap, huge, bsize); + } + else { + mi_heap_stat_decrease(heap, giant, bsize); + } +} +#else +static void mi_stat_huge_free(const mi_page_t* page) { + UNUSED(page); +} +#endif + // ------------------------------------------------------ // Free // ------------------------------------------------------ @@ -300,6 +351,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc // huge page segments are always abandoned and can be freed immediately mi_segment_t* const segment = _mi_page_segment(page); if (segment->page_kind==MI_PAGE_HUGE) { + mi_stat_huge_free(page); _mi_segment_huge_page_free(segment, page, block); return; } @@ -343,7 +395,6 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc } } - // regular free static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block) { @@ -383,6 +434,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) { mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); + mi_stat_free(page, block); _mi_free_block(page, local, block); } @@ -430,19 +482,11 @@ void mi_free(void* p) mi_attr_noexcept mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; -#if (MI_STAT>1) - mi_heap_t* const heap = mi_heap_get_default(); - const size_t bsize = mi_page_usable_block_size(page); - mi_heap_stat_decrease(heap, malloc, bsize); - if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { // huge page stats are accounted for in `_mi_page_retire` - mi_heap_stat_decrease(heap, normal[_mi_bin(bsize)], 1); - } -#endif - if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned if (mi_unlikely(mi_check_is_double_free(page,block))) return; mi_check_padding(page, block); + mi_stat_free(page, block); #if (MI_DEBUG!=0) memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif @@ -513,6 +557,7 @@ void* _mi_externs[] = { (void*)&_mi_page_malloc, (void*)&mi_malloc, (void*)&mi_malloc_small, + (void*)&mi_zalloc_small, (void*)&mi_heap_malloc, (void*)&mi_heap_zalloc, (void*)&mi_heap_malloc_small @@ -584,7 +629,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); memset((uint8_t*)newp + start, 0, newsize - start); } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; @@ -651,7 +696,7 @@ mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_no if (s == NULL) return NULL; size_t n = strlen(s); char* t = (char*)mi_heap_malloc(heap,n+1); - if (t != NULL) memcpy(t, s, n + 1); + if (t != NULL) _mi_memcpy(t, s, n + 1); return t; } @@ -667,7 +712,7 @@ mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_assert_internal(m <= n); char* t = (char*)mi_heap_malloc(heap, m+1); if (t == NULL) return NULL; - memcpy(t, s, m); + _mi_memcpy(t, s, m); t[m] = 0; return t; } @@ -682,7 +727,7 @@ mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { #ifndef PATH_MAX #define PATH_MAX MAX_PATH #endif -#include +#include mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; @@ -746,7 +791,12 @@ but we call `exit` instead (i.e. not returning). #ifdef __cplusplus #include static bool mi_try_new_handler(bool nothrow) { - std::new_handler h = std::get_new_handler(); + #if defined(_MSC_VER) || (__cplusplus >= 201103L) + std::new_handler h = std::get_new_handler(); + #else + std::new_handler h = std::set_new_handler(); + std::set_new_handler(h); + #endif if (h==NULL) { if (!nothrow) throw std::bad_alloc(); return false; diff --git a/kotlin-native/runtime/src/mimalloc/c/arena.c b/kotlin-native/runtime/src/mimalloc/c/arena.c index 73a7e70432f..548654c0d53 100644 --- a/kotlin-native/runtime/src/mimalloc/c/arena.c +++ b/kotlin-native/runtime/src/mimalloc/c/arena.c @@ -1,8 +1,8 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. +"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. -----------------------------------------------------------------------------*/ /* ---------------------------------------------------------------------------- @@ -30,12 +30,13 @@ of 256MiB in practice. #include "mimalloc-atomic.h" #include // memset +#include // ENOMEM -#include "bitmap.inc.c" // atomic bitmap +#include "bitmap.h" // atomic bitmap // os.c -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld); +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* stats); void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats); void _mi_os_free(void* p, size_t size, mi_stats_t* stats); @@ -43,14 +44,14 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_sec void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); /* ----------------------------------------------------------- Arena allocation ----------------------------------------------------------- */ #define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE -#define MI_ARENA_BLOCK_SIZE (8*MI_SEGMENT_ALIGN) // 32MiB -#define MI_ARENA_MAX_OBJ_SIZE (MI_BITMAP_FIELD_BITS * MI_ARENA_BLOCK_SIZE) // 2GiB +#define MI_ARENA_BLOCK_SIZE (4*MI_SEGMENT_ALIGN) // 32MiB #define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 16MiB #define MI_MAX_ARENAS (64) // not more than 256 (since we use 8 bits in the memid) @@ -61,12 +62,12 @@ typedef struct mi_arena_s { size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool is_committed; // is the memory committed - bool is_large; // large OS page allocated - _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks + bool is_committed; // is the memory fully committed? (if so, block_committed == NULL) + bool is_large; // large- or huge OS pages (always committed) + _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed? - mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) + mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) } mi_arena_t; @@ -104,16 +105,11 @@ static size_t mi_block_count_of_size(size_t size) { ----------------------------------------------------------- */ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx) { - const size_t fcount = arena->field_count; size_t idx = mi_atomic_load_acquire(&arena->search_idx); // start from last search - for (size_t visited = 0; visited < fcount; visited++, idx++) { - if (idx >= fcount) idx = 0; // wrap around - // try to atomically claim a range of bits - if (mi_bitmap_try_find_claim_field(arena->blocks_inuse, idx, blocks, bitmap_idx)) { - mi_atomic_store_release(&arena->search_idx, idx); // start search from here next time - return true; - } - } + if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) { + mi_atomic_store_release(&arena->search_idx, idx); // start search from here next time + return true; + }; return false; } @@ -123,16 +119,17 @@ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* ----------------------------------------------------------- */ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t needed_bcount, - bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) + bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { mi_bitmap_index_t bitmap_index; if (!mi_arena_alloc(arena, needed_bcount, &bitmap_index)) return NULL; // claimed it! set the dirty bits (todo: no need for an atomic op here?) - void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE); - *memid = mi_arena_id_create(arena_index, bitmap_index); - *is_zero = mi_bitmap_claim(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); - *large = arena->is_large; + void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE); + *memid = mi_arena_id_create(arena_index, bitmap_index); + *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); + *large = arena->is_large; + *is_pinned = (arena->is_large || arena->is_committed); if (arena->is_committed) { // always committed *commit = true; @@ -140,7 +137,7 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n else if (*commit) { // arena not committed as a whole, but commit requested: ensure commit now bool any_uncommitted; - mi_bitmap_claim(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted); + _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted); if (any_uncommitted) { bool commit_zero; _mi_os_commit(p, needed_bcount * MI_ARENA_BLOCK_SIZE, &commit_zero, tld->stats); @@ -149,25 +146,25 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n } else { // no need to commit, but check if already fully committed - *commit = mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index); + *commit = _mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index); } return p; } -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, - bool* commit, bool* large, bool* is_zero, +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { - mi_assert_internal(commit != NULL && large != NULL && is_zero != NULL && memid != NULL && tld != NULL); + mi_assert_internal(commit != NULL && is_pinned != NULL && is_zero != NULL && memid != NULL && tld != NULL); mi_assert_internal(size > 0); *memid = MI_MEMID_OS; *is_zero = false; + *is_pinned = false; // try to allocate in an arena if the alignment is small enough // and the object is not too large or too small. if (alignment <= MI_SEGMENT_ALIGN && - size <= MI_ARENA_MAX_OBJ_SIZE && - size >= MI_ARENA_MIN_OBJ_SIZE) + size >= MI_ARENA_MIN_OBJ_SIZE && + mi_atomic_load_relaxed(&mi_arena_count) > 0) { const size_t bcount = mi_block_count_of_size(size); const int numa_node = _mi_os_numa_node(tld); // current numa node @@ -180,7 +177,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, if ((arena->numa_node<0 || arena->numa_node==numa_node) && // numa local? (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages { - void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld); + void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld); mi_assert_internal((uintptr_t)p % alignment == 0); if (p != NULL) return p; } @@ -192,7 +189,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, if ((arena->numa_node>=0 && arena->numa_node!=numa_node) && // not numa local! (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages { - void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld); + void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld); mi_assert_internal((uintptr_t)p % alignment == 0); if (p != NULL) return p; } @@ -200,14 +197,20 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, } // finally, fall back to the OS + if (mi_option_is_enabled(mi_option_limit_os_alloc)) { + errno = ENOMEM; + return NULL; + } *is_zero = true; - *memid = MI_MEMID_OS; - return _mi_os_alloc_aligned(size, alignment, *commit, large, tld); + *memid = MI_MEMID_OS; + void* p = _mi_os_alloc_aligned(size, alignment, *commit, large, tld->stats); + if (p != NULL) *is_pinned = *large; + return p; } -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { - return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_zero, memid, tld); + return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_pinned, is_zero, memid, tld); } /* ----------------------------------------------------------- @@ -230,6 +233,8 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s mi_assert_internal(arena_idx < MI_MAX_ARENAS); mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t,&mi_arenas[arena_idx]); mi_assert_internal(arena != NULL); + const size_t blocks = mi_block_count_of_size(size); + // checks if (arena == NULL) { _mi_error_message(EINVAL, "trying to free from non-existent arena: %p, size %zu, memid: 0x%zx\n", p, size, memid); return; @@ -239,9 +244,18 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s _mi_error_message(EINVAL, "trying to free from non-existent arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid); return; } - const size_t blocks = mi_block_count_of_size(size); - bool ones = mi_bitmap_unclaim(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx); - if (!ones) { + // potentially decommit + if (arena->is_committed) { + mi_assert_internal(all_committed); + } + else { + mi_assert_internal(arena->blocks_committed != NULL); + _mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails + _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); + } + // and make it available to others again + bool all_inuse = _mi_bitmap_unclaim_across(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx); + if (!all_inuse) { _mi_error_message(EAGAIN, "trying to free an already freed block: %p, size %zu\n", p, size); return; }; @@ -266,12 +280,64 @@ static bool mi_arena_add(mi_arena_t* arena) { return true; } +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept +{ + if (is_large) { + mi_assert_internal(is_committed); + is_committed = true; + } + + const size_t bcount = mi_block_count_of_size(size); + const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); + const size_t bitmaps = (is_committed ? 2 : 3); + const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); + mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS? + if (arena == NULL) return false; + + arena->block_count = bcount; + arena->field_count = fields; + arena->start = (uint8_t*)start; + arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) + arena->is_large = is_large; + arena->is_zero_init = is_zero; + arena->is_committed = is_committed; + arena->search_idx = 0; + arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap + arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + // the bitmaps are already zero initialized due to os_alloc + // just claim leftover blocks if needed + ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; + mi_assert_internal(post >= 0); + if (post > 0) { + // don't use leftover bits at the end + mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post); + _mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL); + } + + mi_arena_add(arena); + return true; +} + +// Reserve a range of regular OS memory +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept +{ + size = _mi_os_good_alloc_size(size); + bool large = allow_large; + void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &_mi_stats_main); + if (start==NULL) return ENOMEM; + if (!mi_manage_os_memory(start, size, (large || commit), large, true, -1)) { + _mi_os_free_ex(start, size, commit, &_mi_stats_main); + _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); + return ENOMEM; + } + _mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + return 0; +} + /* ----------------------------------------------------------- Reserve a huge page arena. ----------------------------------------------------------- */ -#include // ENOMEM - // reserve at a specific numa node int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept { if (pages==0) return 0; @@ -286,35 +352,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec } _mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages); - size_t bcount = mi_block_count_of_size(hsize); - size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); - size_t asize = sizeof(mi_arena_t) + (2*fields*sizeof(mi_bitmap_field_t)); - mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS? - if (arena == NULL) { + if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); return ENOMEM; } - arena->block_count = bcount; - arena->field_count = fields; - arena->start = (uint8_t*)p; - arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) - arena->is_large = true; - arena->is_zero_init = true; - arena->is_committed = true; - arena->search_idx = 0; - arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap - arena->blocks_committed = NULL; - // the bitmaps are already zero initialized due to os_alloc - // just claim leftover blocks if needed - ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; - mi_assert_internal(post >= 0); - if (post > 0) { - // don't use leftover bits at the end - mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post); - mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL); - } - - mi_arena_add(arena); return 0; } diff --git a/kotlin-native/runtime/src/mimalloc/c/bitmap.c b/kotlin-native/runtime/src/mimalloc/c/bitmap.c new file mode 100644 index 00000000000..d0241adaa85 --- /dev/null +++ b/kotlin-native/runtime/src/mimalloc/c/bitmap.c @@ -0,0 +1,395 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2021 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically, +represeted as an array of fields where each field is a machine word (`uintptr_t`) + +There are two api's; the standard one cannot have sequences that cross +between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). +(this is used in region allocation) + +The `_across` postfixed functions do allow sequences that can cross over +between the fields. (This is used in arena allocation) +---------------------------------------------------------------------------- */ + +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "bitmap.h" + +/* ----------------------------------------------------------- + Bitmap definition +----------------------------------------------------------- */ + +// The bit mask for a given number of blocks at a specified bit index. +static inline uintptr_t mi_bitmap_mask_(size_t count, size_t bitidx) { + mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS); + mi_assert_internal(count > 0); + if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL; + if (count == 0) return 0; + return ((((uintptr_t)1 << count) - 1) << bitidx); +} + + + +/* ----------------------------------------------------------- + Claim a bit sequence atomically +----------------------------------------------------------- */ + +// Try to atomically claim a sequence of `count` bits in a single +// field at `idx` in `bitmap`. Returns `true` on success. +bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx) +{ + mi_assert_internal(bitmap_idx != NULL); + mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); + _Atomic(uintptr_t)* field = &bitmap[idx]; + uintptr_t map = mi_atomic_load_relaxed(field); + if (map==MI_BITMAP_FIELD_FULL) return false; // short cut + + // search for 0-bit sequence of length count + const uintptr_t mask = mi_bitmap_mask_(count, 0); + const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; + +#ifdef MI_HAVE_FAST_BITSCAN + size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible +#else + size_t bitidx = 0; // otherwise start at 0 +#endif + uintptr_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx + + // scan linearly for a free range of zero bits + while (bitidx <= bitidx_max) { + const uintptr_t mapm = map & m; + if (mapm == 0) { // are the mask bits free at bitidx? + mi_assert_internal((m >> bitidx) == mask); // no overflow? + const uintptr_t newmap = map | m; + mi_assert_internal((newmap^map) >> bitidx == mask); + if (!mi_atomic_cas_weak_acq_rel(field, &map, newmap)) { // TODO: use strong cas here? + // no success, another thread claimed concurrently.. keep going (with updated `map`) + continue; + } + else { + // success, we claimed the bits! + *bitmap_idx = mi_bitmap_index_create(idx, bitidx); + return true; + } + } + else { + // on to the next bit range +#ifdef MI_HAVE_FAST_BITSCAN + const size_t shift = (count == 1 ? 1 : mi_bsr(mapm) - bitidx + 1); + mi_assert_internal(shift > 0 && shift <= count); +#else + const size_t shift = 1; +#endif + bitidx += shift; + m <<= shift; + } + } + // no bits found + return false; +} + + +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. +bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) idx = 0; // wrap + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + return true; + } + } + return false; +} + +/* +// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success. +// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never span fields. +bool _mi_bitmap_try_find_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t count, mi_bitmap_index_t* bitmap_idx) { + return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, 0, count, bitmap_idx); +} +*/ + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + // mi_assert_internal((bitmap[idx] & mask) == mask); + uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); + return ((prev & mask) == mask); +} + + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); + uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); + if (any_zero != NULL) *any_zero = ((prev & mask) != mask); + return ((prev & mask) == 0); +} + +// Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one. +static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) { + const size_t idx = mi_bitmap_index_field(bitmap_idx); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]); + if (any_ones != NULL) *any_ones = ((field & mask) != 0); + return ((field & mask) == mask); +} + +bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL); +} + +bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + bool any_ones; + mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); + return any_ones; +} + + +//-------------------------------------------------------------------------- +// the `_across` functions work on bitmaps where sequences can cross over +// between the fields. This is used in arena allocation +//-------------------------------------------------------------------------- + +// Try to atomically claim a sequence of `count` bits starting from the field +// at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success. +static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx) +{ + mi_assert_internal(bitmap_idx != NULL); + + // check initial trailing zeros + _Atomic(uintptr_t)* field = &bitmap[idx]; + uintptr_t map = mi_atomic_load_relaxed(field); + const size_t initial = mi_clz(map); // count of initial zeros starting at idx + mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); + if (initial == 0) return false; + if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields + if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries + + // scan ahead + size_t found = initial; + uintptr_t mask = 0; // mask bits for the final field + while(found < count) { + field++; + map = mi_atomic_load_relaxed(field); + const uintptr_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); + mask = mi_bitmap_mask_(mask_bits, 0); + if ((map & mask) != 0) return false; + found += mask_bits; + } + mi_assert_internal(field < &bitmap[bitmap_fields]); + + // found range of zeros up to the final field; mask contains mask in the final field + // now claim it atomically + _Atomic(uintptr_t)* const final_field = field; + const uintptr_t final_mask = mask; + _Atomic(uintptr_t)* const initial_field = &bitmap[idx]; + const uintptr_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial); + + // initial field + uintptr_t newmap; + field = initial_field; + map = mi_atomic_load_relaxed(field); + do { + newmap = map | initial_mask; + if ((map & initial_mask) != 0) { goto rollback; }; + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + + // intermediate fields + while (++field < final_field) { + newmap = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0); + map = 0; + if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; } + } + + // final field + mi_assert_internal(field == final_field); + map = mi_atomic_load_relaxed(field); + do { + newmap = map | final_mask; + if ((map & final_mask) != 0) { goto rollback; } + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + + // claimed! + *bitmap_idx = mi_bitmap_index_create(idx, MI_BITMAP_FIELD_BITS - initial); + return true; + +rollback: + // roll back intermediate fields + while (--field > initial_field) { + newmap = 0; + map = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0); + mi_assert_internal(mi_atomic_load_relaxed(field) == map); + mi_atomic_store_release(field, newmap); + } + if (field == initial_field) { + map = mi_atomic_load_relaxed(field); + do { + mi_assert_internal((map & initial_mask) == initial_mask); + newmap = map & ~initial_mask; + } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); + } + // retry? (we make a recursive call instead of goto to be able to use const declarations) + if (retries < 4) { + return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx); + } + else { + return false; + } +} + + +// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) { + mi_assert_internal(count > 0); + if (count==1) return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, start_field_idx, count, bitmap_idx); + size_t idx = start_field_idx; + for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) { + if (idx >= bitmap_fields) idx = 0; // wrap + // try to claim inside the field + if (count <= MI_BITMAP_FIELD_BITS) { + if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) { + return true; + } + } + // try to claim across fields + if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) { + return true; + } + } + return false; +} + +// Helper for masks across fields; returns the mid count, post_mask may be 0 +static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) { + UNUSED_RELEASE(bitmap_fields); + const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); + if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { + *pre_mask = mi_bitmap_mask_(count, bitidx); + *mid_mask = 0; + *post_mask = 0; + mi_assert_internal(mi_bitmap_index_field(bitmap_idx) < bitmap_fields); + return 0; + } + else { + const size_t pre_bits = MI_BITMAP_FIELD_BITS - bitidx; + mi_assert_internal(pre_bits < count); + *pre_mask = mi_bitmap_mask_(pre_bits, bitidx); + count -= pre_bits; + const size_t mid_count = (count / MI_BITMAP_FIELD_BITS); + *mid_mask = MI_BITMAP_FIELD_FULL; + count %= MI_BITMAP_FIELD_BITS; + *post_mask = (count==0 ? 0 : mi_bitmap_mask_(count, 0)); + mi_assert_internal(mi_bitmap_index_field(bitmap_idx) + mid_count + (count==0 ? 0 : 1) < bitmap_fields); + return mid_count; + } +} + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + uintptr_t pre_mask; + uintptr_t mid_mask; + uintptr_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_one = true; + _Atomic(uintptr_t)*field = &bitmap[idx]; + uintptr_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); + if ((prev & pre_mask) != pre_mask) all_one = false; + while(mid_count-- > 0) { + prev = mi_atomic_and_acq_rel(field++, ~mid_mask); + if ((prev & mid_mask) != mid_mask) all_one = false; + } + if (post_mask!=0) { + prev = mi_atomic_and_acq_rel(field, ~post_mask); + if ((prev & post_mask) != post_mask) all_one = false; + } + return all_one; +} + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + uintptr_t pre_mask; + uintptr_t mid_mask; + uintptr_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_zero = true; + bool any_zero = false; + _Atomic(uintptr_t)*field = &bitmap[idx]; + uintptr_t prev = mi_atomic_or_acq_rel(field++, pre_mask); + if ((prev & pre_mask) != 0) all_zero = false; + if ((prev & pre_mask) != pre_mask) any_zero = true; + while (mid_count-- > 0) { + prev = mi_atomic_or_acq_rel(field++, mid_mask); + if ((prev & mid_mask) != 0) all_zero = false; + if ((prev & mid_mask) != mid_mask) any_zero = true; + } + if (post_mask!=0) { + prev = mi_atomic_or_acq_rel(field, post_mask); + if ((prev & post_mask) != 0) all_zero = false; + if ((prev & post_mask) != post_mask) any_zero = true; + } + if (pany_zero != NULL) *pany_zero = any_zero; + return all_zero; +} + + +// Returns `true` if all `count` bits were 1. +// `any_ones` is `true` if there was at least one bit set to one. +static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) { + size_t idx = mi_bitmap_index_field(bitmap_idx); + uintptr_t pre_mask; + uintptr_t mid_mask; + uintptr_t post_mask; + size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); + bool all_ones = true; + bool any_ones = false; + _Atomic(uintptr_t)* field = &bitmap[idx]; + uintptr_t prev = mi_atomic_load_relaxed(field++); + if ((prev & pre_mask) != pre_mask) all_ones = false; + if ((prev & pre_mask) != 0) any_ones = true; + while (mid_count-- > 0) { + prev = mi_atomic_load_relaxed(field++); + if ((prev & mid_mask) != mid_mask) all_ones = false; + if ((prev & mid_mask) != 0) any_ones = true; + } + if (post_mask!=0) { + prev = mi_atomic_load_relaxed(field); + if ((prev & post_mask) != post_mask) all_ones = false; + if ((prev & post_mask) != 0) any_ones = true; + } + if (pany_ones != NULL) *pany_ones = any_ones; + return all_ones; +} + +bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + return mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, NULL); +} + +bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { + bool any_ones; + mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, &any_ones); + return any_ones; +} diff --git a/kotlin-native/runtime/src/mimalloc/c/bitmap.h b/kotlin-native/runtime/src/mimalloc/c/bitmap.h new file mode 100644 index 00000000000..dbe6313a93e --- /dev/null +++ b/kotlin-native/runtime/src/mimalloc/c/bitmap.h @@ -0,0 +1,102 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2020 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +/* ---------------------------------------------------------------------------- +Concurrent bitmap that can set/reset sequences of bits atomically, +represeted as an array of fields where each field is a machine word (`uintptr_t`) + +There are two api's; the standard one cannot have sequences that cross +between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). +(this is used in region allocation) + +The `_across` postfixed functions do allow sequences that can cross over +between the fields. (This is used in arena allocation) +---------------------------------------------------------------------------- */ +#pragma once +#ifndef MI_BITMAP_H +#define MI_BITMAP_H + +/* ----------------------------------------------------------- + Bitmap definition +----------------------------------------------------------- */ + +#define MI_BITMAP_FIELD_BITS (8*MI_INTPTR_SIZE) +#define MI_BITMAP_FIELD_FULL (~((uintptr_t)0)) // all bits set + +// An atomic bitmap of `uintptr_t` fields +typedef _Atomic(uintptr_t) mi_bitmap_field_t; +typedef mi_bitmap_field_t* mi_bitmap_t; + +// A bitmap index is the index of the bit in a bitmap. +typedef size_t mi_bitmap_index_t; + +// Create a bit index. +static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) { + mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS); + return (idx*MI_BITMAP_FIELD_BITS) + bitidx; +} + +// Get the field index from a bit index. +static inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) { + return (bitmap_idx / MI_BITMAP_FIELD_BITS); +} + +// Get the bit index in a bitmap field +static inline size_t mi_bitmap_index_bit_in_field(mi_bitmap_index_t bitmap_idx) { + return (bitmap_idx % MI_BITMAP_FIELD_BITS); +} + +// Get the full bit index +static inline size_t mi_bitmap_index_bit(mi_bitmap_index_t bitmap_idx) { + return bitmap_idx; +} + +/* ----------------------------------------------------------- + Claim a bit sequence atomically +----------------------------------------------------------- */ + +// Try to atomically claim a sequence of `count` bits in a single +// field at `idx` in `bitmap`. Returns `true` on success. +bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields. +bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero); + +bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); +bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + + +//-------------------------------------------------------------------------- +// the `_across` functions work on bitmaps where sequences can cross over +// between the fields. This is used in arena allocation +//-------------------------------------------------------------------------- + +// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success. +// Starts at idx, and wraps around to search in all `bitmap_fields` fields. +bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 0 atomically +// Returns `true` if all `count` bits were 1 previously. +bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +// Set `count` bits at `bitmap_idx` to 1 atomically +// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. +bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero); + +bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); +bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx); + +#endif diff --git a/kotlin-native/runtime/src/mimalloc/c/heap.c b/kotlin-native/runtime/src/mimalloc/c/heap.c index bc7b07a8a92..c405c1c3bf7 100644 --- a/kotlin-native/runtime/src/mimalloc/c/heap.c +++ b/kotlin-native/runtime/src/mimalloc/c/heap.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -92,7 +92,7 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); if (mi_page_all_free(page)) { - // no more used blocks, free the page. + // no more used blocks, free the page. // note: this will free retired pages as well. _mi_page_free(page, pq, collect >= MI_FORCE); } @@ -114,10 +114,10 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) { - if (!mi_heap_is_initialized(heap)) return; + if (heap==NULL || !mi_heap_is_initialized(heap)) return; _mi_deferred_free(heap, collect >= MI_FORCE); - // note: never reclaim on collect but leave it to threads that need storage to reclaim + // note: never reclaim on collect but leave it to threads that need storage to reclaim if ( #ifdef NDEBUG collect == MI_FORCE @@ -130,7 +130,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // if all memory is freed by now, all segments should be freed. _mi_abandoned_reclaim_all(heap, &heap->tld->segments); } - + // if abandoning, mark all pages to no longer add to delayed_free if (collect == MI_ABANDON) { mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); @@ -193,7 +193,7 @@ mi_heap_t* mi_heap_new(void) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap==NULL) return NULL; - memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); + _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = bheap->tld; heap->thread_id = _mi_thread_id(); _mi_random_split(&bheap->random, &heap->random); @@ -213,13 +213,14 @@ uintptr_t _mi_heap_random_next(mi_heap_t* heap) { // zero out the page queues static void mi_heap_reset_pages(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); mi_assert_internal(mi_heap_is_initialized(heap)); // TODO: copy full empty heap instead? memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct)); #ifdef MI_MEDIUM_DIRECT memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium)); #endif - memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); + _mi_memcpy_aligned(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); heap->thread_delayed_free = NULL; heap->page_count = 0; } @@ -228,6 +229,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) { static void mi_heap_free(mi_heap_t* heap) { mi_assert(heap != NULL); mi_assert_internal(mi_heap_is_initialized(heap)); + if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (mi_heap_is_backing(heap)) return; // dont free the backing heap // reset default @@ -238,7 +240,7 @@ static void mi_heap_free(mi_heap_t* heap) { // remove ourselves from the thread local heaps list // linear search but we expect the number of heaps to be relatively small mi_heap_t* prev = NULL; - mi_heap_t* curr = heap->tld->heaps; + mi_heap_t* curr = heap->tld->heaps; while (curr != heap && curr != NULL) { prev = curr; curr = curr->next; @@ -272,17 +274,20 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ const size_t bsize = mi_page_block_size(page); if (bsize > MI_LARGE_OBJ_SIZE_MAX) { if (bsize > MI_HUGE_OBJ_SIZE_MAX) { - _mi_stat_decrease(&heap->tld->stats.giant, bsize); + mi_heap_stat_decrease(heap, giant, bsize); } else { - _mi_stat_decrease(&heap->tld->stats.huge, bsize); + mi_heap_stat_decrease(heap, huge, bsize); } } -#if (MI_STAT>1) +#if (MI_STAT) _mi_page_free_collect(page, false); // update used count const size_t inuse = page->used; if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, normal[_mi_bin(bsize)], inuse); + mi_heap_stat_decrease(heap, normal, bsize * inuse); +#if (MI_STAT>1) + mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); +#endif } mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks... #endif @@ -310,7 +315,7 @@ void mi_heap_destroy(mi_heap_t* heap) { mi_assert(mi_heap_is_initialized(heap)); mi_assert(heap->no_reclaim); mi_assert_expensive(mi_heap_is_valid(heap)); - if (!mi_heap_is_initialized(heap)) return; + if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (!heap->no_reclaim) { // don't free in case it may contain reclaimed pages mi_heap_delete(heap); @@ -335,8 +340,8 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { // reduce the size of the delayed frees _mi_heap_delayed_free(from); - - // transfer all pages by appending the queues; this will set a new heap field + + // transfer all pages by appending the queues; this will set a new heap field // so threads may do delayed frees in either heap for a while. // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state // so after this only the new heap will get delayed frees @@ -349,15 +354,17 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { } mi_assert_internal(from->page_count == 0); - // and do outstanding delayed frees in the `from` heap + // and do outstanding delayed frees in the `from` heap // note: be careful here as the `heap` field in all those pages no longer point to `from`, - // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a + // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a // the regular `_mi_free_delayed_block` which is safe. - _mi_heap_delayed_free(from); - mi_assert_internal(from->thread_delayed_free == NULL); + _mi_heap_delayed_free(from); + #if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353 + mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL); + #endif // and reset the `from` heap - mi_heap_reset_pages(from); + mi_heap_reset_pages(from); } // Safe delete a heap without freeing any still allocated blocks in that heap. @@ -366,7 +373,7 @@ void mi_heap_delete(mi_heap_t* heap) mi_assert(heap != NULL); mi_assert(mi_heap_is_initialized(heap)); mi_assert_expensive(mi_heap_is_valid(heap)); - if (!mi_heap_is_initialized(heap)) return; + if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (!mi_heap_is_backing(heap)) { // tranfer still used pages to the backing heap @@ -381,8 +388,9 @@ void mi_heap_delete(mi_heap_t* heap) } mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { + mi_assert(heap != NULL); mi_assert(mi_heap_is_initialized(heap)); - if (!mi_heap_is_initialized(heap)) return NULL; + if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL; mi_assert_expensive(mi_heap_is_valid(heap)); mi_heap_t* old = mi_get_default_heap(); _mi_heap_set_default_direct(heap); @@ -408,7 +416,7 @@ static mi_heap_t* mi_heap_of_block(const void* p) { bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { mi_assert(heap != NULL); - if (!mi_heap_is_initialized(heap)) return false; + if (heap==NULL || !mi_heap_is_initialized(heap)) return false; return (heap == mi_heap_of_block(p)); } @@ -426,7 +434,7 @@ static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { mi_assert(heap != NULL); - if (!mi_heap_is_initialized(heap)) return false; + if (heap==NULL || !mi_heap_is_initialized(heap)) return false; if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers bool found = false; mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void*)p, &found); diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-atomic.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-atomic.h index 3e9d7d5a99a..03795963510 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-atomic.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-atomic.h @@ -1,10 +1,9 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2020 Microsoft Research, Daan Leijen +Copyright (c) 2018-2021 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. -----------------------------------------------------------------------------*/ -// Copyright 2019-2020 JetBrains s.r.o. #pragma once #ifndef MIMALLOC_ATOMIC_H @@ -120,7 +119,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { // MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. #define WIN32_LEAN_AND_MEAN -#include +#include #include #ifdef _WIN64 typedef LONG64 msc_intptr_t; @@ -283,44 +282,52 @@ static inline void mi_atomic_yield(void) { static inline void mi_atomic_yield(void) { YieldProcessor(); } +#elif defined(__SSE2__) +#include +static inline void mi_atomic_yield(void) { + _mm_pause(); +} #elif (defined(__GNUC__) || defined(__clang__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) + (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \ + defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) #if defined(__x86_64__) || defined(__i386__) static inline void mi_atomic_yield(void) { __asm__ volatile ("pause" ::: "memory"); } -#elif defined(__arm__) || defined(__aarch64__) - #if KONAN_MI_MALLOC - #if defined(__arm__) - #include - static inline void mi_atomic_yield(void) { - sched_yield(); - } - #else - static inline void mi_atomic_yield(void) { - asm volatile("yield"); - } - #endif - #else // KONAN_MI_MALLOC - static inline void mi_atomic_yield(void) { - asm volatile("yield"); - } - #endif // KONAN_MI_MALLOC +#elif defined(__aarch64__) +static inline void mi_atomic_yield(void) { + __asm__ volatile("wfe"); +} +#elif (defined(__arm__) && __ARM_ARCH__ >= 7) +static inline void mi_atomic_yield(void) { + __asm__ volatile("yield" ::: "memory"); +} +#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) +static inline void mi_atomic_yield(void) { + __asm__ __volatile__ ("or 27,27,27" ::: "memory"); +} +#elif defined(__armel__) || defined(__ARMEL__) +static inline void mi_atomic_yield(void) { + __asm__ volatile ("nop" ::: "memory"); +} #endif +#elif defined(__sun) +// Fallback for other archs +#include +static inline void mi_atomic_yield(void) { + smt_pause(); +} #elif defined(__wasi__) - #include - static inline void mi_atomic_yield(void) { - sched_yield(); - } +#include +static inline void mi_atomic_yield(void) { + sched_yield(); +} #else - #include - static inline void mi_atomic_yield(void) { - sleep(0); - } +#include +static inline void mi_atomic_yield(void) { + sleep(0); +} #endif + #endif // __MIMALLOC_ATOMIC_H - - - - diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-internal.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-internal.h index 67fd2b5925e..c92fd4bb46c 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-internal.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-internal.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -34,7 +34,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif - // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -53,8 +52,8 @@ uintptr_t _os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c -extern mi_stats_t _mi_stats_main; -extern const mi_page_t _mi_page_empty; +extern mi_decl_cache_align mi_stats_t _mi_stats_main; +extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); bool _mi_preloading(); // true while the C runtime is not ready @@ -66,7 +65,7 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free th size_t _mi_os_good_alloc_size(size_t size); // memory.c -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* id, mi_os_tld_t* tld); +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld); void _mi_mem_free(void* p, size_t size, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld); bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld); @@ -109,7 +108,6 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback fro size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats -uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c" // "heap.c" void _mi_heap_destroy_pages(mi_heap_t* heap); @@ -245,10 +243,9 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) #if defined(_CLOCK_T) // for Illumos #undef _CLOCK_T #endif - static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { // Changed order for armv7 (ULONG_MAX == UINT_MAX, but size_t = unsigned long) -#if defined(__MACH__) && KONAN_MI_MALLOC +#if defined(__APPLE__) && KONAN_MI_MALLOC #if (SIZE_MAX == ULONG_MAX) return __builtin_umull_overflow(count, size, total); #elif (SIZE_MAX == UINT_MAX) @@ -301,7 +298,7 @@ We try to circumvent this in an efficient way: - macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the loader itself calls `malloc` even before the modules are initialized. - OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). -- DragonFly: not yet working. +- DragonFly: the uniqueid use is buggy but kept for reference. ------------------------------------------------------------------------------------------- */ extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap @@ -309,7 +306,7 @@ extern bool _mi_process_is_initialized; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap #if defined(MI_MALLOC_OVERRIDE) -#if defined(__MACH__) // OSX +#if defined(__APPLE__) // macOS #define MI_TLS_SLOT 89 // seems unused? // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) // see @@ -319,7 +316,7 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) #elif defined(__DragonFly__) #warning "mimalloc is not working correctly on DragonFly yet." -#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +//#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) #endif #endif @@ -331,7 +328,7 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { pthread_t self = pthread_self(); #if defined(__DragonFly__) if (self==NULL) { - static mi_heap_t* pheap_main = _mi_heap_main_get(); + mi_heap_t* pheap_main = _mi_heap_main_get(); return &pheap_main; } #endif @@ -340,10 +337,14 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { #elif defined(MI_TLS_PTHREAD) #include extern pthread_key_t _mi_heap_default_key; -#else -extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from #endif +// Default heap to allocate from (if not using TLS- or pthread slots). +// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`). +// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined. +// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356). +extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from + static inline mi_heap_t* mi_get_default_heap(void) { #if defined(MI_TLS_SLOT) mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); @@ -679,13 +680,14 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { int _mi_os_numa_node_get(mi_os_tld_t* tld); size_t _mi_os_numa_node_count_get(void); -extern size_t _mi_numa_node_count; +extern _Atomic(size_t) _mi_numa_node_count; static inline int _mi_os_numa_node(mi_os_tld_t* tld) { - if (mi_likely(_mi_numa_node_count == 1)) return 0; + if (mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1)) return 0; else return _mi_os_numa_node_get(tld); } static inline size_t _mi_os_numa_node_count(void) { - if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count; + const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); + if (mi_likely(count>0)) return count; else return _mi_os_numa_node_count_get(); } @@ -716,8 +718,10 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) +#elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS +#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI #elif defined(__x86_64__) __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) @@ -726,7 +730,12 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { res = tcb[slot]; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); +#if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); + tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits +#else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); +#endif res = tcb[slot]; #endif return res; @@ -737,8 +746,10 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) +#elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS +#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI #elif defined(__x86_64__) __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) @@ -747,13 +758,18 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { tcb[slot] = value; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); +#if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); + tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits +#else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); +#endif tcb[slot] = value; #endif } static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { - #if defined(__MACH__) && KONAN_MI_MALLOC + #if defined(__APPLE__) && KONAN_MI_MALLOC #include #if TARGET_OS_EMBEDDED // iOS/tvOS/watchOS devices. return pthread_mach_thread_np(pthread_self()); @@ -762,8 +778,13 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)mi_tls_slot(0); #endif #else // KONAN_MI_MALLOC - // in all our targets, slot 0 is the pointer to the thread control block - return (uintptr_t)mi_tls_slot(0); + #if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) + // on Android, slot 1 is the thread ID (pointer to pthread internal struct) + return (uintptr_t)mi_tls_slot(1); + #else + // in all our other targets, slot 0 is the pointer to the thread control block + return (uintptr_t)mi_tls_slot(0); + #endif #endif // KONAN_MI_MALLOC } #else @@ -773,5 +794,159 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { } #endif +// ----------------------------------------------------------------------- +// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) +// ----------------------------------------------------------------------- + +#if defined(__GNUC__) + +#include // LONG_MAX +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_clzl(x); +#else + return __builtin_clzll(x); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (INTPTR_MAX == LONG_MAX) + return __builtin_ctzl(x); +#else + return __builtin_ctzll(x); +#endif +} + +#elif defined(_MSC_VER) + +#include // LONG_MAX +#define MI_HAVE_FAST_BITSCAN +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanReverse(&idx, x); +#else + _BitScanReverse64(&idx, x); +#endif + return ((MI_INTPTR_BITS - 1) - idx); +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; + unsigned long idx; +#if (INTPTR_MAX == LONG_MAX) + _BitScanForward(&idx, x); +#else + _BitScanForward64(&idx, x); +#endif + return idx; +} + +#else +static inline size_t mi_ctz32(uint32_t x) { + // de Bruijn multiplication, see + static const unsigned char debruijn[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 + }; + if (x==0) return 32; + return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27]; +} +static inline size_t mi_clz32(uint32_t x) { + // de Bruijn multiplication, see + static const uint8_t debruijn[32] = { + 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, + 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0 + }; + if (x==0) return 32; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27]; +} + +static inline size_t mi_clz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_clz32((uint32_t)x); +#else + size_t count = mi_clz32((uint32_t)(x >> 32)); + if (count < 32) return count; + return (32 + mi_clz32((uint32_t)x)); +#endif +} +static inline size_t mi_ctz(uintptr_t x) { + if (x==0) return MI_INTPTR_BITS; +#if (MI_INTPTR_BITS <= 32) + return mi_ctz32((uint32_t)x); +#else + size_t count = mi_ctz32((uint32_t)x); + if (count < 32) return count; + return (32 + mi_ctz32((uint32_t)(x>>32))); +#endif +} + +#endif + +// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero) +static inline size_t mi_bsr(uintptr_t x) { + return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x)); +} + + +// --------------------------------------------------------------------------------- +// Provide our own `_mi_memcpy` for potential performance optimizations. +// +// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if +// we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support +// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. +// --------------------------------------------------------------------------------- + +#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#include +#include +extern bool _mi_cpu_has_fsrm; +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + if (_mi_cpu_has_fsrm) { + __movsb((unsigned char*)dst, (const unsigned char*)src, n); + } + else { + memcpy(dst, src, n); // todo: use noinline? + } +} +#else +#include +static inline void _mi_memcpy(void* dst, const void* src, size_t n) { + memcpy(dst, src, n); +} +#endif + + +// ------------------------------------------------------------------------------- +// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned +// This is used for example in `mi_realloc`. +// ------------------------------------------------------------------------------- + +#if (__GNUC__ >= 4) || defined(__clang__) +// On GCC/CLang we provide a hint that the pointers are word aligned. +#include +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); + memcpy(adst, asrc, n); +} +#else +// Default fallback on `_mi_memcpy` +static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); + _mi_memcpy(dst, src, n); +} +#endif + #endif diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-new-delete.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-new-delete.h index e9d12fcabca..0617c929503 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-new-delete.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-new-delete.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-override.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-override.h index 43a33ed1223..c5ceb64b588 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-override.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-override.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -24,7 +24,7 @@ not accidentally mix pointers from different allocators). #define free(p) mi_free(p) #define strdup(s) mi_strdup(s) -#define strndup(s) mi_strndup(s) +#define strndup(s,n) mi_strndup(s,n) #define realpath(f,n) mi_realpath(f,n) // Microsoft extensions @@ -33,7 +33,7 @@ not accidentally mix pointers from different allocators). #define _recalloc(p,n,c) mi_recalloc(p,n,c) #define _strdup(s) mi_strdup(s) -#define _strndup(s) mi_strndup(s) +#define _strndup(s,n) mi_strndup(s,n) #define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s)) #define _mbsdup(s) mi_mbsdup(s) #define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v) diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-types.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-types.h index 511271c56e8..f596abea5e7 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-types.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc-types.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include // _Atomic +#include "mimalloc-atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int @@ -265,7 +265,7 @@ typedef enum mi_page_kind_e { typedef struct mi_segment_s { // memory fields size_t memid; // id for the os-level memory manager - bool mem_is_fixed; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) + bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) bool mem_is_committed; // `true` if the whole segment is eagerly committed // segment fields @@ -274,7 +274,7 @@ typedef struct mi_segment_s { struct mi_segment_s* prev; size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) - size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long) size_t used; // count of pages in use (`used <= capacity`) size_t capacity; // count of available pages (`#free + used`) @@ -420,6 +420,7 @@ typedef struct mi_stats_s { mi_stat_count_t segments_abandoned; mi_stat_count_t pages_abandoned; mi_stat_count_t threads; + mi_stat_count_t normal; mi_stat_count_t huge; mi_stat_count_t giant; mi_stat_count_t malloc; @@ -429,10 +430,11 @@ typedef struct mi_stats_s { mi_stat_counter_t commit_calls; mi_stat_counter_t page_no_retire; mi_stat_counter_t searches; + mi_stat_counter_t normal_count; mi_stat_counter_t huge_count; mi_stat_counter_t giant_count; #if MI_STAT>1 - mi_stat_count_t normal[MI_BIN_HUGE+1]; + mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; #endif } mi_stats_t; @@ -451,6 +453,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); #define mi_stat_counter_increase(stat,amount) (void)0 #endif +#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h index 63c338a8e40..6fec51ff798 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 167 // major + 2 digits minor +#define MI_MALLOC_VERSION 171 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -264,6 +264,10 @@ mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; +mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; +mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; + + // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; @@ -302,6 +306,7 @@ typedef enum mi_option_e { mi_option_reset_decommits, mi_option_large_os_pages, // implies eager commit mi_option_reserve_huge_os_pages, + mi_option_reserve_os_memory, mi_option_segment_cache, mi_option_page_reset, mi_option_abandoned_page_reset, @@ -309,8 +314,10 @@ typedef enum mi_option_e { mi_option_eager_commit_delay, mi_option_reset_delay, mi_option_use_numa_nodes, + mi_option_limit_os_alloc, mi_option_os_tag, mi_option_max_errors, + mi_option_max_warnings, _mi_option_last } mi_option_t; diff --git a/kotlin-native/runtime/src/mimalloc/c/init.c b/kotlin-native/runtime/src/mimalloc/c/init.c index 176cd722e26..534d2519b2e 100644 --- a/kotlin-native/runtime/src/mimalloc/c/init.c +++ b/kotlin-native/runtime/src/mimalloc/c/init.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -73,8 +73,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), \ - { 0, 0 }, { 0, 0 }, { 0, 0 }, \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ MI_STAT_COUNT_END_NULL() @@ -87,7 +87,7 @@ const mi_page_t _mi_page_empty = { // may lead to allocation itself on some platforms) // -------------------------------------------------------- -const mi_heap_t _mi_heap_empty = { +mi_decl_cache_align const mi_heap_t _mi_heap_empty = { NULL, MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, @@ -189,7 +189,7 @@ static bool _mi_heap_init(void) { // OS allocated so already zero initialized mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; - memcpy(heap, &_mi_heap_empty, sizeof(*heap)); + _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); heap->thread_id = _mi_thread_id(); _mi_random_init(&heap->random); heap->cookie = _mi_heap_random_next(heap) | 1; @@ -201,7 +201,7 @@ static bool _mi_heap_init(void) { tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; tld->os.stats = &tld->stats; - _mi_heap_set_default_direct(heap); + _mi_heap_set_default_direct(heap); } return false; } @@ -234,16 +234,16 @@ static bool _mi_heap_done(mi_heap_t* heap) { if (heap != &_mi_heap_main) { _mi_heap_collect_abandon(heap); } - + // merge stats - _mi_stats_done(&heap->tld->stats); + _mi_stats_done(&heap->tld->stats); // free if not the main thread if (heap != &_mi_heap_main) { mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); _mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main); } -#if 0 +#if 0 // never free the main thread even in debug mode; if a dll is linked statically with mimalloc, // there may still be delete/free calls after the mi_fls_done is called. Issue #207 else { @@ -284,9 +284,9 @@ static void _mi_thread_done(mi_heap_t* default_heap); // nothing to do as it is done in DllMain #elif defined(_WIN32) && !defined(MI_SHARED_LIB) // use thread local storage keys to detect thread ending - #include + #include #include - #if (_WIN32_WINNT < 0x600) // before Windows Vista + #if (_WIN32_WINNT < 0x600) // before Windows Vista WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); @@ -336,7 +336,7 @@ void mi_thread_init(void) mi_attr_noexcept { // ensure our process has started already mi_process_init(); - + // initialize the thread local default heap // (this will call `_mi_heap_set_default_direct` and thus set the // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) @@ -355,7 +355,7 @@ static void _mi_thread_done(mi_heap_t* heap) { // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... if (heap->thread_id != _mi_thread_id()) return; - + // abandon the thread local heap if (_mi_heap_done(heap)) return; // returns true if already ran } @@ -458,6 +458,22 @@ static void mi_process_load(void) { } } +#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) +#include +mi_decl_cache_align bool _mi_cpu_has_fsrm = false; + +static void mi_detect_cpu_features(void) { + // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + int32_t cpu_info[4]; + __cpuid(cpu_info, 7); + _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see +} +#else +static void mi_detect_cpu_features(void) { + // nothing +} +#endif + // Initialize the process; called by thread_init or the process loader void mi_process_init(void) mi_attr_noexcept { // ensure we are called once @@ -466,6 +482,7 @@ void mi_process_init(void) mi_attr_noexcept { mi_process_setup_auto_thread_done(); _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + mi_detect_cpu_features(); _mi_os_init(); mi_heap_main_init(); #if (MI_DEBUG) @@ -478,6 +495,10 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get(mi_option_reserve_huge_os_pages); mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + } + if (mi_option_is_enabled(mi_option_reserve_os_memory)) { + long ksize = mi_option_get(mi_option_reserve_os_memory); + if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true); } } @@ -494,8 +515,8 @@ static void mi_process_done(void) { FlsSetValue(mi_fls_key, NULL); // don't call main-thread callback FlsFree(mi_fls_key); // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208 #endif - - #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) + + #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) // free all memory if possible on process exit. This is not needed for a stand-alone process // but should be done if mimalloc is statically linked into another shared library which // is repeatedly loaded/unloaded, see issue #281. @@ -505,7 +526,7 @@ static void mi_process_done(void) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { mi_stats_print(NULL); } - mi_allocator_done(); + mi_allocator_done(); _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); os_preloading = true; // don't call the C runtime anymore } diff --git a/kotlin-native/runtime/src/mimalloc/c/options.c b/kotlin-native/runtime/src/mimalloc/c/options.c index b5c50d1b0ad..e53c19bc7d7 100644 --- a/kotlin-native/runtime/src/mimalloc/c/options.c +++ b/kotlin-native/runtime/src/mimalloc/c/options.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -19,7 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = 16; // stop outputting errors after this +static uintptr_t mi_max_error_count = 16; // stop outputting errors after this +static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(); @@ -74,7 +75,8 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(reset_decommits) }, // reset uses MADV_FREE/MADV_DONTNEED #endif { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, + { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { 0, UNINIT, MI_OPTION(reserve_os_memory) }, { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread { 1, UNINIT, MI_OPTION(page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_reset) },// reset free page memory when a thread terminates @@ -86,8 +88,11 @@ static mi_option_desc_t options[_mi_option_last] = #endif { 100, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. + { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_warnings) } // maximum warnings that are output + }; static void mi_option_init(mi_option_desc_t* desc); @@ -105,6 +110,7 @@ void _mi_options_init(void) { } } mi_max_error_count = mi_option_get(mi_option_max_errors); + mi_max_warning_count = mi_option_get(mi_option_max_warnings); } long mi_option_get(mi_option_t option) { @@ -188,7 +194,7 @@ static void mi_out_buf(const char* msg, void* arg) { if (start+n >= MI_MAX_DELAY_OUTPUT) { n = MI_MAX_DELAY_OUTPUT-start-1; } - memcpy(&out_buf[start], msg, n); + _mi_memcpy(&out_buf[start], msg, n); } static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { @@ -245,14 +251,15 @@ static void mi_add_stderr_output() { // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- -static _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings +static _Atomic(uintptr_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings // When overriding malloc, we may recurse into mi_vfprintf if an allocation // inside the C runtime causes another message. static mi_decl_thread bool recurse = false; static bool mi_recurse_enter(void) { - #ifdef MI_TLS_RECURSE_GUARD + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) if (_mi_preloading()) return true; #endif if (recurse) return false; @@ -261,7 +268,7 @@ static bool mi_recurse_enter(void) { } static void mi_recurse_exit(void) { - #ifdef MI_TLS_RECURSE_GUARD + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) if (_mi_preloading()) return; #endif recurse = false; @@ -323,7 +330,7 @@ static void mi_show_error_message(const char* fmt, va_list args) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; - if (mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return; + if (mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return; va_list args; va_start(args,fmt); mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args); @@ -347,7 +354,7 @@ static _Atomic(void*) mi_error_arg; // = NULL static void mi_error_default(int err) { UNUSED(err); -#if (MI_DEBUG>0) +#if (MI_DEBUG>0) if (err==EFAULT) { #ifdef _MSC_VER __debugbreak(); @@ -415,30 +422,30 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { // reliably even when this is invoked before the C runtime is initialized. // i.e. when `_mi_preloading() == true`. // Note: on windows, environment names are not case sensitive. -#include +#include static bool mi_getenv(const char* name, char* result, size_t result_size) { result[0] = 0; size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); return (len > 0 && len < result_size); } #elif !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0) -// On Posix systemsr use `environ` to acces environment variables +// On Posix systemsr use `environ` to acces environment variables // even before the C runtime is initialized. #if defined(__APPLE__) && defined(__has_include) && __has_include() #include static char** mi_get_environ(void) { return (*_NSGetEnviron()); } -#else +#else extern char** environ; static char** mi_get_environ(void) { return environ; } #endif static bool mi_getenv(const char* name, char* result, size_t result_size) { - if (name==NULL) return false; + if (name==NULL) return false; const size_t len = strlen(name); - if (len == 0) return false; + if (len == 0) return false; char** env = mi_get_environ(); if (env == NULL) return false; // compare up to 256 entries @@ -452,7 +459,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } return false; } -#else +#else // fallback: use standard C `getenv` but this cannot be used while initializing the C runtime static bool mi_getenv(const char* name, char* result, size_t result_size) { // cannot call getenv() when still initializing the C runtime. @@ -479,7 +486,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } #endif -static void mi_option_init(mi_option_desc_t* desc) { +static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment char buf[64+1]; mi_strlcpy(buf, "mimalloc_", sizeof(buf)); @@ -503,6 +510,14 @@ static void mi_option_init(mi_option_desc_t* desc) { else { char* end = buf; long value = strtol(buf, &end, 10); + if (desc->option == mi_option_reserve_os_memory) { + // this option is interpreted in KiB to prevent overflow of `long` + if (*end == 'K') { end++; } + else if (*end == 'M') { value *= KiB; end++; } + else if (*end == 'G') { value *= MiB; end++; } + else { value = (value + KiB - 1) / KiB; } + if (*end == 'B') { end++; } + } if (*end == 0) { desc->value = value; desc->init = INITIALIZED; diff --git a/kotlin-native/runtime/src/mimalloc/c/os.c b/kotlin-native/runtime/src/mimalloc/c/os.c index fb47147e708..9e398d13116 100644 --- a/kotlin-native/runtime/src/mimalloc/c/os.c +++ b/kotlin-native/runtime/src/mimalloc/c/os.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_WIN32) -#include +#include #elif defined(__wasi__) // stdlib.h is all we need, and has already been included in mimalloc.h #else @@ -65,8 +65,15 @@ static void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } -static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { - return (sz / alignment) * alignment; +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz / alignment) * alignment); + } } static void* mi_align_down_ptr(void* p, size_t alignment) { @@ -106,7 +113,7 @@ size_t _mi_os_good_alloc_size(size_t size) { else if (size < 8*MiB) align_size = 256*KiB; else if (size < 32*MiB) align_size = 1*MiB; else align_size = 4*MiB; - if (size >= (SIZE_MAX - align_size)) return size; // possible overflow? + if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow? return _mi_align_up(size, align_size); } @@ -261,7 +268,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment return NULL; } // fall through - } + } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) // on modern Windows try use VirtualAlloc2 for aligned allocation @@ -358,7 +365,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro int fd = -1; #if defined(MAP_ALIGNED) // BSD if (try_alignment > 0) { - size_t n = _mi_bsr(try_alignment); + size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); } @@ -421,7 +428,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, 10UL); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations } } } @@ -435,7 +442,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // though since properly aligned allocations will already use large pages if available // in that case -- in particular for our large regions (in `memory.c`). // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvice` anyways. + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. if (allow_large && use_large_os_page(size, try_alignment)) { if (madvise(p, size, MADV_HUGEPAGE) == 0) { *is_large = true; // possibly @@ -465,20 +472,36 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; -// Return a 4MiB aligned address that is probably available -static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { +// Return a 4MiB aligned address that is probably available. +// If this returns NULL, the OS will determine the address but on some OS's that may not be +// properly aligned which can be more costly as it needs to be adjusted afterwards. +// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; +// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses +// in the middle of the 2TiB - 6TiB address range (see issue #372)) + +#define KK_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define KK_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) +#define KK_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) + +static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) +{ if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; + if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. + #if (MI_SECURE>0) + size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. + #endif + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); - if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) - uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area - #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode + if (hint == 0 || hint > KK_HINT_MAX) { // wrap or initialize + uintptr_t init = KK_HINT_BASE; + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); - init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % KK_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > KK_HINT_MAX but that is ok, it is a hint after all } if (hint%try_alignment != 0) return NULL; return (void*)hint; @@ -623,9 +646,9 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { _mi_os_free_ex(p, size, true, stats); } -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld) +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats) { - UNUSED(tld); + UNUSED(tld_stats); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); @@ -721,6 +744,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } + #if defined(MADV_FREE_REUSE) + while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + #endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -747,7 +773,7 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats); } -static bool mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) { +static bool mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) { return mi_os_commitx(addr, size, true, true /* conservative */, is_zero, stats); } @@ -782,10 +808,17 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); - int err = madvise(start, csize, (int)mi_atomic_load_relaxed(&advice)); - if (err != 0 && errno == EINVAL && advice == MADV_FREE) { - // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + #if defined(MADV_FREE_REUSABLE) + #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE + #else + #define KK_MADV_FREE_INITIAL MADV_FREE + #endif + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); + int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; + while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; + if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { + // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } @@ -1090,7 +1123,7 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { /* ---------------------------------------------------------------------------- Support NUMA aware allocation -----------------------------------------------------------------------------*/ -#ifdef _WIN32 +#ifdef _WIN32 static size_t mi_os_numa_nodex() { USHORT numa_node = 0; if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) { @@ -1106,7 +1139,7 @@ static size_t mi_os_numa_nodex() { DWORD pnum = GetCurrentProcessorNumber(); UCHAR nnode = 0; BOOL ok = GetNumaProcessorNode((UCHAR)pnum, &nnode); - if (ok) numa_node = nnode; + if (ok) numa_node = nnode; } return numa_node; } @@ -1169,17 +1202,23 @@ static size_t mi_os_numa_node_countx(void) { } #endif -size_t _mi_numa_node_count = 0; // cache the node count +_Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count size_t _mi_os_numa_node_count_get(void) { - if (mi_unlikely(_mi_numa_node_count <= 0)) { + size_t count = mi_atomic_load_acquire(&_mi_numa_node_count); + if (count <= 0) { long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? - if (ncount <= 0) ncount = (long)mi_os_numa_node_countx(); // or detect dynamically - _mi_numa_node_count = (size_t)(ncount <= 0 ? 1 : ncount); - _mi_verbose_message("using %zd numa regions\n", _mi_numa_node_count); + if (ncount > 0) { + count = (size_t)ncount; + } + else { + count = mi_os_numa_node_countx(); // or detect dynamically + if (count == 0) count = 1; + } + mi_atomic_store_release(&_mi_numa_node_count, count); // save it + _mi_verbose_message("using %zd numa regions\n", count); } - mi_assert_internal(_mi_numa_node_count >= 1); - return _mi_numa_node_count; + return count; } int _mi_os_numa_node_get(mi_os_tld_t* tld) { diff --git a/kotlin-native/runtime/src/mimalloc/c/page-queue.c b/kotlin-native/runtime/src/mimalloc/c/page-queue.c index fc1522a078b..1aeed9d418e 100644 --- a/kotlin-native/runtime/src/mimalloc/c/page-queue.c +++ b/kotlin-native/runtime/src/mimalloc/c/page-queue.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -49,50 +49,6 @@ static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { Bins ----------------------------------------------------------- */ -// Bit scan reverse: return the index of the highest bit. -static inline uint8_t mi_bsr32(uint32_t x); - -#if defined(_MSC_VER) -#include -static inline uint8_t mi_bsr32(uint32_t x) { - uint32_t idx; - _BitScanReverse((DWORD*)&idx, x); - return (uint8_t)idx; -} -#elif defined(__GNUC__) || defined(__clang__) -static inline uint8_t mi_bsr32(uint32_t x) { - return (31 - __builtin_clz(x)); -} -#else -static inline uint8_t mi_bsr32(uint32_t x) { - // de Bruijn multiplication, see - static const uint8_t debruijn[32] = { - 31, 0, 22, 1, 28, 23, 18, 2, 29, 26, 24, 10, 19, 7, 3, 12, - 30, 21, 27, 17, 25, 9, 6, 11, 20, 16, 8, 5, 15, 4, 14, 13, - }; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; - x++; - return debruijn[(x*0x076be629) >> 27]; -} -#endif - -// Bit scan reverse: return the index of the highest bit. -uint8_t _mi_bsr(uintptr_t x) { - if (x == 0) return 0; -#if MI_INTPTR_SIZE==8 - uint32_t hi = (x >> 32); - return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); -#elif MI_INTPTR_SIZE==4 - return mi_bsr32(x); -#else -# error "define bsr for non-32 or 64-bit platforms" -#endif -} - // Return the bin for a given field size. // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", @@ -125,7 +81,7 @@ extern inline uint8_t _mi_bin(size_t size) { #endif wsize--; // find the highest bit - uint8_t b = mi_bsr32((uint32_t)wsize); + uint8_t b = (uint8_t)mi_bsr(wsize); // note: wsize != 0 // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). // - adjust with 3 because we use do not round the first 8 sizes // which each get an exact bin @@ -341,7 +297,7 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue for (mi_page_t* page = append->first; page != NULL; page = page->next) { // inline `mi_page_set_heap` to avoid wrong assertion during absorption; // in this case it is ok to be delayed freeing since both "to" and "from" heap are still alive. - mi_atomic_store_release(&page->xheap, (uintptr_t)heap); + mi_atomic_store_release(&page->xheap, (uintptr_t)heap); // set the flag to delayed free (not overriding NEVER_DELAYED_FREE) which has as a // side effect that it spins until any DELAYED_FREEING is finished. This ensures // that after appending only the new heap will be used for delayed free operations. diff --git a/kotlin-native/runtime/src/mimalloc/c/page.c b/kotlin-native/runtime/src/mimalloc/c/page.c index d0cabfa4214..d5df1d6e961 100644 --- a/kotlin-native/runtime/src/mimalloc/c/page.c +++ b/kotlin-native/runtime/src/mimalloc/c/page.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -124,7 +124,7 @@ bool _mi_page_is_valid(mi_page_t* page) { void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { mi_thread_free_t tfreex; mi_delayed_t old_delay; - mi_thread_free_t tfree; + mi_thread_free_t tfree; do { tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS; tfreex = mi_tf_set_delayed(tfree, delay); @@ -376,7 +376,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { _mi_segment_page_free(page, force, segments_tld); } -#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX +#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX #define MI_RETIRE_CYCLES (8) // Retire a page with no more used blocks @@ -566,7 +566,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) if (page->capacity >= page->reserved) return; size_t page_size; - //uint8_t* page_start = + //uint8_t* page_start = _mi_page_start(_mi_page_segment(page), page, &page_size); mi_stat_counter_increase(tld->stats.pages_extended, 1); @@ -687,7 +687,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p page = mi_page_fresh(heap, pq); if (page == NULL && first_try) { // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again - page = mi_page_queue_find_free_ex(heap, pq, false); + page = mi_page_queue_find_free_ex(heap, pq, false); } } else { @@ -705,14 +705,17 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { mi_page_queue_t* pq = mi_page_queue(heap,size); mi_page_t* page = pq->first; if (page != NULL) { - if ((MI_SECURE >= 3) && page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) { - // in secure mode, we extend half the time to increase randomness + #if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness + if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) { mi_page_extend_free(heap, page, heap->tld); mi_assert_internal(mi_page_immediate_available(page)); } - else { + else + #endif + { _mi_page_free_collect(page,false); } + if (mi_page_immediate_available(page)) { page->retire_expire = 0; return page; // fast path @@ -785,7 +788,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept { // huge allocation? - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE) )) { if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); @@ -830,7 +833,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept } if (mi_unlikely(page == NULL)) { // out of memory - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); return NULL; } diff --git a/kotlin-native/runtime/src/mimalloc/c/random.c b/kotlin-native/runtime/src/mimalloc/c/random.c index b9485ea094f..255bede4db3 100644 --- a/kotlin-native/runtime/src/mimalloc/c/random.c +++ b/kotlin-native/runtime/src/mimalloc/c/random.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -115,7 +115,7 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { memset(ctx_new, 0, sizeof(*ctx_new)); - memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); ctx_new->input[12] = 0; ctx_new->input[13] = 0; ctx_new->input[14] = (uint32_t)nonce; @@ -244,7 +244,7 @@ static bool os_random_buf(void* buf, size_t buf_len) { #endif #if defined(_WIN32) -#include +#include #elif defined(__APPLE__) #include #else diff --git a/kotlin-native/runtime/src/mimalloc/c/region.c b/kotlin-native/runtime/src/mimalloc/c/region.c index b6d0da31508..4d9a47728e6 100644 --- a/kotlin-native/runtime/src/mimalloc/c/region.c +++ b/kotlin-native/runtime/src/mimalloc/c/region.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -16,8 +16,8 @@ We need this memory layer between the raw OS calls because of: 1. on `sbrk` like systems (like WebAssembly) we need our own memory maps in order to reuse memory effectively. 2. It turns out that for large objects, between 1MiB and 32MiB (?), the cost of - an OS allocation/free is still (much) too expensive relative to the accesses - in that object :-( (`malloc-large` tests this). This means we need a cheaper + an OS allocation/free is still (much) too expensive relative to the accesses + in that object :-( (`malloc-large` tests this). This means we need a cheaper way to reuse memory. 3. This layer allows for NUMA aware allocation. @@ -37,7 +37,7 @@ Possible issues: #include // memset -#include "bitmap.inc.c" +#include "bitmap.h" // Internal raw OS interface size_t _mi_os_large_page_size(); @@ -50,14 +50,14 @@ bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats); // arena.c void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats); -void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); -void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); +void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); // Constants #if (MI_INTPTR_SIZE==8) -#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map +#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map #elif (MI_INTPTR_SIZE==4) #define MI_HEAP_REGION_MAX_SIZE (3 * GiB) // ~ KiB for the region map #else @@ -70,14 +70,15 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo #define MI_REGION_SIZE (MI_SEGMENT_SIZE * MI_BITMAP_FIELD_BITS) // 256MiB (64MiB on 32 bits) #define MI_REGION_MAX (MI_HEAP_REGION_MAX_SIZE / MI_REGION_SIZE) // 1024 (48 on 32 bits) #define MI_REGION_MAX_OBJ_BLOCKS (MI_REGION_MAX_BLOCKS/4) // 64MiB -#define MI_REGION_MAX_OBJ_SIZE (MI_REGION_MAX_OBJ_BLOCKS*MI_SEGMENT_SIZE) +#define MI_REGION_MAX_OBJ_SIZE (MI_REGION_MAX_OBJ_BLOCKS*MI_SEGMENT_SIZE) -// Region info +// Region info typedef union mi_region_info_u { - uintptr_t value; + uintptr_t value; struct { bool valid; // initialized? - bool is_large; // allocated in fixed large/huge OS pages + bool is_large:1; // allocated in fixed large/huge OS pages + bool is_pinned:1; // pinned memory cannot be decommitted short numa_node; // the associated NUMA node (where -1 means no associated node) } x; } mi_region_info_t; @@ -87,7 +88,7 @@ typedef union mi_region_info_u { // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. typedef struct mem_region_s { _Atomic(uintptr_t) info; // mi_region_info_t.value - _Atomic(void*) start; // start of the memory area + _Atomic(void*) start; // start of the memory area mi_bitmap_field_t in_use; // bit per in-use block mi_bitmap_field_t dirty; // track if non-zero per block mi_bitmap_field_t commit; // track if committed per block @@ -100,7 +101,7 @@ typedef struct mem_region_s { static mem_region_t regions[MI_REGION_MAX]; // Allocated regions -static _Atomic(uintptr_t) regions_count; // = 0; +static _Atomic(uintptr_t) regions_count; // = 0; /* ---------------------------------------------------------------------------- @@ -135,7 +136,7 @@ bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { static void* mi_region_blocks_start(const mem_region_t* region, mi_bitmap_index_t bit_idx) { uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t, &((mem_region_t*)region)->start); mi_assert_internal(start != NULL); - return (start + (bit_idx * MI_SEGMENT_SIZE)); + return (start + (bit_idx * MI_SEGMENT_SIZE)); } static size_t mi_memid_create(mem_region_t* region, mi_bitmap_index_t bit_idx) { @@ -177,8 +178,9 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, bool region_commit = (commit && mi_option_is_enabled(mi_option_eager_region_commit)); bool region_large = (commit && allow_large); bool is_zero = false; + bool is_pinned = false; size_t arena_memid = 0; - void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, ®ion_commit, ®ion_large, &is_zero, &arena_memid, tld); + void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, ®ion_commit, ®ion_large, &is_pinned, &is_zero, &arena_memid, tld); if (start == NULL) return false; mi_assert_internal(!(region_large && !allow_large)); mi_assert_internal(!region_large || region_commit); @@ -200,14 +202,15 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, mi_atomic_store_release(&r->commit, (region_commit ? MI_BITMAP_FIELD_FULL : 0)); mi_atomic_store_release(&r->reset, (uintptr_t)0); *bit_idx = 0; - mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL); + _mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL); mi_atomic_store_ptr_release(void,&r->start, start); - // and share it + // and share it mi_region_info_t info; info.value = 0; // initialize the full union to zero info.x.valid = true; info.x.is_large = region_large; + info.x.is_pinned = is_pinned; info.x.numa_node = (short)_mi_os_numa_node(tld); mi_atomic_store_release(&r->info, info.value); // now make it available to others *region = r; @@ -239,7 +242,7 @@ static bool mi_region_is_suitable(const mem_region_t* region, int numa_node, boo static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld) { - // try all regions for a free slot + // try all regions for a free slot const size_t count = mi_atomic_load_relaxed(®ions_count); // monotonic, so ok to be relaxed size_t idx = tld->region_idx; // Or start at 0 to reuse low addresses? Starting at 0 seems to increase latency though for (size_t visited = 0; visited < count; visited++, idx++) { @@ -248,7 +251,7 @@ static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, // if this region suits our demand (numa node matches, large OS page matches) if (mi_region_is_suitable(r, numa_node, allow_large)) { // then try to atomically claim a segment(s) in this region - if (mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) { + if (_mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) { tld->region_idx = idx; // remember the last found position *region = r; return true; @@ -259,64 +262,65 @@ static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, } -static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { mi_assert_internal(blocks <= MI_BITMAP_FIELD_BITS); mem_region_t* region; mi_bitmap_index_t bit_idx; const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld)); // try to claim in existing regions - if (!mi_region_try_claim(numa_node, blocks, *is_large, ®ion, &bit_idx, tld)) { + if (!mi_region_try_claim(numa_node, blocks, *large, ®ion, &bit_idx, tld)) { // otherwise try to allocate a fresh region and claim in there - if (!mi_region_try_alloc_os(blocks, *commit, *is_large, ®ion, &bit_idx, tld)) { + if (!mi_region_try_alloc_os(blocks, *commit, *large, ®ion, &bit_idx, tld)) { // out of regions or memory return NULL; } } - + // ------------------------------------------------ // found a region and claimed `blocks` at `bit_idx`, initialize them now mi_assert_internal(region != NULL); - mi_assert_internal(mi_bitmap_is_claimed(®ion->in_use, 1, blocks, bit_idx)); + mi_assert_internal(_mi_bitmap_is_claimed(®ion->in_use, 1, blocks, bit_idx)); mi_region_info_t info; info.value = mi_atomic_load_acquire(®ion->info); uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ion->start); - mi_assert_internal(!(info.x.is_large && !*is_large)); + mi_assert_internal(!(info.x.is_large && !*large)); mi_assert_internal(start != NULL); - *is_zero = mi_bitmap_claim(®ion->dirty, 1, blocks, bit_idx, NULL); - *is_large = info.x.is_large; - *memid = mi_memid_create(region, bit_idx); + *is_zero = _mi_bitmap_claim(®ion->dirty, 1, blocks, bit_idx, NULL); + *large = info.x.is_large; + *is_pinned = info.x.is_pinned; + *memid = mi_memid_create(region, bit_idx); void* p = start + (mi_bitmap_index_bit_in_field(bit_idx) * MI_SEGMENT_SIZE); // commit if (*commit) { // ensure commit bool any_uncommitted; - mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, &any_uncommitted); + _mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, &any_uncommitted); if (any_uncommitted) { - mi_assert_internal(!info.x.is_large); + mi_assert_internal(!info.x.is_large && !info.x.is_pinned); bool commit_zero = false; if (!_mi_mem_commit(p, blocks * MI_SEGMENT_SIZE, &commit_zero, tld)) { // failed to commit! unclaim and return mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx); return NULL; } - if (commit_zero) *is_zero = true; + if (commit_zero) *is_zero = true; } } else { // no need to commit, but check if already fully committed - *commit = mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx); - } - mi_assert_internal(!*commit || mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx)); + *commit = _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx); + } + mi_assert_internal(!*commit || _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx)); // unreset reset blocks - if (mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)) { + if (_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)) { // some blocks are still reset - mi_assert_internal(!info.x.is_large); - mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit || mi_option_get(mi_option_eager_commit_delay) > 0); + mi_assert_internal(!info.x.is_large && !info.x.is_pinned); + mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit || mi_option_get(mi_option_eager_commit_delay) > 0); mi_bitmap_unclaim(®ion->reset, 1, blocks, bit_idx); if (*commit || !mi_option_is_enabled(mi_option_reset_decommits)) { // only if needed bool reset_zero = false; @@ -324,14 +328,14 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo if (reset_zero) *is_zero = true; } } - mi_assert_internal(!mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)); - + mi_assert_internal(!_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)); + #if (MI_DEBUG>=2) if (*commit) { ((uint8_t*)p)[0] = 0; } #endif - - // and return the allocation - mi_assert_internal(p != NULL); + + // and return the allocation + mi_assert_internal(p != NULL); return p; } @@ -342,14 +346,15 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo // Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`. // (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`) -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { mi_assert_internal(memid != NULL && tld != NULL); mi_assert_internal(size > 0); *memid = 0; *is_zero = false; + *is_pinned = false; bool default_large = false; - if (large==NULL) large = &default_large; // ensure `large != NULL` + if (large==NULL) large = &default_large; // ensure `large != NULL` if (size == 0) return NULL; size = _mi_align_up(size, _mi_os_page_size()); @@ -358,14 +363,14 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l size_t arena_memid; const size_t blocks = mi_region_block_count(size); if (blocks <= MI_REGION_MAX_OBJ_BLOCKS && alignment <= MI_SEGMENT_ALIGN) { - p = mi_region_try_alloc(blocks, commit, large, is_zero, memid, tld); + p = mi_region_try_alloc(blocks, commit, large, is_pinned, is_zero, memid, tld); if (p == NULL) { _mi_warning_message("unable to allocate from region: size %zu\n", size); } } if (p == NULL) { // and otherwise fall back to the OS - p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_zero, &arena_memid, tld); + p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_pinned, is_zero, &arena_memid, tld); *memid = mi_memid_create_from_arena(arena_memid); } @@ -390,7 +395,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re if (p==NULL) return; if (size==0) return; size = _mi_align_up(size, _mi_os_page_size()); - + size_t arena_memid = 0; mi_bitmap_index_t bit_idx; mem_region_t* region; @@ -413,21 +418,21 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re // committed? if (full_commit && (size % MI_SEGMENT_SIZE) == 0) { - mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, NULL); + _mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, NULL); } if (any_reset) { // set the is_reset bits if any pages were reset - mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, NULL); + _mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, NULL); } // reset the blocks to reduce the working set. - if (!info.x.is_large && mi_option_is_enabled(mi_option_segment_reset) + if (!info.x.is_large && !info.x.is_pinned && mi_option_is_enabled(mi_option_segment_reset) && (mi_option_is_enabled(mi_option_eager_commit) || - mi_option_is_enabled(mi_option_reset_decommits))) // cannot reset halfway committed segments, use only `option_page_reset` instead + mi_option_is_enabled(mi_option_reset_decommits))) // cannot reset halfway committed segments, use only `option_page_reset` instead { bool any_unreset; - mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, &any_unreset); + _mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, &any_unreset); if (any_unreset) { _mi_abandoned_await_readers(); // ensure no more pending write (in case reset = decommit) _mi_mem_reset(p, blocks * MI_SEGMENT_SIZE, tld); diff --git a/kotlin-native/runtime/src/mimalloc/c/segment.c b/kotlin-native/runtime/src/mimalloc/c/segment.c index b3585443fc0..5ee4a599d6c 100644 --- a/kotlin-native/runtime/src/mimalloc/c/segment.c +++ b/kotlin-native/runtime/src/mimalloc/c/segment.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -237,7 +237,7 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t* static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld) { mi_assert_internal(page->is_committed); if (!mi_option_is_enabled(mi_option_page_reset)) return; - if (segment->mem_is_fixed || page->segment_in_use || !page->is_committed || page->is_reset) return; + if (segment->mem_is_pinned || page->segment_in_use || !page->is_committed || page->is_reset) return; size_t psize; void* start = mi_segment_raw_page_start(segment, page, &psize); page->is_reset = true; @@ -250,8 +250,8 @@ static bool mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, { mi_assert_internal(page->is_reset); mi_assert_internal(page->is_committed); - mi_assert_internal(!segment->mem_is_fixed); - if (segment->mem_is_fixed || !page->is_committed || !page->is_reset) return true; + mi_assert_internal(!segment->mem_is_pinned); + if (segment->mem_is_pinned || !page->is_committed || !page->is_reset) return true; page->is_reset = false; size_t psize; uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); @@ -290,7 +290,7 @@ static void mi_pages_reset_add(mi_segment_t* segment, mi_page_t* page, mi_segmen mi_assert_expensive(!mi_pages_reset_contains(page, tld)); mi_assert_internal(_mi_page_segment(page)==segment); if (!mi_option_is_enabled(mi_option_page_reset)) return; - if (segment->mem_is_fixed || page->segment_in_use || !page->is_committed || page->is_reset) return; + if (segment->mem_is_pinned || page->segment_in_use || !page->is_committed || page->is_reset) return; if (mi_option_get(mi_option_reset_delay) == 0) { // reset immediately? @@ -330,7 +330,7 @@ static void mi_pages_reset_remove(mi_page_t* page, mi_segments_tld_t* tld) { } static void mi_pages_reset_remove_all_in_segment(mi_segment_t* segment, bool force_reset, mi_segments_tld_t* tld) { - if (segment->mem_is_fixed) return; // never reset in huge OS pages + if (segment->mem_is_pinned) return; // never reset in huge OS pages for (size_t i = 0; i < segment->capacity; i++) { mi_page_t* page = &segment->pages[i]; if (!page->segment_in_use && page->is_committed && !page->is_reset) { @@ -385,11 +385,13 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_ psize -= segment->segment_info_size; } - if (MI_SECURE > 1 || (MI_SECURE == 1 && page->segment_idx == segment->capacity - 1)) { - // secure == 1: the last page has an os guard page at the end - // secure > 1: every page has an os guard page +#if (MI_SECURE > 1) // every page has an os guard page + psize -= _mi_os_page_size(); +#elif (MI_SECURE==1) // the last page has an os guard page at the end + if (page->segment_idx == segment->capacity - 1) { psize -= _mi_os_page_size(); } +#endif if (page_size != NULL) *page_size = psize; mi_assert_internal(page->xblock_size == 0 || _mi_ptr_page(p) == page); @@ -464,7 +466,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se segment->thread_id = 0; mi_segments_track_size(-((long)segment_size),tld); if (MI_SECURE != 0) { - mi_assert_internal(!segment->mem_is_fixed); + mi_assert_internal(!segment->mem_is_pinned); mi_segment_protect(segment, false, tld->os); // ensure no more guard pages are set } @@ -593,7 +595,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ else { if (MI_SECURE!=0) { - mi_assert_internal(!segment->mem_is_fixed); + mi_assert_internal(!segment->mem_is_pinned); mi_segment_protect(segment, false, tld->os); // reset protection if the page kind differs } // different page kinds; unreset any reset pages, and unprotect @@ -624,26 +626,28 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ // Allocate the segment from the OS size_t memid; bool mem_large = (!eager_delayed && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy - segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_zero, &memid, os_tld); + bool is_pinned = false; + segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld); if (segment == NULL) return NULL; // failed to allocate if (!commit) { // ensure the initial info is committed + mi_assert_internal(!mem_large && !is_pinned); bool commit_zero = false; bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os); if (commit_zero) is_zero = true; if (!ok) { // commit failed; we cannot touch the memory: free the segment directly and return `NULL` _mi_mem_free(segment, MI_SEGMENT_SIZE, memid, false, false, os_tld); - return NULL; + return NULL; } } segment->memid = memid; - segment->mem_is_fixed = mem_large; - segment->mem_is_committed = commit; + segment->mem_is_pinned = (mem_large || is_pinned); + segment->mem_is_committed = commit; mi_segments_track_size((long)segment_size, tld); } mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); - mi_assert_internal(segment->mem_is_fixed ? segment->mem_is_committed : true); + mi_assert_internal(segment->mem_is_pinned ? segment->mem_is_committed : true); mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan if (!pages_still_good) { // zero the segment info (but not the `mem` fields) @@ -728,8 +732,8 @@ static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg mi_pages_reset_remove(page, tld); // check commit if (!page->is_committed) { - mi_assert_internal(!segment->mem_is_fixed); - mi_assert_internal(!page->is_reset); + mi_assert_internal(!segment->mem_is_pinned); + mi_assert_internal(!page->is_reset); size_t psize; uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); bool is_zero = false; @@ -745,8 +749,8 @@ static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg segment->used++; // check reset if (page->is_reset) { - mi_assert_internal(!segment->mem_is_fixed); - bool ok = mi_page_unreset(segment, page, 0, tld); + mi_assert_internal(!segment->mem_is_pinned); + bool ok = mi_page_unreset(segment, page, 0, tld); if (!ok) { page->segment_in_use = false; segment->used--; @@ -892,7 +896,7 @@ static mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = static mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL // Maintain these for debug purposes (these counts may be a bit off) -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count; +static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count; static mi_decl_cache_align _Atomic(uintptr_t) abandoned_visited_count; // We also maintain a count of current readers of the abandoned list @@ -1330,13 +1334,6 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block page->is_zero = false; mi_assert(page->used == 0); mi_tld_t* tld = heap->tld; - const size_t bsize = mi_page_usable_block_size(page); - if (bsize > MI_HUGE_OBJ_SIZE_MAX) { - _mi_stat_decrease(&tld->stats.giant, bsize); - } - else { - _mi_stat_decrease(&tld->stats.huge, bsize); - } mi_segments_track_size((long)segment->segment_size, &tld->segments); _mi_segment_page_free(page, true, &tld->segments); } diff --git a/kotlin-native/runtime/src/mimalloc/c/static.c b/kotlin-native/runtime/src/mimalloc/c/static.c index 39b417ad5ea..4cc87747027 100644 --- a/kotlin-native/runtime/src/mimalloc/c/static.c +++ b/kotlin-native/runtime/src/mimalloc/c/static.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -23,6 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "stats.c" #include "random.c" #include "os.c" +#include "bitmap.c" #include "arena.c" #include "region.c" #include "segment.c" diff --git a/kotlin-native/runtime/src/mimalloc/c/stats.c b/kotlin-native/runtime/src/mimalloc/c/stats.c index 9280917b3e4..5848333f2c5 100644 --- a/kotlin-native/runtime/src/mimalloc/c/stats.c +++ b/kotlin-native/runtime/src/mimalloc/c/stats.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution. @@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file static bool mi_is_in_main(void* stat) { return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main - && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); + && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); } static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { @@ -51,7 +51,7 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { } } -void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { +void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { if (mi_is_in_main(stat)) { mi_atomic_addi64_relaxed( &stat->count, 1 ); mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount ); @@ -103,6 +103,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_add(&stats->malloc, &src->malloc, 1); mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); + mi_stat_add(&stats->normal, &src->normal, 1); mi_stat_add(&stats->huge, &src->huge, 1); mi_stat_add(&stats->giant, &src->giant, 1); @@ -112,12 +113,13 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); mi_stat_counter_add(&stats->searches, &src->searches, 1); + mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); mi_stat_counter_add(&stats->giant_count, &src->giant_count, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { - if (src->normal[i].allocated > 0 || src->normal[i].freed > 0) { - mi_stat_add(&stats->normal[i], &src->normal[i], 1); + if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { + mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i], 1); } } #endif @@ -127,7 +129,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { Display statistics ----------------------------------------------------------- */ -// unit > 0 : size in binary bytes +// unit > 0 : size in binary bytes // unit == 0: count as decimal // unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { @@ -149,7 +151,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s", whole, frac1, magnitude, suffix); + snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } @@ -170,6 +172,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_print_amount(stat->peak, unit, out, arg); mi_print_amount(stat->allocated, unit, out, arg); mi_print_amount(stat->freed, unit, out, arg); + mi_print_amount(stat->current, unit, out, arg); mi_print_amount(unit, 1, out, arg); mi_print_count(stat->allocated, unit, out, arg); if (stat->allocated > stat->freed) @@ -181,6 +184,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_print_amount(stat->peak, -1, out, arg); mi_print_amount(stat->allocated, -1, out, arg); mi_print_amount(stat->freed, -1, out, arg); + mi_print_amount(stat->current, -1, out, arg); if (unit==-1) { _mi_fprintf(out, arg, "%22s", ""); } @@ -196,6 +200,8 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t else { mi_print_amount(stat->peak, 1, out, arg); mi_print_amount(stat->allocated, 1, out, arg); + _mi_fprintf(out, arg, "%11s", " "); // no freed + mi_print_amount(stat->current, 1, out, arg); _mi_fprintf(out, arg, "\n"); } } @@ -215,11 +221,11 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* static void mi_print_header(mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count "); + _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); } #if MI_STAT>1 -static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) { +static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; for (size_t i = 0; i <= max; i++) { @@ -227,12 +233,9 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin found = true; int64_t unit = _mi_bin_size((uint8_t)i); snprintf(buf, 64, "%s %3lu", fmt, (long)i); - mi_stat_add(all, &bins[i], unit); mi_stat_print(&bins[i], buf, unit, out, arg); } } - //snprintf(buf, 64, "%s all", fmt); - //mi_stat_print(all, buf, 1); if (found) { _mi_fprintf(out, arg, "\n"); mi_print_header(out, arg); @@ -289,19 +292,21 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) // and print using that mi_print_header(out,arg); #if MI_STAT>1 - mi_stat_count_t normal = { 0,0,0,0 }; - mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out,arg); - mi_stat_print(&normal, "normal", 1, out, arg); + mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg); + #endif + #if MI_STAT + mi_stat_print(&stats->normal, "normal", (stats->normal_count.count == 0 ? 1 : -(stats->normal.allocated / stats->normal_count.count)), out, arg); mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg); mi_stat_print(&stats->giant, "giant", (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out, arg); mi_stat_count_t total = { 0,0,0,0 }; - mi_stat_add(&total, &normal, 1); + mi_stat_add(&total, &stats->normal, 1); mi_stat_add(&total, &stats->huge, 1); mi_stat_add(&total, &stats->giant, 1); mi_stat_print(&total, "total", 1, out, arg); - _mi_fprintf(out, arg, "malloc requested: "); - mi_print_amount(stats->malloc.allocated, 1, out, arg); - _mi_fprintf(out, arg, "\n\n"); + #endif + #if MI_STAT>1 + mi_stat_print(&stats->malloc, "malloc req", 1, out, arg); + _mi_fprintf(out, arg, "\n"); #endif mi_stat_print(&stats->reserved, "reserved", 1, out, arg); mi_stat_print(&stats->committed, "committed", 1, out, arg); @@ -388,7 +393,7 @@ void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { // Basic timer for convenience; use milli-seconds to avoid doubles // ---------------------------------------------------------------- #ifdef _WIN32 -#include +#include static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { static LARGE_INTEGER mfreq; // = 0 if (mfreq.QuadPart == 0LL) { @@ -443,7 +448,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { // -------------------------------------------------------- #if defined(_WIN32) -#include +#include #include #pragma comment(lib,"psapi.lib") @@ -474,12 +479,12 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec *page_faults = (size_t)info.PageFaultCount; } -#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__HAIKU__) +#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) #include #include #include -#if defined(__APPLE__) && defined(__MACH__) +#if defined(__APPLE__) #include #endif @@ -515,7 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } -#elif defined(__APPLE__) && defined(__MACH__) +#elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;