azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Updated mimalloc to version 1.7.2

Browse Source
This commit is contained in:
Elena Lepilkina
2021-08-31 18:42:50 +03:00
committed by Space
parent e454c824d5
commit 8ff960fbe3
25 changed files with 1367 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/mimalloc/c/alloc-aligned.c
+6 -6
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc.h"
#include "mimalloc-internal.h"
#include <string.h> // memset, memcpy
#include <string.h> // memset
// ------------------------------------------------------
// Aligned Allocation
@@ -21,7 +21,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
// try if there is a small block available with just the right alignment
const size_t padsize = size + MI_PADDING_SIZE;
if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) {
@@ -46,7 +46,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
return p;
}
// otherwise over-allocate
void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero);
if (p == NULL) return NULL;
@@ -55,7 +55,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask);
mi_assert_internal(adjust <= alignment);
void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust));
if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) );
return aligned_p;
@@ -137,7 +137,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne
memset((uint8_t*)newp + start, 0, newsize - start);
}
}
memcpy(newp, p, (newsize > size ? size : newsize));
_mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
mi_free(p); // only free if successful
}
return newp;
kotlin-native/runtime/src/mimalloc/c/alloc-override-osx.c
+65 -43
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -37,7 +37,6 @@ terms of the MIT license. A copy of the license can be found in the file
extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import));
#endif
/* ------------------------------------------------------
malloc zone members
------------------------------------------------------ */
@@ -67,7 +66,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) {
static void zone_free(malloc_zone_t* zone, void* p) {
UNUSED(zone);
return mi_free(p);
mi_free(p);
}
static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) {
@@ -192,63 +191,85 @@ static malloc_zone_t* mi_get_default_zone()
}
}
static void __attribute__((constructor)) _mi_macos_override_malloc()
{
static malloc_introspection_t intro;
memset(&intro, 0, sizeof(intro));
static malloc_introspection_t mi_introspect = {
.enumerator = &intro_enumerator,
.good_size = &intro_good_size,
.check = &intro_check,
.print = &intro_print,
.log = &intro_log,
.force_lock = &intro_force_lock,
.force_unlock = &intro_force_unlock,
#if defined(MAC_OS_X_VERSION_10_6) && \
MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
.zone_locked = &intro_zone_locked,
.statistics = &intro_statistics,
#endif
};
intro.enumerator = &intro_enumerator;
intro.good_size = &intro_good_size;
intro.check = &intro_check;
intro.print = &intro_print;
intro.log = &intro_log;
intro.force_lock = &intro_force_lock;
intro.force_unlock = &intro_force_unlock;
static malloc_zone_t mi_malloc_zone = {
.size = &zone_size,
.zone_name = "mimalloc",
.introspect = &mi_introspect,
.malloc = &zone_malloc,
.calloc = &zone_calloc,
.valloc = &zone_valloc,
.free = &zone_free,
.realloc = &zone_realloc,
.destroy = &zone_destroy,
.batch_malloc = &zone_batch_malloc,
.batch_free = &zone_batch_free,
#if defined(MAC_OS_X_VERSION_10_6) && \
MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
// switch to version 9 on OSX 10.6 to support memalign.
.version = 9,
.memalign = &zone_memalign,
.free_definite_size = &zone_free_definite_size,
.pressure_relief = &zone_pressure_relief,
#else
.version = 4,
#endif
};
static malloc_zone_t zone;
memset(&zone, 0, sizeof(zone));
zone.version = 4;
zone.zone_name = "mimalloc";
zone.size = &zone_size;
zone.introspect = &intro;
zone.malloc = &zone_malloc;
zone.calloc = &zone_calloc;
zone.valloc = &zone_valloc;
zone.free = &zone_free;
zone.realloc = &zone_realloc;
zone.destroy = &zone_destroy;
zone.batch_malloc = &zone_batch_malloc;
zone.batch_free = &zone_batch_free;
#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE)
static malloc_zone_t *mi_malloc_default_zone(void) {
return &mi_malloc_zone;
}
// TODO: should use the macros in alloc-override but they aren't available here.
__attribute__((used)) static struct {
const void *replacement;
const void *target;
} replace_malloc_default_zone[] __attribute__((section("__DATA, __interpose"))) = {
{ (const void*)mi_malloc_default_zone, (const void*)malloc_default_zone },
};
#endif
static void __attribute__((constructor(0))) _mi_macos_override_malloc() {
malloc_zone_t* purgeable_zone = NULL;
#if defined(MAC_OS_X_VERSION_10_6) && \
MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
// switch to version 9 on OSX 10.6 to support memalign.
zone.version = 9;
zone.memalign = &zone_memalign;
zone.free_definite_size = &zone_free_definite_size;
zone.pressure_relief = &zone_pressure_relief;
intro.zone_locked = &intro_zone_locked;
intro.statistics = &intro_statistics;
// force the purgeable zone to exist to avoid strange bugs
if (malloc_default_purgeable_zone) {
purgeable_zone = malloc_default_purgeable_zone();
}
#endif
// Register our zone
malloc_zone_register(&zone);
// Register our zone.
// thomcc: I think this is still needed to put us in the zone list.
malloc_zone_register(&mi_malloc_zone);
// Unregister the default zone, this makes our zone the new default
// as that was the last registered.
malloc_zone_t *default_zone = mi_get_default_zone();
malloc_zone_unregister(default_zone);
// thomcc: Unsure if the next test is *always* false or just false in the
// cases I've tried. I'm also unsure if the code inside is needed. at all
if (default_zone != &mi_malloc_zone) {
malloc_zone_unregister(default_zone);
// Reregister the default zone so free and realloc in that zone keep working.
malloc_zone_register(default_zone);
// Reregister the default zone so free and realloc in that zone keep working.
malloc_zone_register(default_zone);
}
// Unregister, and re-register the purgeable_zone to avoid bugs if it occurs
// earlier than the default zone.
@@ -256,7 +277,8 @@ static void __attribute__((constructor)) _mi_macos_override_malloc()
malloc_zone_unregister(purgeable_zone);
malloc_zone_register(purgeable_zone);
}
}
#endif // MI_MALLOC_OVERRIDE
#endif
#endif // KONAN_MI_MALLOC
kotlin-native/runtime/src/mimalloc/c/alloc-override.c
+53 -45
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -15,18 +15,18 @@ terms of the MIT license. A copy of the license can be found in the file
#error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)"
#endif
#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__MACH__) && !defined(MI_INTERPOSE)))
#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_INTERPOSE)))
// ------------------------------------------------------
// Override system malloc
// ------------------------------------------------------
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
// use aliasing to alias the exported function to one of our `mi_` functions
#if (defined(__GNUC__) && __GNUC__ >= 9)
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun)))
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun)));
#else
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")))
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")));
#endif
#define MI_FORWARD1(fun,x) MI_FORWARD(fun)
#define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)
@@ -62,6 +62,13 @@ terms of the MIT license. A copy of the license can be found in the file
MI_INTERPOSE_MI(posix_memalign),
MI_INTERPOSE_MI(reallocf),
MI_INTERPOSE_MI(valloc),
#ifndef MI_OSX_ZONE
// some code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity <https://github.com/nneonneo/osx-10.9-opensource/blob/master/objc4-551.1/runtime/hashtable2.mm>)
MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us
#else
// We interpose malloc_default_zone in alloc-override-osx.c
MI_INTERPOSE_MI(free),
#endif
// some code allocates from a zone but deallocates using plain free :-( (like NxHashResizeToCapacity <https://github.com/nneonneo/osx-10.9-opensource/blob/master/objc4-551.1/runtime/hashtable2.mm>)
MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us
};
@@ -70,13 +77,13 @@ terms of the MIT license. A copy of the license can be found in the file
// we just override new/delete which does work in a static library.
#else
// On all other systems forward to our API
void* malloc(size_t size) MI_FORWARD1(mi_malloc, size);
void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n);
void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize);
void free(void* p) MI_FORWARD0(mi_free, p);
void* malloc(size_t size) MI_FORWARD1(mi_malloc, size)
void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n)
void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize)
void free(void* p) MI_FORWARD0(mi_free, p)
#endif
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
#pragma GCC visibility push(default)
#endif
@@ -91,18 +98,18 @@ terms of the MIT license. A copy of the license can be found in the file
// see <https://en.cppreference.com/w/cpp/memory/new/operator_new>
// ------------------------------------------------------
#include <new>
void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p);
void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p);
void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p)
void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p)
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n);
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n);
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n)
void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); }
void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); }
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n);
void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n);
void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n)
#endif
#if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))
@@ -123,30 +130,30 @@ terms of the MIT license. A copy of the license can be found in the file
// used by GCC and CLang).
// See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
// ------------------------------------------------------
void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete
void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[]
void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete
void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[]
void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n)
void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
typedef struct mi_nothrow_s { } mi_nothrow_t;
typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t;
#if (MI_INTPTR_SIZE==8)
void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit
void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit
void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
#elif (MI_INTPTR_SIZE==4)
void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit
void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit
void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit
void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit
void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al)
void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
@@ -165,13 +172,13 @@ extern "C" {
// Posix & Unix functions definitions
// ------------------------------------------------------
void cfree(void* p) MI_FORWARD0(mi_free, p);
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize);
size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p);
void cfree(void* p) MI_FORWARD0(mi_free, p)
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)
#if !defined(__ANDROID__)
size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p);
size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)
#else
size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p);
size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
#endif
// no forwarding here due to aliasing/name mangling issues
@@ -182,22 +189,23 @@ void* memalign(size_t alignment, size_t size) { return mi_memali
int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
// on some glibc `aligned_alloc` is declared `static inline` so we cannot override it (e.g. Conda). This happens
// when _GLIBCXX_HAVE_ALIGNED_ALLOC is not defined. However, in those cases it will use `memalign`, `posix_memalign`,
// or `_aligned_malloc` and we can avoid overriding it ourselves.
// We should always override if using C compilation. (issue #276)
#if _GLIBCXX_HAVE_ALIGNED_ALLOC || !defined(__cplusplus)
// `aligned_alloc` is only available when __USE_ISOC11 is defined.
// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot
// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.
// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it
// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves.
#if __USE_ISOC11
void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
#endif
#if defined(__GLIBC__) && defined(__linux__)
// forward __libc interface (needed for glibc-based Linux distributions)
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size);
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size);
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size);
void __libc_free(void* p) MI_FORWARD0(mi_free,p);
void __libc_cfree(void* p) MI_FORWARD0(mi_free,p);
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size)
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size)
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size)
void __libc_free(void* p) MI_FORWARD0(mi_free,p)
void __libc_cfree(void* p) MI_FORWARD0(mi_free,p)
void* __libc_valloc(size_t size) { return mi_valloc(size); }
void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); }
@@ -209,7 +217,7 @@ void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(a
}
#endif
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
#pragma GCC visibility pop
#endif
kotlin-native/runtime/src/mimalloc/c/alloc-posix.c
+4 -4
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018,2019, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -17,7 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file
// ------------------------------------------------------
#include <errno.h>
#include <string.h> // memcpy
#include <string.h> // memset
#include <stdlib.h> // getenv
#ifdef _MSC_VER
@@ -77,7 +77,7 @@ mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
}
mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL;
if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL;
if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
void* p = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment));
mi_assert_internal(((uintptr_t)p % alignment) == 0);
@@ -103,7 +103,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex
size_t size = (len+1)*sizeof(unsigned short);
unsigned short* p = (unsigned short*)mi_malloc(size);
if (p != NULL) {
memcpy(p,s,size);
_mi_memcpy(p,s,size);
}
return p;
}
kotlin-native/runtime/src/mimalloc/c/alloc.c
+71 -21
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc-internal.h"
#include "mimalloc-atomic.h"
#include <string.h> // memset, memcpy, strlen
#include <string.h> // memset, strlen
#include <stdlib.h> // malloc, exit
#define MI_IN_ALLOC_C
@@ -23,27 +23,34 @@ terms of the MIT license. A copy of the license can be found in the file
// Fall back to generic allocation only if the list is empty.
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept {
mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size);
mi_block_t* block = page->free;
mi_block_t* const block = page->free;
if (mi_unlikely(block == NULL)) {
return _mi_malloc_generic(heap, size);
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
// pop from the free list
page->free = mi_block_next(page, block);
page->used++;
page->free = mi_block_next(page, block);
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
#if (MI_DEBUG>0)
if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); }
#elif (MI_SECURE!=0)
block->next = 0; // don't leak internal data
#endif
#if (MI_STAT>1)
#if (MI_STAT>0)
const size_t bsize = mi_page_usable_block_size(page);
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_increase(heap, normal, bsize);
mi_heap_stat_counter_increase(heap, normal_count, 1);
#if (MI_STAT>1)
const size_t bin = _mi_bin(bsize);
mi_heap_stat_increase(heap, normal[bin], 1);
mi_heap_stat_increase(heap, normal_bins[bin], 1);
#endif
}
#endif
#if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST)
mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page));
ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE));
@@ -54,6 +61,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes
for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; }
#endif
return block;
}
@@ -282,6 +290,49 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co
}
#endif
// only maintain stats for smaller objects if requested
#if (MI_STAT>0)
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
#if (MI_STAT < 2)
UNUSED(block);
#endif
mi_heap_t* const heap = mi_heap_get_default();
const size_t bsize = mi_page_usable_block_size(page);
#if (MI_STAT>1)
const size_t usize = mi_page_usable_size_of(page, block);
mi_heap_stat_decrease(heap, malloc, usize);
#endif
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, normal, bsize);
#if (MI_STAT > 1)
mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1);
#endif
}
}
#else
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
UNUSED(page); UNUSED(block);
}
#endif
#if (MI_STAT>0)
// maintain stats for huge objects
static void mi_stat_huge_free(const mi_page_t* page) {
mi_heap_t* const heap = mi_heap_get_default();
const size_t bsize = mi_page_block_size(page); // to match stats in `page.c:mi_page_huge_alloc`
if (bsize <= MI_HUGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, huge, bsize);
}
else {
mi_heap_stat_decrease(heap, giant, bsize);
}
}
#else
static void mi_stat_huge_free(const mi_page_t* page) {
UNUSED(page);
}
#endif
// ------------------------------------------------------
// Free
// ------------------------------------------------------
@@ -300,6 +351,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
// huge page segments are always abandoned and can be freed immediately
mi_segment_t* const segment = _mi_page_segment(page);
if (segment->page_kind==MI_PAGE_HUGE) {
mi_stat_huge_free(page);
_mi_segment_huge_page_free(segment, page, block);
return;
}
@@ -343,7 +395,6 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
}
}
// regular free
static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block)
{
@@ -383,6 +434,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p
static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) {
mi_page_t* const page = _mi_segment_page_of(segment, p);
mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p);
mi_stat_free(page, block);
_mi_free_block(page, local, block);
}
@@ -430,19 +482,11 @@ void mi_free(void* p) mi_attr_noexcept
mi_page_t* const page = _mi_segment_page_of(segment, p);
mi_block_t* const block = (mi_block_t*)p;
#if (MI_STAT>1)
mi_heap_t* const heap = mi_heap_get_default();
const size_t bsize = mi_page_usable_block_size(page);
mi_heap_stat_decrease(heap, malloc, bsize);
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { // huge page stats are accounted for in `_mi_page_retire`
mi_heap_stat_decrease(heap, normal[_mi_bin(bsize)], 1);
}
#endif
if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks
// local, and not full or aligned
if (mi_unlikely(mi_check_is_double_free(page,block))) return;
mi_check_padding(page, block);
mi_stat_free(page, block);
#if (MI_DEBUG!=0)
memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
#endif
@@ -513,6 +557,7 @@ void* _mi_externs[] = {
(void*)&_mi_page_malloc,
(void*)&mi_malloc,
(void*)&mi_malloc_small,
(void*)&mi_zalloc_small,
(void*)&mi_heap_malloc,
(void*)&mi_heap_zalloc,
(void*)&mi_heap_malloc_small
@@ -584,7 +629,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero)
size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
memset((uint8_t*)newp + start, 0, newsize - start);
}
memcpy(newp, p, (newsize > size ? size : newsize));
_mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
mi_free(p); // only free if successful
}
return newp;
@@ -651,7 +696,7 @@ mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_no
if (s == NULL) return NULL;
size_t n = strlen(s);
char* t = (char*)mi_heap_malloc(heap,n+1);
if (t != NULL) memcpy(t, s, n + 1);
if (t != NULL) _mi_memcpy(t, s, n + 1);
return t;
}
@@ -667,7 +712,7 @@ mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n)
mi_assert_internal(m <= n);
char* t = (char*)mi_heap_malloc(heap, m+1);
if (t == NULL) return NULL;
memcpy(t, s, m);
_mi_memcpy(t, s, m);
t[m] = 0;
return t;
}
@@ -682,7 +727,7 @@ mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept {
#ifndef PATH_MAX
#define PATH_MAX MAX_PATH
#endif
#include <Windows.h>
#include <windows.h>
mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept {
// todo: use GetFullPathNameW to allow longer file names
char buf[PATH_MAX];
@@ -746,7 +791,12 @@ but we call `exit` instead (i.e. not returning).
#ifdef __cplusplus
#include <new>
static bool mi_try_new_handler(bool nothrow) {
std::new_handler h = std::get_new_handler();
#if defined(_MSC_VER) || (__cplusplus >= 201103L)
std::new_handler h = std::get_new_handler();
#else
std::new_handler h = std::set_new_handler();
std::set_new_handler(h);
#endif
if (h==NULL) {
if (!nothrow) throw std::bad_alloc();
return false;
kotlin-native/runtime/src/mimalloc/c/arena.c
+109 -68
View File
@@ -1,8 +1,8 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2019, Microsoft Research, Daan Leijen
Copyright (c) 2019-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
-----------------------------------------------------------------------------*/
/* ----------------------------------------------------------------------------
@@ -30,12 +30,13 @@ of 256MiB in practice.
#include "mimalloc-atomic.h"
#include <string.h> // memset
#include <errno.h> // ENOMEM
#include "bitmap.inc.c" // atomic bitmap
#include "bitmap.h" // atomic bitmap
// os.c
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld);
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* stats);
void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats);
void _mi_os_free(void* p, size_t size, mi_stats_t* stats);
@@ -43,14 +44,14 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_sec
void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
/* -----------------------------------------------------------
Arena allocation
----------------------------------------------------------- */
#define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE
#define MI_ARENA_BLOCK_SIZE (8*MI_SEGMENT_ALIGN) // 32MiB
#define MI_ARENA_MAX_OBJ_SIZE (MI_BITMAP_FIELD_BITS * MI_ARENA_BLOCK_SIZE) // 2GiB
#define MI_ARENA_BLOCK_SIZE (4*MI_SEGMENT_ALIGN) // 32MiB
#define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_BLOCK_SIZE/2) // 16MiB
#define MI_MAX_ARENAS (64) // not more than 256 (since we use 8 bits in the memid)
@@ -61,12 +62,12 @@ typedef struct mi_arena_s {
size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`)
int numa_node; // associated NUMA node
bool is_zero_init; // is the arena zero initialized?
bool is_committed; // is the memory committed
bool is_large; // large OS page allocated
_Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks
bool is_committed; // is the memory fully committed? (if so, block_committed == NULL)
bool is_large; // large- or huge OS pages (always committed)
_Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks
mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?
mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed?
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
} mi_arena_t;
@@ -104,16 +105,11 @@ static size_t mi_block_count_of_size(size_t size) {
----------------------------------------------------------- */
static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx)
{
const size_t fcount = arena->field_count;
size_t idx = mi_atomic_load_acquire(&arena->search_idx); // start from last search
for (size_t visited = 0; visited < fcount; visited++, idx++) {
if (idx >= fcount) idx = 0; // wrap around
// try to atomically claim a range of bits
if (mi_bitmap_try_find_claim_field(arena->blocks_inuse, idx, blocks, bitmap_idx)) {
mi_atomic_store_release(&arena->search_idx, idx); // start search from here next time
return true;
}
}
if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) {
mi_atomic_store_release(&arena->search_idx, idx); // start search from here next time
return true;
};
return false;
}
@@ -123,16 +119,17 @@ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t*
----------------------------------------------------------- */
static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t needed_bcount,
bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{
mi_bitmap_index_t bitmap_index;
if (!mi_arena_alloc(arena, needed_bcount, &bitmap_index)) return NULL;
// claimed it! set the dirty bits (todo: no need for an atomic op here?)
void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE);
*memid = mi_arena_id_create(arena_index, bitmap_index);
*is_zero = mi_bitmap_claim(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL);
*large = arena->is_large;
void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE);
*memid = mi_arena_id_create(arena_index, bitmap_index);
*is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL);
*large = arena->is_large;
*is_pinned = (arena->is_large || arena->is_committed);
if (arena->is_committed) {
// always committed
*commit = true;
@@ -140,7 +137,7 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n
else if (*commit) {
// arena not committed as a whole, but commit requested: ensure commit now
bool any_uncommitted;
mi_bitmap_claim(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
_mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
if (any_uncommitted) {
bool commit_zero;
_mi_os_commit(p, needed_bcount * MI_ARENA_BLOCK_SIZE, &commit_zero, tld->stats);
@@ -149,25 +146,25 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n
}
else {
// no need to commit, but check if already fully committed
*commit = mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index);
*commit = _mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index);
}
return p;
}
void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
bool* commit, bool* large, bool* is_zero,
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero,
size_t* memid, mi_os_tld_t* tld)
{
mi_assert_internal(commit != NULL && large != NULL && is_zero != NULL && memid != NULL && tld != NULL);
mi_assert_internal(commit != NULL && is_pinned != NULL && is_zero != NULL && memid != NULL && tld != NULL);
mi_assert_internal(size > 0);
*memid = MI_MEMID_OS;
*is_zero = false;
*is_pinned = false;
// try to allocate in an arena if the alignment is small enough
// and the object is not too large or too small.
if (alignment <= MI_SEGMENT_ALIGN &&
size <= MI_ARENA_MAX_OBJ_SIZE &&
size >= MI_ARENA_MIN_OBJ_SIZE)
size >= MI_ARENA_MIN_OBJ_SIZE &&
mi_atomic_load_relaxed(&mi_arena_count) > 0)
{
const size_t bcount = mi_block_count_of_size(size);
const int numa_node = _mi_os_numa_node(tld); // current numa node
@@ -180,7 +177,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
if ((arena->numa_node<0 || arena->numa_node==numa_node) && // numa local?
(*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages
{
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld);
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld);
mi_assert_internal((uintptr_t)p % alignment == 0);
if (p != NULL) return p;
}
@@ -192,7 +189,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
if ((arena->numa_node>=0 && arena->numa_node!=numa_node) && // not numa local!
(*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages
{
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld);
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_pinned, is_zero, memid, tld);
mi_assert_internal((uintptr_t)p % alignment == 0);
if (p != NULL) return p;
}
@@ -200,14 +197,20 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
}
// finally, fall back to the OS
if (mi_option_is_enabled(mi_option_limit_os_alloc)) {
errno = ENOMEM;
return NULL;
}
*is_zero = true;
*memid = MI_MEMID_OS;
return _mi_os_alloc_aligned(size, alignment, *commit, large, tld);
*memid = MI_MEMID_OS;
void* p = _mi_os_alloc_aligned(size, alignment, *commit, large, tld->stats);
if (p != NULL) *is_pinned = *large;
return p;
}
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{
return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_zero, memid, tld);
return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_pinned, is_zero, memid, tld);
}
/* -----------------------------------------------------------
@@ -230,6 +233,8 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s
mi_assert_internal(arena_idx < MI_MAX_ARENAS);
mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t,&mi_arenas[arena_idx]);
mi_assert_internal(arena != NULL);
const size_t blocks = mi_block_count_of_size(size);
// checks
if (arena == NULL) {
_mi_error_message(EINVAL, "trying to free from non-existent arena: %p, size %zu, memid: 0x%zx\n", p, size, memid);
return;
@@ -239,9 +244,18 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s
_mi_error_message(EINVAL, "trying to free from non-existent arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
return;
}
const size_t blocks = mi_block_count_of_size(size);
bool ones = mi_bitmap_unclaim(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx);
if (!ones) {
// potentially decommit
if (arena->is_committed) {
mi_assert_internal(all_committed);
}
else {
mi_assert_internal(arena->blocks_committed != NULL);
_mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails
_mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx);
}
// and make it available to others again
bool all_inuse = _mi_bitmap_unclaim_across(arena->blocks_inuse, arena->field_count, blocks, bitmap_idx);
if (!all_inuse) {
_mi_error_message(EAGAIN, "trying to free an already freed block: %p, size %zu\n", p, size);
return;
};
@@ -266,12 +280,64 @@ static bool mi_arena_add(mi_arena_t* arena) {
return true;
}
bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept
{
if (is_large) {
mi_assert_internal(is_committed);
is_committed = true;
}
const size_t bcount = mi_block_count_of_size(size);
const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS);
const size_t bitmaps = (is_committed ? 2 : 3);
const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t));
mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS?
if (arena == NULL) return false;
arena->block_count = bcount;
arena->field_count = fields;
arena->start = (uint8_t*)start;
arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1)
arena->is_large = is_large;
arena->is_zero_init = is_zero;
arena->is_committed = is_committed;
arena->search_idx = 0;
arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap
arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap
// the bitmaps are already zero initialized due to os_alloc
// just claim leftover blocks if needed
ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount;
mi_assert_internal(post >= 0);
if (post > 0) {
// don't use leftover bits at the end
mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post);
_mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL);
}
mi_arena_add(arena);
return true;
}
// Reserve a range of regular OS memory
int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept
{
size = _mi_os_good_alloc_size(size);
bool large = allow_large;
void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &_mi_stats_main);
if (start==NULL) return ENOMEM;
if (!mi_manage_os_memory(start, size, (large || commit), large, true, -1)) {
_mi_os_free_ex(start, size, commit, &_mi_stats_main);
_mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024));
return ENOMEM;
}
_mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : "");
return 0;
}
/* -----------------------------------------------------------
Reserve a huge page arena.
----------------------------------------------------------- */
#include <errno.h> // ENOMEM
// reserve at a specific numa node
int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept {
if (pages==0) return 0;
@@ -286,35 +352,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec
}
_mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages);
size_t bcount = mi_block_count_of_size(hsize);
size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS);
size_t asize = sizeof(mi_arena_t) + (2*fields*sizeof(mi_bitmap_field_t));
mi_arena_t* arena = (mi_arena_t*)_mi_os_alloc(asize, &_mi_stats_main); // TODO: can we avoid allocating from the OS?
if (arena == NULL) {
if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) {
_mi_os_free_huge_pages(p, hsize, &_mi_stats_main);
return ENOMEM;
}
arena->block_count = bcount;
arena->field_count = fields;
arena->start = (uint8_t*)p;
arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1)
arena->is_large = true;
arena->is_zero_init = true;
arena->is_committed = true;
arena->search_idx = 0;
arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap
arena->blocks_committed = NULL;
// the bitmaps are already zero initialized due to os_alloc
// just claim leftover blocks if needed
ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount;
mi_assert_internal(post >= 0);
if (post > 0) {
// don't use leftover bits at the end
mi_bitmap_index_t postidx = mi_bitmap_index_create(fields - 1, MI_BITMAP_FIELD_BITS - post);
mi_bitmap_claim(arena->blocks_inuse, fields, post, postidx, NULL);
}
mi_arena_add(arena);
return 0;
}
kotlin-native/runtime/src/mimalloc/c/bitmap.c
+395
View File
@@ -0,0 +1,395 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2019-2021 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
-----------------------------------------------------------------------------*/
/* ----------------------------------------------------------------------------
Concurrent bitmap that can set/reset sequences of bits atomically,
represeted as an array of fields where each field is a machine word (`uintptr_t`)
There are two api's; the standard one cannot have sequences that cross
between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS).
(this is used in region allocation)
The `_across` postfixed functions do allow sequences that can cross over
between the fields. (This is used in arena allocation)
---------------------------------------------------------------------------- */
#include "mimalloc.h"
#include "mimalloc-internal.h"
#include "bitmap.h"
/* -----------------------------------------------------------
Bitmap definition
----------------------------------------------------------- */
// The bit mask for a given number of blocks at a specified bit index.
static inline uintptr_t mi_bitmap_mask_(size_t count, size_t bitidx) {
mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS);
mi_assert_internal(count > 0);
if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL;
if (count == 0) return 0;
return ((((uintptr_t)1 << count) - 1) << bitidx);
}
/* -----------------------------------------------------------
Claim a bit sequence atomically
----------------------------------------------------------- */
// Try to atomically claim a sequence of `count` bits in a single
// field at `idx` in `bitmap`. Returns `true` on success.
bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx)
{
mi_assert_internal(bitmap_idx != NULL);
mi_assert_internal(count <= MI_BITMAP_FIELD_BITS);
_Atomic(uintptr_t)* field = &bitmap[idx];
uintptr_t map = mi_atomic_load_relaxed(field);
if (map==MI_BITMAP_FIELD_FULL) return false; // short cut
// search for 0-bit sequence of length count
const uintptr_t mask = mi_bitmap_mask_(count, 0);
const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count;
#ifdef MI_HAVE_FAST_BITSCAN
size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible
#else
size_t bitidx = 0; // otherwise start at 0
#endif
uintptr_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx
// scan linearly for a free range of zero bits
while (bitidx <= bitidx_max) {
const uintptr_t mapm = map & m;
if (mapm == 0) { // are the mask bits free at bitidx?
mi_assert_internal((m >> bitidx) == mask); // no overflow?
const uintptr_t newmap = map | m;
mi_assert_internal((newmap^map) >> bitidx == mask);
if (!mi_atomic_cas_weak_acq_rel(field, &map, newmap)) { // TODO: use strong cas here?
// no success, another thread claimed concurrently.. keep going (with updated `map`)
continue;
}
else {
// success, we claimed the bits!
*bitmap_idx = mi_bitmap_index_create(idx, bitidx);
return true;
}
}
else {
// on to the next bit range
#ifdef MI_HAVE_FAST_BITSCAN
const size_t shift = (count == 1 ? 1 : mi_bsr(mapm) - bitidx + 1);
mi_assert_internal(shift > 0 && shift <= count);
#else
const size_t shift = 1;
#endif
bitidx += shift;
m <<= shift;
}
}
// no bits found
return false;
}
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields.
bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) {
size_t idx = start_field_idx;
for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) {
if (idx >= bitmap_fields) idx = 0; // wrap
if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {
return true;
}
}
return false;
}
/*
// Find `count` bits of 0 and set them to 1 atomically; returns `true` on success.
// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never span fields.
bool _mi_bitmap_try_find_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t count, mi_bitmap_index_t* bitmap_idx) {
return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, 0, count, bitmap_idx);
}
*/
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously.
bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
// mi_assert_internal((bitmap[idx] & mask) == mask);
uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask);
return ((prev & mask) == mask);
}
// Set `count` bits at `bitmap_idx` to 1 atomically
// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.
bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
//mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0);
uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask);
if (any_zero != NULL) *any_zero = ((prev & mask) != mask);
return ((prev & mask) == 0);
}
// Returns `true` if all `count` bits were 1. `any_ones` is `true` if there was at least one bit set to one.
static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
const uintptr_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields);
uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]);
if (any_ones != NULL) *any_ones = ((field & mask) != 0);
return ((field & mask) == mask);
}
bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
return mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, NULL);
}
bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
bool any_ones;
mi_bitmap_is_claimedx(bitmap, bitmap_fields, count, bitmap_idx, &any_ones);
return any_ones;
}
//--------------------------------------------------------------------------
// the `_across` functions work on bitmaps where sequences can cross over
// between the fields. This is used in arena allocation
//--------------------------------------------------------------------------
// Try to atomically claim a sequence of `count` bits starting from the field
// at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success.
static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx)
{
mi_assert_internal(bitmap_idx != NULL);
// check initial trailing zeros
_Atomic(uintptr_t)* field = &bitmap[idx];
uintptr_t map = mi_atomic_load_relaxed(field);
const size_t initial = mi_clz(map); // count of initial zeros starting at idx
mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS);
if (initial == 0) return false;
if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields
if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries
// scan ahead
size_t found = initial;
uintptr_t mask = 0; // mask bits for the final field
while(found < count) {
field++;
map = mi_atomic_load_relaxed(field);
const uintptr_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found));
mask = mi_bitmap_mask_(mask_bits, 0);
if ((map & mask) != 0) return false;
found += mask_bits;
}
mi_assert_internal(field < &bitmap[bitmap_fields]);
// found range of zeros up to the final field; mask contains mask in the final field
// now claim it atomically
_Atomic(uintptr_t)* const final_field = field;
const uintptr_t final_mask = mask;
_Atomic(uintptr_t)* const initial_field = &bitmap[idx];
const uintptr_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial);
// initial field
uintptr_t newmap;
field = initial_field;
map = mi_atomic_load_relaxed(field);
do {
newmap = map | initial_mask;
if ((map & initial_mask) != 0) { goto rollback; };
} while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));
// intermediate fields
while (++field < final_field) {
newmap = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0);
map = 0;
if (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)) { goto rollback; }
}
// final field
mi_assert_internal(field == final_field);
map = mi_atomic_load_relaxed(field);
do {
newmap = map | final_mask;
if ((map & final_mask) != 0) { goto rollback; }
} while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));
// claimed!
*bitmap_idx = mi_bitmap_index_create(idx, MI_BITMAP_FIELD_BITS - initial);
return true;
rollback:
// roll back intermediate fields
while (--field > initial_field) {
newmap = 0;
map = mi_bitmap_mask_(MI_BITMAP_FIELD_BITS, 0);
mi_assert_internal(mi_atomic_load_relaxed(field) == map);
mi_atomic_store_release(field, newmap);
}
if (field == initial_field) {
map = mi_atomic_load_relaxed(field);
do {
mi_assert_internal((map & initial_mask) == initial_mask);
newmap = map & ~initial_mask;
} while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));
}
// retry? (we make a recursive call instead of goto to be able to use const declarations)
if (retries < 4) {
return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx);
}
else {
return false;
}
}
// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) {
mi_assert_internal(count > 0);
if (count==1) return _mi_bitmap_try_find_from_claim(bitmap, bitmap_fields, start_field_idx, count, bitmap_idx);
size_t idx = start_field_idx;
for (size_t visited = 0; visited < bitmap_fields; visited++, idx++) {
if (idx >= bitmap_fields) idx = 0; // wrap
// try to claim inside the field
if (count <= MI_BITMAP_FIELD_BITS) {
if (_mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx)) {
return true;
}
}
// try to claim across fields
if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) {
return true;
}
}
return false;
}
// Helper for masks across fields; returns the mid count, post_mask may be 0
static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) {
UNUSED_RELEASE(bitmap_fields);
const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx);
if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) {
*pre_mask = mi_bitmap_mask_(count, bitidx);
*mid_mask = 0;
*post_mask = 0;
mi_assert_internal(mi_bitmap_index_field(bitmap_idx) < bitmap_fields);
return 0;
}
else {
const size_t pre_bits = MI_BITMAP_FIELD_BITS - bitidx;
mi_assert_internal(pre_bits < count);
*pre_mask = mi_bitmap_mask_(pre_bits, bitidx);
count -= pre_bits;
const size_t mid_count = (count / MI_BITMAP_FIELD_BITS);
*mid_mask = MI_BITMAP_FIELD_FULL;
count %= MI_BITMAP_FIELD_BITS;
*post_mask = (count==0 ? 0 : mi_bitmap_mask_(count, 0));
mi_assert_internal(mi_bitmap_index_field(bitmap_idx) + mid_count + (count==0 ? 0 : 1) < bitmap_fields);
return mid_count;
}
}
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously.
bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
size_t idx = mi_bitmap_index_field(bitmap_idx);
uintptr_t pre_mask;
uintptr_t mid_mask;
uintptr_t post_mask;
size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);
bool all_one = true;
_Atomic(uintptr_t)*field = &bitmap[idx];
uintptr_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask);
if ((prev & pre_mask) != pre_mask) all_one = false;
while(mid_count-- > 0) {
prev = mi_atomic_and_acq_rel(field++, ~mid_mask);
if ((prev & mid_mask) != mid_mask) all_one = false;
}
if (post_mask!=0) {
prev = mi_atomic_and_acq_rel(field, ~post_mask);
if ((prev & post_mask) != post_mask) all_one = false;
}
return all_one;
}
// Set `count` bits at `bitmap_idx` to 1 atomically
// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.
bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) {
size_t idx = mi_bitmap_index_field(bitmap_idx);
uintptr_t pre_mask;
uintptr_t mid_mask;
uintptr_t post_mask;
size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);
bool all_zero = true;
bool any_zero = false;
_Atomic(uintptr_t)*field = &bitmap[idx];
uintptr_t prev = mi_atomic_or_acq_rel(field++, pre_mask);
if ((prev & pre_mask) != 0) all_zero = false;
if ((prev & pre_mask) != pre_mask) any_zero = true;
while (mid_count-- > 0) {
prev = mi_atomic_or_acq_rel(field++, mid_mask);
if ((prev & mid_mask) != 0) all_zero = false;
if ((prev & mid_mask) != mid_mask) any_zero = true;
}
if (post_mask!=0) {
prev = mi_atomic_or_acq_rel(field, post_mask);
if ((prev & post_mask) != 0) all_zero = false;
if ((prev & post_mask) != post_mask) any_zero = true;
}
if (pany_zero != NULL) *pany_zero = any_zero;
return all_zero;
}
// Returns `true` if all `count` bits were 1.
// `any_ones` is `true` if there was at least one bit set to one.
static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) {
size_t idx = mi_bitmap_index_field(bitmap_idx);
uintptr_t pre_mask;
uintptr_t mid_mask;
uintptr_t post_mask;
size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask);
bool all_ones = true;
bool any_ones = false;
_Atomic(uintptr_t)* field = &bitmap[idx];
uintptr_t prev = mi_atomic_load_relaxed(field++);
if ((prev & pre_mask) != pre_mask) all_ones = false;
if ((prev & pre_mask) != 0) any_ones = true;
while (mid_count-- > 0) {
prev = mi_atomic_load_relaxed(field++);
if ((prev & mid_mask) != mid_mask) all_ones = false;
if ((prev & mid_mask) != 0) any_ones = true;
}
if (post_mask!=0) {
prev = mi_atomic_load_relaxed(field);
if ((prev & post_mask) != post_mask) all_ones = false;
if ((prev & post_mask) != 0) any_ones = true;
}
if (pany_ones != NULL) *pany_ones = any_ones;
return all_ones;
}
bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
return mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, NULL);
}
bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
bool any_ones;
mi_bitmap_is_claimedx_across(bitmap, bitmap_fields, count, bitmap_idx, &any_ones);
return any_ones;
}
kotlin-native/runtime/src/mimalloc/c/bitmap.h
+102
View File
@@ -0,0 +1,102 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2019-2020 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
-----------------------------------------------------------------------------*/
/* ----------------------------------------------------------------------------
Concurrent bitmap that can set/reset sequences of bits atomically,
represeted as an array of fields where each field is a machine word (`uintptr_t`)
There are two api's; the standard one cannot have sequences that cross
between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS).
(this is used in region allocation)
The `_across` postfixed functions do allow sequences that can cross over
between the fields. (This is used in arena allocation)
---------------------------------------------------------------------------- */
#pragma once
#ifndef MI_BITMAP_H
#define MI_BITMAP_H
/* -----------------------------------------------------------
Bitmap definition
----------------------------------------------------------- */
#define MI_BITMAP_FIELD_BITS (8*MI_INTPTR_SIZE)
#define MI_BITMAP_FIELD_FULL (~((uintptr_t)0)) // all bits set
// An atomic bitmap of `uintptr_t` fields
typedef _Atomic(uintptr_t) mi_bitmap_field_t;
typedef mi_bitmap_field_t* mi_bitmap_t;
// A bitmap index is the index of the bit in a bitmap.
typedef size_t mi_bitmap_index_t;
// Create a bit index.
static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) {
mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS);
return (idx*MI_BITMAP_FIELD_BITS) + bitidx;
}
// Get the field index from a bit index.
static inline size_t mi_bitmap_index_field(mi_bitmap_index_t bitmap_idx) {
return (bitmap_idx / MI_BITMAP_FIELD_BITS);
}
// Get the bit index in a bitmap field
static inline size_t mi_bitmap_index_bit_in_field(mi_bitmap_index_t bitmap_idx) {
return (bitmap_idx % MI_BITMAP_FIELD_BITS);
}
// Get the full bit index
static inline size_t mi_bitmap_index_bit(mi_bitmap_index_t bitmap_idx) {
return bitmap_idx;
}
/* -----------------------------------------------------------
Claim a bit sequence atomically
----------------------------------------------------------- */
// Try to atomically claim a sequence of `count` bits in a single
// field at `idx` in `bitmap`. Returns `true` on success.
bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_t count, mi_bitmap_index_t* bitmap_idx);
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
// For now, `count` can be at most MI_BITMAP_FIELD_BITS and will never cross fields.
bool _mi_bitmap_try_find_from_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx);
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously.
bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
// Set `count` bits at `bitmap_idx` to 1 atomically
// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.
bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero);
bool _mi_bitmap_is_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
//--------------------------------------------------------------------------
// the `_across` functions work on bitmaps where sequences can cross over
// between the fields. This is used in arena allocation
//--------------------------------------------------------------------------
// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx);
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously.
bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
// Set `count` bits at `bitmap_idx` to 1 atomically
// Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit.
bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero);
bool _mi_bitmap_is_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
bool _mi_bitmap_is_any_claimed_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx);
#endif
kotlin-native/runtime/src/mimalloc/c/heap.c
+32 -24
View File
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -92,7 +92,7 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t
mi_collect_t collect = *((mi_collect_t*)arg_collect);
_mi_page_free_collect(page, collect >= MI_FORCE);
if (mi_page_all_free(page)) {
// no more used blocks, free the page.
// no more used blocks, free the page.
// note: this will free retired pages as well.
_mi_page_free(page, pq, collect >= MI_FORCE);
}
@@ -114,10 +114,10 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq
static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
{
if (!mi_heap_is_initialized(heap)) return;
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
_mi_deferred_free(heap, collect >= MI_FORCE);
// note: never reclaim on collect but leave it to threads that need storage to reclaim
// note: never reclaim on collect but leave it to threads that need storage to reclaim
if (
#ifdef NDEBUG
collect == MI_FORCE
@@ -130,7 +130,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
// if all memory is freed by now, all segments should be freed.
_mi_abandoned_reclaim_all(heap, &heap->tld->segments);
}
// if abandoning, mark all pages to no longer add to delayed_free
if (collect == MI_ABANDON) {
mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
@@ -193,7 +193,7 @@ mi_heap_t* mi_heap_new(void) {
mi_heap_t* bheap = mi_heap_get_backing();
mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
if (heap==NULL) return NULL;
memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t));
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = bheap->tld;
heap->thread_id = _mi_thread_id();
_mi_random_split(&bheap->random, &heap->random);
@@ -213,13 +213,14 @@ uintptr_t _mi_heap_random_next(mi_heap_t* heap) {
// zero out the page queues
static void mi_heap_reset_pages(mi_heap_t* heap) {
mi_assert_internal(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap));
// TODO: copy full empty heap instead?
memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct));
#ifdef MI_MEDIUM_DIRECT
memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium));
#endif
memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages));
_mi_memcpy_aligned(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages));
heap->thread_delayed_free = NULL;
heap->page_count = 0;
}
@@ -228,6 +229,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
static void mi_heap_free(mi_heap_t* heap) {
mi_assert(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (mi_heap_is_backing(heap)) return; // dont free the backing heap
// reset default
@@ -238,7 +240,7 @@ static void mi_heap_free(mi_heap_t* heap) {
// remove ourselves from the thread local heaps list
// linear search but we expect the number of heaps to be relatively small
mi_heap_t* prev = NULL;
mi_heap_t* curr = heap->tld->heaps;
mi_heap_t* curr = heap->tld->heaps;
while (curr != heap && curr != NULL) {
prev = curr;
curr = curr->next;
@@ -272,17 +274,20 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
const size_t bsize = mi_page_block_size(page);
if (bsize > MI_LARGE_OBJ_SIZE_MAX) {
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&heap->tld->stats.giant, bsize);
mi_heap_stat_decrease(heap, giant, bsize);
}
else {
_mi_stat_decrease(&heap->tld->stats.huge, bsize);
mi_heap_stat_decrease(heap, huge, bsize);
}
}
#if (MI_STAT>1)
#if (MI_STAT)
_mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used;
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap, normal[_mi_bin(bsize)], inuse);
mi_heap_stat_decrease(heap, normal, bsize * inuse);
#if (MI_STAT>1)
mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse);
#endif
}
mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks...
#endif
@@ -310,7 +315,7 @@ void mi_heap_destroy(mi_heap_t* heap) {
mi_assert(mi_heap_is_initialized(heap));
mi_assert(heap->no_reclaim);
mi_assert_expensive(mi_heap_is_valid(heap));
if (!mi_heap_is_initialized(heap)) return;
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (!heap->no_reclaim) {
// don't free in case it may contain reclaimed pages
mi_heap_delete(heap);
@@ -335,8 +340,8 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
// reduce the size of the delayed frees
_mi_heap_delayed_free(from);
// transfer all pages by appending the queues; this will set a new heap field
// transfer all pages by appending the queues; this will set a new heap field
// so threads may do delayed frees in either heap for a while.
// note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state
// so after this only the new heap will get delayed frees
@@ -349,15 +354,17 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
}
mi_assert_internal(from->page_count == 0);
// and do outstanding delayed frees in the `from` heap
// and do outstanding delayed frees in the `from` heap
// note: be careful here as the `heap` field in all those pages no longer point to `from`,
// turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a
// turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a
// the regular `_mi_free_delayed_block` which is safe.
_mi_heap_delayed_free(from);
mi_assert_internal(from->thread_delayed_free == NULL);
_mi_heap_delayed_free(from);
#if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353
mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL);
#endif
// and reset the `from` heap
mi_heap_reset_pages(from);
mi_heap_reset_pages(from);
}
// Safe delete a heap without freeing any still allocated blocks in that heap.
@@ -366,7 +373,7 @@ void mi_heap_delete(mi_heap_t* heap)
mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap));
mi_assert_expensive(mi_heap_is_valid(heap));
if (!mi_heap_is_initialized(heap)) return;
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (!mi_heap_is_backing(heap)) {
// tranfer still used pages to the backing heap
@@ -381,8 +388,9 @@ void mi_heap_delete(mi_heap_t* heap)
}
mi_heap_t* mi_heap_set_default(mi_heap_t* heap) {
mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap));
if (!mi_heap_is_initialized(heap)) return NULL;
if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL;
mi_assert_expensive(mi_heap_is_valid(heap));
mi_heap_t* old = mi_get_default_heap();
_mi_heap_set_default_direct(heap);
@@ -408,7 +416,7 @@ static mi_heap_t* mi_heap_of_block(const void* p) {
bool mi_heap_contains_block(mi_heap_t* heap, const void* p) {
mi_assert(heap != NULL);
if (!mi_heap_is_initialized(heap)) return false;
if (heap==NULL || !mi_heap_is_initialized(heap)) return false;
return (heap == mi_heap_of_block(p));
}
@@ -426,7 +434,7 @@ static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
bool mi_heap_check_owned(mi_heap_t* heap, const void* p) {
mi_assert(heap != NULL);
if (!mi_heap_is_initialized(heap)) return false;
if (heap==NULL || !mi_heap_is_initialized(heap)) return false;
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers
bool found = false;
mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void*)p, &found);
kotlin-native/runtime/src/mimalloc/c/include/mimalloc-atomic.h
+40 -33
View File
@@ -1,10 +1,9 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018,2020 Microsoft Research, Daan Leijen
Copyright (c) 2018-2021 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
-----------------------------------------------------------------------------*/
// Copyright 2019-2020 JetBrains s.r.o.
#pragma once
#ifndef MIMALLOC_ATOMIC_H
@@ -120,7 +119,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics.
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <windows.h>
#include <intrin.h>
#ifdef _WIN64
typedef LONG64 msc_intptr_t;
@@ -283,44 +282,52 @@ static inline void mi_atomic_yield(void) {
static inline void mi_atomic_yield(void) {
YieldProcessor();
}
#elif defined(__SSE2__)
#include <emmintrin.h>
static inline void mi_atomic_yield(void) {
_mm_pause();
}
#elif (defined(__GNUC__) || defined(__clang__)) && \
(defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))
(defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \
defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__))
#if defined(__x86_64__) || defined(__i386__)
static inline void mi_atomic_yield(void) {
__asm__ volatile ("pause" ::: "memory");
}
#elif defined(__arm__) || defined(__aarch64__)
#if KONAN_MI_MALLOC
#if defined(__arm__)
#include <sched.h>
static inline void mi_atomic_yield(void) {
sched_yield();
}
#else
static inline void mi_atomic_yield(void) {
asm volatile("yield");
}
#endif
#else // KONAN_MI_MALLOC
static inline void mi_atomic_yield(void) {
asm volatile("yield");
}
#endif // KONAN_MI_MALLOC
#elif defined(__aarch64__)
static inline void mi_atomic_yield(void) {
__asm__ volatile("wfe");
}
#elif (defined(__arm__) && __ARM_ARCH__ >= 7)
static inline void mi_atomic_yield(void) {
__asm__ volatile("yield" ::: "memory");
}
#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)
static inline void mi_atomic_yield(void) {
__asm__ __volatile__ ("or 27,27,27" ::: "memory");
}
#elif defined(__armel__) || defined(__ARMEL__)
static inline void mi_atomic_yield(void) {
__asm__ volatile ("nop" ::: "memory");
}
#endif
#elif defined(__sun)
// Fallback for other archs
#include <synch.h>
static inline void mi_atomic_yield(void) {
smt_pause();
}
#elif defined(__wasi__)
#include <sched.h>
static inline void mi_atomic_yield(void) {
sched_yield();
}
#include <sched.h>
static inline void mi_atomic_yield(void) {
sched_yield();
}
#else
#include <unistd.h>
static inline void mi_atomic_yield(void) {
sleep(0);
}
#include <unistd.h>
static inline void mi_atomic_yield(void) {
sleep(0);
}
#endif
#endif // __MIMALLOC_ATOMIC_H
kotlin-native/runtime/src/mimalloc/c/include/mimalloc-internal.h
+197 -22
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -34,7 +34,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_cache_align
#endif
// "options.c"
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
@@ -53,8 +52,8 @@ uintptr_t _os_random_weak(uintptr_t extra_seed);
static inline uintptr_t _mi_random_shuffle(uintptr_t x);
// init.c
extern mi_stats_t _mi_stats_main;
extern const mi_page_t _mi_page_empty;
extern mi_decl_cache_align mi_stats_t _mi_stats_main;
extern mi_decl_cache_align const mi_page_t _mi_page_empty;
bool _mi_is_main_thread(void);
bool _mi_preloading(); // true while the C runtime is not ready
@@ -66,7 +65,7 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free th
size_t _mi_os_good_alloc_size(size_t size);
// memory.c
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* id, mi_os_tld_t* tld);
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld);
void _mi_mem_free(void* p, size_t size, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld);
bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld);
@@ -109,7 +108,6 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback fro
size_t _mi_bin_size(uint8_t bin); // for stats
uint8_t _mi_bin(size_t size); // for stats
uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c"
// "heap.c"
void _mi_heap_destroy_pages(mi_heap_t* heap);
@@ -245,10 +243,9 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size)
#if defined(_CLOCK_T) // for Illumos
#undef _CLOCK_T
#endif
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
// Changed order for armv7 (ULONG_MAX == UINT_MAX, but size_t = unsigned long)
#if defined(__MACH__) && KONAN_MI_MALLOC
#if defined(__APPLE__) && KONAN_MI_MALLOC
#if (SIZE_MAX == ULONG_MAX)
return __builtin_umull_overflow(count, size, total);
#elif (SIZE_MAX == UINT_MAX)
@@ -301,7 +298,7 @@ We try to circumvent this in an efficient way:
- macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the
loader itself calls `malloc` even before the modules are initialized.
- OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS).
- DragonFly: not yet working.
- DragonFly: the uniqueid use is buggy but kept for reference.
------------------------------------------------------------------------------------------- */
extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
@@ -309,7 +306,7 @@ extern bool _mi_process_is_initialized;
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
#if defined(MI_MALLOC_OVERRIDE)
#if defined(__MACH__) // OSX
#if defined(__APPLE__) // macOS
#define MI_TLS_SLOT 89 // seems unused?
// other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)
// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
@@ -319,7 +316,7 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea
#define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24)
#elif defined(__DragonFly__)
#warning "mimalloc is not working correctly on DragonFly yet."
#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) <https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/lib/libthread_xu/thread/thr_private.h#L458>
//#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) <https://github.com/DragonFlyBSD/DragonFlyBSD/blob/master/lib/libthread_xu/thread/thr_private.h#L458>
#endif
#endif
@@ -331,7 +328,7 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) {
pthread_t self = pthread_self();
#if defined(__DragonFly__)
if (self==NULL) {
static mi_heap_t* pheap_main = _mi_heap_main_get();
mi_heap_t* pheap_main = _mi_heap_main_get();
return &pheap_main;
}
#endif
@@ -340,10 +337,14 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) {
#elif defined(MI_TLS_PTHREAD)
#include <pthread.h>
extern pthread_key_t _mi_heap_default_key;
#else
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
#endif
// Default heap to allocate from (if not using TLS- or pthread slots).
// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`).
// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined.
// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356).
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
static inline mi_heap_t* mi_get_default_heap(void) {
#if defined(MI_TLS_SLOT)
mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT);
@@ -679,13 +680,14 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
int _mi_os_numa_node_get(mi_os_tld_t* tld);
size_t _mi_os_numa_node_count_get(void);
extern size_t _mi_numa_node_count;
extern _Atomic(size_t) _mi_numa_node_count;
static inline int _mi_os_numa_node(mi_os_tld_t* tld) {
if (mi_likely(_mi_numa_node_count == 1)) return 0;
if (mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1)) return 0;
else return _mi_os_numa_node_get(tld);
}
static inline size_t _mi_os_numa_node_count(void) {
if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count;
const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count);
if (mi_likely(count>0)) return count;
else return _mi_os_numa_node_count_get();
}
@@ -716,8 +718,10 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
const size_t ofs = (slot*sizeof(void*));
#if defined(__i386__)
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS
#elif defined(__MACH__) && defined(__x86_64__)
#elif defined(__APPLE__) && defined(__x86_64__)
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
#elif defined(__x86_64__)
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
#elif defined(__arm__)
@@ -726,7 +730,12 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
res = tcb[slot];
#elif defined(__aarch64__)
void** tcb; UNUSED(ofs);
#if defined(__APPLE__) // M1, issue #343
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb));
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits
#else
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
#endif
res = tcb[slot];
#endif
return res;
@@ -737,8 +746,10 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
const size_t ofs = (slot*sizeof(void*));
#if defined(__i386__)
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
#elif defined(__MACH__) && defined(__x86_64__)
#elif defined(__APPLE__) && defined(__x86_64__)
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
__asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
#elif defined(__x86_64__)
__asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
#elif defined(__arm__)
@@ -747,13 +758,18 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
tcb[slot] = value;
#elif defined(__aarch64__)
void** tcb; UNUSED(ofs);
#if defined(__APPLE__) // M1, issue #343
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb));
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits
#else
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
#endif
tcb[slot] = value;
#endif
}
static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
#if defined(__MACH__) && KONAN_MI_MALLOC
#if defined(__APPLE__) && KONAN_MI_MALLOC
#include <TargetConditionals.h>
#if TARGET_OS_EMBEDDED // iOS/tvOS/watchOS devices.
return pthread_mach_thread_np(pthread_self());
@@ -762,8 +778,13 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
return (uintptr_t)mi_tls_slot(0);
#endif
#else // KONAN_MI_MALLOC
// in all our targets, slot 0 is the pointer to the thread control block
return (uintptr_t)mi_tls_slot(0);
#if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__))
// on Android, slot 1 is the thread ID (pointer to pthread internal struct)
return (uintptr_t)mi_tls_slot(1);
#else
// in all our other targets, slot 0 is the pointer to the thread control block
return (uintptr_t)mi_tls_slot(0);
#endif
#endif // KONAN_MI_MALLOC
}
#else
@@ -773,5 +794,159 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
}
#endif
// -----------------------------------------------------------------------
// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero)
// -----------------------------------------------------------------------
#if defined(__GNUC__)
#include <limits.h> // LONG_MAX
#define MI_HAVE_FAST_BITSCAN
static inline size_t mi_clz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (INTPTR_MAX == LONG_MAX)
return __builtin_clzl(x);
#else
return __builtin_clzll(x);
#endif
}
static inline size_t mi_ctz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (INTPTR_MAX == LONG_MAX)
return __builtin_ctzl(x);
#else
return __builtin_ctzll(x);
#endif
}
#elif defined(_MSC_VER)
#include <limits.h> // LONG_MAX
#define MI_HAVE_FAST_BITSCAN
static inline size_t mi_clz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
unsigned long idx;
#if (INTPTR_MAX == LONG_MAX)
_BitScanReverse(&idx, x);
#else
_BitScanReverse64(&idx, x);
#endif
return ((MI_INTPTR_BITS - 1) - idx);
}
static inline size_t mi_ctz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
unsigned long idx;
#if (INTPTR_MAX == LONG_MAX)
_BitScanForward(&idx, x);
#else
_BitScanForward64(&idx, x);
#endif
return idx;
}
#else
static inline size_t mi_ctz32(uint32_t x) {
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
static const unsigned char debruijn[32] = {
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
};
if (x==0) return 32;
return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27];
}
static inline size_t mi_clz32(uint32_t x) {
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
static const uint8_t debruijn[32] = {
31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
};
if (x==0) return 32;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27];
}
static inline size_t mi_clz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (MI_INTPTR_BITS <= 32)
return mi_clz32((uint32_t)x);
#else
size_t count = mi_clz32((uint32_t)(x >> 32));
if (count < 32) return count;
return (32 + mi_clz32((uint32_t)x));
#endif
}
static inline size_t mi_ctz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (MI_INTPTR_BITS <= 32)
return mi_ctz32((uint32_t)x);
#else
size_t count = mi_ctz32((uint32_t)x);
if (count < 32) return count;
return (32 + mi_ctz32((uint32_t)(x>>32)));
#endif
}
#endif
// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero)
static inline size_t mi_bsr(uintptr_t x) {
return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x));
}
// ---------------------------------------------------------------------------------
// Provide our own `_mi_memcpy` for potential performance optimizations.
//
// For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if
// we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support
// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253.
// ---------------------------------------------------------------------------------
#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
#include <intrin.h>
#include <string.h>
extern bool _mi_cpu_has_fsrm;
static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
if (_mi_cpu_has_fsrm) {
__movsb((unsigned char*)dst, (const unsigned char*)src, n);
}
else {
memcpy(dst, src, n); // todo: use noinline?
}
}
#else
#include <string.h>
static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
memcpy(dst, src, n);
}
#endif
// -------------------------------------------------------------------------------
// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned
// This is used for example in `mi_realloc`.
// -------------------------------------------------------------------------------
#if (__GNUC__ >= 4) || defined(__clang__)
// On GCC/CLang we provide a hint that the pointers are word aligned.
#include <string.h>
static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
memcpy(adst, asrc, n);
}
#else
// Default fallback on `_mi_memcpy`
static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
_mi_memcpy(dst, src, n);
}
#endif
#endif
kotlin-native/runtime/src/mimalloc/c/include/mimalloc-new-delete.h
+1 -1
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018,2019 Microsoft Research, Daan Leijen
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
kotlin-native/runtime/src/mimalloc/c/include/mimalloc-override.h
+3 -3
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018,2019 Microsoft Research, Daan Leijen
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -24,7 +24,7 @@ not accidentally mix pointers from different allocators).
#define free(p) mi_free(p)
#define strdup(s) mi_strdup(s)
#define strndup(s) mi_strndup(s)
#define strndup(s,n) mi_strndup(s,n)
#define realpath(f,n) mi_realpath(f,n)
// Microsoft extensions
@@ -33,7 +33,7 @@ not accidentally mix pointers from different allocators).
#define _recalloc(p,n,c) mi_recalloc(p,n,c)
#define _strdup(s) mi_strdup(s)
#define _strndup(s) mi_strndup(s)
#define _strndup(s,n) mi_strndup(s,n)
#define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s))
#define _mbsdup(s) mi_mbsdup(s)
#define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v)
kotlin-native/runtime/src/mimalloc/c/include/mimalloc-types.h
+8 -5
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stddef.h> // ptrdiff_t
#include <stdint.h> // uintptr_t, uint16_t, etc
#include <mimalloc-atomic.h> // _Atomic
#include "mimalloc-atomic.h" // _Atomic
#ifdef _MSC_VER
#pragma warning(disable:4214) // bitfield is not int
@@ -265,7 +265,7 @@ typedef enum mi_page_kind_e {
typedef struct mi_segment_s {
// memory fields
size_t memid; // id for the os-level memory manager
bool mem_is_fixed; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
bool mem_is_committed; // `true` if the whole segment is eagerly committed
// segment fields
@@ -274,7 +274,7 @@ typedef struct mi_segment_s {
struct mi_segment_s* prev;
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long)
size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long)
size_t used; // count of pages in use (`used <= capacity`)
size_t capacity; // count of available pages (`#free + used`)
@@ -420,6 +420,7 @@ typedef struct mi_stats_s {
mi_stat_count_t segments_abandoned;
mi_stat_count_t pages_abandoned;
mi_stat_count_t threads;
mi_stat_count_t normal;
mi_stat_count_t huge;
mi_stat_count_t giant;
mi_stat_count_t malloc;
@@ -429,10 +430,11 @@ typedef struct mi_stats_s {
mi_stat_counter_t commit_calls;
mi_stat_counter_t page_no_retire;
mi_stat_counter_t searches;
mi_stat_counter_t normal_count;
mi_stat_counter_t huge_count;
mi_stat_counter_t giant_count;
#if MI_STAT>1
mi_stat_count_t normal[MI_BIN_HUGE+1];
mi_stat_count_t normal_bins[MI_BIN_HUGE+1];
#endif
} mi_stats_t;
@@ -451,6 +453,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
#define mi_stat_counter_increase(stat,amount) (void)0
#endif
#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)
kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h
+9 -2
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MIMALLOC_H
#define MIMALLOC_H
#define MI_MALLOC_VERSION 167 // major + 2 digits minor
#define MI_MALLOC_VERSION 171 // major + 2 digits minor
// ------------------------------------------------------
// Compiler specific attributes
@@ -264,6 +264,10 @@ mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
// deprecated
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
@@ -302,6 +306,7 @@ typedef enum mi_option_e {
mi_option_reset_decommits,
mi_option_large_os_pages, // implies eager commit
mi_option_reserve_huge_os_pages,
mi_option_reserve_os_memory,
mi_option_segment_cache,
mi_option_page_reset,
mi_option_abandoned_page_reset,
@@ -309,8 +314,10 @@ typedef enum mi_option_e {
mi_option_eager_commit_delay,
mi_option_reset_delay,
mi_option_use_numa_nodes,
mi_option_limit_os_alloc,
mi_option_os_tag,
mi_option_max_errors,
mi_option_max_warnings,
_mi_option_last
} mi_option_t;
kotlin-native/runtime/src/mimalloc/c/init.c
+37 -16
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -73,8 +73,8 @@ const mi_page_t _mi_page_empty = {
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \
MI_STAT_COUNT_END_NULL()
@@ -87,7 +87,7 @@ const mi_page_t _mi_page_empty = {
// may lead to allocation itself on some platforms)
// --------------------------------------------------------
const mi_heap_t _mi_heap_empty = {
mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
NULL,
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY,
@@ -189,7 +189,7 @@ static bool _mi_heap_init(void) {
// OS allocated so already zero initialized
mi_tld_t* tld = &td->tld;
mi_heap_t* heap = &td->heap;
memcpy(heap, &_mi_heap_empty, sizeof(*heap));
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap));
heap->thread_id = _mi_thread_id();
_mi_random_init(&heap->random);
heap->cookie = _mi_heap_random_next(heap) | 1;
@@ -201,7 +201,7 @@ static bool _mi_heap_init(void) {
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
_mi_heap_set_default_direct(heap);
_mi_heap_set_default_direct(heap);
}
return false;
}
@@ -234,16 +234,16 @@ static bool _mi_heap_done(mi_heap_t* heap) {
if (heap != &_mi_heap_main) {
_mi_heap_collect_abandon(heap);
}
// merge stats
_mi_stats_done(&heap->tld->stats);
_mi_stats_done(&heap->tld->stats);
// free if not the main thread
if (heap != &_mi_heap_main) {
mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());
_mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main);
}
#if 0
#if 0
// never free the main thread even in debug mode; if a dll is linked statically with mimalloc,
// there may still be delete/free calls after the mi_fls_done is called. Issue #207
else {
@@ -284,9 +284,9 @@ static void _mi_thread_done(mi_heap_t* default_heap);
// nothing to do as it is done in DllMain
#elif defined(_WIN32) && !defined(MI_SHARED_LIB)
// use thread local storage keys to detect thread ending
#include <Windows.h>
#include <windows.h>
#include <fibersapi.h>
#if (_WIN32_WINNT < 0x600) // before Windows Vista
#if (_WIN32_WINNT < 0x600) // before Windows Vista
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
@@ -336,7 +336,7 @@ void mi_thread_init(void) mi_attr_noexcept
{
// ensure our process has started already
mi_process_init();
// initialize the thread local default heap
// (this will call `_mi_heap_set_default_direct` and thus set the
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
@@ -355,7 +355,7 @@ static void _mi_thread_done(mi_heap_t* heap) {
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...
if (heap->thread_id != _mi_thread_id()) return;
// abandon the thread local heap
if (_mi_heap_done(heap)) return; // returns true if already ran
}
@@ -458,6 +458,22 @@ static void mi_process_load(void) {
}
}
#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
#include <intrin.h>
mi_decl_cache_align bool _mi_cpu_has_fsrm = false;
static void mi_detect_cpu_features(void) {
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
int32_t cpu_info[4];
__cpuid(cpu_info, 7);
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
}
#else
static void mi_detect_cpu_features(void) {
// nothing
}
#endif
// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
// ensure we are called once
@@ -466,6 +482,7 @@ void mi_process_init(void) mi_attr_noexcept {
mi_process_setup_auto_thread_done();
_mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
mi_detect_cpu_features();
_mi_os_init();
mi_heap_main_init();
#if (MI_DEBUG)
@@ -478,6 +495,10 @@ void mi_process_init(void) mi_attr_noexcept {
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);
}
if (mi_option_is_enabled(mi_option_reserve_os_memory)) {
long ksize = mi_option_get(mi_option_reserve_os_memory);
if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true);
}
}
@@ -494,8 +515,8 @@ static void mi_process_done(void) {
FlsSetValue(mi_fls_key, NULL); // don't call main-thread callback
FlsFree(mi_fls_key); // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208
#endif
#if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB)
#if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB)
// free all memory if possible on process exit. This is not needed for a stand-alone process
// but should be done if mimalloc is statically linked into another shared library which
// is repeatedly loaded/unloaded, see issue #281.
@@ -505,7 +526,7 @@ static void mi_process_done(void) {
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
mi_stats_print(NULL);
}
mi_allocator_done();
mi_allocator_done();
_mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id);
os_preloading = true; // don't call the C runtime anymore
}
kotlin-native/runtime/src/mimalloc/c/options.c
+32 -17
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -19,7 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file
#endif
static uintptr_t mi_max_error_count = 16; // stop outputting errors after this
static uintptr_t mi_max_error_count = 16; // stop outputting errors after this
static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this
static void mi_add_stderr_output();
@@ -74,7 +75,8 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(reset_decommits) }, // reset uses MADV_FREE/MADV_DONTNEED
#endif
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{ 0, UNINIT, MI_OPTION(reserve_os_memory) },
{ 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
{ 1, UNINIT, MI_OPTION(page_reset) }, // reset page memory on free
{ 0, UNINIT, MI_OPTION(abandoned_page_reset) },// reset free page memory when a thread terminates
@@ -86,8 +88,11 @@ static mi_option_desc_t options[_mi_option_last] =
#endif
{ 100, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
{ 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output
{ 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
{ 16, UNINIT, MI_OPTION(max_warnings) } // maximum warnings that are output
};
static void mi_option_init(mi_option_desc_t* desc);
@@ -105,6 +110,7 @@ void _mi_options_init(void) {
}
}
mi_max_error_count = mi_option_get(mi_option_max_errors);
mi_max_warning_count = mi_option_get(mi_option_max_warnings);
}
long mi_option_get(mi_option_t option) {
@@ -188,7 +194,7 @@ static void mi_out_buf(const char* msg, void* arg) {
if (start+n >= MI_MAX_DELAY_OUTPUT) {
n = MI_MAX_DELAY_OUTPUT-start-1;
}
memcpy(&out_buf[start], msg, n);
_mi_memcpy(&out_buf[start], msg, n);
}
static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
@@ -245,14 +251,15 @@ static void mi_add_stderr_output() {
// --------------------------------------------------------
// Messages, all end up calling `_mi_fputs`.
// --------------------------------------------------------
static _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings
static _Atomic(uintptr_t) error_count; // = 0; // when >= max_error_count stop emitting errors
static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings
// When overriding malloc, we may recurse into mi_vfprintf if an allocation
// inside the C runtime causes another message.
static mi_decl_thread bool recurse = false;
static bool mi_recurse_enter(void) {
#ifdef MI_TLS_RECURSE_GUARD
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
if (_mi_preloading()) return true;
#endif
if (recurse) return false;
@@ -261,7 +268,7 @@ static bool mi_recurse_enter(void) {
}
static void mi_recurse_exit(void) {
#ifdef MI_TLS_RECURSE_GUARD
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
if (_mi_preloading()) return;
#endif
recurse = false;
@@ -323,7 +330,7 @@ static void mi_show_error_message(const char* fmt, va_list args) {
void _mi_warning_message(const char* fmt, ...) {
if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
if (mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return;
if (mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args);
@@ -347,7 +354,7 @@ static _Atomic(void*) mi_error_arg; // = NULL
static void mi_error_default(int err) {
UNUSED(err);
#if (MI_DEBUG>0)
#if (MI_DEBUG>0)
if (err==EFAULT) {
#ifdef _MSC_VER
__debugbreak();
@@ -415,30 +422,30 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) {
// reliably even when this is invoked before the C runtime is initialized.
// i.e. when `_mi_preloading() == true`.
// Note: on windows, environment names are not case sensitive.
#include <Windows.h>
#include <windows.h>
static bool mi_getenv(const char* name, char* result, size_t result_size) {
result[0] = 0;
size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
return (len > 0 && len < result_size);
}
#elif !defined(MI_USE_ENVIRON) || (MI_USE_ENVIRON!=0)
// On Posix systemsr use `environ` to acces environment variables
// On Posix systemsr use `environ` to acces environment variables
// even before the C runtime is initialized.
#if defined(__APPLE__) && defined(__has_include) && __has_include(<crt_externs.h>)
#include <crt_externs.h>
static char** mi_get_environ(void) {
return (*_NSGetEnviron());
}
#else
#else
extern char** environ;
static char** mi_get_environ(void) {
return environ;
}
#endif
static bool mi_getenv(const char* name, char* result, size_t result_size) {
if (name==NULL) return false;
if (name==NULL) return false;
const size_t len = strlen(name);
if (len == 0) return false;
if (len == 0) return false;
char** env = mi_get_environ();
if (env == NULL) return false;
// compare up to 256 entries
@@ -452,7 +459,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) {
}
return false;
}
#else
#else
// fallback: use standard C `getenv` but this cannot be used while initializing the C runtime
static bool mi_getenv(const char* name, char* result, size_t result_size) {
// cannot call getenv() when still initializing the C runtime.
@@ -479,7 +486,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) {
}
#endif
static void mi_option_init(mi_option_desc_t* desc) {
static void mi_option_init(mi_option_desc_t* desc) {
// Read option value from the environment
char buf[64+1];
mi_strlcpy(buf, "mimalloc_", sizeof(buf));
@@ -503,6 +510,14 @@ static void mi_option_init(mi_option_desc_t* desc) {
else {
char* end = buf;
long value = strtol(buf, &end, 10);
if (desc->option == mi_option_reserve_os_memory) {
// this option is interpreted in KiB to prevent overflow of `long`
if (*end == 'K') { end++; }
else if (*end == 'M') { value *= KiB; end++; }
else if (*end == 'G') { value *= MiB; end++; }
else { value = (value + KiB - 1) / KiB; }
if (*end == 'B') { end++; }
}
if (*end == 0) {
desc->value = value;
desc->init = INITIALIZED;
kotlin-native/runtime/src/mimalloc/c/os.c
+71 -32
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file
#if defined(_WIN32)
#include <Windows.h>
#include <windows.h>
#elif defined(__wasi__)
// stdlib.h is all we need, and has already been included in mimalloc.h
#else
@@ -65,8 +65,15 @@ static void* mi_align_up_ptr(void* p, size_t alignment) {
return (void*)_mi_align_up((uintptr_t)p, alignment);
}
static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
return (sz / alignment) * alignment;
static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
mi_assert_internal(alignment != 0);
uintptr_t mask = alignment - 1;
if ((alignment & mask) == 0) { // power of two?
return (sz & ~mask);
}
else {
return ((sz / alignment) * alignment);
}
}
static void* mi_align_down_ptr(void* p, size_t alignment) {
@@ -106,7 +113,7 @@ size_t _mi_os_good_alloc_size(size_t size) {
else if (size < 8*MiB) align_size = 256*KiB;
else if (size < 32*MiB) align_size = 1*MiB;
else align_size = 4*MiB;
if (size >= (SIZE_MAX - align_size)) return size; // possible overflow?
if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow?
return _mi_align_up(size, align_size);
}
@@ -261,7 +268,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
return NULL;
}
// fall through
}
}
#endif
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
// on modern Windows try use VirtualAlloc2 for aligned allocation
@@ -358,7 +365,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
int fd = -1;
#if defined(MAP_ALIGNED) // BSD
if (try_alignment > 0) {
size_t n = _mi_bsr(try_alignment);
size_t n = mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_ALIGNED(n);
}
@@ -421,7 +428,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#endif
if (large_only) return p;
if (p == NULL) {
mi_atomic_store_release(&large_page_try_ok, 10UL); // on error, don't try again for the next N allocations
mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations
}
}
}
@@ -435,7 +442,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// though since properly aligned allocations will already use large pages if available
// in that case -- in particular for our large regions (in `memory.c`).
// However, some systems only allow THP if called with explicit `madvise`, so
// when large OS pages are enabled for mimalloc, we call `madvice` anyways.
// when large OS pages are enabled for mimalloc, we call `madvise` anyways.
if (allow_large && use_large_os_page(size, try_alignment)) {
if (madvise(p, size, MADV_HUGEPAGE) == 0) {
*is_large = true; // possibly
@@ -465,20 +472,36 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED)))
static mi_decl_cache_align _Atomic(uintptr_t) aligned_base;
// Return a 4MiB aligned address that is probably available
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
// Return a 4MiB aligned address that is probably available.
// If this returns NULL, the OS will determine the address but on some OS's that may not be
// properly aligned which can be more costly as it needs to be adjusted afterwards.
// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization;
// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses
// in the middle of the 2TiB - 6TiB address range (see issue #372))
#define KK_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start
#define KK_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes)
#define KK_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages)
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size)
{
if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL;
if ((size%MI_SEGMENT_SIZE) != 0) return NULL;
if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096.
#if (MI_SECURE>0)
size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
#endif
uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages)
uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
if (hint == 0 || hint > KK_HINT_MAX) { // wrap or initialize
uintptr_t init = KK_HINT_BASE;
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_heap_random_next(mi_get_default_heap());
init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)); // (randomly 20 bits)*4MiB == 0 to 4TiB
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % KK_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
#endif
uintptr_t expected = hint + size;
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all
hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > KK_HINT_MAX but that is ok, it is a hint after all
}
if (hint%try_alignment != 0) return NULL;
return (void*)hint;
@@ -623,9 +646,9 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
_mi_os_free_ex(p, size, true, stats);
}
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld)
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats)
{
UNUSED(tld);
UNUSED(tld_stats);
if (size == 0) return NULL;
size = _mi_os_good_alloc_size(size);
alignment = _mi_align_up(alignment, _mi_os_page_size());
@@ -721,6 +744,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
// for commit, just change the protection
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
if (err != 0) { err = errno; }
#if defined(MADV_FREE_REUSE)
while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; }
#endif
}
#else
err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
@@ -747,7 +773,7 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) {
return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats);
}
static bool mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
static bool mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, true /* conservative */, is_zero, stats);
}
@@ -782,10 +808,17 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
if (p != start) return false;
#else
#if defined(MADV_FREE)
static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE);
int err = madvise(start, csize, (int)mi_atomic_load_relaxed(&advice));
if (err != 0 && errno == EINVAL && advice == MADV_FREE) {
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
#if defined(MADV_FREE_REUSABLE)
#define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE
#else
#define KK_MADV_FREE_INITIAL MADV_FREE
#endif
static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL);
int oadvice = (int)mi_atomic_load_relaxed(&advice);
int err;
while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; };
if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) {
// if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on
mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED);
err = madvise(start, csize, MADV_DONTNEED);
}
@@ -1090,7 +1123,7 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
/* ----------------------------------------------------------------------------
Support NUMA aware allocation
-----------------------------------------------------------------------------*/
#ifdef _WIN32
#ifdef _WIN32
static size_t mi_os_numa_nodex() {
USHORT numa_node = 0;
if (pGetCurrentProcessorNumberEx != NULL && pGetNumaProcessorNodeEx != NULL) {
@@ -1106,7 +1139,7 @@ static size_t mi_os_numa_nodex() {
DWORD pnum = GetCurrentProcessorNumber();
UCHAR nnode = 0;
BOOL ok = GetNumaProcessorNode((UCHAR)pnum, &nnode);
if (ok) numa_node = nnode;
if (ok) numa_node = nnode;
}
return numa_node;
}
@@ -1169,17 +1202,23 @@ static size_t mi_os_numa_node_countx(void) {
}
#endif
size_t _mi_numa_node_count = 0; // cache the node count
_Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count
size_t _mi_os_numa_node_count_get(void) {
if (mi_unlikely(_mi_numa_node_count <= 0)) {
size_t count = mi_atomic_load_acquire(&_mi_numa_node_count);
if (count <= 0) {
long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly?
if (ncount <= 0) ncount = (long)mi_os_numa_node_countx(); // or detect dynamically
_mi_numa_node_count = (size_t)(ncount <= 0 ? 1 : ncount);
_mi_verbose_message("using %zd numa regions\n", _mi_numa_node_count);
if (ncount > 0) {
count = (size_t)ncount;
}
else {
count = mi_os_numa_node_countx(); // or detect dynamically
if (count == 0) count = 1;
}
mi_atomic_store_release(&_mi_numa_node_count, count); // save it
_mi_verbose_message("using %zd numa regions\n", count);
}
mi_assert_internal(_mi_numa_node_count >= 1);
return _mi_numa_node_count;
return count;
}
int _mi_os_numa_node_get(mi_os_tld_t* tld) {
kotlin-native/runtime/src/mimalloc/c/page-queue.c
+3 -47
View File
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -49,50 +49,6 @@ static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
Bins
----------------------------------------------------------- */
// Bit scan reverse: return the index of the highest bit.
static inline uint8_t mi_bsr32(uint32_t x);
#if defined(_MSC_VER)
#include <intrin.h>
static inline uint8_t mi_bsr32(uint32_t x) {
uint32_t idx;
_BitScanReverse((DWORD*)&idx, x);
return (uint8_t)idx;
}
#elif defined(__GNUC__) || defined(__clang__)
static inline uint8_t mi_bsr32(uint32_t x) {
return (31 - __builtin_clz(x));
}
#else
static inline uint8_t mi_bsr32(uint32_t x) {
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
static const uint8_t debruijn[32] = {
31, 0, 22, 1, 28, 23, 18, 2, 29, 26, 24, 10, 19, 7, 3, 12,
30, 21, 27, 17, 25, 9, 6, 11, 20, 16, 8, 5, 15, 4, 14, 13,
};
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x++;
return debruijn[(x*0x076be629) >> 27];
}
#endif
// Bit scan reverse: return the index of the highest bit.
uint8_t _mi_bsr(uintptr_t x) {
if (x == 0) return 0;
#if MI_INTPTR_SIZE==8
uint32_t hi = (x >> 32);
return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi));
#elif MI_INTPTR_SIZE==4
return mi_bsr32(x);
#else
# error "define bsr for non-32 or 64-bit platforms"
#endif
}
// Return the bin for a given field size.
// Returns MI_BIN_HUGE if the size is too large.
// We use `wsize` for the size in "machine word sizes",
@@ -125,7 +81,7 @@ extern inline uint8_t _mi_bin(size_t size) {
#endif
wsize--;
// find the highest bit
uint8_t b = mi_bsr32((uint32_t)wsize);
uint8_t b = (uint8_t)mi_bsr(wsize); // note: wsize != 0
// and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
// - adjust with 3 because we use do not round the first 8 sizes
// which each get an exact bin
@@ -341,7 +297,7 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue
for (mi_page_t* page = append->first; page != NULL; page = page->next) {
// inline `mi_page_set_heap` to avoid wrong assertion during absorption;
// in this case it is ok to be delayed freeing since both "to" and "from" heap are still alive.
mi_atomic_store_release(&page->xheap, (uintptr_t)heap);
mi_atomic_store_release(&page->xheap, (uintptr_t)heap);
// set the flag to delayed free (not overriding NEVER_DELAYED_FREE) which has as a
// side effect that it spins until any DELAYED_FREEING is finished. This ensures
// that after appending only the new heap will be used for delayed free operations.
kotlin-native/runtime/src/mimalloc/c/page.c
+13 -10
View File
@@ -1,5 +1,5 @@
/*----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -124,7 +124,7 @@ bool _mi_page_is_valid(mi_page_t* page) {
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) {
mi_thread_free_t tfreex;
mi_delayed_t old_delay;
mi_thread_free_t tfree;
mi_thread_free_t tfree;
do {
tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS;
tfreex = mi_tf_set_delayed(tfree, delay);
@@ -376,7 +376,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
_mi_segment_page_free(page, force, segments_tld);
}
#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX
#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX
#define MI_RETIRE_CYCLES (8)
// Retire a page with no more used blocks
@@ -566,7 +566,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
if (page->capacity >= page->reserved) return;
size_t page_size;
//uint8_t* page_start =
//uint8_t* page_start =
_mi_page_start(_mi_page_segment(page), page, &page_size);
mi_stat_counter_increase(tld->stats.pages_extended, 1);
@@ -687,7 +687,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
page = mi_page_fresh(heap, pq);
if (page == NULL && first_try) {
// out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
page = mi_page_queue_find_free_ex(heap, pq, false);
page = mi_page_queue_find_free_ex(heap, pq, false);
}
}
else {
@@ -705,14 +705,17 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
mi_page_queue_t* pq = mi_page_queue(heap,size);
mi_page_t* page = pq->first;
if (page != NULL) {
if ((MI_SECURE >= 3) && page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) {
// in secure mode, we extend half the time to increase randomness
#if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness
if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) {
mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page));
}
else {
else
#endif
{
_mi_page_free_collect(page,false);
}
if (mi_page_immediate_available(page)) {
page->retire_expire = 0;
return page; // fast path
@@ -785,7 +788,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept {
// huge allocation?
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE) )) {
if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
_mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
@@ -830,7 +833,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
}
if (mi_unlikely(page == NULL)) { // out of memory
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
_mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
return NULL;
}
kotlin-native/runtime/src/mimalloc/c/random.c
+3 -3
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2019, Microsoft Research, Daan Leijen
Copyright (c) 2019-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -115,7 +115,7 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no
static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) {
memset(ctx_new, 0, sizeof(*ctx_new));
memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));
_mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input));
ctx_new->input[12] = 0;
ctx_new->input[13] = 0;
ctx_new->input[14] = (uint32_t)nonce;
@@ -244,7 +244,7 @@ static bool os_random_buf(void* buf, size_t buf_len) {
#endif
#if defined(_WIN32)
#include <Windows.h>
#include <windows.h>
#elif defined(__APPLE__)
#include <mach/mach_time.h>
#else
kotlin-native/runtime/src/mimalloc/c/region.c
+57 -52
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2019, Microsoft Research, Daan Leijen
Copyright (c) 2019-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -16,8 +16,8 @@ We need this memory layer between the raw OS calls because of:
1. on `sbrk` like systems (like WebAssembly) we need our own memory maps in order
to reuse memory effectively.
2. It turns out that for large objects, between 1MiB and 32MiB (?), the cost of
an OS allocation/free is still (much) too expensive relative to the accesses
in that object :-( (`malloc-large` tests this). This means we need a cheaper
an OS allocation/free is still (much) too expensive relative to the accesses
in that object :-( (`malloc-large` tests this). This means we need a cheaper
way to reuse memory.
3. This layer allows for NUMA aware allocation.
@@ -37,7 +37,7 @@ Possible issues:
#include <string.h> // memset
#include "bitmap.inc.c"
#include "bitmap.h"
// Internal raw OS interface
size_t _mi_os_large_page_size();
@@ -50,14 +50,14 @@ bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
// arena.c
void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats);
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
// Constants
#if (MI_INTPTR_SIZE==8)
#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map
#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map
#elif (MI_INTPTR_SIZE==4)
#define MI_HEAP_REGION_MAX_SIZE (3 * GiB) // ~ KiB for the region map
#else
@@ -70,14 +70,15 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo
#define MI_REGION_SIZE (MI_SEGMENT_SIZE * MI_BITMAP_FIELD_BITS) // 256MiB (64MiB on 32 bits)
#define MI_REGION_MAX (MI_HEAP_REGION_MAX_SIZE / MI_REGION_SIZE) // 1024 (48 on 32 bits)
#define MI_REGION_MAX_OBJ_BLOCKS (MI_REGION_MAX_BLOCKS/4) // 64MiB
#define MI_REGION_MAX_OBJ_SIZE (MI_REGION_MAX_OBJ_BLOCKS*MI_SEGMENT_SIZE)
#define MI_REGION_MAX_OBJ_SIZE (MI_REGION_MAX_OBJ_BLOCKS*MI_SEGMENT_SIZE)
// Region info
// Region info
typedef union mi_region_info_u {
uintptr_t value;
uintptr_t value;
struct {
bool valid; // initialized?
bool is_large; // allocated in fixed large/huge OS pages
bool is_large:1; // allocated in fixed large/huge OS pages
bool is_pinned:1; // pinned memory cannot be decommitted
short numa_node; // the associated NUMA node (where -1 means no associated node)
} x;
} mi_region_info_t;
@@ -87,7 +88,7 @@ typedef union mi_region_info_u {
// a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block.
typedef struct mem_region_s {
_Atomic(uintptr_t) info; // mi_region_info_t.value
_Atomic(void*) start; // start of the memory area
_Atomic(void*) start; // start of the memory area
mi_bitmap_field_t in_use; // bit per in-use block
mi_bitmap_field_t dirty; // track if non-zero per block
mi_bitmap_field_t commit; // track if committed per block
@@ -100,7 +101,7 @@ typedef struct mem_region_s {
static mem_region_t regions[MI_REGION_MAX];
// Allocated regions
static _Atomic(uintptr_t) regions_count; // = 0;
static _Atomic(uintptr_t) regions_count; // = 0;
/* ----------------------------------------------------------------------------
@@ -135,7 +136,7 @@ bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
static void* mi_region_blocks_start(const mem_region_t* region, mi_bitmap_index_t bit_idx) {
uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t, &((mem_region_t*)region)->start);
mi_assert_internal(start != NULL);
return (start + (bit_idx * MI_SEGMENT_SIZE));
return (start + (bit_idx * MI_SEGMENT_SIZE));
}
static size_t mi_memid_create(mem_region_t* region, mi_bitmap_index_t bit_idx) {
@@ -177,8 +178,9 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large,
bool region_commit = (commit && mi_option_is_enabled(mi_option_eager_region_commit));
bool region_large = (commit && allow_large);
bool is_zero = false;
bool is_pinned = false;
size_t arena_memid = 0;
void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, &region_commit, &region_large, &is_zero, &arena_memid, tld);
void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, &region_commit, &region_large, &is_pinned, &is_zero, &arena_memid, tld);
if (start == NULL) return false;
mi_assert_internal(!(region_large && !allow_large));
mi_assert_internal(!region_large || region_commit);
@@ -200,14 +202,15 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large,
mi_atomic_store_release(&r->commit, (region_commit ? MI_BITMAP_FIELD_FULL : 0));
mi_atomic_store_release(&r->reset, (uintptr_t)0);
*bit_idx = 0;
mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL);
_mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL);
mi_atomic_store_ptr_release(void,&r->start, start);
// and share it
// and share it
mi_region_info_t info;
info.value = 0; // initialize the full union to zero
info.x.valid = true;
info.x.is_large = region_large;
info.x.is_pinned = is_pinned;
info.x.numa_node = (short)_mi_os_numa_node(tld);
mi_atomic_store_release(&r->info, info.value); // now make it available to others
*region = r;
@@ -239,7 +242,7 @@ static bool mi_region_is_suitable(const mem_region_t* region, int numa_node, boo
static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld)
{
// try all regions for a free slot
// try all regions for a free slot
const size_t count = mi_atomic_load_relaxed(&regions_count); // monotonic, so ok to be relaxed
size_t idx = tld->region_idx; // Or start at 0 to reuse low addresses? Starting at 0 seems to increase latency though
for (size_t visited = 0; visited < count; visited++, idx++) {
@@ -248,7 +251,7 @@ static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large,
// if this region suits our demand (numa node matches, large OS page matches)
if (mi_region_is_suitable(r, numa_node, allow_large)) {
// then try to atomically claim a segment(s) in this region
if (mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) {
if (_mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) {
tld->region_idx = idx; // remember the last found position
*region = r;
return true;
@@ -259,64 +262,65 @@ static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large,
}
static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{
mi_assert_internal(blocks <= MI_BITMAP_FIELD_BITS);
mem_region_t* region;
mi_bitmap_index_t bit_idx;
const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld));
// try to claim in existing regions
if (!mi_region_try_claim(numa_node, blocks, *is_large, &region, &bit_idx, tld)) {
if (!mi_region_try_claim(numa_node, blocks, *large, &region, &bit_idx, tld)) {
// otherwise try to allocate a fresh region and claim in there
if (!mi_region_try_alloc_os(blocks, *commit, *is_large, &region, &bit_idx, tld)) {
if (!mi_region_try_alloc_os(blocks, *commit, *large, &region, &bit_idx, tld)) {
// out of regions or memory
return NULL;
}
}
// ------------------------------------------------
// found a region and claimed `blocks` at `bit_idx`, initialize them now
mi_assert_internal(region != NULL);
mi_assert_internal(mi_bitmap_is_claimed(&region->in_use, 1, blocks, bit_idx));
mi_assert_internal(_mi_bitmap_is_claimed(&region->in_use, 1, blocks, bit_idx));
mi_region_info_t info;
info.value = mi_atomic_load_acquire(&region->info);
uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,&region->start);
mi_assert_internal(!(info.x.is_large && !*is_large));
mi_assert_internal(!(info.x.is_large && !*large));
mi_assert_internal(start != NULL);
*is_zero = mi_bitmap_claim(&region->dirty, 1, blocks, bit_idx, NULL);
*is_large = info.x.is_large;
*memid = mi_memid_create(region, bit_idx);
*is_zero = _mi_bitmap_claim(&region->dirty, 1, blocks, bit_idx, NULL);
*large = info.x.is_large;
*is_pinned = info.x.is_pinned;
*memid = mi_memid_create(region, bit_idx);
void* p = start + (mi_bitmap_index_bit_in_field(bit_idx) * MI_SEGMENT_SIZE);
// commit
if (*commit) {
// ensure commit
bool any_uncommitted;
mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, &any_uncommitted);
_mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, &any_uncommitted);
if (any_uncommitted) {
mi_assert_internal(!info.x.is_large);
mi_assert_internal(!info.x.is_large && !info.x.is_pinned);
bool commit_zero = false;
if (!_mi_mem_commit(p, blocks * MI_SEGMENT_SIZE, &commit_zero, tld)) {
// failed to commit! unclaim and return
mi_bitmap_unclaim(&region->in_use, 1, blocks, bit_idx);
return NULL;
}
if (commit_zero) *is_zero = true;
if (commit_zero) *is_zero = true;
}
}
else {
// no need to commit, but check if already fully committed
*commit = mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx);
}
mi_assert_internal(!*commit || mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx));
*commit = _mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx);
}
mi_assert_internal(!*commit || _mi_bitmap_is_claimed(&region->commit, 1, blocks, bit_idx));
// unreset reset blocks
if (mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx)) {
if (_mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx)) {
// some blocks are still reset
mi_assert_internal(!info.x.is_large);
mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit || mi_option_get(mi_option_eager_commit_delay) > 0);
mi_assert_internal(!info.x.is_large && !info.x.is_pinned);
mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit || mi_option_get(mi_option_eager_commit_delay) > 0);
mi_bitmap_unclaim(&region->reset, 1, blocks, bit_idx);
if (*commit || !mi_option_is_enabled(mi_option_reset_decommits)) { // only if needed
bool reset_zero = false;
@@ -324,14 +328,14 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo
if (reset_zero) *is_zero = true;
}
}
mi_assert_internal(!mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx));
mi_assert_internal(!_mi_bitmap_is_any_claimed(&region->reset, 1, blocks, bit_idx));
#if (MI_DEBUG>=2)
if (*commit) { ((uint8_t*)p)[0] = 0; }
#endif
// and return the allocation
mi_assert_internal(p != NULL);
// and return the allocation
mi_assert_internal(p != NULL);
return p;
}
@@ -342,14 +346,15 @@ static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* is_large, bo
// Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`.
// (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`)
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{
mi_assert_internal(memid != NULL && tld != NULL);
mi_assert_internal(size > 0);
*memid = 0;
*is_zero = false;
*is_pinned = false;
bool default_large = false;
if (large==NULL) large = &default_large; // ensure `large != NULL`
if (large==NULL) large = &default_large; // ensure `large != NULL`
if (size == 0) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
@@ -358,14 +363,14 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l
size_t arena_memid;
const size_t blocks = mi_region_block_count(size);
if (blocks <= MI_REGION_MAX_OBJ_BLOCKS && alignment <= MI_SEGMENT_ALIGN) {
p = mi_region_try_alloc(blocks, commit, large, is_zero, memid, tld);
p = mi_region_try_alloc(blocks, commit, large, is_pinned, is_zero, memid, tld);
if (p == NULL) {
_mi_warning_message("unable to allocate from region: size %zu\n", size);
}
}
if (p == NULL) {
// and otherwise fall back to the OS
p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_zero, &arena_memid, tld);
p = _mi_arena_alloc_aligned(size, alignment, commit, large, is_pinned, is_zero, &arena_memid, tld);
*memid = mi_memid_create_from_arena(arena_memid);
}
@@ -390,7 +395,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re
if (p==NULL) return;
if (size==0) return;
size = _mi_align_up(size, _mi_os_page_size());
size_t arena_memid = 0;
mi_bitmap_index_t bit_idx;
mem_region_t* region;
@@ -413,21 +418,21 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re
// committed?
if (full_commit && (size % MI_SEGMENT_SIZE) == 0) {
mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, NULL);
_mi_bitmap_claim(&region->commit, 1, blocks, bit_idx, NULL);
}
if (any_reset) {
// set the is_reset bits if any pages were reset
mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, NULL);
_mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, NULL);
}
// reset the blocks to reduce the working set.
if (!info.x.is_large && mi_option_is_enabled(mi_option_segment_reset)
if (!info.x.is_large && !info.x.is_pinned && mi_option_is_enabled(mi_option_segment_reset)
&& (mi_option_is_enabled(mi_option_eager_commit) ||
mi_option_is_enabled(mi_option_reset_decommits))) // cannot reset halfway committed segments, use only `option_page_reset` instead
mi_option_is_enabled(mi_option_reset_decommits))) // cannot reset halfway committed segments, use only `option_page_reset` instead
{
bool any_unreset;
mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, &any_unreset);
_mi_bitmap_claim(&region->reset, 1, blocks, bit_idx, &any_unreset);
if (any_unreset) {
_mi_abandoned_await_readers(); // ensure no more pending write (in case reset = decommit)
_mi_mem_reset(p, blocks * MI_SEGMENT_SIZE, tld);
kotlin-native/runtime/src/mimalloc/c/segment.c
+25 -28
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -237,7 +237,7 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t*
static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld) {
mi_assert_internal(page->is_committed);
if (!mi_option_is_enabled(mi_option_page_reset)) return;
if (segment->mem_is_fixed || page->segment_in_use || !page->is_committed || page->is_reset) return;
if (segment->mem_is_pinned || page->segment_in_use || !page->is_committed || page->is_reset) return;
size_t psize;
void* start = mi_segment_raw_page_start(segment, page, &psize);
page->is_reset = true;
@@ -250,8 +250,8 @@ static bool mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size,
{
mi_assert_internal(page->is_reset);
mi_assert_internal(page->is_committed);
mi_assert_internal(!segment->mem_is_fixed);
if (segment->mem_is_fixed || !page->is_committed || !page->is_reset) return true;
mi_assert_internal(!segment->mem_is_pinned);
if (segment->mem_is_pinned || !page->is_committed || !page->is_reset) return true;
page->is_reset = false;
size_t psize;
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
@@ -290,7 +290,7 @@ static void mi_pages_reset_add(mi_segment_t* segment, mi_page_t* page, mi_segmen
mi_assert_expensive(!mi_pages_reset_contains(page, tld));
mi_assert_internal(_mi_page_segment(page)==segment);
if (!mi_option_is_enabled(mi_option_page_reset)) return;
if (segment->mem_is_fixed || page->segment_in_use || !page->is_committed || page->is_reset) return;
if (segment->mem_is_pinned || page->segment_in_use || !page->is_committed || page->is_reset) return;
if (mi_option_get(mi_option_reset_delay) == 0) {
// reset immediately?
@@ -330,7 +330,7 @@ static void mi_pages_reset_remove(mi_page_t* page, mi_segments_tld_t* tld) {
}
static void mi_pages_reset_remove_all_in_segment(mi_segment_t* segment, bool force_reset, mi_segments_tld_t* tld) {
if (segment->mem_is_fixed) return; // never reset in huge OS pages
if (segment->mem_is_pinned) return; // never reset in huge OS pages
for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i];
if (!page->segment_in_use && page->is_committed && !page->is_reset) {
@@ -385,11 +385,13 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_
psize -= segment->segment_info_size;
}
if (MI_SECURE > 1 || (MI_SECURE == 1 && page->segment_idx == segment->capacity - 1)) {
// secure == 1: the last page has an os guard page at the end
// secure > 1: every page has an os guard page
#if (MI_SECURE > 1) // every page has an os guard page
psize -= _mi_os_page_size();
#elif (MI_SECURE==1) // the last page has an os guard page at the end
if (page->segment_idx == segment->capacity - 1) {
psize -= _mi_os_page_size();
}
#endif
if (page_size != NULL) *page_size = psize;
mi_assert_internal(page->xblock_size == 0 || _mi_ptr_page(p) == page);
@@ -464,7 +466,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
segment->thread_id = 0;
mi_segments_track_size(-((long)segment_size),tld);
if (MI_SECURE != 0) {
mi_assert_internal(!segment->mem_is_fixed);
mi_assert_internal(!segment->mem_is_pinned);
mi_segment_protect(segment, false, tld->os); // ensure no more guard pages are set
}
@@ -593,7 +595,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
else
{
if (MI_SECURE!=0) {
mi_assert_internal(!segment->mem_is_fixed);
mi_assert_internal(!segment->mem_is_pinned);
mi_segment_protect(segment, false, tld->os); // reset protection if the page kind differs
}
// different page kinds; unreset any reset pages, and unprotect
@@ -624,26 +626,28 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
// Allocate the segment from the OS
size_t memid;
bool mem_large = (!eager_delayed && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy
segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_zero, &memid, os_tld);
bool is_pinned = false;
segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld);
if (segment == NULL) return NULL; // failed to allocate
if (!commit) {
// ensure the initial info is committed
mi_assert_internal(!mem_large && !is_pinned);
bool commit_zero = false;
bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os);
if (commit_zero) is_zero = true;
if (!ok) {
// commit failed; we cannot touch the memory: free the segment directly and return `NULL`
_mi_mem_free(segment, MI_SEGMENT_SIZE, memid, false, false, os_tld);
return NULL;
return NULL;
}
}
segment->memid = memid;
segment->mem_is_fixed = mem_large;
segment->mem_is_committed = commit;
segment->mem_is_pinned = (mem_large || is_pinned);
segment->mem_is_committed = commit;
mi_segments_track_size((long)segment_size, tld);
}
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
mi_assert_internal(segment->mem_is_fixed ? segment->mem_is_committed : true);
mi_assert_internal(segment->mem_is_pinned ? segment->mem_is_committed : true);
mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan
if (!pages_still_good) {
// zero the segment info (but not the `mem` fields)
@@ -728,8 +732,8 @@ static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg
mi_pages_reset_remove(page, tld);
// check commit
if (!page->is_committed) {
mi_assert_internal(!segment->mem_is_fixed);
mi_assert_internal(!page->is_reset);
mi_assert_internal(!segment->mem_is_pinned);
mi_assert_internal(!page->is_reset);
size_t psize;
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
bool is_zero = false;
@@ -745,8 +749,8 @@ static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg
segment->used++;
// check reset
if (page->is_reset) {
mi_assert_internal(!segment->mem_is_fixed);
bool ok = mi_page_unreset(segment, page, 0, tld);
mi_assert_internal(!segment->mem_is_pinned);
bool ok = mi_page_unreset(segment, page, 0, tld);
if (!ok) {
page->segment_in_use = false;
segment->used--;
@@ -892,7 +896,7 @@ static mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // =
static mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL
// Maintain these for debug purposes (these counts may be a bit off)
static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count;
static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count;
static mi_decl_cache_align _Atomic(uintptr_t) abandoned_visited_count;
// We also maintain a count of current readers of the abandoned list
@@ -1330,13 +1334,6 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block
page->is_zero = false;
mi_assert(page->used == 0);
mi_tld_t* tld = heap->tld;
const size_t bsize = mi_page_usable_block_size(page);
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&tld->stats.giant, bsize);
}
else {
_mi_stat_decrease(&tld->stats.huge, bsize);
}
mi_segments_track_size((long)segment->segment_size, &tld->segments);
_mi_segment_page_free(page, true, &tld->segments);
}
kotlin-native/runtime/src/mimalloc/c/static.c
+2 -1
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -23,6 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "stats.c"
#include "random.c"
#include "os.c"
#include "bitmap.c"
#include "arena.c"
#include "region.c"
#include "segment.c"
kotlin-native/runtime/src/mimalloc/c/stats.c
+29 -24
View File
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"licenses/third_party/mimalloc_LICENSE.txt" at the root of this distribution.
@@ -21,7 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file
static bool mi_is_in_main(void* stat) {
return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main
&& (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));
&& (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));
}
static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
@@ -51,7 +51,7 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
}
}
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
if (mi_is_in_main(stat)) {
mi_atomic_addi64_relaxed( &stat->count, 1 );
mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount );
@@ -103,6 +103,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
mi_stat_add(&stats->malloc, &src->malloc, 1);
mi_stat_add(&stats->segments_cache, &src->segments_cache, 1);
mi_stat_add(&stats->normal, &src->normal, 1);
mi_stat_add(&stats->huge, &src->huge, 1);
mi_stat_add(&stats->giant, &src->giant, 1);
@@ -112,12 +113,13 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1);
mi_stat_counter_add(&stats->searches, &src->searches, 1);
mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1);
mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1);
mi_stat_counter_add(&stats->giant_count, &src->giant_count, 1);
#if MI_STAT>1
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
if (src->normal[i].allocated > 0 || src->normal[i].freed > 0) {
mi_stat_add(&stats->normal[i], &src->normal[i], 1);
if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) {
mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i], 1);
}
}
#endif
@@ -127,7 +129,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
Display statistics
----------------------------------------------------------- */
// unit > 0 : size in binary bytes
// unit > 0 : size in binary bytes
// unit == 0: count as decimal
// unit < 0 : count in binary
static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) {
@@ -149,7 +151,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
const int64_t tens = (n / (divider/10));
const long whole = (long)(tens/10);
const long frac1 = (long)(tens%10);
snprintf(buf, len, "%ld.%ld %s%s", whole, frac1, magnitude, suffix);
snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix);
}
_mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf);
}
@@ -170,6 +172,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
mi_print_amount(stat->peak, unit, out, arg);
mi_print_amount(stat->allocated, unit, out, arg);
mi_print_amount(stat->freed, unit, out, arg);
mi_print_amount(stat->current, unit, out, arg);
mi_print_amount(unit, 1, out, arg);
mi_print_count(stat->allocated, unit, out, arg);
if (stat->allocated > stat->freed)
@@ -181,6 +184,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
mi_print_amount(stat->peak, -1, out, arg);
mi_print_amount(stat->allocated, -1, out, arg);
mi_print_amount(stat->freed, -1, out, arg);
mi_print_amount(stat->current, -1, out, arg);
if (unit==-1) {
_mi_fprintf(out, arg, "%22s", "");
}
@@ -196,6 +200,8 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
else {
mi_print_amount(stat->peak, 1, out, arg);
mi_print_amount(stat->allocated, 1, out, arg);
_mi_fprintf(out, arg, "%11s", " "); // no freed
mi_print_amount(stat->current, 1, out, arg);
_mi_fprintf(out, arg, "\n");
}
}
@@ -215,11 +221,11 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char*
static void mi_print_header(mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count ");
_mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count ");
}
#if MI_STAT>1
static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) {
static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out, void* arg) {
bool found = false;
char buf[64];
for (size_t i = 0; i <= max; i++) {
@@ -227,12 +233,9 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin
found = true;
int64_t unit = _mi_bin_size((uint8_t)i);
snprintf(buf, 64, "%s %3lu", fmt, (long)i);
mi_stat_add(all, &bins[i], unit);
mi_stat_print(&bins[i], buf, unit, out, arg);
}
}
//snprintf(buf, 64, "%s all", fmt);
//mi_stat_print(all, buf, 1);
if (found) {
_mi_fprintf(out, arg, "\n");
mi_print_header(out, arg);
@@ -289,19 +292,21 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
// and print using that
mi_print_header(out,arg);
#if MI_STAT>1
mi_stat_count_t normal = { 0,0,0,0 };
mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out,arg);
mi_stat_print(&normal, "normal", 1, out, arg);
mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg);
#endif
#if MI_STAT
mi_stat_print(&stats->normal, "normal", (stats->normal_count.count == 0 ? 1 : -(stats->normal.allocated / stats->normal_count.count)), out, arg);
mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out, arg);
mi_stat_print(&stats->giant, "giant", (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out, arg);
mi_stat_count_t total = { 0,0,0,0 };
mi_stat_add(&total, &normal, 1);
mi_stat_add(&total, &stats->normal, 1);
mi_stat_add(&total, &stats->huge, 1);
mi_stat_add(&total, &stats->giant, 1);
mi_stat_print(&total, "total", 1, out, arg);
_mi_fprintf(out, arg, "malloc requested: ");
mi_print_amount(stats->malloc.allocated, 1, out, arg);
_mi_fprintf(out, arg, "\n\n");
#endif
#if MI_STAT>1
mi_stat_print(&stats->malloc, "malloc req", 1, out, arg);
_mi_fprintf(out, arg, "\n");
#endif
mi_stat_print(&stats->reserved, "reserved", 1, out, arg);
mi_stat_print(&stats->committed, "committed", 1, out, arg);
@@ -388,7 +393,7 @@ void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
// Basic timer for convenience; use milli-seconds to avoid doubles
// ----------------------------------------------------------------
#ifdef _WIN32
#include <Windows.h>
#include <windows.h>
static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) {
static LARGE_INTEGER mfreq; // = 0
if (mfreq.QuadPart == 0LL) {
@@ -443,7 +448,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) {
// --------------------------------------------------------
#if defined(_WIN32)
#include <Windows.h>
#include <windows.h>
#include <psapi.h>
#pragma comment(lib,"psapi.lib")
@@ -474,12 +479,12 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec
*page_faults = (size_t)info.PageFaultCount;
}
#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__HAIKU__)
#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)
#include <stdio.h>
#include <unistd.h>
#include <sys/resource.h>
#if defined(__APPLE__) && defined(__MACH__)
#if defined(__APPLE__)
#include <mach/mach.h>
#endif
@@ -515,7 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec
while (get_next_area_info(tid.team, &c, &mem) == B_OK) {
*peak_rss += mem.ram_size;
}
#elif defined(__APPLE__) && defined(__MACH__)
#elif defined(__APPLE__)
*peak_rss = rusage.ru_maxrss; // BSD reports in bytes
struct mach_task_basic_info info;
mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;