[K/N] Track current heap size ^KT-55364
This commit is contained in:
committed by
Space Cloud
parent
18b6351d5d
commit
cb7698a8c2
@@ -175,6 +175,8 @@ bitcode {
|
|||||||
sourceSets {
|
sourceSets {
|
||||||
main {}
|
main {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
compilerArgs.add("-DKONAN_MI_MALLOC=1")
|
||||||
}
|
}
|
||||||
|
|
||||||
module("exceptionsSupport") {
|
module("exceptionsSupport") {
|
||||||
|
|||||||
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
#include "CustomLogging.hpp"
|
#include "CustomLogging.hpp"
|
||||||
|
|
||||||
|
#include "GCApi.hpp"
|
||||||
|
|
||||||
// These functions are just stubs to make the existing object creation
|
// These functions are just stubs to make the existing object creation
|
||||||
// infrastructure link correctly, but if they are ever called, something went
|
// infrastructure link correctly, but if they are ever called, something went
|
||||||
// wrong.
|
// wrong.
|
||||||
@@ -16,12 +18,16 @@ void* allocateInObjectPool(size_t size) noexcept {
|
|||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
void freeInObjectPool(void* ptr) noexcept {
|
void freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||||
CustomAllocWarning("static freeInObjectPool(%p) not supported", ptr);
|
CustomAllocWarning("static freeInObjectPool(%p, %zu) not supported", ptr, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
void initObjectPool() noexcept {}
|
void initObjectPool() noexcept {}
|
||||||
void compactObjectPoolInMainThread() noexcept {}
|
void compactObjectPoolInMainThread() noexcept {}
|
||||||
void compactObjectPoolInCurrentThread() noexcept {}
|
void compactObjectPoolInCurrentThread() noexcept {}
|
||||||
|
|
||||||
|
size_t allocatedBytes() noexcept {
|
||||||
|
return alloc::GetAllocatedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace kotlin
|
} // namespace kotlin
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ ExtraObjectPage::ExtraObjectPage() noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void ExtraObjectPage::Destroy() noexcept {
|
void ExtraObjectPage::Destroy() noexcept {
|
||||||
std_support::free(this);
|
Free(this, EXTRA_OBJECT_PAGE_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept {
|
mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept {
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) {
|
|||||||
EXPECT_EQ(prev + sizeof(Cell), cur);
|
EXPECT_EQ(prev + sizeof(Cell), cur);
|
||||||
prev = cur;
|
prev = cur;
|
||||||
}
|
}
|
||||||
free(page);
|
page->Destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
|
TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
|
||||||
@@ -43,7 +43,7 @@ TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) {
|
|||||||
Queue finalizerQueue;
|
Queue finalizerQueue;
|
||||||
EXPECT_FALSE(page->Sweep(finalizerQueue));
|
EXPECT_FALSE(page->Sweep(finalizerQueue));
|
||||||
EXPECT_EQ(finalizerQueue.size(), size_t(0));
|
EXPECT_EQ(finalizerQueue.size(), size_t(0));
|
||||||
free(page);
|
page->Destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
|
TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
|
||||||
@@ -58,7 +58,7 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) {
|
|||||||
Queue finalizerQueue;
|
Queue finalizerQueue;
|
||||||
EXPECT_FALSE(page->Sweep(finalizerQueue));
|
EXPECT_FALSE(page->Sweep(finalizerQueue));
|
||||||
EXPECT_EQ(finalizerQueue.size(), size_t(0));
|
EXPECT_EQ(finalizerQueue.size(), size_t(0));
|
||||||
free(page);
|
page->Destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ FixedBlockPage* FixedBlockPage::Create(uint32_t blockSize) noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void FixedBlockPage::Destroy() noexcept {
|
void FixedBlockPage::Destroy() noexcept {
|
||||||
std_support::free(this);
|
Free(this, FIXED_BLOCK_PAGE_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) {
|
FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
|
|
||||||
#include "GCApi.hpp"
|
#include "GCApi.hpp"
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
|
|
||||||
#include "ConcurrentMarkAndSweep.hpp"
|
#include "ConcurrentMarkAndSweep.hpp"
|
||||||
@@ -13,6 +14,12 @@
|
|||||||
#include "KAssert.h"
|
#include "KAssert.h"
|
||||||
#include "ObjectFactory.hpp"
|
#include "ObjectFactory.hpp"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
std::atomic<size_t> allocatedBytesCounter;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
namespace kotlin::alloc {
|
namespace kotlin::alloc {
|
||||||
|
|
||||||
bool TryResetMark(void* ptr) noexcept {
|
bool TryResetMark(void* ptr) noexcept {
|
||||||
@@ -83,7 +90,17 @@ void* SafeAlloc(uint64_t size) noexcept {
|
|||||||
konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes. Aborting.\n", size);
|
konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes. Aborting.\n", size);
|
||||||
konan::abort();
|
konan::abort();
|
||||||
}
|
}
|
||||||
|
allocatedBytesCounter.fetch_add(static_cast<size_t>(size), std::memory_order_relaxed);
|
||||||
return memory;
|
return memory;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Free(void* ptr, size_t size) noexcept {
|
||||||
|
std_support::free(ptr);
|
||||||
|
allocatedBytesCounter.fetch_sub(static_cast<size_t>(size), std::memory_order_relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t GetAllocatedBytes() noexcept {
|
||||||
|
return allocatedBytesCounter.load(std::memory_order_relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace kotlin::alloc
|
} // namespace kotlin::alloc
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ bool TryResetMark(void* ptr) noexcept;
|
|||||||
bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
|
bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectCell>& finalizerQueue) noexcept;
|
||||||
|
|
||||||
void* SafeAlloc(uint64_t size) noexcept;
|
void* SafeAlloc(uint64_t size) noexcept;
|
||||||
|
void Free(void* ptr, size_t size) noexcept;
|
||||||
|
|
||||||
|
size_t GetAllocatedBytes() noexcept;
|
||||||
|
|
||||||
} // namespace kotlin::alloc
|
} // namespace kotlin::alloc
|
||||||
|
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ AtomicStack<ExtraObjectCell> Heap::SweepExtraObjects(gc::GCHandle gcHandle) noex
|
|||||||
while ((page = usedExtraObjectPages_.Pop())) {
|
while ((page = usedExtraObjectPages_.Pop())) {
|
||||||
if (!page->Sweep(finalizerQueue)) {
|
if (!page->Sweep(finalizerQueue)) {
|
||||||
CustomAllocInfo("SweepExtraObjects free(%p)", page);
|
CustomAllocInfo("SweepExtraObjects free(%p)", page);
|
||||||
free(page);
|
page->Destroy();
|
||||||
} else {
|
} else {
|
||||||
extraObjectPages_.Push(page);
|
extraObjectPages_.Push(page);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ NextFitPage* NextFitPage::Create(uint32_t cellCount) noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void NextFitPage::Destroy() noexcept {
|
void NextFitPage::Destroy() noexcept {
|
||||||
std_support::free(this);
|
Free(this, NEXT_FIT_PAGE_SIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
NextFitPage::NextFitPage(uint32_t cellCount) noexcept : curBlock_(cells_) {
|
NextFitPage::NextFitPage(uint32_t cellCount) noexcept : curBlock_(cells_) {
|
||||||
|
|||||||
@@ -18,11 +18,13 @@ SingleObjectPage* SingleObjectPage::Create(uint64_t cellCount) noexcept {
|
|||||||
CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount);
|
CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount);
|
||||||
RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage");
|
RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage");
|
||||||
uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t);
|
uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t);
|
||||||
return new (SafeAlloc(size)) SingleObjectPage();
|
auto* page = new (SafeAlloc(size)) SingleObjectPage();
|
||||||
|
page->size_ = size;
|
||||||
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SingleObjectPage::Destroy() noexcept {
|
void SingleObjectPage::Destroy() noexcept {
|
||||||
std_support::free(this);
|
Free(this, size_);
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t* SingleObjectPage::Data() noexcept {
|
uint8_t* SingleObjectPage::Data() noexcept {
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ private:
|
|||||||
friend class AtomicStack<SingleObjectPage>;
|
friend class AtomicStack<SingleObjectPage>;
|
||||||
SingleObjectPage* next_;
|
SingleObjectPage* next_;
|
||||||
bool isAllocated_ = false;
|
bool isAllocated_ = false;
|
||||||
|
size_t size_;
|
||||||
struct alignas(8) {
|
struct alignas(8) {
|
||||||
uint8_t data_[];
|
uint8_t data_[];
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -105,6 +105,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
|
|||||||
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
|
return allocatedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
||||||
return impl_->gcScheduler().config();
|
return impl_->gcScheduler().config();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ namespace gc {
|
|||||||
class Allocator {
|
class Allocator {
|
||||||
public:
|
public:
|
||||||
void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); }
|
void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); }
|
||||||
static void Free(void* instance) noexcept { freeInObjectPool(instance); }
|
static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); }
|
||||||
};
|
};
|
||||||
|
|
||||||
template <typename BaseAllocator, typename GCThreadData>
|
template <typename BaseAllocator, typename GCThreadData>
|
||||||
@@ -37,7 +37,7 @@ public:
|
|||||||
return base_.Alloc(size);
|
return base_.Alloc(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void Free(void* instance) noexcept { BaseAllocator::Free(instance); }
|
static void Free(void* instance, size_t size) noexcept { BaseAllocator::Free(instance, size); }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
BaseAllocator base_;
|
BaseAllocator base_;
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ public:
|
|||||||
size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
|
size_t GetTotalHeapObjectsSizeUnsafe() const noexcept;
|
||||||
size_t GetExtraObjectsCountUnsafe() const noexcept;
|
size_t GetExtraObjectsCountUnsafe() const noexcept;
|
||||||
size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
|
size_t GetTotalExtraObjectsSizeUnsafe() const noexcept;
|
||||||
|
size_t GetTotalHeapObjectsSizeBytes() const noexcept;
|
||||||
|
|
||||||
gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
|
gc::GCSchedulerConfig& gcSchedulerConfig() noexcept;
|
||||||
|
|
||||||
|
|||||||
@@ -80,6 +80,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
|
|||||||
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
|
return allocatedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
||||||
return impl_->gcScheduler().config();
|
return impl_->gcScheduler().config();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,6 +93,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept {
|
|||||||
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept {
|
||||||
|
return allocatedBytes();
|
||||||
|
}
|
||||||
|
|
||||||
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept {
|
||||||
return impl_->gcScheduler().config();
|
return impl_->gcScheduler().config();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1123,7 +1123,7 @@ void processFinalizerQueue(MemoryState* state) {
|
|||||||
state->containers->erase(container);
|
state->containers->erase(container);
|
||||||
#endif
|
#endif
|
||||||
CONTAINER_DESTROY_EVENT(state, container)
|
CONTAINER_DESTROY_EVENT(state, container)
|
||||||
freeInObjectPool(container);
|
freeInObjectPool(container, 0);
|
||||||
atomicAdd(&allocCount, -1);
|
atomicAdd(&allocCount, -1);
|
||||||
}
|
}
|
||||||
RuntimeAssert(state->finalizerQueueSize == 0, "Queue must be empty here");
|
RuntimeAssert(state->finalizerQueueSize == 0, "Queue must be empty here");
|
||||||
@@ -1164,7 +1164,7 @@ void scheduleDestroyContainer(MemoryState* state, ContainerHeader* container) {
|
|||||||
processFinalizerQueue(state);
|
processFinalizerQueue(state);
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
freeInObjectPool(container);
|
freeInObjectPool(container), 0;
|
||||||
atomicAdd(&allocCount, -1);
|
atomicAdd(&allocCount, -1);
|
||||||
CONTAINER_DESTROY_EVENT(state, container);
|
CONTAINER_DESTROY_EVENT(state, container);
|
||||||
#endif
|
#endif
|
||||||
@@ -3142,7 +3142,7 @@ void ArenaContainer::Deinit() {
|
|||||||
while (chunk != nullptr) {
|
while (chunk != nullptr) {
|
||||||
auto toRemove = chunk;
|
auto toRemove = chunk;
|
||||||
chunk = chunk->next;
|
chunk = chunk->next;
|
||||||
freeInObjectPool(toRemove);
|
freeInObjectPool(toRemove, 0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,13 +12,15 @@ namespace kotlin {
|
|||||||
|
|
||||||
void initObjectPool() noexcept;
|
void initObjectPool() noexcept;
|
||||||
void* allocateInObjectPool(size_t size) noexcept;
|
void* allocateInObjectPool(size_t size) noexcept;
|
||||||
void freeInObjectPool(void* ptr) noexcept;
|
void freeInObjectPool(void* ptr, size_t size) noexcept;
|
||||||
// Instruct the allocator to free unused resources.
|
// Instruct the allocator to free unused resources.
|
||||||
void compactObjectPoolInCurrentThread() noexcept;
|
void compactObjectPoolInCurrentThread() noexcept;
|
||||||
// Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread.
|
// Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread.
|
||||||
// May do nothing if the main thread is not an event loop.
|
// May do nothing if the main thread is not an event loop.
|
||||||
void compactObjectPoolInMainThread() noexcept;
|
void compactObjectPoolInMainThread() noexcept;
|
||||||
|
|
||||||
|
size_t allocatedBytes() noexcept;
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
struct ObjectPoolAllocator {
|
struct ObjectPoolAllocator {
|
||||||
using value_type = T;
|
using value_type = T;
|
||||||
@@ -36,7 +38,7 @@ struct ObjectPoolAllocator {
|
|||||||
|
|
||||||
T* allocate(std::size_t n) noexcept { return static_cast<T*>(allocateInObjectPool(n * sizeof(T))); }
|
T* allocate(std::size_t n) noexcept { return static_cast<T*>(allocateInObjectPool(n * sizeof(T))); }
|
||||||
|
|
||||||
void deallocate(T* p, std::size_t n) noexcept { freeInObjectPool(p); }
|
void deallocate(T* p, std::size_t n) noexcept { freeInObjectPool(p, n * sizeof(T)); }
|
||||||
};
|
};
|
||||||
|
|
||||||
template <typename T, typename U>
|
template <typename T, typename U>
|
||||||
|
|||||||
@@ -373,6 +373,10 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, s
|
|||||||
mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
|
mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
|
||||||
mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
|
mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
|
||||||
|
|
||||||
|
#if KONAN_MI_MALLOC
|
||||||
|
mi_decl_nodiscard mi_decl_export size_t mi_allocated_size(void) mi_attr_noexcept;
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -103,6 +103,13 @@ static mem_region_t regions[MI_REGION_MAX];
|
|||||||
// Allocated regions
|
// Allocated regions
|
||||||
static _Atomic(uintptr_t) regions_count; // = 0;
|
static _Atomic(uintptr_t) regions_count; // = 0;
|
||||||
|
|
||||||
|
#if KONAN_MI_MALLOC
|
||||||
|
static _Atomic(size_t) allocated_size = 0;
|
||||||
|
|
||||||
|
size_t mi_allocated_size(void) mi_attr_noexcept {
|
||||||
|
return mi_atomic_load_relaxed(&allocated_size);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
/* ----------------------------------------------------------------------------
|
/* ----------------------------------------------------------------------------
|
||||||
Utility functions
|
Utility functions
|
||||||
@@ -378,6 +385,9 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l
|
|||||||
mi_assert_internal((uintptr_t)p % alignment == 0);
|
mi_assert_internal((uintptr_t)p % alignment == 0);
|
||||||
#if (MI_DEBUG>=2)
|
#if (MI_DEBUG>=2)
|
||||||
if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed
|
if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed
|
||||||
|
#endif
|
||||||
|
#if KONAN_MI_MALLOC
|
||||||
|
mi_atomic_add_relaxed(&allocated_size, size);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
return p;
|
return p;
|
||||||
@@ -443,6 +453,9 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re
|
|||||||
bool all_unclaimed = mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx);
|
bool all_unclaimed = mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx);
|
||||||
mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed);
|
mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed);
|
||||||
}
|
}
|
||||||
|
#if KONAN_MI_MALLOC
|
||||||
|
mi_atomic_sub_relaxed(&allocated_size, size);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ namespace internal {
|
|||||||
// uses `Allocator` to allocate and free memory.
|
// uses `Allocator` to allocate and free memory.
|
||||||
// Precondition on `Allocator`: must allocate with alignment at least `DataAlignment`.
|
// Precondition on `Allocator`: must allocate with alignment at least `DataAlignment`.
|
||||||
// TODO: Consider merging with `MultiSourceQueue` somehow.
|
// TODO: Consider merging with `MultiSourceQueue` somehow.
|
||||||
template <size_t DataAlignment, typename Allocator>
|
template <size_t DataAlignment, typename Allocator, typename DataSizeProvider>
|
||||||
class ObjectFactoryStorage : private Pinned {
|
class ObjectFactoryStorage : private Pinned {
|
||||||
static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment");
|
static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment");
|
||||||
|
|
||||||
@@ -38,8 +38,9 @@ class ObjectFactoryStorage : private Pinned {
|
|||||||
class Deleter {
|
class Deleter {
|
||||||
public:
|
public:
|
||||||
void operator()(T* instance) noexcept {
|
void operator()(T* instance) noexcept {
|
||||||
|
auto size = instance->GetAllocatedSize();
|
||||||
instance->~T();
|
instance->~T();
|
||||||
Allocator::Free(instance);
|
Allocator::Free(instance, size);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -81,6 +82,12 @@ public:
|
|||||||
return *static_cast<T*>(Data());
|
return *static_cast<T*>(Data());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t GetAllocatedSize() noexcept {
|
||||||
|
auto dataSize = DataSizeProvider::GetDataSize(Data());
|
||||||
|
auto totalSize = GetSizeForDataSize(dataSize);
|
||||||
|
return totalSize;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
friend class ObjectFactoryStorage;
|
friend class ObjectFactoryStorage;
|
||||||
|
|
||||||
@@ -447,8 +454,34 @@ class ObjectFactory : private Pinned {
|
|||||||
alignas(kObjectAlignment) ArrayHeader array;
|
alignas(kObjectAlignment) ArrayHeader array;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
|
||||||
|
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
|
||||||
|
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
|
||||||
|
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
|
||||||
|
// at about half of uint64_t max.
|
||||||
|
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
|
||||||
|
// Note: array body is aligned, but for size computation it is enough to align the sum.
|
||||||
|
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DataSizeProvider {
|
||||||
|
static size_t GetDataSize(void* data) noexcept {
|
||||||
|
ObjHeader* object = &static_cast<HeapObjHeader*>(data)->object;
|
||||||
|
RuntimeAssert(object->heap(), "Object must be a heap object");
|
||||||
|
const auto* typeInfo = object->type_info();
|
||||||
|
if (typeInfo->IsArray()) {
|
||||||
|
return ArrayAllocatedDataSize(typeInfo, object->array()->count_);
|
||||||
|
} else {
|
||||||
|
return ObjectAllocatedDataSize(typeInfo);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
public:
|
public:
|
||||||
using Storage = internal::ObjectFactoryStorage<kObjectAlignment, Allocator>;
|
using Storage = internal::ObjectFactoryStorage<kObjectAlignment, Allocator, DataSizeProvider>;
|
||||||
|
|
||||||
class NodeRef {
|
class NodeRef {
|
||||||
public:
|
public:
|
||||||
@@ -561,19 +594,6 @@ public:
|
|||||||
void ClearForTests() noexcept { producer_.ClearForTests(); }
|
void ClearForTests() noexcept { producer_.ClearForTests(); }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept {
|
|
||||||
size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader);
|
|
||||||
return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment);
|
|
||||||
}
|
|
||||||
|
|
||||||
static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept {
|
|
||||||
// -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped
|
|
||||||
// at about half of uint64_t max.
|
|
||||||
uint64_t membersSize = static_cast<uint64_t>(-typeInfo->instanceSize_) * count;
|
|
||||||
// Note: array body is aligned, but for size computation it is enough to align the sum.
|
|
||||||
return AlignUp<uint64_t>(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment);
|
|
||||||
}
|
|
||||||
|
|
||||||
typename Storage::Producer producer_;
|
typename Storage::Producer producer_;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -29,8 +29,12 @@ namespace {
|
|||||||
|
|
||||||
using SimpleAllocator = gc::Allocator;
|
using SimpleAllocator = gc::Allocator;
|
||||||
|
|
||||||
|
struct DataSizeProvider {
|
||||||
|
static size_t GetDataSize(void* data) noexcept { return 0; }
|
||||||
|
};
|
||||||
|
|
||||||
template <size_t DataAlignment>
|
template <size_t DataAlignment>
|
||||||
using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator>;
|
using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage<DataAlignment, SimpleAllocator, DataSizeProvider>;
|
||||||
|
|
||||||
using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
|
using ObjectFactoryStorageRegular = ObjectFactoryStorage<alignof(void*)>;
|
||||||
|
|
||||||
@@ -782,11 +786,11 @@ public:
|
|||||||
~MockAllocator();
|
~MockAllocator();
|
||||||
|
|
||||||
MOCK_METHOD(void*, Alloc, (size_t));
|
MOCK_METHOD(void*, Alloc, (size_t));
|
||||||
MOCK_METHOD(void, Free, (void*));
|
MOCK_METHOD(void, Free, (void*, size_t));
|
||||||
|
|
||||||
void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); }
|
void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); }
|
||||||
|
|
||||||
void DefaultFree(void* instance) { freeInObjectPool(instance); }
|
void DefaultFree(void* instance, size_t size) { freeInObjectPool(instance, size); }
|
||||||
};
|
};
|
||||||
|
|
||||||
class GlobalMockAllocator {
|
class GlobalMockAllocator {
|
||||||
@@ -796,9 +800,9 @@ public:
|
|||||||
return instance_->Alloc(size);
|
return instance_->Alloc(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void Free(void* instance) {
|
static void Free(void* instance, size_t size) {
|
||||||
RuntimeAssert(instance_ != nullptr, "Global allocator must be set");
|
RuntimeAssert(instance_ != nullptr, "Global allocator must be set");
|
||||||
instance_->Free(instance);
|
instance_->Free(instance, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void SetMockAllocator(MockAllocator* instance) {
|
static void SetMockAllocator(MockAllocator* instance) {
|
||||||
@@ -820,7 +824,7 @@ private:
|
|||||||
MockAllocator::MockAllocator() {
|
MockAllocator::MockAllocator() {
|
||||||
GlobalMockAllocator::SetMockAllocator(this);
|
GlobalMockAllocator::SetMockAllocator(this);
|
||||||
ON_CALL(*this, Alloc(_)).WillByDefault([this](size_t size) { return DefaultAlloc(size); });
|
ON_CALL(*this, Alloc(_)).WillByDefault([this](size_t size) { return DefaultAlloc(size); });
|
||||||
ON_CALL(*this, Free(_)).WillByDefault([this](void* instance) { DefaultFree(instance); });
|
ON_CALL(*this, Free(_, _)).WillByDefault([this](void* instance, size_t size) { DefaultFree(instance, size); });
|
||||||
}
|
}
|
||||||
|
|
||||||
MockAllocator::~MockAllocator() {
|
MockAllocator::~MockAllocator() {
|
||||||
@@ -886,7 +890,7 @@ TEST(ObjectFactoryTest, CreateObject) {
|
|||||||
++it;
|
++it;
|
||||||
EXPECT_THAT(it, iter.end());
|
EXPECT_THAT(it, iter.end());
|
||||||
|
|
||||||
EXPECT_CALL(allocator, Free(allocAddress));
|
EXPECT_CALL(allocator, Free(allocAddress, allocSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, CreateObjectArray) {
|
TEST(ObjectFactoryTest, CreateObjectArray) {
|
||||||
@@ -921,7 +925,7 @@ TEST(ObjectFactoryTest, CreateObjectArray) {
|
|||||||
++it;
|
++it;
|
||||||
EXPECT_THAT(it, iter.end());
|
EXPECT_THAT(it, iter.end());
|
||||||
|
|
||||||
EXPECT_CALL(allocator, Free(allocAddress));
|
EXPECT_CALL(allocator, Free(allocAddress, allocSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, CreateCharArray) {
|
TEST(ObjectFactoryTest, CreateCharArray) {
|
||||||
@@ -956,7 +960,7 @@ TEST(ObjectFactoryTest, CreateCharArray) {
|
|||||||
++it;
|
++it;
|
||||||
EXPECT_THAT(it, iter.end());
|
EXPECT_THAT(it, iter.end());
|
||||||
|
|
||||||
EXPECT_CALL(allocator, Free(allocAddress));
|
EXPECT_CALL(allocator, Free(allocAddress, allocSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, Erase) {
|
TEST(ObjectFactoryTest, Erase) {
|
||||||
@@ -979,7 +983,7 @@ TEST(ObjectFactoryTest, Erase) {
|
|||||||
auto iter = objectFactory.LockForIter();
|
auto iter = objectFactory.LockForIter();
|
||||||
for (auto it = iter.begin(); it != iter.end();) {
|
for (auto it = iter.begin(); it != iter.end();) {
|
||||||
if (it->GetObjHeader()->type_info()->IsArray()) {
|
if (it->GetObjHeader()->type_info()->IsArray()) {
|
||||||
EXPECT_CALL(allocator, Free(_));
|
EXPECT_CALL(allocator, Free(_, _));
|
||||||
iter.EraseAndAdvance(it);
|
iter.EraseAndAdvance(it);
|
||||||
testing::Mock::VerifyAndClearExpectations(&allocator);
|
testing::Mock::VerifyAndClearExpectations(&allocator);
|
||||||
} else {
|
} else {
|
||||||
@@ -996,7 +1000,7 @@ TEST(ObjectFactoryTest, Erase) {
|
|||||||
}
|
}
|
||||||
EXPECT_THAT(count, 10);
|
EXPECT_THAT(count, 10);
|
||||||
}
|
}
|
||||||
EXPECT_CALL(allocator, Free(_)).Times(10);
|
EXPECT_CALL(allocator, Free(_, _)).Times(10);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, Move) {
|
TEST(ObjectFactoryTest, Move) {
|
||||||
@@ -1045,7 +1049,7 @@ TEST(ObjectFactoryTest, Move) {
|
|||||||
EXPECT_THAT(count, 10);
|
EXPECT_THAT(count, 10);
|
||||||
}
|
}
|
||||||
|
|
||||||
EXPECT_CALL(allocator, Free(_)).Times(20);
|
EXPECT_CALL(allocator, Free(_, _)).Times(20);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, RunFinalizers) {
|
TEST(ObjectFactoryTest, RunFinalizers) {
|
||||||
@@ -1080,7 +1084,7 @@ TEST(ObjectFactoryTest, RunFinalizers) {
|
|||||||
finalizerQueue.Finalize();
|
finalizerQueue.Finalize();
|
||||||
// Hooks called before `FinalizerQueue` destructor.
|
// Hooks called before `FinalizerQueue` destructor.
|
||||||
testing::Mock::VerifyAndClearExpectations(&finalizerHooks.finalizerHook());
|
testing::Mock::VerifyAndClearExpectations(&finalizerHooks.finalizerHook());
|
||||||
EXPECT_CALL(allocator, Free(_)).Times(10);
|
EXPECT_CALL(allocator, Free(_, _)).Times(10);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(ObjectFactoryTest, ConcurrentPublish) {
|
TEST(ObjectFactoryTest, ConcurrentPublish) {
|
||||||
@@ -1124,5 +1128,5 @@ TEST(ObjectFactoryTest, ConcurrentPublish) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
|
EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
|
||||||
EXPECT_CALL(allocator, Free(_)).Times(kThreadCount);
|
EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
|||||||
return mi_calloc_aligned(1, size, kObjectAlignment);
|
return mi_calloc_aligned(1, size, kObjectAlignment);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::freeInObjectPool(void* ptr) noexcept {
|
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||||
mi_free(ptr);
|
mi_free(ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,4 +67,8 @@ void kotlin::compactObjectPoolInMainThread() noexcept {
|
|||||||
scheduledCompactOnMainThread.clear();
|
scheduledCompactOnMainThread.clear();
|
||||||
});
|
});
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t kotlin::allocatedBytes() noexcept {
|
||||||
|
return mi_allocated_size();
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,10 @@
|
|||||||
|
|
||||||
#include "ObjectAlloc.hpp"
|
#include "ObjectAlloc.hpp"
|
||||||
|
|
||||||
|
#ifndef KONAN_NO_THREADS
|
||||||
|
#include <atomic>
|
||||||
|
#endif
|
||||||
|
|
||||||
#if KONAN_INTERNAL_DLMALLOC
|
#if KONAN_INTERNAL_DLMALLOC
|
||||||
extern "C" void* dlcalloc(size_t, size_t);
|
extern "C" void* dlcalloc(size_t, size_t);
|
||||||
extern "C" void dlfree(void*);
|
extern "C" void dlfree(void*);
|
||||||
@@ -20,17 +24,45 @@ extern "C" void dlfree(void*);
|
|||||||
|
|
||||||
using namespace kotlin;
|
using namespace kotlin;
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
#ifndef KONAN_NO_THREADS
|
||||||
|
std::atomic<size_t> allocatedBytesCounter = 0;
|
||||||
|
#else
|
||||||
|
size_t allocatedBytesCounter = 0;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
void kotlin::initObjectPool() noexcept {}
|
void kotlin::initObjectPool() noexcept {}
|
||||||
|
|
||||||
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
void* kotlin::allocateInObjectPool(size_t size) noexcept {
|
||||||
|
#ifndef KONAN_NO_THREADS
|
||||||
|
allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed);
|
||||||
|
#else
|
||||||
|
allocatedBytesCounter += size;
|
||||||
|
#endif
|
||||||
// TODO: Check that alignment to kObjectAlignment is satisfied.
|
// TODO: Check that alignment to kObjectAlignment is satisfied.
|
||||||
return callocImpl(1, size);
|
return callocImpl(1, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::freeInObjectPool(void* ptr) noexcept {
|
void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept {
|
||||||
|
#ifndef KONAN_NO_THREADS
|
||||||
|
allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed);
|
||||||
|
#else
|
||||||
|
allocatedBytesCounter -= size;
|
||||||
|
#endif
|
||||||
freeImpl(ptr);
|
freeImpl(ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInCurrentThread() noexcept {}
|
void kotlin::compactObjectPoolInCurrentThread() noexcept {}
|
||||||
|
|
||||||
void kotlin::compactObjectPoolInMainThread() noexcept {}
|
void kotlin::compactObjectPoolInMainThread() noexcept {}
|
||||||
|
|
||||||
|
size_t kotlin::allocatedBytes() noexcept {
|
||||||
|
#ifndef KONAN_NO_THREADS
|
||||||
|
return allocatedBytesCounter.load();
|
||||||
|
#else
|
||||||
|
return allocatedBytesCounter;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user