diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 7869ba75746..53a8c68fa9d 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -175,6 +175,8 @@ bitcode { sourceSets { main {} } + + compilerArgs.add("-DKONAN_MI_MALLOC=1") } module("exceptionsSupport") { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp index b0cb87a1dee..c39f2fb8ab2 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Allocator.cpp @@ -5,6 +5,8 @@ #include "CustomLogging.hpp" +#include "GCApi.hpp" + // These functions are just stubs to make the existing object creation // infrastructure link correctly, but if they are ever called, something went // wrong. @@ -16,12 +18,16 @@ void* allocateInObjectPool(size_t size) noexcept { return nullptr; } -void freeInObjectPool(void* ptr) noexcept { - CustomAllocWarning("static freeInObjectPool(%p) not supported", ptr); +void freeInObjectPool(void* ptr, size_t size) noexcept { + CustomAllocWarning("static freeInObjectPool(%p, %zu) not supported", ptr, size); } void initObjectPool() noexcept {} void compactObjectPoolInMainThread() noexcept {} void compactObjectPoolInCurrentThread() noexcept {} +size_t allocatedBytes() noexcept { + return alloc::GetAllocatedBytes(); +} + } // namespace kotlin diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp index 7a652e9d60a..663bf5c7146 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPage.cpp @@ -32,7 +32,7 @@ ExtraObjectPage::ExtraObjectPage() noexcept { } void ExtraObjectPage::Destroy() noexcept { - std_support::free(this); + Free(this, EXTRA_OBJECT_PAGE_SIZE); } mm::ExtraObjectData* ExtraObjectPage::TryAllocate() noexcept { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp index 6e0fb16a5d5..e9408241887 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/ExtraObjectPageTest.cpp @@ -35,7 +35,7 @@ TEST(CustomAllocTest, ExtraObjectPageConsequtiveAlloc) { EXPECT_EQ(prev + sizeof(Cell), cur); prev = cur; } - free(page); + page->Destroy(); } TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) { @@ -43,7 +43,7 @@ TEST(CustomAllocTest, ExtraObjectPageSweepEmptyPage) { Queue finalizerQueue; EXPECT_FALSE(page->Sweep(finalizerQueue)); EXPECT_EQ(finalizerQueue.size(), size_t(0)); - free(page); + page->Destroy(); } TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) { @@ -58,7 +58,7 @@ TEST(CustomAllocTest, ExtraObjectPageSweepFullFinalizedPage) { Queue finalizerQueue; EXPECT_FALSE(page->Sweep(finalizerQueue)); EXPECT_EQ(finalizerQueue.size(), size_t(0)); - free(page); + page->Destroy(); } } // namespace diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp index 2fae4a792a5..5c96ad00990 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/FixedBlockPage.cpp @@ -23,7 +23,7 @@ FixedBlockPage* FixedBlockPage::Create(uint32_t blockSize) noexcept { } void FixedBlockPage::Destroy() noexcept { - std_support::free(this); + Free(this, FIXED_BLOCK_PAGE_SIZE); } FixedBlockPage::FixedBlockPage(uint32_t blockSize) noexcept : blockSize_(blockSize) { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 3f83d25347b..35bcb6d916c 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -5,6 +5,7 @@ #include "GCApi.hpp" +#include #include #include "ConcurrentMarkAndSweep.hpp" @@ -13,6 +14,12 @@ #include "KAssert.h" #include "ObjectFactory.hpp" +namespace { + +std::atomic allocatedBytesCounter; + +} + namespace kotlin::alloc { bool TryResetMark(void* ptr) noexcept { @@ -83,7 +90,17 @@ void* SafeAlloc(uint64_t size) noexcept { konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 "bytes. Aborting.\n", size); konan::abort(); } + allocatedBytesCounter.fetch_add(static_cast(size), std::memory_order_relaxed); return memory; } +void Free(void* ptr, size_t size) noexcept { + std_support::free(ptr); + allocatedBytesCounter.fetch_sub(static_cast(size), std::memory_order_relaxed); +} + +size_t GetAllocatedBytes() noexcept { + return allocatedBytesCounter.load(std::memory_order_relaxed); +} + } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp index 1d0354b670a..9de000c8cba 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.hpp @@ -22,6 +22,9 @@ bool TryResetMark(void* ptr) noexcept; bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack& finalizerQueue) noexcept; void* SafeAlloc(uint64_t size) noexcept; +void Free(void* ptr, size_t size) noexcept; + +size_t GetAllocatedBytes() noexcept; } // namespace kotlin::alloc diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp index 2a9075e0fa7..22523a9eda4 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/Heap.cpp @@ -56,7 +56,7 @@ AtomicStack Heap::SweepExtraObjects(gc::GCHandle gcHandle) noex while ((page = usedExtraObjectPages_.Pop())) { if (!page->Sweep(finalizerQueue)) { CustomAllocInfo("SweepExtraObjects free(%p)", page); - free(page); + page->Destroy(); } else { extraObjectPages_.Push(page); } diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp index 6ddb52384fe..56766e166b4 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/NextFitPage.cpp @@ -21,7 +21,7 @@ NextFitPage* NextFitPage::Create(uint32_t cellCount) noexcept { } void NextFitPage::Destroy() noexcept { - std_support::free(this); + Free(this, NEXT_FIT_PAGE_SIZE); } NextFitPage::NextFitPage(uint32_t cellCount) noexcept : curBlock_(cells_) { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp index 896c6922243..55a3b1e2f3d 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.cpp @@ -18,11 +18,13 @@ SingleObjectPage* SingleObjectPage::Create(uint64_t cellCount) noexcept { CustomAllocInfo("SingleObjectPage::Create(%" PRIu64 ")", cellCount); RuntimeAssert(cellCount > NEXT_FIT_PAGE_MAX_BLOCK_SIZE, "blockSize too small for SingleObjectPage"); uint64_t size = sizeof(SingleObjectPage) + cellCount * sizeof(uint64_t); - return new (SafeAlloc(size)) SingleObjectPage(); + auto* page = new (SafeAlloc(size)) SingleObjectPage(); + page->size_ = size; + return page; } void SingleObjectPage::Destroy() noexcept { - std_support::free(this); + Free(this, size_); } uint8_t* SingleObjectPage::Data() noexcept { diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp index c42500bf233..7781f1feb2c 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/SingleObjectPage.hpp @@ -29,6 +29,7 @@ private: friend class AtomicStack; SingleObjectPage* next_; bool isAllocated_ = false; + size_t size_; struct alignas(8) { uint8_t data_[]; }; diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 418660efdf7..72e6c82715f 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -105,6 +105,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); } +size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { + return allocatedBytes(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp index 123ae478c72..858506a02d9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp @@ -19,7 +19,7 @@ namespace gc { class Allocator { public: void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); } - static void Free(void* instance) noexcept { freeInObjectPool(instance); } + static void Free(void* instance, size_t size) noexcept { freeInObjectPool(instance, size); } }; template @@ -37,7 +37,7 @@ public: return base_.Alloc(size); } - static void Free(void* instance) noexcept { BaseAllocator::Free(instance); } + static void Free(void* instance, size_t size) noexcept { BaseAllocator::Free(instance, size); } private: BaseAllocator base_; diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index 953232d167f..1f8ce32ece9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -63,6 +63,7 @@ public: size_t GetTotalHeapObjectsSizeUnsafe() const noexcept; size_t GetExtraObjectsCountUnsafe() const noexcept; size_t GetTotalExtraObjectsSizeUnsafe() const noexcept; + size_t GetTotalHeapObjectsSizeBytes() const noexcept; gc::GCSchedulerConfig& gcSchedulerConfig() noexcept; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 6dc3051bea4..05f3bb57834 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -80,6 +80,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); } +size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { + return allocatedBytes(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index dd80b22548a..48fd63ea14f 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -93,6 +93,10 @@ size_t gc::GC::GetTotalExtraObjectsSizeUnsafe() const noexcept { return mm::GlobalData::Instance().extraObjectDataFactory().GetTotalObjectsSizeUnsafe(); } +size_t gc::GC::GetTotalHeapObjectsSizeBytes() const noexcept { + return allocatedBytes(); +} + gc::GCSchedulerConfig& gc::GC::gcSchedulerConfig() noexcept { return impl_->gcScheduler().config(); } diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index faecb7408a2..44e19fb9478 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -1123,7 +1123,7 @@ void processFinalizerQueue(MemoryState* state) { state->containers->erase(container); #endif CONTAINER_DESTROY_EVENT(state, container) - freeInObjectPool(container); + freeInObjectPool(container, 0); atomicAdd(&allocCount, -1); } RuntimeAssert(state->finalizerQueueSize == 0, "Queue must be empty here"); @@ -1164,7 +1164,7 @@ void scheduleDestroyContainer(MemoryState* state, ContainerHeader* container) { processFinalizerQueue(state); } #else - freeInObjectPool(container); + freeInObjectPool(container), 0; atomicAdd(&allocCount, -1); CONTAINER_DESTROY_EVENT(state, container); #endif @@ -3142,7 +3142,7 @@ void ArenaContainer::Deinit() { while (chunk != nullptr) { auto toRemove = chunk; chunk = chunk->next; - freeInObjectPool(toRemove); + freeInObjectPool(toRemove, 0); } } diff --git a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp index 279deb484aa..a196b968f63 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp @@ -12,13 +12,15 @@ namespace kotlin { void initObjectPool() noexcept; void* allocateInObjectPool(size_t size) noexcept; -void freeInObjectPool(void* ptr) noexcept; +void freeInObjectPool(void* ptr, size_t size) noexcept; // Instruct the allocator to free unused resources. void compactObjectPoolInCurrentThread() noexcept; // Platform dependent. Schedule `compactObjectPoolInCurrentThread` on the main thread. // May do nothing if the main thread is not an event loop. void compactObjectPoolInMainThread() noexcept; +size_t allocatedBytes() noexcept; + template struct ObjectPoolAllocator { using value_type = T; @@ -36,7 +38,7 @@ struct ObjectPoolAllocator { T* allocate(std::size_t n) noexcept { return static_cast(allocateInObjectPool(n * sizeof(T))); } - void deallocate(T* p, std::size_t n) noexcept { freeInObjectPool(p); } + void deallocate(T* p, std::size_t n) noexcept { freeInObjectPool(p, n * sizeof(T)); } }; template diff --git a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h index 6fec51ff798..9b04b13f5e9 100644 --- a/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h +++ b/kotlin-native/runtime/src/mimalloc/c/include/mimalloc.h @@ -373,6 +373,10 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, s mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); +#if KONAN_MI_MALLOC +mi_decl_nodiscard mi_decl_export size_t mi_allocated_size(void) mi_attr_noexcept; +#endif + #ifdef __cplusplus } #endif diff --git a/kotlin-native/runtime/src/mimalloc/c/region.c b/kotlin-native/runtime/src/mimalloc/c/region.c index 4d9a47728e6..4d9d6fed28c 100644 --- a/kotlin-native/runtime/src/mimalloc/c/region.c +++ b/kotlin-native/runtime/src/mimalloc/c/region.c @@ -103,6 +103,13 @@ static mem_region_t regions[MI_REGION_MAX]; // Allocated regions static _Atomic(uintptr_t) regions_count; // = 0; +#if KONAN_MI_MALLOC +static _Atomic(size_t) allocated_size = 0; + +size_t mi_allocated_size(void) mi_attr_noexcept { + return mi_atomic_load_relaxed(&allocated_size); +} +#endif /* ---------------------------------------------------------------------------- Utility functions @@ -378,6 +385,9 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* l mi_assert_internal((uintptr_t)p % alignment == 0); #if (MI_DEBUG>=2) if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed +#endif +#if KONAN_MI_MALLOC + mi_atomic_add_relaxed(&allocated_size, size); #endif } return p; @@ -443,6 +453,9 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re bool all_unclaimed = mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx); mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed); } +#if KONAN_MI_MALLOC + mi_atomic_sub_relaxed(&allocated_size, size); +#endif } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index f05118809b9..ba66c436abb 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -30,7 +30,7 @@ namespace internal { // uses `Allocator` to allocate and free memory. // Precondition on `Allocator`: must allocate with alignment at least `DataAlignment`. // TODO: Consider merging with `MultiSourceQueue` somehow. -template +template class ObjectFactoryStorage : private Pinned { static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment"); @@ -38,8 +38,9 @@ class ObjectFactoryStorage : private Pinned { class Deleter { public: void operator()(T* instance) noexcept { + auto size = instance->GetAllocatedSize(); instance->~T(); - Allocator::Free(instance); + Allocator::Free(instance, size); } }; @@ -81,6 +82,12 @@ public: return *static_cast(Data()); } + size_t GetAllocatedSize() noexcept { + auto dataSize = DataSizeProvider::GetDataSize(Data()); + auto totalSize = GetSizeForDataSize(dataSize); + return totalSize; + } + private: friend class ObjectFactoryStorage; @@ -447,8 +454,34 @@ class ObjectFactory : private Pinned { alignas(kObjectAlignment) ArrayHeader array; }; + static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { + size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); + return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); + } + + static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { + // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped + // at about half of uint64_t max. + uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; + // Note: array body is aligned, but for size computation it is enough to align the sum. + return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + } + + struct DataSizeProvider { + static size_t GetDataSize(void* data) noexcept { + ObjHeader* object = &static_cast(data)->object; + RuntimeAssert(object->heap(), "Object must be a heap object"); + const auto* typeInfo = object->type_info(); + if (typeInfo->IsArray()) { + return ArrayAllocatedDataSize(typeInfo, object->array()->count_); + } else { + return ObjectAllocatedDataSize(typeInfo); + } + } + }; + public: - using Storage = internal::ObjectFactoryStorage; + using Storage = internal::ObjectFactoryStorage; class NodeRef { public: @@ -561,19 +594,6 @@ public: void ClearForTests() noexcept { producer_.ClearForTests(); } private: - static size_t ObjectAllocatedDataSize(const TypeInfo* typeInfo) noexcept { - size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); - return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); - } - - static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { - // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped - // at about half of uint64_t max. - uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; - // Note: array body is aligned, but for size computation it is enough to align the sum. - return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); - } - typename Storage::Producer producer_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index 840dedc1580..36ec7b6c95f 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -29,8 +29,12 @@ namespace { using SimpleAllocator = gc::Allocator; +struct DataSizeProvider { + static size_t GetDataSize(void* data) noexcept { return 0; } +}; + template -using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; +using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; using ObjectFactoryStorageRegular = ObjectFactoryStorage; @@ -782,11 +786,11 @@ public: ~MockAllocator(); MOCK_METHOD(void*, Alloc, (size_t)); - MOCK_METHOD(void, Free, (void*)); + MOCK_METHOD(void, Free, (void*, size_t)); void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); } - void DefaultFree(void* instance) { freeInObjectPool(instance); } + void DefaultFree(void* instance, size_t size) { freeInObjectPool(instance, size); } }; class GlobalMockAllocator { @@ -796,9 +800,9 @@ public: return instance_->Alloc(size); } - static void Free(void* instance) { + static void Free(void* instance, size_t size) { RuntimeAssert(instance_ != nullptr, "Global allocator must be set"); - instance_->Free(instance); + instance_->Free(instance, size); } static void SetMockAllocator(MockAllocator* instance) { @@ -820,7 +824,7 @@ private: MockAllocator::MockAllocator() { GlobalMockAllocator::SetMockAllocator(this); ON_CALL(*this, Alloc(_)).WillByDefault([this](size_t size) { return DefaultAlloc(size); }); - ON_CALL(*this, Free(_)).WillByDefault([this](void* instance) { DefaultFree(instance); }); + ON_CALL(*this, Free(_, _)).WillByDefault([this](void* instance, size_t size) { DefaultFree(instance, size); }); } MockAllocator::~MockAllocator() { @@ -886,7 +890,7 @@ TEST(ObjectFactoryTest, CreateObject) { ++it; EXPECT_THAT(it, iter.end()); - EXPECT_CALL(allocator, Free(allocAddress)); + EXPECT_CALL(allocator, Free(allocAddress, allocSize)); } TEST(ObjectFactoryTest, CreateObjectArray) { @@ -921,7 +925,7 @@ TEST(ObjectFactoryTest, CreateObjectArray) { ++it; EXPECT_THAT(it, iter.end()); - EXPECT_CALL(allocator, Free(allocAddress)); + EXPECT_CALL(allocator, Free(allocAddress, allocSize)); } TEST(ObjectFactoryTest, CreateCharArray) { @@ -956,7 +960,7 @@ TEST(ObjectFactoryTest, CreateCharArray) { ++it; EXPECT_THAT(it, iter.end()); - EXPECT_CALL(allocator, Free(allocAddress)); + EXPECT_CALL(allocator, Free(allocAddress, allocSize)); } TEST(ObjectFactoryTest, Erase) { @@ -979,7 +983,7 @@ TEST(ObjectFactoryTest, Erase) { auto iter = objectFactory.LockForIter(); for (auto it = iter.begin(); it != iter.end();) { if (it->GetObjHeader()->type_info()->IsArray()) { - EXPECT_CALL(allocator, Free(_)); + EXPECT_CALL(allocator, Free(_, _)); iter.EraseAndAdvance(it); testing::Mock::VerifyAndClearExpectations(&allocator); } else { @@ -996,7 +1000,7 @@ TEST(ObjectFactoryTest, Erase) { } EXPECT_THAT(count, 10); } - EXPECT_CALL(allocator, Free(_)).Times(10); + EXPECT_CALL(allocator, Free(_, _)).Times(10); } TEST(ObjectFactoryTest, Move) { @@ -1045,7 +1049,7 @@ TEST(ObjectFactoryTest, Move) { EXPECT_THAT(count, 10); } - EXPECT_CALL(allocator, Free(_)).Times(20); + EXPECT_CALL(allocator, Free(_, _)).Times(20); } TEST(ObjectFactoryTest, RunFinalizers) { @@ -1080,7 +1084,7 @@ TEST(ObjectFactoryTest, RunFinalizers) { finalizerQueue.Finalize(); // Hooks called before `FinalizerQueue` destructor. testing::Mock::VerifyAndClearExpectations(&finalizerHooks.finalizerHook()); - EXPECT_CALL(allocator, Free(_)).Times(10); + EXPECT_CALL(allocator, Free(_, _)).Times(10); } TEST(ObjectFactoryTest, ConcurrentPublish) { @@ -1124,5 +1128,5 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { } EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); - EXPECT_CALL(allocator, Free(_)).Times(kThreadCount); + EXPECT_CALL(allocator, Free(_, _)).Times(kThreadCount); } diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp index 17293ec726f..6627c96710a 100644 --- a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp @@ -44,7 +44,7 @@ void* kotlin::allocateInObjectPool(size_t size) noexcept { return mi_calloc_aligned(1, size, kObjectAlignment); } -void kotlin::freeInObjectPool(void* ptr) noexcept { +void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept { mi_free(ptr); } @@ -67,4 +67,8 @@ void kotlin::compactObjectPoolInMainThread() noexcept { scheduledCompactOnMainThread.clear(); }); #endif -} \ No newline at end of file +} + +size_t kotlin::allocatedBytes() noexcept { + return mi_allocated_size(); +} diff --git a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp index 65e431a7a70..7e91582b2a5 100644 --- a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp +++ b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp @@ -5,6 +5,10 @@ #include "ObjectAlloc.hpp" +#ifndef KONAN_NO_THREADS +#include +#endif + #if KONAN_INTERNAL_DLMALLOC extern "C" void* dlcalloc(size_t, size_t); extern "C" void dlfree(void*); @@ -20,17 +24,45 @@ extern "C" void dlfree(void*); using namespace kotlin; +namespace { + +#ifndef KONAN_NO_THREADS +std::atomic allocatedBytesCounter = 0; +#else +size_t allocatedBytesCounter = 0; +#endif + +} // namespace + void kotlin::initObjectPool() noexcept {} void* kotlin::allocateInObjectPool(size_t size) noexcept { +#ifndef KONAN_NO_THREADS + allocatedBytesCounter.fetch_add(size, std::memory_order_relaxed); +#else + allocatedBytesCounter += size; +#endif // TODO: Check that alignment to kObjectAlignment is satisfied. return callocImpl(1, size); } -void kotlin::freeInObjectPool(void* ptr) noexcept { +void kotlin::freeInObjectPool(void* ptr, size_t size) noexcept { +#ifndef KONAN_NO_THREADS + allocatedBytesCounter.fetch_sub(size, std::memory_order_relaxed); +#else + allocatedBytesCounter -= size; +#endif freeImpl(ptr); } void kotlin::compactObjectPoolInCurrentThread() noexcept {} -void kotlin::compactObjectPoolInMainThread() noexcept {} \ No newline at end of file +void kotlin::compactObjectPoolInMainThread() noexcept {} + +size_t kotlin::allocatedBytes() noexcept { +#ifndef KONAN_NO_THREADS + return allocatedBytesCounter.load(); +#else + return allocatedBytesCounter; +#endif +}