diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 9ce1ca6aae7..8007ce68f8f 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -130,8 +130,13 @@ bitcode { includeRuntime() } - create("std_alloc") - create("opt_alloc") + create("std_alloc") { + includeRuntime() + } + + create("opt_alloc") { + includeRuntime() + } create("exceptionsSupport", file("src/exceptions_support")) { includeRuntime() diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp index 995688f7762..6226c03b73a 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.hpp @@ -67,7 +67,7 @@ public: class ThreadData : private Pinned { public: using ObjectData = ConcurrentMarkAndSweep::ObjectData; - using Allocator = AllocatorWithGC; + using Allocator = AllocatorWithGC; explicit ThreadData(ConcurrentMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : gc_(gc), gcScheduler_(gcScheduler) {} @@ -80,7 +80,7 @@ public: void OnOOM(size_t size) noexcept; - Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } + Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } private: ConcurrentMarkAndSweep& gc_; diff --git a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp index d1ef7b05796..123ae478c72 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/Allocator.hpp @@ -7,15 +7,19 @@ #include -#include "std_support/CStdlib.hpp" +#include "ObjectAlloc.hpp" namespace kotlin { namespace gc { -class AlignedAllocator { +// TODO: Try to move from custom allocator interface to standard one. +// Currently Free method is in the way: it is static to avoid keeping allocator state in +// unique_ptr's deleter in ObjectFactory. + +class Allocator { public: - void* Alloc(size_t size, size_t alignment) noexcept { return std_support::aligned_calloc(alignment, 1, size); } - static void Free(void* instance) noexcept { std_support::free(instance); } + void* Alloc(size_t size) noexcept { return allocateInObjectPool(size); } + static void Free(void* instance) noexcept { freeInObjectPool(instance); } }; template @@ -23,14 +27,14 @@ class AllocatorWithGC { public: AllocatorWithGC(BaseAllocator base, GCThreadData& gc) noexcept : base_(std::move(base)), gc_(gc) {} - void* Alloc(size_t size, size_t alignment) noexcept { + void* Alloc(size_t size) noexcept { gc_.SafePointAllocation(size); - if (void* ptr = base_.Alloc(size, alignment)) { + if (void* ptr = base_.Alloc(size)) { return ptr; } // Tell GC that we failed to allocate, and try one more time. gc_.OnOOM(size); - return base_.Alloc(size, alignment); + return base_.Alloc(size); } static void Free(void* instance) noexcept { BaseAllocator::Free(instance); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp index 5c3fecf2f6f..44de2b131c7 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/AllocatorTest.cpp @@ -18,14 +18,14 @@ namespace { class MockAllocator { public: - MOCK_METHOD(void*, Alloc, (size_t, size_t)); + MOCK_METHOD(void*, Alloc, (size_t)); }; class MockAllocatorWrapper { public: MockAllocator& operator*() { return *mock_; } - void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); } + void* Alloc(size_t size) { return mock_->Alloc(size); } private: std_support::unique_ptr> mock_ = std_support::make_unique>(); @@ -41,52 +41,49 @@ public: TEST(AllocatorWithGCTest, AllocateWithoutOOM) { constexpr size_t size = 256; - constexpr size_t alignment = 8; void* nonNull = reinterpret_cast(1); MockAllocatorWrapper baseAllocator; testing::StrictMock gc; { testing::InSequence seq; EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); EXPECT_CALL(gc, OnOOM(_)).Times(0); } gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); + void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nonNull); } TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { constexpr size_t size = 256; - constexpr size_t alignment = 8; void* nonNull = reinterpret_cast(1); MockAllocatorWrapper baseAllocator; testing::StrictMock gc; { testing::InSequence seq; EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); EXPECT_CALL(gc, OnOOM(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nonNull)); } gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); + void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nonNull); } TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { constexpr size_t size = 256; - constexpr size_t alignment = 8; MockAllocatorWrapper baseAllocator; testing::StrictMock gc; { testing::InSequence seq; EXPECT_CALL(gc, SafePointAllocation(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); EXPECT_CALL(gc, OnOOM(size)); - EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(*baseAllocator, Alloc(size)).WillOnce(testing::Return(nullptr)); } gc::AllocatorWithGC allocator(std::move(baseAllocator), gc); - void* ptr = allocator.Alloc(size, alignment); + void* ptr = allocator.Alloc(size); EXPECT_THAT(ptr, nullptr); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index 52c2106f5f3..68b1be572f0 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -54,7 +54,7 @@ struct ObjectFactoryTraits { State state = State::kUnmarked; }; - using Allocator = gc::AlignedAllocator; + using Allocator = gc::Allocator; }; using ObjectFactory = mm::ObjectFactory; @@ -268,7 +268,7 @@ private: kotlin::ScopedMemoryInit memoryInit; FinalizerHooksTestSupport finalizerHooks_; ObjectFactory objectFactory_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gc::AlignedAllocator()}; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_{objectFactory_, gc::Allocator()}; ExtraObjectsDataFactory extraObjectFactory_; ExtraObjectsDataFactory::ThreadQueue extraObjectFactoryThreadQueue_{extraObjectFactory_}; std_support::vector finalizers_; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp index 64623848f66..5f1bf00677f 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/NoOpGC.hpp @@ -28,7 +28,7 @@ class NoOpGC : private Pinned { public: class ObjectData {}; - using Allocator = AlignedAllocator; + using Allocator = gc::Allocator; class ThreadData : private Pinned { public: diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp index a928e6674a9..ceef89f5745 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.hpp @@ -68,7 +68,7 @@ public: class ThreadData : private Pinned { public: using ObjectData = SameThreadMarkAndSweep::ObjectData; - using Allocator = AllocatorWithGC; + using Allocator = AllocatorWithGC; ThreadData(SameThreadMarkAndSweep& gc, mm::ThreadData& threadData, GCSchedulerThreadData& gcScheduler) noexcept : gc_(gc), gcScheduler_(gcScheduler) {} @@ -82,7 +82,7 @@ public: void OnOOM(size_t size) noexcept; - Allocator CreateAllocator() noexcept { return Allocator(AlignedAllocator(), *this); } + Allocator CreateAllocator() noexcept { return Allocator(gc::Allocator(), *this); } private: diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index b1b2e5cea2b..8ffb9c113b2 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -46,6 +46,7 @@ #include "MemoryPrivate.hpp" #include "Mutex.hpp" #include "Natives.h" +#include "ObjectAlloc.hpp" #include "ObjectTraversal.hpp" #include "Porting.h" #include "Runtime.h" @@ -89,6 +90,8 @@ ALWAYS_INLINE bool IsStrictMemoryModel() noexcept { return CurrentMemoryModel == MemoryModel::kStrict; } +inline constexpr ObjectPoolAllocator objectAllocator; + typedef uint32_t container_size_t; // Granularity of arena container chunks. @@ -1074,7 +1077,7 @@ ContainerHeader* allocContainer(MemoryState* state, size_t size) { if (state != nullptr) state->allocSinceLastGc += size; #endif - result = new (std_support::calloc(1, alignUp(size, kObjectAlignment))) ContainerHeader(); + result = new (allocateInObjectPool(alignUp(size, kObjectAlignment))) ContainerHeader(); atomicAdd(&allocCount, 1); } if (state != nullptr) { @@ -1115,7 +1118,7 @@ void processFinalizerQueue(MemoryState* state) { state->containers->erase(container); #endif CONTAINER_DESTROY_EVENT(state, container) - std_support::free(container); + freeInObjectPool(container); atomicAdd(&allocCount, -1); } RuntimeAssert(state->finalizerQueueSize == 0, "Queue must be empty here"); @@ -1156,7 +1159,7 @@ void scheduleDestroyContainer(MemoryState* state, ContainerHeader* container) { processFinalizerQueue(state); } #else - std_support::free(container); + freeInObjectPool(container); atomicAdd(&allocCount, -1); CONTAINER_DESTROY_EVENT(state, container); #endif @@ -1767,7 +1770,7 @@ inline ArenaContainer* initedArena(ObjHeader** auxSlot) { auto frame = asFrameOverlay(auxSlot); auto arena = reinterpret_cast(frame->arena); if (!arena) { - arena = new (std_support::kalloc) ArenaContainer(); + arena = std_support::allocator_new(objectAllocator); MEMORY_LOG("Initializing arena in %p\n", frame) arena->Init(); frame->arena = arena; @@ -3150,7 +3153,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { } #endif - MetaObjHeader* meta = new (std_support::kalloc) MetaObjHeader(); + MetaObjHeader* meta = std_support::allocator_new(objectAllocator); meta->typeInfo_ = typeInfo; #if KONAN_NO_THREADS *location = reinterpret_cast(meta); @@ -3158,7 +3161,7 @@ MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { TypeInfo* old = __sync_val_compare_and_swap(location, typeInfo, reinterpret_cast(meta)); if (old != typeInfo) { // Someone installed a new meta-object since the check. - std_support::free(meta); + std_support::allocator_delete(objectAllocator, meta); meta = reinterpret_cast(old); } #endif @@ -3178,7 +3181,7 @@ void ObjHeader::destroyMetaObject(ObjHeader* object) { Kotlin_ObjCExport_detachAndReleaseAssociatedObject(meta->associatedObject_); #endif - std_support::free(meta); + std_support::allocator_delete(objectAllocator, meta); } void ObjectContainer::Init(MemoryState* state, const TypeInfo* typeInfo) { @@ -3228,7 +3231,7 @@ void ArenaContainer::Deinit() { while (chunk != nullptr) { auto toRemove = chunk; chunk = chunk->next; - std_support::free(toRemove); + freeInObjectPool(toRemove); } } @@ -3236,7 +3239,7 @@ bool ArenaContainer::allocContainer(container_size_t minSize) { auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk); size = alignUp(size, kContainerAlignment); // TODO: keep simple cache of container chunks. - ContainerChunk* result = new (std_support::calloc(1, size)) ContainerChunk(); + ContainerChunk* result = new (allocateInObjectPool(size)) ContainerChunk(); RuntimeCheck(result != nullptr, "Cannot alloc memory"); if (result == nullptr) return false; result->next = currentChunk_; diff --git a/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp new file mode 100644 index 00000000000..0ae08ef40db --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/ObjectAlloc.hpp @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include +#include + +namespace kotlin { + +void initObjectPool() noexcept; +void* allocateInObjectPool(size_t size) noexcept; +void freeInObjectPool(void* ptr) noexcept; + +template +struct ObjectPoolAllocator { + using value_type = T; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using propagate_on_container_move_assignment = std::true_type; + using is_always_equal = std::true_type; + + constexpr ObjectPoolAllocator() noexcept = default; + + constexpr ObjectPoolAllocator(const ObjectPoolAllocator&) noexcept = default; + + template + constexpr ObjectPoolAllocator(const ObjectPoolAllocator&) noexcept {} + + T* allocate(std::size_t n) noexcept { return static_cast(allocateInObjectPool(n * sizeof(T))); } + + void deallocate(T* p, std::size_t n) noexcept { freeInObjectPool(p); } +}; + +template +constexpr bool operator==(const ObjectPoolAllocator&, const ObjectPoolAllocator&) noexcept { + return true; +} + +template +constexpr bool operator!=(const ObjectPoolAllocator&, const ObjectPoolAllocator&) noexcept { + return false; +} + +} diff --git a/kotlin-native/runtime/src/main/cpp/Runtime.cpp b/kotlin-native/runtime/src/main/cpp/Runtime.cpp index 65e999769e0..af485cddb31 100644 --- a/kotlin-native/runtime/src/main/cpp/Runtime.cpp +++ b/kotlin-native/runtime/src/main/cpp/Runtime.cpp @@ -21,6 +21,7 @@ #include "KAssert.h" #include "Memory.h" #include "ObjCExportInit.h" +#include "ObjectAlloc.hpp" #include "Porting.h" #include "Runtime.h" #include "RuntimePrivate.hpp" @@ -102,6 +103,7 @@ volatile GlobalRuntimeStatus globalRuntimeStatus = kGlobalRuntimeUninitialized; RuntimeState* initRuntime() { SetKonanTerminateHandler(); + initObjectPool(); RuntimeState* result = new (std_support::kalloc) RuntimeState(); if (!result) return kInvalidRuntime; RuntimeCheck(!isValidRuntime(), "No active runtimes allowed"); diff --git a/kotlin-native/runtime/src/main/cpp/std_support/CStdlib.cpp b/kotlin-native/runtime/src/main/cpp/std_support/CStdlib.cpp index 7a85ce708f8..fb591098411 100644 --- a/kotlin-native/runtime/src/main/cpp/std_support/CStdlib.cpp +++ b/kotlin-native/runtime/src/main/cpp/std_support/CStdlib.cpp @@ -6,6 +6,7 @@ #include "std_support/CStdlib.hpp" #include +#include #include using namespace kotlin; @@ -22,18 +23,13 @@ extern "C" void dlfree(void*); #define realloc_impl dlrealloc #define free_impl dlfree #else -extern "C" void* konan_malloc_impl(size_t); -extern "C" void* konan_aligned_alloc_impl(size_t, size_t); -extern "C" void* konan_calloc_impl(size_t, size_t); -extern "C" void* konan_aligned_calloc_impl(size_t, size_t, size_t); -extern "C" void* konan_realloc_impl(void*, size_t); -extern "C" void konan_free_impl(void*); -#define malloc_impl konan_malloc_impl -#define aligned_alloc_impl konan_aligned_alloc_impl -#define calloc_impl konan_calloc_impl -#define aligned_calloc_impl konan_aligned_calloc_impl -#define realloc_impl konan_realloc_impl -#define free_impl konan_free_impl +#define malloc_impl std::malloc +// TODO: Consider using std::aligned_alloc +#define aligned_alloc_impl(alignment, size) std::malloc(size) +#define calloc_impl std::calloc +#define aligned_calloc_impl(alignment, num, size) std::calloc(num, size) +#define realloc_impl std::realloc +#define free_impl std::free #endif void* std_support::malloc(std::size_t size) noexcept { diff --git a/kotlin-native/runtime/src/mimalloc/c/os.c b/kotlin-native/runtime/src/mimalloc/c/os.c index 9e398d13116..9a7b5d2ab19 100644 --- a/kotlin-native/runtime/src/mimalloc/c/os.c +++ b/kotlin-native/runtime/src/mimalloc/c/os.c @@ -1204,7 +1204,11 @@ static size_t mi_os_numa_node_countx(void) { _Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count +#if defined(KONAN_MI_MALLOC) +__attribute__((annotate("no_external_calls_check"))) size_t _mi_os_numa_node_count_get(void) { +#else size_t _mi_os_numa_node_count_get(void) { +#endif size_t count = mi_atomic_load_acquire(&_mi_numa_node_count); if (count <= 0) { long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp index 26b68dee67b..0525f3f97ba 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataFactory.hpp @@ -6,10 +6,11 @@ #ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H #define RUNTIME_MM_EXTRA_OBJECT_DATA_REGISTRY_H +#include "ExtraObjectData.hpp" #include "Memory.h" #include "MultiSourceQueue.hpp" +#include "ObjectAlloc.hpp" #include "ThreadRegistry.hpp" -#include "ExtraObjectData.hpp" namespace kotlin { namespace mm { @@ -17,7 +18,7 @@ namespace mm { // Registry for extra data, attached to some kotlin objects: weak refs, associated objects, ... class ExtraObjectDataFactory : Pinned { using Mutex = SpinLock; - using Queue = MultiSourceQueue; + using Queue = MultiSourceQueue>; public: class ThreadQueue : public Queue::Producer { public: diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index d25a27c8ad6..d2b8c90c962 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -27,6 +27,7 @@ namespace internal { // A queue that is constructed by collecting subqueues from several `Producer`s. // This is essentially a heterogeneous `MultiSourceQueue` on top of a singly linked list that // uses `Allocator` to allocate and free memory. +// Precondition on `Allocator`: must allocate with alignment at least `DataAlignment`. // TODO: Consider merging with `MultiSourceQueue` somehow. template class ObjectFactoryStorage : private Pinned { @@ -52,11 +53,10 @@ public: public: ~Node() = default; - constexpr static std::pair GetSizeAndAlignmentForDataSize(size_t dataSize) noexcept { + constexpr static size_t GetSizeForDataSize(size_t dataSize) noexcept { size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); - size_t totalAlignment = std::max(alignof(Node), DataAlignment); - size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment); - return std::make_pair(totalSize, totalAlignment); + size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, DataAlignment); + return totalSize; } static Node& FromData(void* data) noexcept { @@ -86,16 +86,16 @@ public: Node() noexcept = default; static unique_ptr Create(Allocator& allocator, size_t dataSize) noexcept { - auto [totalSize, totalAlignment] = GetSizeAndAlignmentForDataSize(dataSize); + auto totalSize = GetSizeForDataSize(dataSize); RuntimeAssert( DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize, DataOffset()); - void* ptr = allocator.Alloc(totalSize, totalAlignment); + void* ptr = allocator.Alloc(totalSize); if (!ptr) { konan::consoleErrorf("Out of memory trying to allocate %zu bytes. Aborting.\n", totalSize); konan::abort(); } - RuntimeAssert(IsAligned(ptr, totalAlignment), "Allocator returned unaligned to %zu pointer %p", totalAlignment, ptr); + RuntimeAssert(IsAligned(ptr, DataAlignment), "Allocator returned unaligned to %zu pointer %p", DataAlignment, ptr); return unique_ptr(new (ptr) Node()); } @@ -103,6 +103,7 @@ public: // There's some more data of an unknown (at compile-time) size here, but it cannot be represented // with C++ members. }; + static_assert(alignof(Node) <= DataAlignment, "DataAlignment must be greater than Node alignment"); class Producer : private MoveOnly { public: @@ -515,7 +516,7 @@ public: static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); size_t allocSize = ObjectAllocatedDataSize(typeInfo); - return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first; + return Storage::Node::GetSizeForDataSize(allocSize); } ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept { @@ -532,7 +533,7 @@ public: static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept { RuntimeAssert(typeInfo->IsArray(), "Must be an array"); size_t allocSize = ArrayAllocatedDataSize(typeInfo, count); - return Storage::Node::GetSizeAndAlignmentForDataSize(allocSize).first; + return Storage::Node::GetSizeForDataSize(allocSize); } ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index bcc5cf08d3b..81a957bfd9b 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -27,7 +27,7 @@ using testing::_; namespace { -using SimpleAllocator = gc::AlignedAllocator; +using SimpleAllocator = gc::Allocator; template using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; @@ -781,19 +781,19 @@ public: MockAllocator(); ~MockAllocator(); - MOCK_METHOD(void*, Alloc, (size_t, size_t)); + MOCK_METHOD(void*, Alloc, (size_t)); MOCK_METHOD(void, Free, (void*)); - void* DefaultAlloc(size_t size, size_t alignment) { return std_support::aligned_calloc(alignment, 1, size); } + void* DefaultAlloc(size_t size) { return allocateInObjectPool(size); } - void DefaultFree(void* instance) { std_support::free(instance); } + void DefaultFree(void* instance) { freeInObjectPool(instance); } }; class GlobalMockAllocator { public: - void* Alloc(size_t size, size_t alignment) { + void* Alloc(size_t size) { RuntimeAssert(instance_ != nullptr, "Global allocator must be set"); - return instance_->Alloc(size, alignment); + return instance_->Alloc(size); } static void Free(void* instance) { @@ -819,7 +819,7 @@ private: MockAllocator::MockAllocator() { GlobalMockAllocator::SetMockAllocator(this); - ON_CALL(*this, Alloc(_, _)).WillByDefault([this](size_t size, size_t alignment) { return DefaultAlloc(size, alignment); }); + ON_CALL(*this, Alloc(_)).WillByDefault([this](size_t size) { return DefaultAlloc(size); }); ON_CALL(*this, Free(_)).WillByDefault([this](void* instance) { DefaultFree(instance); }); } @@ -862,9 +862,9 @@ TEST(ObjectFactoryTest, CreateObject) { size_t allocSize = 0; void* allocAddress = nullptr; - EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + EXPECT_CALL(allocator, Alloc(_)).WillOnce([&](size_t size) { allocSize = size; - allocAddress = allocator.DefaultAlloc(size, alignment); + allocAddress = allocator.DefaultAlloc(size); return allocAddress; }); auto* object = threadQueue.CreateObject(type.typeInfo()); @@ -897,9 +897,9 @@ TEST(ObjectFactoryTest, CreateObjectArray) { size_t allocSize = 0; void* allocAddress = nullptr; - EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + EXPECT_CALL(allocator, Alloc(_)).WillOnce([&](size_t size) { allocSize = size; - allocAddress = allocator.DefaultAlloc(size, alignment); + allocAddress = allocator.DefaultAlloc(size); return allocAddress; }); auto* array = threadQueue.CreateArray(theArrayTypeInfo, 3); @@ -932,9 +932,9 @@ TEST(ObjectFactoryTest, CreateCharArray) { size_t allocSize = 0; void* allocAddress = nullptr; - EXPECT_CALL(allocator, Alloc(_, _)).WillOnce([&](size_t size, size_t alignment) { + EXPECT_CALL(allocator, Alloc(_)).WillOnce([&](size_t size) { allocSize = size; - allocAddress = allocator.DefaultAlloc(size, alignment); + allocAddress = allocator.DefaultAlloc(size); return allocAddress; }); auto* array = threadQueue.CreateArray(theCharArrayTypeInfo, 3); @@ -966,7 +966,7 @@ TEST(ObjectFactoryTest, Erase) { ObjectFactory objectFactory; ObjectFactory::ThreadQueue threadQueue(objectFactory, GlobalMockAllocator()); - EXPECT_CALL(allocator, Alloc(_, _)).Times(20); + EXPECT_CALL(allocator, Alloc(_)).Times(20); for (int i = 0; i < 10; ++i) { threadQueue.CreateObject(objectType.typeInfo()); threadQueue.CreateArray(theArrayTypeInfo, 3); @@ -1007,7 +1007,7 @@ TEST(ObjectFactoryTest, Move) { ObjectFactory::ThreadQueue threadQueue(objectFactory, GlobalMockAllocator()); ObjectFactory::FinalizerQueue finalizerQueue; - EXPECT_CALL(allocator, Alloc(_, _)).Times(20); + EXPECT_CALL(allocator, Alloc(_)).Times(20); for (int i = 0; i < 10; ++i) { threadQueue.CreateObject(objectType.typeInfo()); threadQueue.CreateArray(theArrayTypeInfo, 3); @@ -1059,7 +1059,7 @@ TEST(ObjectFactoryTest, RunFinalizers) { ObjectFactory::FinalizerQueue finalizerQueue; std_support::vector objects; - EXPECT_CALL(allocator, Alloc(_, _)).Times(10); + EXPECT_CALL(allocator, Alloc(_)).Times(10); for (int i = 0; i < 10; ++i) { objects.push_back(threadQueue.CreateObject(objectType.typeInfo())); } @@ -1095,7 +1095,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { std::mutex expectedMutex; std_support::vector expected; - EXPECT_CALL(allocator, Alloc(_, _)).Times(kThreadCount); + EXPECT_CALL(allocator, Alloc(_)).Times(kThreadCount); for (int i = 0; i < kThreadCount; ++i) { threads.emplace_back([&type, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() { ObjectFactory::ThreadQueue threadQueue(objectFactory, GlobalMockAllocator()); diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/AllocImpl.cpp b/kotlin-native/runtime/src/opt_alloc/cpp/AllocImpl.cpp deleted file mode 100644 index 00a36bb5811..00000000000 --- a/kotlin-native/runtime/src/opt_alloc/cpp/AllocImpl.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include - -extern "C" { -void* mi_malloc(size_t size); -void* mi_malloc_aligned(size_t size, size_t alignment); -void* mi_calloc(size_t count, size_t size); -void* mi_calloc_aligned(size_t count, size_t size, size_t alignment); -void* mi_realloc(void* ptr, size_t size); -void mi_free(void* ptr); -} - -extern "C" void* konan_malloc_impl(size_t size) { - return mi_malloc(size); -} - -extern "C" void* konan_aligned_alloc_impl(size_t alignment, size_t size) { - return mi_malloc_aligned(size, alignment); -} - -extern "C" void* konan_calloc_impl(size_t num, size_t size) { - return mi_calloc(num, size); -} - -extern "C" void* konan_aligned_calloc_impl(size_t alignment, size_t num, size_t size) { - return mi_calloc_aligned(num, size, alignment); -} - -extern "C" void* konan_realloc_impl(void* ptr, size_t size) { - return mi_realloc(ptr, size); -} - -extern "C" void konan_free_impl(void* ptr) { - return mi_free(ptr); -} diff --git a/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp new file mode 100644 index 00000000000..020dafad9af --- /dev/null +++ b/kotlin-native/runtime/src/opt_alloc/cpp/ObjectAlloc.cpp @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ObjectAlloc.hpp" + +#include "../../mimalloc/c/include/mimalloc.h" +#include "Alignment.hpp" + +using namespace kotlin; + +void kotlin::initObjectPool() noexcept { + mi_thread_init(); +} + +void* kotlin::allocateInObjectPool(size_t size) noexcept { + return mi_calloc_aligned(1, size, kObjectAlignment); +} + +void kotlin::freeInObjectPool(void* ptr) noexcept { + mi_free(ptr); +} diff --git a/kotlin-native/runtime/src/std_alloc/cpp/AllocImpl.cpp b/kotlin-native/runtime/src/std_alloc/cpp/AllocImpl.cpp deleted file mode 100644 index 88d0cf9282e..00000000000 --- a/kotlin-native/runtime/src/std_alloc/cpp/AllocImpl.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include - -extern "C" void* konan_malloc_impl(size_t size) { - return ::malloc(size); -} - -extern "C" void* konan_aligned_alloc_impl(size_t alignment, size_t size) { - return ::malloc(size); -} - -extern "C" void* konan_calloc_impl(size_t num, size_t size) { - return ::calloc(num, size); -} - -extern "C" void* konan_aligned_calloc_impl(size_t alignment, size_t num, size_t size) { - return ::calloc(num, size); -} - -extern "C" void* konan_realloc_impl(void* ptr, size_t size) { - return ::realloc(ptr, size); -} - -extern "C" void konan_free_impl(void* ptr) { - return ::free(ptr); -} diff --git a/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp new file mode 100644 index 00000000000..3f64d87a853 --- /dev/null +++ b/kotlin-native/runtime/src/std_alloc/cpp/ObjectAlloc.cpp @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ObjectAlloc.hpp" + +#if KONAN_INTERNAL_DLMALLOC +extern "C" void* dlcalloc(size_t, size_t); +extern "C" void dlfree(void*); + +#define callocImpl dlcalloc +#define freeImpl dlfree +#else +#include + +#define callocImpl ::calloc +#define freeImpl ::free +#endif + +using namespace kotlin; + +void kotlin::initObjectPool() noexcept {} + +void* kotlin::allocateInObjectPool(size_t size) noexcept { + // TODO: Check that alignment to kObjectAlignment is satisfied. + return callocImpl(1, size); +} + +void kotlin::freeInObjectPool(void* ptr) noexcept { + freeImpl(ptr); +}