diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index 7659b38c3ea..e24ef92592d 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -3441,6 +3441,24 @@ standaloneTest("stress_gc_allocations") { flags = ['-tr', '-opt-in=kotlin.native.internal.InternalForKotlinNative'] } +standaloneTest("array_out_of_memory") { + source = "runtime/memory/array_out_of_memory.kt" + flags = ['-tr'] + switch(project.target.architecture) { + case Architecture.X64: + case Architecture.ARM64: + break; + case Architecture.X86: + case Architecture.ARM32: + case Architecture.MIPS32: + case Architecture.MIPSEL32: + case Architecture.WASM32: + expectedExitStatusChecker = { it != 0 } + outputChecker = { s -> s.contains("Out of memory trying to allocate") } + break; + } +} + standaloneTest("mpp1") { source = "codegen/mpp/mpp1.kt" flags = ['-tr', '-Xmulti-platform'] diff --git a/kotlin-native/backend.native/tests/runtime/memory/array_out_of_memory.kt b/kotlin-native/backend.native/tests/runtime/memory/array_out_of_memory.kt new file mode 100644 index 00000000000..216918ceebf --- /dev/null +++ b/kotlin-native/backend.native/tests/runtime/memory/array_out_of_memory.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +import kotlin.test.* + +fun testArrayAllocation(size: Int) { + val arr = IntArray(size) + // Force a write into the memory. + // TODO: How to make sure the optimizer never deletes this write? + arr[size - 1] = 42 + assertEquals(42, arr[size - 1]) +} + +@Test +fun sanity() { + // Should always succeed everywhere + testArrayAllocation(1 shl 10) +} + +@Test +fun test() { + // Will fail on 32 bits. + testArrayAllocation(1 shl 30) +} diff --git a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp index 107752e7f3b..114efb334c9 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/GCImpl.cpp @@ -54,7 +54,7 @@ ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeIn return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); } -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) { +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); } diff --git a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp index d583b3cad74..742dcb35fe9 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/GC.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/GC.hpp @@ -42,7 +42,7 @@ public: void ClearForTests() noexcept; ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; - ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements); + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept; void OnStoppedForGC() noexcept; void OnSuspendForGC() noexcept; diff --git a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp index 1a717998b92..dfca326cd84 100644 --- a/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/noop/cpp/GCImpl.cpp @@ -44,7 +44,7 @@ ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeIn return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); } -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) { +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); } diff --git a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp index a10c025ff5e..fbb33f6c974 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/GCImpl.cpp @@ -57,7 +57,7 @@ ALWAYS_INLINE ObjHeader* gc::GC::ThreadData::CreateObject(const TypeInfo* typeIn return impl_->objectFactoryThreadQueue().CreateObject(typeInfo); } -ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) { +ALWAYS_INLINE ArrayHeader* gc::GC::ThreadData::CreateArray(const TypeInfo* typeInfo, uint32_t elements) noexcept { return impl_->objectFactoryThreadQueue().CreateArray(typeInfo, elements); } diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index 92106f59630..e5576c64697 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -17,6 +17,7 @@ #include #include +#include #include // for offsetof #include @@ -32,6 +33,7 @@ #endif #include "KAssert.h" +#include "Alignment.hpp" #include "Atomic.h" #include "Cleaner.h" #include "CompilerConstants.hpp" @@ -92,7 +94,7 @@ ALWAYS_INLINE bool IsStrictMemoryModel() noexcept { inline constexpr ObjectPoolAllocator objectAllocator; -typedef uint32_t container_size_t; +using container_size_t = size_t; // Granularity of arena container chunks. constexpr container_size_t kContainerAlignment = 1024; @@ -913,7 +915,7 @@ class ArenaContainer { // Places an array of certain type in this container. Note that array_type_info // is type info for an array, not for an individual element. Also note that exactly // same operation could be used to place strings. - ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count); + ArrayHeader* PlaceArray(const TypeInfo* array_type_info, uint32_t count); ObjHeader** getSlot(); @@ -962,23 +964,23 @@ inline ContainerHeader* clearRemoved(ContainerHeader* container) { reinterpret_cast(container) & ~static_cast(1)); } -inline container_size_t alignUp(container_size_t size, int alignment) { - return (size + alignment - 1) & ~(alignment - 1); -} - inline ContainerHeader* realShareableContainer(ContainerHeader* container) { RuntimeAssert(container->shareable(), "Only makes sense on shareable objects"); return containerFor(reinterpret_cast(container + 1)); } -inline uint32_t arrayObjectSize(const TypeInfo* typeInfo, uint32_t count) { - // Note: array body is aligned, but for size computation it is enough to align the sum. +inline uint64_t arrayObjectSize(const TypeInfo* typeInfo, uint32_t count) { static_assert(kObjectAlignment % alignof(KLong) == 0, ""); static_assert(kObjectAlignment % alignof(KDouble) == 0, ""); - return alignUp(sizeof(ArrayHeader) - typeInfo->instanceSize_ * count, kObjectAlignment); + // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped + // at about half of uint64_t max. + uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; + // Note: array body is aligned, but for size computation it is enough to align the sum. + return AlignUp(sizeof(ArrayHeader) + membersSize, kObjectAlignment); } -inline uint32_t arrayObjectSize(const ArrayHeader* obj) { +inline container_size_t arrayObjectSize(const ArrayHeader* obj) { + // Only used for already allocated arrays. Cannov overflow size_t. return arrayObjectSize(obj->type_info(), obj->count_); } @@ -990,7 +992,7 @@ inline container_size_t objectSize(const ObjHeader* obj) { arrayObjectSize(obj->array()) : type_info->instanceSize_); - return alignUp(size, kObjectAlignment); + return kotlin::AlignUp(size, kObjectAlignment); } template @@ -1079,7 +1081,7 @@ ContainerHeader* allocContainer(MemoryState* state, size_t size) { if (state != nullptr) state->allocSinceLastGc += size; #endif - result = new (allocateInObjectPool(alignUp(size, kObjectAlignment))) ContainerHeader(); + result = new (allocateInObjectPool(kotlin::AlignUp(size, kObjectAlignment))) ContainerHeader(); atomicAdd(&allocCount, 1); } if (state != nullptr) { @@ -1780,11 +1782,11 @@ inline ArenaContainer* initedArena(ObjHeader** auxSlot) { return arena; } -inline size_t containerSize(const ContainerHeader* container) { - size_t result = 0; +inline container_size_t containerSize(const ContainerHeader* container) { + container_size_t result = 0; const ObjHeader* obj = reinterpret_cast(container + 1); for (uint32_t object = 0; object < container->objectCount(); object++) { - size_t size = objectSize(obj); + container_size_t size = objectSize(obj); result += size; obj = reinterpret_cast(reinterpret_cast(obj) + size); } @@ -3188,7 +3190,7 @@ void ObjHeader::destroyMetaObject(ObjHeader* object) { void ObjectContainer::Init(MemoryState* state, const TypeInfo* typeInfo) { RuntimeAssert(typeInfo->instanceSize_ >= 0, "Must be an object"); - uint32_t allocSize = sizeof(ContainerHeader) + typeInfo->instanceSize_; + container_size_t allocSize = sizeof(ContainerHeader) + typeInfo->instanceSize_; header_ = allocContainer(state, allocSize); RuntimeCheck(header_ != nullptr, "Cannot alloc memory"); // One object in this container, no need to set. @@ -3201,8 +3203,12 @@ void ObjectContainer::Init(MemoryState* state, const TypeInfo* typeInfo) { void ArrayContainer::Init(MemoryState* state, const TypeInfo* typeInfo, uint32_t elements) { RuntimeAssert(typeInfo->instanceSize_ < 0, "Must be an array"); - uint32_t allocSize = - sizeof(ContainerHeader) + arrayObjectSize(typeInfo, elements); + uint64_t dataSize = arrayObjectSize(typeInfo, elements); + uint64_t allocSize = sizeof(ContainerHeader) + dataSize; + if (allocSize > std::numeric_limits::max()) { + konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 " bytes. Aborting.\n", allocSize); + konan::abort(); + } header_ = allocContainer(state, allocSize); RuntimeCheck(header_ != nullptr, "Cannot alloc memory"); // One object in this container, no need to set. @@ -3211,7 +3217,7 @@ void ArrayContainer::Init(MemoryState* state, const TypeInfo* typeInfo, uint32_t // header->refCount_ is zero initialized by allocContainer(). GetPlace()->count_ = elements; SetHeader(GetPlace()->obj(), typeInfo); - OBJECT_ALLOC_EVENT(memoryState, arrayObjectSize(typeInfo, elements), GetPlace()->obj()) + OBJECT_ALLOC_EVENT(memoryState, dataSize, GetPlace()->obj()) } // TODO: store arena containers in some reuseable data structure, similar to @@ -3239,7 +3245,7 @@ void ArenaContainer::Deinit() { bool ArenaContainer::allocContainer(container_size_t minSize) { auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk); - size = alignUp(size, kContainerAlignment); + size = kotlin::AlignUp(size, kContainerAlignment); // TODO: keep simple cache of container chunks. ContainerChunk* result = new (allocateInObjectPool(size)) ContainerChunk(); RuntimeCheck(result != nullptr, "Cannot alloc memory"); @@ -3254,7 +3260,7 @@ bool ArenaContainer::allocContainer(container_size_t minSize) { } void* ArenaContainer::place(container_size_t size) { - size = alignUp(size, kObjectAlignment); + size = kotlin::AlignUp(size, kObjectAlignment); // Fast path. if (current_ + size < end_) { void* result = current_; @@ -3282,7 +3288,7 @@ ObjHeader** ArenaContainer::getSlot() { ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - uint32_t size = type_info->instanceSize_; + container_size_t size = type_info->instanceSize_; ObjHeader* result = reinterpret_cast(place(size)); if (!result) { return nullptr; @@ -3295,12 +3301,16 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t count) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); - container_size_t size = arrayObjectSize(type_info, count); + uint64_t size = arrayObjectSize(type_info, count); + if (size > std::numeric_limits::max()) { + konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 " bytes. Aborting.\n", size); + konan::abort(); + } ArrayHeader* result = reinterpret_cast(place(size)); if (!result) { return nullptr; } - OBJECT_ALLOC_EVENT(memoryState, arrayObjectSize(type_info, count), result->obj()) + OBJECT_ALLOC_EVENT(memoryState, size, result->obj()) currentChunk_->asHeader()->incObjectCount(); setHeader(result->obj(), type_info); result->count_ = count; diff --git a/kotlin-native/runtime/src/main/cpp/Alignment.hpp b/kotlin-native/runtime/src/main/cpp/Alignment.hpp index bb97e1b1fcc..4bfa10d5ccd 100644 --- a/kotlin-native/runtime/src/main/cpp/Alignment.hpp +++ b/kotlin-native/runtime/src/main/cpp/Alignment.hpp @@ -13,7 +13,8 @@ namespace kotlin { constexpr size_t kObjectAlignment = 8; -constexpr inline size_t AlignUp(size_t size, size_t alignment) { +template +constexpr T AlignUp(T size, T alignment) { return (size + alignment - 1) & ~(alignment - 1); } diff --git a/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp b/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp index 26b313784e8..be6e38a0794 100644 --- a/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp +++ b/kotlin-native/runtime/src/main/cpp/std_support/Optional.hpp @@ -3,6 +3,8 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ +#pragma once + #if __has_include() #include #elif __has_include() @@ -15,4 +17,4 @@ inline constexpr auto nullopt = std::experimental::nullopt; } // namespace std #else #error "No " -#endif \ No newline at end of file +#endif diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 51d3106e0ce..f05118809b9 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -7,12 +7,12 @@ #define RUNTIME_MM_OBJECT_FACTORY_H #include +#include #include #include #include #include "Alignment.hpp" -#include "Exceptions.h" #include "FinalizerHooks.hpp" #include "Memory.h" #include "Mutex.hpp" @@ -54,9 +54,9 @@ public: public: ~Node() = default; - constexpr static size_t GetSizeForDataSize(size_t dataSize) noexcept { - size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); - size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, DataAlignment); + constexpr static uint64_t GetSizeForDataSize(uint64_t dataSize) noexcept { + uint64_t dataSizeAligned = AlignUp(dataSize, DataAlignment); + uint64_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, DataAlignment); return totalSize; } @@ -86,14 +86,19 @@ public: Node() noexcept = default; - static unique_ptr Create(Allocator& allocator, size_t dataSize) noexcept { + static unique_ptr Create(Allocator& allocator, uint64_t dataSize) noexcept { auto totalSize = GetSizeForDataSize(dataSize); - RuntimeAssert( - DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize, - DataOffset()); - void* ptr = allocator.Alloc(totalSize); + void* ptr = nullptr; + if (totalSize <= std::numeric_limits::max()) { + RuntimeAssert( + DataOffset() + dataSize <= totalSize, "totalSize %" PRIu64 " is not enough to fit data %" PRIu64 " at offset %zu", totalSize, dataSize, + DataOffset()); + ptr = allocator.Alloc(totalSize); + } if (!ptr) { - konan::consoleErrorf("Out of memory trying to allocate %zu bytes. Aborting.\n", totalSize); + // TODO: This should throw OutOfMemoryError in the future if we add hard memory limits instead + // of limiting at virtual address space boundary. + konan::consoleErrorf("Out of memory trying to allocate %" PRIu64 " bytes. Aborting.\n", totalSize); konan::abort(); } RuntimeAssert(IsAligned(ptr, DataAlignment), "Allocator returned unaligned to %zu pointer %p", DataAlignment, ptr); @@ -134,7 +139,7 @@ public: size_t size() const noexcept { return size_; } - Node& Insert(size_t dataSize) noexcept { + Node& Insert(uint64_t dataSize) noexcept { AssertCorrect(); auto node = Node::Create(allocator_, dataSize); auto* nodePtr = node.get(); @@ -513,7 +518,8 @@ public: static size_t ObjectAllocatedSize(const TypeInfo* typeInfo) noexcept { RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - size_t allocSize = ObjectAllocatedDataSize(typeInfo); + // Only used for already allocated objects. Cannot overflow size_t + auto allocSize = ObjectAllocatedDataSize(typeInfo); return Storage::Node::GetSizeForDataSize(allocSize); } @@ -530,21 +536,14 @@ public: static size_t ArrayAllocatedSize(const TypeInfo* typeInfo, uint32_t count) noexcept { RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - size_t allocSize = ArrayAllocatedDataSize(typeInfo, count); + // Only used for already allocated arrays. Cannot overflow size_t + auto allocSize = ArrayAllocatedDataSize(typeInfo, count); return Storage::Node::GetSizeForDataSize(allocSize); } - ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) { + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - size_t allocSize = ArrayAllocatedDataSize(typeInfo, count); - // On 32-bit systems, overflow can happen in several places along - // the size calculation. If overflow did not happen in the - // multiplication checked in the previous call, but any of the - // subsequent additions overflowed, then the overflowed value will - // be small compared to the number of entries in the array. - if (Storage::Node::GetSizeForDataSize(allocSize) < count) { - ThrowOutOfMemoryError(); - } + auto allocSize = ArrayAllocatedDataSize(typeInfo, count); auto& node = producer_.Insert(allocSize); auto* heapArray = new (node.Data()) HeapArrayHeader(); auto* array = &heapArray->array; @@ -567,13 +566,12 @@ public: return AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); } - static size_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { - size_t membersSize; - if (__builtin_mul_overflow(static_cast(-typeInfo->instanceSize_), count, &membersSize)) { - ThrowOutOfMemoryError(); - } + static uint64_t ArrayAllocatedDataSize(const TypeInfo* typeInfo, uint32_t count) noexcept { + // -(int32_t min) * uint32_t max cannot overflow uint64_t. And are capped + // at about half of uint64_t max. + uint64_t membersSize = static_cast(-typeInfo->instanceSize_) * count; // Note: array body is aligned, but for size computation it is enough to align the sum. - return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + return AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); } typename Storage::Producer producer_; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index ebf6e9efde8..b72df3f89fa 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -62,7 +62,7 @@ OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) RETURN_OBJ(object); } -OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) { +OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. auto* array = threadData->gc().CreateArray(typeInfo, static_cast(elements)); diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp index 867322885ab..9e06fa89b16 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -27,7 +27,7 @@ void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept; OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept; OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; -OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements); +OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept; // This does not take into account how much storage did the underlying allocator (malloc/mimalloc) reserved. size_t GetAllocatedHeapSize(ObjHeader* object) noexcept;