diff --git a/kotlin-native/runtime/src/main/cpp/Alignment.hpp b/kotlin-native/runtime/src/main/cpp/Alignment.hpp new file mode 100644 index 00000000000..bb97e1b1fcc --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/Alignment.hpp @@ -0,0 +1,39 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_ALIGNMENT_H +#define RUNTIME_ALIGNMENT_H + +#include +#include + +namespace kotlin { + +constexpr size_t kObjectAlignment = 8; + +constexpr inline size_t AlignUp(size_t size, size_t alignment) { + return (size + alignment - 1) & ~(alignment - 1); +} + +inline void* AlignUp(void* ptr, size_t alignment) { + static_assert(sizeof(void*) == sizeof(size_t), "size_t size must be equal to pointer size for this to work"); + return reinterpret_cast(AlignUp(reinterpret_cast(ptr), alignment)); +} + +constexpr inline bool IsValidAlignment(size_t alignment) { + return alignment != 0 && (alignment & (alignment - 1)) == 0; +} + +constexpr inline bool IsAligned(size_t size, size_t alignment) { + return size % alignment == 0; +} + +inline bool IsAligned(void* ptr, size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; +} + +} // namespace kotlin + +#endif // RUNTIME_ALIGNMENT_H diff --git a/kotlin-native/runtime/src/main/cpp/AlignmentTest.cpp b/kotlin-native/runtime/src/main/cpp/AlignmentTest.cpp new file mode 100644 index 00000000000..cd70996470c --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/AlignmentTest.cpp @@ -0,0 +1,157 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Alignment.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "Types.h" + +using namespace kotlin; + +namespace { + +template +class NamedTestWithParam : public testing::TestWithParam> { +public: + using Param = std::tuple; + + static std::string Print(const testing::TestParamInfo& info) { return std::string(std::get<0>(info.param)); } + + template + static const typename std::tuple_element::type& Get() { + const auto& param = testing::TestWithParam::GetParam(); + return std::get(param); + } +}; + +#define INSTANTIATE_NAMED_TEST(testName, ...) INSTANTIATE_TEST_SUITE_P(, testName, testing::Values(__VA_ARGS__), &testName::Print) + +} // namespace + +using IsValidAlignmentTest = NamedTestWithParam; + +TEST_P(IsValidAlignmentTest, Test) { + const auto& alignment = Get<0>(); + const auto& expected = Get<1>(); + EXPECT_THAT(IsValidAlignment(alignment), expected); +} + +INSTANTIATE_NAMED_TEST( + IsValidAlignmentTest, + std::make_tuple("0", 0, false), + std::make_tuple("1", 1, true), + std::make_tuple("2", 2, true), + std::make_tuple("3", 3, false), + std::make_tuple("4", 4, true), + std::make_tuple("5", 5, false), + std::make_tuple("6", 6, false), + std::make_tuple("7", 7, false), + std::make_tuple("8", 8, true), + std::make_tuple("9", 9, false), + std::make_tuple("10", 10, false), + std::make_tuple("11", 11, false), + std::make_tuple("12", 12, false), + std::make_tuple("13", 13, false), + std::make_tuple("14", 14, false), + std::make_tuple("15", 15, false), + std::make_tuple("16", 16, true), + std::make_tuple("int", alignof(int), true), + std::make_tuple("ptr", alignof(void*), true), + std::make_tuple("max", alignof(std::max_align_t), true)); + +using IsAlignedSizeTest = NamedTestWithParam; + +TEST_P(IsAlignedSizeTest, Test) { + const auto& size = Get<0>(); + const auto& alignment = Get<1>(); + const auto& expected = Get<2>(); + EXPECT_THAT(IsAligned(size, alignment), expected); +} + +INSTANTIATE_NAMED_TEST( + IsAlignedSizeTest, + std::make_tuple("1_1", 1, 1, true), + std::make_tuple("2_1", 2, 1, true), + std::make_tuple("3_1", 3, 1, true), + std::make_tuple("4_1", 4, 1, true), + std::make_tuple("1_2", 1, 2, false), + std::make_tuple("2_2", 2, 2, true), + std::make_tuple("3_2", 3, 2, false), + std::make_tuple("4_2", 4, 2, true)); + +using IsAlignedPointerTest = NamedTestWithParam; + +TEST_P(IsAlignedPointerTest, Test) { + const auto& ptr = Get<0>(); + const auto& alignment = Get<1>(); + const auto& expected = Get<2>(); + EXPECT_THAT(IsAligned(reinterpret_cast(ptr), alignment), expected); +} + +INSTANTIATE_NAMED_TEST( + IsAlignedPointerTest, + std::make_tuple("0_1", 0, 1, true), + std::make_tuple("1_1", 1, 1, true), + std::make_tuple("2_1", 2, 1, true), + std::make_tuple("3_1", 3, 1, true), + std::make_tuple("4_1", 4, 1, true), + std::make_tuple("0_2", 0, 2, true), + std::make_tuple("1_2", 1, 2, false), + std::make_tuple("2_2", 2, 2, true), + std::make_tuple("3_2", 3, 2, false), + std::make_tuple("4_2", 4, 2, true)); + +using AlignUpSizeTest = NamedTestWithParam; + +TEST_P(AlignUpSizeTest, Test) { + const auto& size = Get<0>(); + const auto& alignment = Get<1>(); + const auto& expected = Get<2>(); + EXPECT_THAT(AlignUp(size, alignment), expected); +} + +INSTANTIATE_NAMED_TEST( + AlignUpSizeTest, + std::make_tuple("1_1", 1, 1, 1), + std::make_tuple("2_1", 2, 1, 2), + std::make_tuple("3_1", 3, 1, 3), + std::make_tuple("4_1", 4, 1, 4), + std::make_tuple("1_2", 1, 2, 2), + std::make_tuple("2_2", 2, 2, 2), + std::make_tuple("3_2", 3, 2, 4), + std::make_tuple("4_2", 4, 2, 4)); + +using AlignUpPointerTest = NamedTestWithParam; + +TEST_P(AlignUpPointerTest, Test) { + const auto& ptr = Get<0>(); + const auto& alignment = Get<1>(); + const auto& expected = Get<2>(); + EXPECT_THAT(AlignUp(reinterpret_cast(ptr), alignment), reinterpret_cast(expected)); +} + +INSTANTIATE_NAMED_TEST( + AlignUpPointerTest, + std::make_tuple("0_1", 0, 1, 0), + std::make_tuple("1_1", 1, 1, 1), + std::make_tuple("2_1", 2, 1, 2), + std::make_tuple("3_1", 3, 1, 3), + std::make_tuple("4_1", 4, 1, 4), + std::make_tuple("0_2", 0, 2, 0), + std::make_tuple("1_2", 1, 2, 2), + std::make_tuple("2_2", 2, 2, 2), + std::make_tuple("3_2", 3, 2, 4), + std::make_tuple("4_2", 4, 2, 4)); + +TEST(AlignmentTest, ObjectAlignment) { + static_assert(IsValidAlignment(kObjectAlignment), "kObjectAlignment must be a valid alignment"); + static_assert(kObjectAlignment % alignof(KLong) == 0, ""); + static_assert(kObjectAlignment % alignof(KDouble) == 0, ""); +} diff --git a/kotlin-native/runtime/src/main/cpp/Alloc.h b/kotlin-native/runtime/src/main/cpp/Alloc.h index fca43491c32..8c853edf93f 100644 --- a/kotlin-native/runtime/src/main/cpp/Alloc.h +++ b/kotlin-native/runtime/src/main/cpp/Alloc.h @@ -29,6 +29,10 @@ inline void* konanAllocMemory(size_t size) { return konan::calloc(1, size); } +inline void* konanAllocAlignedMemory(size_t size, size_t alignment) { + return konan::calloc_aligned(1, size, alignment); +} + inline void konanFreeMemory(void* memory) { konan::free(memory); } diff --git a/kotlin-native/runtime/src/main/cpp/TypeInfo.h b/kotlin-native/runtime/src/main/cpp/TypeInfo.h index 669962de6ce..ef3c302f6fd 100644 --- a/kotlin-native/runtime/src/main/cpp/TypeInfo.h +++ b/kotlin-native/runtime/src/main/cpp/TypeInfo.h @@ -157,6 +157,8 @@ struct TypeInfo { inline VTableElement* vtable() { return reinterpret_cast(this + 1); } + + inline bool IsArray() const { return instanceSize_ < 0; } #endif }; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index e0b0e0a80e2..cc06903da16 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -6,6 +6,7 @@ #ifndef RUNTIME_MM_GLOBAL_DATA_H #define RUNTIME_MM_GLOBAL_DATA_H +#include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" #include "StableRefRegistry.hpp" #include "ThreadRegistry.hpp" @@ -19,9 +20,10 @@ class GlobalData : private Pinned { public: static GlobalData& Instance() noexcept { return instance_; } - ThreadRegistry& threadRegistry() { return threadRegistry_; } - GlobalsRegistry& globalsRegistry() { return globalsRegistry_; } - StableRefRegistry& stableRefRegistry() { return stableRefRegistry_; } + ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; } + GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } + StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; } + ObjectFactory& objectFactory() noexcept { return objectFactory_; } private: GlobalData(); @@ -32,6 +34,7 @@ private: ThreadRegistry threadRegistry_; GlobalsRegistry globalsRegistry_; StableRefRegistry stableRefRegistry_; + ObjectFactory objectFactory_; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 184736dc66d..8eb8ef9e4fd 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -5,6 +5,7 @@ #include "Memory.h" +#include "Exceptions.h" #include "GlobalsRegistry.hpp" #include "KAssert.h" #include "Porting.h" @@ -74,6 +75,22 @@ extern "C" void RestoreMemory(MemoryState*) { // TODO: Remove when legacy MM is gone. } +extern "C" RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo); + RETURN_OBJ(object); +} + +extern "C" OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elements) { + if (elements < 0) { + ThrowIllegalArgumentException(); + } + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast(elements)); + // `ArrayHeader` and `ObjHeader` are expected to be compatible. + RETURN_OBJ(reinterpret_cast(array)); +} + extern "C" OBJ_GETTER(InitSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); // TODO: This should only be called if singleton is actually created here. It's possible that the diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp new file mode 100644 index 00000000000..33b6bb9b497 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp @@ -0,0 +1,61 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ObjectFactory.hpp" + +#include "Alignment.hpp" +#include "Alloc.h" +#include "GlobalData.hpp" +#include "Types.h" + +using namespace kotlin; + +ObjHeader* mm::ObjectFactory::ThreadQueue::CreateObject(const TypeInfo* typeInfo) noexcept { + RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); + size_t allocSize = typeInfo->instanceSize_; + auto& node = producer_.Insert(allocSize); + auto* object = static_cast(node.Data()); + object->typeInfoOrMeta_ = const_cast(typeInfo); + return object; +} + +ArrayHeader* mm::ObjectFactory::ThreadQueue::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { + RuntimeAssert(typeInfo->IsArray(), "Must be an array"); + uint32_t arraySize = static_cast(-typeInfo->instanceSize_) * count; + // Note: array body is aligned, but for size computation it is enough to align the sum. + size_t allocSize = AlignUp(sizeof(ArrayHeader) + arraySize, kObjectAlignment); + auto& node = producer_.Insert(allocSize); + auto* array = static_cast(node.Data()); + array->typeInfoOrMeta_ = const_cast(typeInfo); + array->count_ = count; + return array; +} + +bool mm::ObjectFactory::Iterator::IsArray() noexcept { + // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can + // be always casted to the other. + auto* object = static_cast((*iterator_).Data()); + return object->type_info()->IsArray(); +} + +ObjHeader* mm::ObjectFactory::Iterator::GetObjHeader() noexcept { + auto* object = static_cast((*iterator_).Data()); + RuntimeAssert(!object->type_info()->IsArray(), "Must not be an array"); + return object; +} + +ArrayHeader* mm::ObjectFactory::Iterator::GetArrayHeader() noexcept { + auto* array = static_cast((*iterator_).Data()); + RuntimeAssert(array->type_info()->IsArray(), "Must be an array"); + return array; +} + +mm::ObjectFactory::ObjectFactory() noexcept = default; +mm::ObjectFactory::~ObjectFactory() = default; + +// static +mm::ObjectFactory& mm::ObjectFactory::Instance() noexcept { + return GlobalData::Instance().objectFactory(); +} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp new file mode 100644 index 00000000000..cb5968e9831 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -0,0 +1,316 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_OBJECT_FACTORY_H +#define RUNTIME_MM_OBJECT_FACTORY_H + +#include +#include +#include + +#include "Alignment.hpp" +#include "Alloc.h" +#include "CppSupport.hpp" +#include "Memory.h" +#include "Mutex.hpp" +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +namespace internal { + +// A queue that is constructed by collecting subqueues from several `Producer`s. +// This is essentially a heterogeneous `MultiSourceQueue` on top of a singly linked list that +// uses `konanAllocMemory` and `konanFreeMemory` +// TODO: Consider merging with `MultiSourceQueue` somehow. +template +class ObjectFactoryStorage : private Pinned { + static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment"); + +public: + // This class does not know its size at compile-time. + class Node : private Pinned { + constexpr static size_t DataOffset() noexcept { return AlignUp(sizeof(Node), DataAlignment); } + + public: + ~Node() = default; + + static void operator delete(void* ptr) noexcept { konanFreeMemory(ptr); } + + // Note: This can only be trivially destructible data, as nobody can invoke its destructor. + void* Data() noexcept { + constexpr size_t kDataOffset = DataOffset(); + void* ptr = reinterpret_cast(this) + kDataOffset; + RuntimeAssert(IsAligned(ptr, DataAlignment), "Data=%p is not aligned to %zu", ptr, DataAlignment); + return ptr; + } + + // It's a caller responsibility to know if the underlying data is `T`. + template + T& Data() noexcept { + return *static_cast(Data()); + } + + private: + friend class ObjectFactoryStorage; + + Node() noexcept = default; + + static void* operator new(size_t size, size_t dataSize) noexcept { + size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); + size_t totalAlignment = std::max(alignof(Node), DataAlignment); + size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment); + RuntimeAssert( + DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize, + DataOffset()); + void* ptr = konanAllocAlignedMemory(totalSize, totalAlignment); + if (!ptr) { + // TODO: Try doing GC first. + konan::consoleErrorf("Out of memory trying to allocate %zu. Aborting.\n", totalSize); + konan::abort(); + } + RuntimeAssert(IsAligned(ptr, totalAlignment), "Allocator returned unaligned to %zu pointer %p", totalAlignment, ptr); + return ptr; + } + + std::unique_ptr next_; + // There's some more data of an unknown (at compile-time) size here, but it cannot be represented + // with C++ members. + }; + + class Producer : private MoveOnly { + public: + explicit Producer(ObjectFactoryStorage& owner) noexcept : owner_(owner) {} + + ~Producer() { Publish(); } + + Node& Insert(size_t dataSize) noexcept { + AssertCorrect(); + auto* nodePtr = new (dataSize) Node(); + std::unique_ptr node(nodePtr); + if (!root_) { + RuntimeAssert(last_ == nullptr, "Unsynchronized root_ and last_"); + root_ = std::move(node); + } else { + RuntimeAssert(last_ != nullptr, "Unsynchronized root_ and last_"); + last_->next_ = std::move(node); + } + + last_ = nodePtr; + RuntimeAssert(root_ != nullptr, "Must not be empty"); + AssertCorrect(); + return *nodePtr; + } + + template + Node& Insert(Args&&... args) noexcept { + static_assert(alignof(T) <= DataAlignment, "Cannot insert type with alignment bigger than DataAlignment"); + static_assert(std_support::is_trivially_destructible_v, "Type must be trivially destructible"); + auto& node = Insert(sizeof(T)); + new (node.Data()) T(std::forward(args)...); + return node; + } + + // Merge `this` queue with owning `ObjectFactoryStorage`. + // `this` will have empty queue after the call. + // This call is performed without heap allocations. TODO: Test that no allocations are happening. + void Publish() noexcept { + AssertCorrect(); + if (!root_) { + return; + } + + std::lock_guard guard(owner_.mutex_); + + owner_.AssertCorrectUnsafe(); + + if (!owner_.root_) { + owner_.root_ = std::move(root_); + } else { + owner_.last_->next_ = std::move(root_); + } + + owner_.last_ = last_; + last_ = nullptr; + + RuntimeAssert(root_ == nullptr, "Must be empty"); + AssertCorrect(); + RuntimeAssert(owner_.root_ != nullptr, "Must not be empty"); + owner_.AssertCorrectUnsafe(); + } + + private: + friend class ObjectFactoryStorage; + + ALWAYS_INLINE void AssertCorrect() const noexcept { + if (root_ == nullptr) { + RuntimeAssert(last_ == nullptr, "last_ must be null"); + } else { + RuntimeAssert(last_ != nullptr, "last_ must not be null"); + RuntimeAssert(last_->next_ == nullptr, "last_ must not have next"); + } + } + + ObjectFactoryStorage& owner_; // weak + std::unique_ptr root_; + Node* last_ = nullptr; + }; + + class Iterator { + public: + Node& operator*() noexcept { return *node_; } + Node* operator->() noexcept { return node_; } + + Iterator& operator++() noexcept { + previousNode_ = node_; + node_ = node_->next_.get(); + return *this; + } + + bool operator==(const Iterator& rhs) const noexcept { return node_ == rhs.node_; } + + bool operator!=(const Iterator& rhs) const noexcept { return node_ != rhs.node_; } + + private: + friend class ObjectFactoryStorage; + + Iterator(Node* previousNode, Node* node) noexcept : previousNode_(previousNode), node_(node) {} + + Node* previousNode_; // Kept for `Iterable::EraseAndAdvance`. + Node* node_; + }; + + class Iterable : private MoveOnly { + public: + explicit Iterable(ObjectFactoryStorage& owner) noexcept : owner_(owner), guard_(owner_.mutex_) {} + + Iterator begin() noexcept { return Iterator(nullptr, owner_.root_.get()); } + Iterator end() noexcept { return Iterator(owner_.last_, nullptr); } + + void EraseAndAdvance(Iterator& iterator) noexcept { iterator.node_ = owner_.EraseUnsafe(iterator.previousNode_); } + + private: + ObjectFactoryStorage& owner_; // weak + std::unique_lock guard_; + }; + + // Lock `ObjectFactoryStorage` for safe iteration. + Iterable Iter() noexcept { return Iterable(*this); } + +private: + // Expects `mutex_` to be held by the current thread. + Node* EraseUnsafe(Node* previousNode) noexcept { + RuntimeAssert(root_ != nullptr, "Must not be empty"); + AssertCorrectUnsafe(); + + if (previousNode == nullptr) { + // Deleting the root. + root_ = std::move(root_->next_); + if (!root_) { + last_ = nullptr; + } + AssertCorrectUnsafe(); + return root_.get(); + } + + auto node = std::move(previousNode->next_); + previousNode->next_ = std::move(node->next_); + if (!previousNode->next_) { + last_ = previousNode; + } + + AssertCorrectUnsafe(); + return previousNode->next_.get(); + } + + // Expects `mutex_` to be held by the current thread. + ALWAYS_INLINE void AssertCorrectUnsafe() const noexcept { + if (root_ == nullptr) { + RuntimeAssert(last_ == nullptr, "last_ must be null"); + } else { + RuntimeAssert(last_ != nullptr, "last_ must not be null"); + RuntimeAssert(last_->next_ == nullptr, "last_ must not have next"); + } + } + + std::unique_ptr root_; + Node* last_ = nullptr; + SpinLock mutex_; +}; + +} // namespace internal + +class ObjectFactory : private Pinned { +public: + using Storage = internal::ObjectFactoryStorage; + + class ThreadQueue : private MoveOnly { + public: + explicit ThreadQueue(ObjectFactory& owner) noexcept : producer_(owner.storage_) {} + + ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept; + + void Publish() noexcept { producer_.Publish(); } + + private: + Storage::Producer producer_; + }; + + class Iterator { + public: + Storage::Node& operator*() noexcept { return *iterator_; } + + Iterator& operator++() noexcept { + ++iterator_; + return *this; + } + + bool operator==(const Iterator& rhs) const noexcept { return iterator_ == rhs.iterator_; } + + bool operator!=(const Iterator& rhs) const noexcept { return iterator_ != rhs.iterator_; } + + bool IsArray() noexcept; + + ObjHeader* GetObjHeader() noexcept; + ArrayHeader* GetArrayHeader() noexcept; + + private: + friend class ObjectFactory; + + explicit Iterator(Storage::Iterator iterator) noexcept : iterator_(std::move(iterator)) {} + + Storage::Iterator iterator_; + }; + + class Iterable { + public: + Iterable(ObjectFactory& owner) noexcept : iter_(owner.storage_.Iter()) {} + + Iterator begin() noexcept { return Iterator(iter_.begin()); } + Iterator end() noexcept { return Iterator(iter_.end()); } + + void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); } + + private: + Storage::Iterable iter_; + }; + + ObjectFactory() noexcept; + ~ObjectFactory(); + + static ObjectFactory& Instance() noexcept; + + Iterable Iter() noexcept { return Iterable(*this); } + +private: + Storage storage_; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_OBJECT_FACTORY_H diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp new file mode 100644 index 00000000000..f3c4a6a0e54 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -0,0 +1,573 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ObjectFactory.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "CppSupport.hpp" +#include "TestSupport.hpp" + +using namespace kotlin; + +template +using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; + +using ObjectFactoryStorageRegular = ObjectFactoryStorage; + +namespace { + +template +std::vector Collect(ObjectFactoryStorage& storage) { + std::vector result; + for (auto& node : storage.Iter()) { + result.push_back(node.Data()); + } + return result; +} + +template +std::vector Collect(ObjectFactoryStorage& storage) { + std::vector result; + for (auto& node : storage.Iter()) { + result.push_back(*static_cast(node.Data())); + } + return result; +} + +struct MoveOnlyImpl : private MoveOnly { + MoveOnlyImpl(int value1, int value2) : value1(value1), value2(value2) {} + + int value1; + int value2; +}; + +struct PinnedImpl : private Pinned { + PinnedImpl(int value1, int value2, int value3) : value1(value1), value2(value2), value3(value3) {} + + int value1; + int value2; + int value3; +}; + +struct MaxAlignedData { + explicit MaxAlignedData(int value) : value(value) {} + + std::max_align_t padding; + int value; +}; + +} // namespace + +TEST(ObjectFactoryStorageTest, Empty) { + ObjectFactoryStorageRegular storage; + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(ObjectFactoryStorageTest, DoNotPublish) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(ObjectFactoryStorageTest, Publish) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer1(storage); + ObjectFactoryStorageRegular::Producer producer2(storage); + + producer1.Insert(1); + producer1.Insert(2); + producer2.Insert(10); + producer2.Insert(20); + + producer1.Publish(); + producer2.Publish(); + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(1, 2, 10, 20)); +} + +TEST(ObjectFactoryStorageTest, PublishDifferentTypes) { + ObjectFactoryStorage storage; + ObjectFactoryStorage::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + producer.Insert(3, 4); + producer.Insert(5, 6, 7); + producer.Insert(8); + + producer.Publish(); + + auto actual = storage.Iter(); + auto it = actual.begin(); + EXPECT_THAT(it->Data(), 1); + ++it; + EXPECT_THAT(it->Data(), 2); + ++it; + auto& moveOnly = it->Data(); + EXPECT_THAT(moveOnly.value1, 3); + EXPECT_THAT(moveOnly.value2, 4); + ++it; + auto& pinned = it->Data(); + EXPECT_THAT(pinned.value1, 5); + EXPECT_THAT(pinned.value2, 6); + EXPECT_THAT(pinned.value3, 7); + ++it; + auto& maxAlign = it->Data(); + EXPECT_THAT(maxAlign.value, 8); + ++it; + EXPECT_THAT(it, actual.end()); +} + +TEST(ObjectFactoryStorageTest, PublishSeveralTimes) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + // Add 2 elements and publish. + producer.Insert(1); + producer.Insert(2); + producer.Publish(); + + // Add another element and publish. + producer.Insert(3); + producer.Publish(); + + // Publish without adding elements. + producer.Publish(); + + // Add yet another two elements and publish. + producer.Insert(4); + producer.Insert(5); + producer.Publish(); + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(ObjectFactoryStorageTest, PublishInDestructor) { + ObjectFactoryStorageRegular storage; + + { + ObjectFactoryStorageRegular::Producer producer(storage); + producer.Insert(1); + producer.Insert(2); + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(1, 2)); +} + +TEST(ObjectFactoryStorageTest, EraseFirst) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + producer.Insert(3); + + producer.Publish(); + + { + auto iter = storage.Iter(); + for (auto it = iter.begin(); it != iter.end();) { + if (it->Data() == 1) { + iter.EraseAndAdvance(it); + } else { + ++it; + } + } + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(2, 3)); +} + +TEST(ObjectFactoryStorageTest, EraseMiddle) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + producer.Insert(3); + + producer.Publish(); + + { + auto iter = storage.Iter(); + for (auto it = iter.begin(); it != iter.end();) { + if (it->Data() == 2) { + iter.EraseAndAdvance(it); + } else { + ++it; + } + } + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(1, 3)); +} + +TEST(ObjectFactoryStorageTest, EraseLast) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + producer.Insert(3); + + producer.Publish(); + + { + auto iter = storage.Iter(); + for (auto it = iter.begin(); it != iter.end();) { + if (it->Data() == 3) { + iter.EraseAndAdvance(it); + } else { + ++it; + } + } + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::ElementsAre(1, 2)); +} + +TEST(ObjectFactoryStorageTest, EraseAll) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + producer.Insert(2); + producer.Insert(3); + + producer.Publish(); + + { + auto iter = storage.Iter(); + for (auto it = iter.begin(); it != iter.end();) { + iter.EraseAndAdvance(it); + } + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) { + ObjectFactoryStorageRegular storage; + ObjectFactoryStorageRegular::Producer producer(storage); + + producer.Insert(1); + + producer.Publish(); + + { + auto iter = storage.Iter(); + auto it = iter.begin(); + iter.EraseAndAdvance(it); + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(ObjectFactoryStorageTest, ConcurrentPublish) { + ObjectFactoryStorageRegular storage; + constexpr int kThreadCount = kDefaultThreadCount; + std::atomic canStart(false); + std::atomic readyCount(0); + std::vector threads; + std::vector expected; + + for (int i = 0; i < kThreadCount; ++i) { + expected.push_back(i); + threads.emplace_back([i, &storage, &canStart, &readyCount]() { + ObjectFactoryStorageRegular::Producer producer(storage); + producer.Insert(i); + ++readyCount; + while (!canStart) { + } + producer.Publish(); + }); + } + + while (readyCount < kThreadCount) { + } + canStart = true; + for (auto& t : threads) { + t.join(); + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); +} + +TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) { + ObjectFactoryStorageRegular storage; + constexpr int kStartCount = 50; + constexpr int kThreadCount = kDefaultThreadCount; + + std::vector expectedBefore; + std::vector expectedAfter; + ObjectFactoryStorageRegular::Producer producer(storage); + for (int i = 0; i < kStartCount; ++i) { + expectedBefore.push_back(i); + expectedAfter.push_back(i); + producer.Insert(i); + } + producer.Publish(); + + std::atomic canStart(false); + std::atomic readyCount(0); + std::atomic startedCount(0); + std::vector threads; + for (int i = 0; i < kThreadCount; ++i) { + int j = i + kStartCount; + expectedAfter.push_back(j); + threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() { + ObjectFactoryStorageRegular::Producer producer(storage); + producer.Insert(j); + ++readyCount; + while (!canStart) { + } + ++startedCount; + producer.Publish(); + }); + } + + std::vector actualBefore; + { + auto iter = storage.Iter(); + while (readyCount < kThreadCount) { + } + canStart = true; + while (startedCount < kThreadCount) { + } + + for (auto& node : iter) { + int element = *static_cast(node.Data()); + actualBefore.push_back(element); + } + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore)); + + auto actualAfter = Collect(storage); + + EXPECT_THAT(actualAfter, testing::UnorderedElementsAreArray(expectedAfter)); +} + +TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { + ObjectFactoryStorageRegular storage; + constexpr int kStartCount = 50; + constexpr int kThreadCount = kDefaultThreadCount; + + std::vector expectedAfter; + ObjectFactoryStorageRegular::Producer producer(storage); + for (int i = 0; i < kStartCount; ++i) { + if (i % 2 == 0) { + expectedAfter.push_back(i); + } + producer.Insert(i); + } + producer.Publish(); + + std::atomic canStart(false); + std::atomic readyCount(0); + std::atomic startedCount(0); + std::vector threads; + for (int i = 0; i < kThreadCount; ++i) { + int j = i + kStartCount; + expectedAfter.push_back(j); + threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() { + ObjectFactoryStorageRegular::Producer producer(storage); + producer.Insert(j); + ++readyCount; + while (!canStart) { + } + ++startedCount; + producer.Publish(); + }); + } + + { + auto iter = storage.Iter(); + while (readyCount < kThreadCount) { + } + canStart = true; + while (startedCount < kThreadCount) { + } + + for (auto it = iter.begin(); it != iter.end();) { + if (it->Data() % 2 != 0) { + iter.EraseAndAdvance(it); + } else { + ++it; + } + } + } + + for (auto& t : threads) { + t.join(); + } + + auto actual = Collect(storage); + + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expectedAfter)); +} + +using mm::ObjectFactory; + +namespace { + +std::unique_ptr MakeObjectTypeInfo(int32_t size) { + auto typeInfo = std_support::make_unique(); + typeInfo->typeInfo_ = typeInfo.get(); + typeInfo->instanceSize_ = size; + return typeInfo; +} + +std::unique_ptr MakeArrayTypeInfo(int32_t elementSize) { + auto typeInfo = std_support::make_unique(); + typeInfo->typeInfo_ = typeInfo.get(); + typeInfo->instanceSize_ = -elementSize; + return typeInfo; +} + +} // namespace + +TEST(ObjectFactoryTest, CreateObject) { + auto typeInfo = MakeObjectTypeInfo(24); + ObjectFactory objectFactory; + ObjectFactory::ThreadQueue threadQueue(objectFactory); + + auto* object = threadQueue.CreateObject(typeInfo.get()); + threadQueue.Publish(); + + auto iter = objectFactory.Iter(); + auto it = iter.begin(); + EXPECT_FALSE(it.IsArray()); + EXPECT_THAT(it.GetObjHeader(), object); + ++it; + EXPECT_THAT(it, iter.end()); +} + +TEST(ObjectFactoryTest, CreateArray) { + auto typeInfo = MakeArrayTypeInfo(24); + ObjectFactory objectFactory; + ObjectFactory::ThreadQueue threadQueue(objectFactory); + + auto* array = threadQueue.CreateArray(typeInfo.get(), 3); + threadQueue.Publish(); + + auto iter = objectFactory.Iter(); + auto it = iter.begin(); + EXPECT_TRUE(it.IsArray()); + EXPECT_THAT(it.GetArrayHeader(), array); + ++it; + EXPECT_THAT(it, iter.end()); +} + +TEST(ObjectFactoryTest, Erase) { + auto objectTypeInfo = MakeObjectTypeInfo(24); + auto arrayTypeInfo = MakeArrayTypeInfo(24); + ObjectFactory objectFactory; + ObjectFactory::ThreadQueue threadQueue(objectFactory); + + for (int i = 0; i < 10; ++i) { + threadQueue.CreateObject(objectTypeInfo.get()); + threadQueue.CreateArray(arrayTypeInfo.get(), 3); + } + + threadQueue.Publish(); + + { + auto iter = objectFactory.Iter(); + for (auto it = iter.begin(); it != iter.end();) { + if (it.IsArray()) { + iter.EraseAndAdvance(it); + } else { + ++it; + } + } + } + + { + auto iter = objectFactory.Iter(); + int count = 0; + for (auto it = iter.begin(); it != iter.end(); ++it, ++count) { + EXPECT_FALSE(it.IsArray()); + } + EXPECT_THAT(count, 10); + } +} + +TEST(ObjectFactoryTest, ConcurrentPublish) { + auto typeInfo = MakeObjectTypeInfo(24); + ObjectFactory objectFactory; + constexpr int kThreadCount = kDefaultThreadCount; + std::atomic canStart(false); + std::atomic readyCount(0); + std::vector threads; + std::mutex expectedMutex; + std::vector expected; + + for (int i = 0; i < kThreadCount; ++i) { + threads.emplace_back([&typeInfo, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() { + ObjectFactory::ThreadQueue threadQueue(objectFactory); + auto* object = threadQueue.CreateObject(typeInfo.get()); + { + std::lock_guard guard(expectedMutex); + expected.push_back(object); + } + ++readyCount; + while (!canStart) { + } + threadQueue.Publish(); + }); + } + + while (readyCount < kThreadCount) { + } + canStart = true; + for (auto& t : threads) { + t.join(); + } + + auto iter = objectFactory.Iter(); + std::vector actual; + for (auto it = iter.begin(); it != iter.end(); ++it) { + actual.push_back(it.GetObjHeader()); + } + + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); +} diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index bc79217b2cd..dd3cc74c1c6 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -45,14 +45,6 @@ static void destroyMetaObject(TypeInfo** location) { extern "C" { -RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { - TODO(); -} - -OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements) { - TODO(); -} - OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { TODO(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index b5f18dc1853..5e09933297f 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -9,6 +9,7 @@ #include #include +#include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" #include "StableRefRegistry.hpp" #include "ThreadLocalStorage.hpp" @@ -26,7 +27,8 @@ public: threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()), - state_(ThreadState::kRunnable) {} + state_(ThreadState::kRunnable), + objectFactoryThreadQueue_(ObjectFactory::Instance()) {} ~ThreadData() = default; @@ -42,12 +44,15 @@ public: ThreadState setState(ThreadState state) noexcept { return state_.exchange(state); } + ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + private: const pthread_t threadId_; GlobalsRegistry::ThreadQueue globalsThreadQueue_; ThreadLocalStorage tls_; StableRefRegistry::ThreadQueue stableRefThreadQueue_; std::atomic state_; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_; }; } // namespace mm