diff --git a/kotlin-native/runtime/src/main/cpp/CppSupport.hpp b/kotlin-native/runtime/src/main/cpp/CppSupport.hpp index 9f04f544c41..e423ab56d6c 100644 --- a/kotlin-native/runtime/src/main/cpp/CppSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/CppSupport.hpp @@ -44,6 +44,8 @@ template constexpr bool is_nothrow_move_constructible_v = std::is_nothrow_move_constructible::value; template constexpr bool is_nothrow_move_assignable_v = std::is_nothrow_move_assignable::value; +template +constexpr bool is_standard_layout_v = std::is_standard_layout::value; } // namespace std_support } // namespace kotlin diff --git a/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp b/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp new file mode 100644 index 00000000000..06cbdaee316 --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/SingleLockList.hpp @@ -0,0 +1,118 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_SINGLE_LOCK_LIST_H +#define RUNTIME_SINGLE_LOCK_LIST_H + +#include +#include +#include + +#include "CppSupport.hpp" +#include "Mutex.hpp" +#include "Utils.hpp" + +namespace kotlin { + +// TODO: Consider different locking mechanisms. +template +class SingleLockList : private Pinned { +public: + class Node : Pinned { + public: + Value* Get() noexcept { return &value; } + + private: + friend class SingleLockList; + + template + Node(Args... args) noexcept : value(args...) {} + + Value value; + std::unique_ptr next; + Node* previous = nullptr; // weak + }; + + class Iterator { + public: + explicit Iterator(Node* node) noexcept : node_(node) {} + + Value& operator*() noexcept { return node_->value; } + + Iterator& operator++() noexcept { + node_ = node_->next.get(); + return *this; + } + + bool operator==(const Iterator& rhs) const noexcept { return node_ == rhs.node_; } + + bool operator!=(const Iterator& rhs) const noexcept { return node_ != rhs.node_; } + + private: + Node* node_; + }; + + class Iterable : private MoveOnly { + public: + explicit Iterable(SingleLockList* list) noexcept : list_(list), guard_(list->mutex_) {} + + Iterator begin() noexcept { return Iterator(list_->root_.get()); } + + Iterator end() noexcept { return Iterator(nullptr); } + + private: + SingleLockList* list_; + std::unique_lock guard_; + }; + + template + Node* Emplace(Args... args) noexcept { + auto* nodePtr = new Node(args...); + std::unique_ptr node(nodePtr); + LockGuard guard(mutex_); + if (root_) { + root_->previous = node.get(); + } + node->next = std::move(root_); + root_ = std::move(node); + return nodePtr; + } + + // Using `node` including its referred `Value` after `Erase` is undefined behaviour. + void Erase(Node* node) noexcept { + LockGuard guard(mutex_); + if (root_.get() == node) { + root_ = std::move(node->next); + if (root_) { + root_->previous = nullptr; + } + return; + } + auto* previous = node->previous; + RuntimeAssert(previous != nullptr, "Only the root node doesn't have the previous node"); + auto ownedNode = std::move(previous->next); + previous->next = std::move(node->next); + if (auto& next = previous->next) { + next->previous = previous; + } + } + + // Returned value locks `this` to perform safe iteration. `this` unlocks when + // `Iterable` gets out of scope. Example usage: + // for (auto& value: list.Iter()) { + // // Do something with `value`, there's a guarantee that it'll not be + // // destroyed mid-iteration. + // } + // // At this point `list` is unlocked. + Iterable Iter() noexcept { return Iterable(this); } + +private: + std::unique_ptr root_; + Mutex mutex_; +}; + +} // namespace kotlin + +#endif // RUNTIME_SINGLE_LOCK_LIST_H diff --git a/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp b/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp new file mode 100644 index 00000000000..2fd1d93b338 --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp @@ -0,0 +1,297 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "SingleLockList.hpp" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace kotlin; + +namespace { + +using IntList = SingleLockList; + +} // namespace + +TEST(SingleLockListTest, Emplace) { + IntList list; + constexpr int kFirst = 1; + constexpr int kSecond = 2; + constexpr int kThird = 3; + auto* firstNode = list.Emplace(kFirst); + auto* secondNode = list.Emplace(kSecond); + auto* thirdNode = list.Emplace(kThird); + int* first = firstNode->Get(); + int* second = secondNode->Get(); + int* third = thirdNode->Get(); + EXPECT_THAT(*first, kFirst); + EXPECT_THAT(*second, kSecond); + EXPECT_THAT(*third, kThird); +} + +TEST(SingleLockListTest, EmplaceAndIter) { + IntList list; + constexpr int kFirst = 1; + constexpr int kSecond = 2; + constexpr int kThird = 3; + list.Emplace(kFirst); + list.Emplace(kSecond); + list.Emplace(kThird); + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::ElementsAre(kThird, kSecond, kFirst)); +} + +TEST(SingleLockListTest, EmplaceEraseAndIter) { + IntList list; + constexpr int kFirst = 1; + constexpr int kSecond = 2; + constexpr int kThird = 3; + list.Emplace(kFirst); + auto* secondNode = list.Emplace(kSecond); + list.Emplace(kThird); + list.Erase(secondNode); + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::ElementsAre(kThird, kFirst)); +} + +TEST(SingleLockListTest, IterEmpty) { + IntList list; + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(SingleLockListTest, EraseToEmptyEmplaceAndIter) { + IntList list; + constexpr int kFirst = 1; + constexpr int kSecond = 2; + constexpr int kThird = 3; + constexpr int kFourth = 4; + auto* firstNode = list.Emplace(kFirst); + auto* secondNode = list.Emplace(kSecond); + list.Erase(firstNode); + list.Erase(secondNode); + list.Emplace(kThird); + list.Emplace(kFourth); + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::ElementsAre(kFourth, kThird)); +} + +TEST(SingleLockListTest, ConcurrentEmplace) { + IntList list; + constexpr int kThreadCount = 100; + std::atomic canStart(false); + std::vector threads; + std::vector expected; + for (int i = 0; i < kThreadCount; ++i) { + expected.push_back(i); + threads.emplace_back([i, &list, &canStart]() { + while (!canStart) { + } + list.Emplace(i); + }); + } + + canStart = true; + for (auto& t : threads) { + t.join(); + } + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); +} + +TEST(SingleLockListTest, ConcurrentErase) { + IntList list; + constexpr int kThreadCount = 100; + std::vector items; + for (int i = 0; i < kThreadCount; ++i) { + items.push_back(list.Emplace(i)); + } + + std::atomic canStart(false); + std::vector threads; + for (auto* item : items) { + threads.emplace_back([item, &list, &canStart]() { + while (!canStart) { + } + list.Erase(item); + }); + } + + canStart = true; + for (auto& t : threads) { + t.join(); + } + + std::vector actual; + for (int element : list.Iter()) { + actual.push_back(element); + } + + EXPECT_THAT(actual, testing::IsEmpty()); +} + +TEST(SingleLockListTest, IterWhileConcurrentEmplace) { + IntList list; + constexpr int kStartCount = 50; + constexpr int kThreadCount = 100; + + std::deque expectedBefore; + std::vector expectedAfter; + for (int i = 0; i < kStartCount; ++i) { + expectedBefore.push_front(i); + expectedAfter.push_back(i); + list.Emplace(i); + } + + std::atomic canStart(false); + std::atomic startedCount(0); + std::vector threads; + for (int i = 0; i < kThreadCount; ++i) { + int j = i + kStartCount; + expectedAfter.push_back(j); + threads.emplace_back([j, &list, &canStart, &startedCount]() { + while (!canStart) { + } + ++startedCount; + list.Emplace(j); + }); + } + + std::vector actualBefore; + { + auto iter = list.Iter(); + canStart = true; + while (startedCount < kThreadCount) { + } + + for (int element : iter) { + actualBefore.push_back(element); + } + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore)); + + std::vector actualAfter; + for (int element : list.Iter()) { + actualAfter.push_back(element); + } + + EXPECT_THAT(actualAfter, testing::UnorderedElementsAreArray(expectedAfter)); +} + +TEST(SingleLockListTest, IterWhileConcurrentErase) { + IntList list; + constexpr int kThreadCount = 100; + + std::deque expectedBefore; + std::vector items; + for (int i = 0; i < kThreadCount; ++i) { + expectedBefore.push_front(i); + items.push_back(list.Emplace(i)); + } + + std::atomic canStart(false); + std::atomic startedCount(0); + std::vector threads; + for (auto* item : items) { + threads.emplace_back([item, &list, &canStart, &startedCount]() { + while (!canStart) { + } + ++startedCount; + list.Erase(item); + }); + } + + std::vector actualBefore; + { + auto iter = list.Iter(); + canStart = true; + while (startedCount < kThreadCount) { + } + + for (int element : iter) { + actualBefore.push_back(element); + } + } + + for (auto& t : threads) { + t.join(); + } + + EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore)); + + std::vector actualAfter; + for (int element : list.Iter()) { + actualAfter.push_back(element); + } + + EXPECT_THAT(actualAfter, testing::IsEmpty()); +} + +namespace { + +class PinnedType : private Pinned { +public: + PinnedType(int value) : value_(value) {} + + int value() const { return value_; } + +private: + int value_; +}; + +} // namespace + +TEST(SingleLockListTest, PinnedType) { + SingleLockList list; + constexpr int kFirst = 1; + + auto* itemNode = list.Emplace(kFirst); + PinnedType* item = itemNode->Get(); + EXPECT_THAT(item->value(), kFirst); + + list.Erase(itemNode); + + std::vector actualAfter; + for (auto& element : list.Iter()) { + actualAfter.push_back(&element); + } + + EXPECT_THAT(actualAfter, testing::IsEmpty()); +} diff --git a/kotlin-native/runtime/src/main/cpp/Utils.hpp b/kotlin-native/runtime/src/main/cpp/Utils.hpp index 1f10f53082a..645c2924e88 100644 --- a/kotlin-native/runtime/src/main/cpp/Utils.hpp +++ b/kotlin-native/runtime/src/main/cpp/Utils.hpp @@ -6,6 +6,8 @@ #ifndef RUNTIME_UTILS_H #define RUNTIME_UTILS_H +#include "CppSupport.hpp" + namespace kotlin { // A helper for implementing classes with disabled copy constructor and copy assignment. @@ -52,6 +54,25 @@ protected: ~Pinned() = default; }; +// Given +// struct SomeWrapper { +// SomeType value; +// ... // (possibly) some other fields +// }; +// allows to cast from `SomeValue*` to `SomeWrapper*` as no-op. It only works +// if `SomeWrapper` is standard layout and `value` is the first non-static data member. +// See https://en.cppreference.com/w/cpp/language/data_members#Standard_layout +// +// Useful for exporting SomeType under a different name (e.g. exporting inner C++ class +// as public C struct). +#define wrapper_cast(Wrapper, inner, field) \ + /* With -O2 the lambda is replaced with a cast in the bitcode. */ \ + [inner]() { \ + static_assert(std_support::is_standard_layout_v, #Wrapper " must be standard layout"); \ + static_assert(offsetof(Wrapper, field) == 0, #field " must be at 0 offset"); \ + return reinterpret_cast(inner); \ + }() + } // namespace kotlin #endif // RUNTIME_UTILS_H diff --git a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp index 5ba0aea917c..78f39f520f7 100644 --- a/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/UtilsTest.cpp @@ -48,3 +48,27 @@ TEST(UtilsTest, PinnedImpl) { static_assert(!std_support::is_move_assignable_v, "Must not be move assignable"); static_assert(sizeof(PinnedImpl) == sizeof(A), "Must not increase size"); } + +namespace { + +struct Wrapper { + A wrapped; +}; + +struct WrapperOverPinned { + PinnedImpl wrapped; +}; + +} // namespace + +TEST(UtilsTest, WrapperCast) { + A value; + Wrapper* wrapper = wrapper_cast(Wrapper, &value, wrapped); + EXPECT_EQ(&value, &wrapper->wrapped); +} + +TEST(UtilsTest, WrapperOverPinnedCast) { + PinnedImpl value; + WrapperOverPinned* wrapper = wrapper_cast(WrapperOverPinned, &value, wrapped); + EXPECT_EQ(&value, &wrapper->wrapped); +} diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.cpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.cpp new file mode 100644 index 00000000000..73347e58e6f --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.cpp @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "GlobalData.hpp" + +using namespace kotlin; + +mm::GlobalData::GlobalData() = default; +mm::GlobalData::~GlobalData() = default; + +// static +mm::GlobalData mm::GlobalData::instance_; diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp new file mode 100644 index 00000000000..a75f67ce962 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_GLOBAL_DATA_H +#define RUNTIME_MM_GLOBAL_DATA_H + +#include "ThreadRegistry.hpp" +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +// Global (de)initialization is undefined in C++. Use single global singleton to define it for simplicity. +class GlobalData : private Pinned { +public: + static GlobalData& Instance() noexcept { return instance_; } + + ThreadRegistry& threadRegistry() { return threadRegistry_; } + +private: + GlobalData(); + ~GlobalData(); + + static GlobalData instance_; + + ThreadRegistry threadRegistry_; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_GLOBAL_DATA_H diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp new file mode 100644 index 00000000000..be1b12e85a5 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Memory.h" + +#include "ThreadData.hpp" +#include "ThreadRegistry.hpp" +#include "Utils.hpp" + +using namespace kotlin; + +extern "C" struct MemoryState { + mm::ThreadRegistry::Node data; + // Do not add any other fields: this struct is just a wrapper around ThreadDataNode. +}; + +namespace { + +ALWAYS_INLINE MemoryState* ToMemoryState(mm::ThreadRegistry::Node* data) { + return wrapper_cast(MemoryState, data, data); +} + +ALWAYS_INLINE mm::ThreadRegistry::Node* FromMemoryState(MemoryState* state) { + return &state->data; +} + +} // namespace + +extern "C" MemoryState* InitMemory(bool firstRuntime) { + return ToMemoryState(mm::ThreadRegistry::Instance().RegisterCurrentThread()); +} + +extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) { + mm::ThreadRegistry::Instance().Unregister(FromMemoryState(state)); +} diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index c0b6f7b2019..57bd6ab0bed 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -47,16 +47,8 @@ static void destroyMetaObject(TypeInfo** location) { extern "C" { -MemoryState* InitMemory(bool firstRuntime) { - RuntimeCheck(false, "Unimplemented"); -} - -void DeinitMemory(MemoryState*, bool destroyRuntime) { - RuntimeCheck(false, "Unimplemented"); -} - -void RestoreMemory(MemoryState* memoryState) { - RuntimeCheck(false, "Unimplemented"); +void RestoreMemory(MemoryState*) { + // TODO: Remove this function when legacy MM is gone. } RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp new file mode 100644 index 00000000000..3dac2cbd733 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_THREAD_DATA_H +#define RUNTIME_MM_THREAD_DATA_H + +#include + +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +// `ThreadData` is supposed to be thread local singleton. +// Pin it in memory to prevent accidental copying. +class ThreadData final : private Pinned { +public: + ThreadData(pthread_t threadId) noexcept : threadId_(threadId) {} + + ~ThreadData() = default; + + pthread_t threadId() const noexcept { return threadId_; } + +private: + const pthread_t threadId_; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_THREAD_DATA_H diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp new file mode 100644 index 00000000000..f0fe50a8aa5 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp @@ -0,0 +1,39 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ThreadRegistry.hpp" + +#include "GlobalData.hpp" +#include "ThreadData.hpp" + +using namespace kotlin; + +// static +mm::ThreadRegistry& mm::ThreadRegistry::Instance() noexcept { + return mm::GlobalData::Instance().threadRegistry(); +} + +mm::ThreadRegistry::Node* mm::ThreadRegistry::RegisterCurrentThread() noexcept { + auto* threadDataNode = list_.Emplace(pthread_self()); + ThreadData*& currentData = currentThreadData_; + RuntimeAssert(currentData == nullptr, "This thread already had some data assigned to it."); + currentData = threadDataNode->Get(); + return threadDataNode; +} + +void mm::ThreadRegistry::Unregister(Node* threadDataNode) noexcept { + list_.Erase(threadDataNode); + // Do not touch `currentThreadData_` as TLS may already have been deallocated. +} + +mm::ThreadRegistry::Iterable mm::ThreadRegistry::Iter() noexcept { + return list_.Iter(); +} + +mm::ThreadRegistry::ThreadRegistry() = default; +mm::ThreadRegistry::~ThreadRegistry() = default; + +// static +thread_local mm::ThreadData* mm::ThreadRegistry::currentThreadData_ = nullptr; diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp new file mode 100644 index 00000000000..58b494b9ced --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp @@ -0,0 +1,53 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_THREAD_REGISTRY_H +#define RUNTIME_MM_THREAD_REGISTRY_H + +#include + +#include "SingleLockList.hpp" +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +class ThreadData; + +class ThreadRegistry final : private Pinned { +public: + using Node = SingleLockList::Node; + using Iterable = SingleLockList::Iterable; + + static ThreadRegistry& Instance() noexcept; + + Node* RegisterCurrentThread() noexcept; + + // `ThreadData` associated with `threadDataNode` cannot be used after this call. + void Unregister(Node* threadDataNode) noexcept; + + // Locks `ThreadRegistry` for safe iteration. + Iterable Iter() noexcept; + + // Try not to use it very often, as (1) thread local access can be slow on some platforms, + // (2) TLS gets deallocated before our thread destruction hooks run. + // Using this after `Unregister` for the thread has been called is undefined behaviour. + ThreadData* CurrentThreadData() const noexcept { return currentThreadData_; } + +private: + friend class GlobalData; + + ThreadRegistry(); + ~ThreadRegistry(); + + static thread_local ThreadData* currentThreadData_; + + SingleLockList list_; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_THREAD_REGISTRY_H diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp new file mode 100644 index 00000000000..11fd9b033eb --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ThreadRegistry.hpp" + +#include +#include + +#include "gtest/gtest.h" + +#include "ThreadData.hpp" + +using namespace kotlin; + +TEST(ThreadRegistryTest, RegisterCurrentThread) { + std::thread t([]() { + auto* node = mm::ThreadRegistry::Instance().RegisterCurrentThread(); + auto* threadData = node->Get(); + EXPECT_EQ(pthread_self(), threadData->threadId()); + EXPECT_EQ(threadData, mm::ThreadRegistry::Instance().CurrentThreadData()); + }); + t.join(); +}