Thread registry (#4518)

kotlin-native/runtime/src/main/cpp/CppSupport.hpp

@@ -44,6 +44,8 @@ template <typename T>
 constexpr bool is_nothrow_move_constructible_v = std::is_nothrow_move_constructible<T>::value;
 template <typename T>
 constexpr bool is_nothrow_move_assignable_v = std::is_nothrow_move_assignable<T>::value;
+template <typename T>
+constexpr bool is_standard_layout_v = std::is_standard_layout<T>::value;
 
 } // namespace std_support
 } // namespace kotlin

kotlin-native/runtime/src/main/cpp/SingleLockList.hpp

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_SINGLE_LOCK_LIST_H
+#define RUNTIME_SINGLE_LOCK_LIST_H
+
+#include <cstddef>
+#include <memory>
+#include <mutex>
+
+#include "CppSupport.hpp"
+#include "Mutex.hpp"
+#include "Utils.hpp"
+
+namespace kotlin {
+
+// TODO: Consider different locking mechanisms.
+template <typename Value, typename Mutex = SimpleMutex>
+class SingleLockList : private Pinned {
+public:
+    class Node : Pinned {
+    public:
+        Value* Get() noexcept { return &value; }
+
+    private:
+        friend class SingleLockList;
+
+        template <typename... Args>
+        Node(Args... args) noexcept : value(args...) {}
+
+        Value value;
+        std::unique_ptr<Node> next;
+        Node* previous = nullptr; // weak
+    };
+
+    class Iterator {
+    public:
+        explicit Iterator(Node* node) noexcept : node_(node) {}
+
+        Value& operator*() noexcept { return node_->value; }
+
+        Iterator& operator++() noexcept {
+            node_ = node_->next.get();
+            return *this;
+        }
+
+        bool operator==(const Iterator& rhs) const noexcept { return node_ == rhs.node_; }
+
+        bool operator!=(const Iterator& rhs) const noexcept { return node_ != rhs.node_; }
+
+    private:
+        Node* node_;
+    };
+
+    class Iterable : private MoveOnly {
+    public:
+        explicit Iterable(SingleLockList* list) noexcept : list_(list), guard_(list->mutex_) {}
+
+        Iterator begin() noexcept { return Iterator(list_->root_.get()); }
+
+        Iterator end() noexcept { return Iterator(nullptr); }
+
+    private:
+        SingleLockList* list_;
+        std::unique_lock<Mutex> guard_;
+    };
+
+    template <typename... Args>
+    Node* Emplace(Args... args) noexcept {
+        auto* nodePtr = new Node(args...);
+        std::unique_ptr<Node> node(nodePtr);
+        LockGuard<Mutex> guard(mutex_);
+        if (root_) {
+            root_->previous = node.get();
+        }
+        node->next = std::move(root_);
+        root_ = std::move(node);
+        return nodePtr;
+    }
+
+    // Using `node` including its referred `Value` after `Erase` is undefined behaviour.
+    void Erase(Node* node) noexcept {
+        LockGuard<Mutex> guard(mutex_);
+        if (root_.get() == node) {
+            root_ = std::move(node->next);
+            if (root_) {
+                root_->previous = nullptr;
+            }
+            return;
+        }
+        auto* previous = node->previous;
+        RuntimeAssert(previous != nullptr, "Only the root node doesn't have the previous node");
+        auto ownedNode = std::move(previous->next);
+        previous->next = std::move(node->next);
+        if (auto& next = previous->next) {
+            next->previous = previous;
+        }
+    }
+
+    // Returned value locks `this` to perform safe iteration. `this` unlocks when
+    // `Iterable` gets out of scope. Example usage:
+    // for (auto& value: list.Iter()) {
+    //    // Do something with `value`, there's a guarantee that it'll not be
+    //    // destroyed mid-iteration.
+    // }
+    // // At this point `list` is unlocked.
+    Iterable Iter() noexcept { return Iterable(this); }
+
+private:
+    std::unique_ptr<Node> root_;
+    Mutex mutex_;
+};
+
+} // namespace kotlin
+
+#endif // RUNTIME_SINGLE_LOCK_LIST_H

kotlin-native/runtime/src/main/cpp/SingleLockListTest.cpp

@@ -0,0 +1,297 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "SingleLockList.hpp"
+
+#include <atomic>
+#include <deque>
+#include <thread>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace kotlin;
+
+namespace {
+
+using IntList = SingleLockList<int>;
+
+} // namespace
+
+TEST(SingleLockListTest, Emplace) {
+    IntList list;
+    constexpr int kFirst = 1;
+    constexpr int kSecond = 2;
+    constexpr int kThird = 3;
+    auto* firstNode = list.Emplace(kFirst);
+    auto* secondNode = list.Emplace(kSecond);
+    auto* thirdNode = list.Emplace(kThird);
+    int* first = firstNode->Get();
+    int* second = secondNode->Get();
+    int* third = thirdNode->Get();
+    EXPECT_THAT(*first, kFirst);
+    EXPECT_THAT(*second, kSecond);
+    EXPECT_THAT(*third, kThird);
+}
+
+TEST(SingleLockListTest, EmplaceAndIter) {
+    IntList list;
+    constexpr int kFirst = 1;
+    constexpr int kSecond = 2;
+    constexpr int kThird = 3;
+    list.Emplace(kFirst);
+    list.Emplace(kSecond);
+    list.Emplace(kThird);
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::ElementsAre(kThird, kSecond, kFirst));
+}
+
+TEST(SingleLockListTest, EmplaceEraseAndIter) {
+    IntList list;
+    constexpr int kFirst = 1;
+    constexpr int kSecond = 2;
+    constexpr int kThird = 3;
+    list.Emplace(kFirst);
+    auto* secondNode = list.Emplace(kSecond);
+    list.Emplace(kThird);
+    list.Erase(secondNode);
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::ElementsAre(kThird, kFirst));
+}
+
+TEST(SingleLockListTest, IterEmpty) {
+    IntList list;
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::IsEmpty());
+}
+
+TEST(SingleLockListTest, EraseToEmptyEmplaceAndIter) {
+    IntList list;
+    constexpr int kFirst = 1;
+    constexpr int kSecond = 2;
+    constexpr int kThird = 3;
+    constexpr int kFourth = 4;
+    auto* firstNode = list.Emplace(kFirst);
+    auto* secondNode = list.Emplace(kSecond);
+    list.Erase(firstNode);
+    list.Erase(secondNode);
+    list.Emplace(kThird);
+    list.Emplace(kFourth);
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::ElementsAre(kFourth, kThird));
+}
+
+TEST(SingleLockListTest, ConcurrentEmplace) {
+    IntList list;
+    constexpr int kThreadCount = 100;
+    std::atomic<bool> canStart(false);
+    std::vector<std::thread> threads;
+    std::vector<int> expected;
+    for (int i = 0; i < kThreadCount; ++i) {
+        expected.push_back(i);
+        threads.emplace_back([i, &list, &canStart]() {
+            while (!canStart) {
+            }
+            list.Emplace(i);
+        });
+    }
+
+    canStart = true;
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected));
+}
+
+TEST(SingleLockListTest, ConcurrentErase) {
+    IntList list;
+    constexpr int kThreadCount = 100;
+    std::vector<IntList::Node*> items;
+    for (int i = 0; i < kThreadCount; ++i) {
+        items.push_back(list.Emplace(i));
+    }
+
+    std::atomic<bool> canStart(false);
+    std::vector<std::thread> threads;
+    for (auto* item : items) {
+        threads.emplace_back([item, &list, &canStart]() {
+            while (!canStart) {
+            }
+            list.Erase(item);
+        });
+    }
+
+    canStart = true;
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    std::vector<int> actual;
+    for (int element : list.Iter()) {
+        actual.push_back(element);
+    }
+
+    EXPECT_THAT(actual, testing::IsEmpty());
+}
+
+TEST(SingleLockListTest, IterWhileConcurrentEmplace) {
+    IntList list;
+    constexpr int kStartCount = 50;
+    constexpr int kThreadCount = 100;
+
+    std::deque<int> expectedBefore;
+    std::vector<int> expectedAfter;
+    for (int i = 0; i < kStartCount; ++i) {
+        expectedBefore.push_front(i);
+        expectedAfter.push_back(i);
+        list.Emplace(i);
+    }
+
+    std::atomic<bool> canStart(false);
+    std::atomic<int> startedCount(0);
+    std::vector<std::thread> threads;
+    for (int i = 0; i < kThreadCount; ++i) {
+        int j = i + kStartCount;
+        expectedAfter.push_back(j);
+        threads.emplace_back([j, &list, &canStart, &startedCount]() {
+            while (!canStart) {
+            }
+            ++startedCount;
+            list.Emplace(j);
+        });
+    }
+
+    std::vector<int> actualBefore;
+    {
+        auto iter = list.Iter();
+        canStart = true;
+        while (startedCount < kThreadCount) {
+        }
+
+        for (int element : iter) {
+            actualBefore.push_back(element);
+        }
+    }
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore));
+
+    std::vector<int> actualAfter;
+    for (int element : list.Iter()) {
+        actualAfter.push_back(element);
+    }
+
+    EXPECT_THAT(actualAfter, testing::UnorderedElementsAreArray(expectedAfter));
+}
+
+TEST(SingleLockListTest, IterWhileConcurrentErase) {
+    IntList list;
+    constexpr int kThreadCount = 100;
+
+    std::deque<int> expectedBefore;
+    std::vector<IntList::Node*> items;
+    for (int i = 0; i < kThreadCount; ++i) {
+        expectedBefore.push_front(i);
+        items.push_back(list.Emplace(i));
+    }
+
+    std::atomic<bool> canStart(false);
+    std::atomic<int> startedCount(0);
+    std::vector<std::thread> threads;
+    for (auto* item : items) {
+        threads.emplace_back([item, &list, &canStart, &startedCount]() {
+            while (!canStart) {
+            }
+            ++startedCount;
+            list.Erase(item);
+        });
+    }
+
+    std::vector<int> actualBefore;
+    {
+        auto iter = list.Iter();
+        canStart = true;
+        while (startedCount < kThreadCount) {
+        }
+
+        for (int element : iter) {
+            actualBefore.push_back(element);
+        }
+    }
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore));
+
+    std::vector<int> actualAfter;
+    for (int element : list.Iter()) {
+        actualAfter.push_back(element);
+    }
+
+    EXPECT_THAT(actualAfter, testing::IsEmpty());
+}
+
+namespace {
+
+class PinnedType : private Pinned {
+public:
+    PinnedType(int value) : value_(value) {}
+
+    int value() const { return value_; }
+
+private:
+    int value_;
+};
+
+} // namespace
+
+TEST(SingleLockListTest, PinnedType) {
+    SingleLockList<PinnedType> list;
+    constexpr int kFirst = 1;
+
+    auto* itemNode = list.Emplace(kFirst);
+    PinnedType* item = itemNode->Get();
+    EXPECT_THAT(item->value(), kFirst);
+
+    list.Erase(itemNode);
+
+    std::vector<PinnedType*> actualAfter;
+    for (auto& element : list.Iter()) {
+        actualAfter.push_back(&element);
+    }
+
+    EXPECT_THAT(actualAfter, testing::IsEmpty());
+}

kotlin-native/runtime/src/main/cpp/Utils.hpp

@@ -6,6 +6,8 @@
 #ifndef RUNTIME_UTILS_H
 #define RUNTIME_UTILS_H
 
+#include "CppSupport.hpp"
+
 namespace kotlin {
 
 // A helper for implementing classes with disabled copy constructor and copy assignment.
@@ -52,6 +54,25 @@ protected:
     ~Pinned() = default;
 };
 
+// Given
+// struct SomeWrapper {
+//     SomeType value;
+//     ... // (possibly) some other fields
+// };
+// allows to cast from `SomeValue*` to `SomeWrapper*` as no-op. It only works
+// if `SomeWrapper` is standard layout and `value` is the first non-static data member.
+// See https://en.cppreference.com/w/cpp/language/data_members#Standard_layout
+//
+// Useful for exporting SomeType under a different name (e.g. exporting inner C++ class
+// as public C struct).
+#define wrapper_cast(Wrapper, inner, field) \
+    /* With -O2 the lambda is replaced with a cast in the bitcode. */ \
+    [inner]() { \
+        static_assert(std_support::is_standard_layout_v<Wrapper>, #Wrapper " must be standard layout"); \
+        static_assert(offsetof(Wrapper, field) == 0, #field " must be at 0 offset"); \
+        return reinterpret_cast<Wrapper*>(inner); \
+    }()
+
 } // namespace kotlin
 
 #endif // RUNTIME_UTILS_H

kotlin-native/runtime/src/main/cpp/UtilsTest.cpp

@@ -48,3 +48,27 @@ TEST(UtilsTest, PinnedImpl) {
     static_assert(!std_support::is_move_assignable_v<PinnedImpl>, "Must not be move assignable");
     static_assert(sizeof(PinnedImpl) == sizeof(A), "Must not increase size");
 }
+
+namespace {
+
+struct Wrapper {
+    A wrapped;
+};
+
+struct WrapperOverPinned {
+    PinnedImpl wrapped;
+};
+
+} // namespace
+
+TEST(UtilsTest, WrapperCast) {
+    A value;
+    Wrapper* wrapper = wrapper_cast(Wrapper, &value, wrapped);
+    EXPECT_EQ(&value, &wrapper->wrapped);
+}
+
+TEST(UtilsTest, WrapperOverPinnedCast) {
+    PinnedImpl value;
+    WrapperOverPinned* wrapper = wrapper_cast(WrapperOverPinned, &value, wrapped);
+    EXPECT_EQ(&value, &wrapper->wrapped);
+}

kotlin-native/runtime/src/mm/cpp/GlobalData.cpp

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "GlobalData.hpp"
+
+using namespace kotlin;
+
+mm::GlobalData::GlobalData() = default;
+mm::GlobalData::~GlobalData() = default;
+
+// static
+mm::GlobalData mm::GlobalData::instance_;

kotlin-native/runtime/src/mm/cpp/GlobalData.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_GLOBAL_DATA_H
+#define RUNTIME_MM_GLOBAL_DATA_H
+
+#include "ThreadRegistry.hpp"
+#include "Utils.hpp"
+
+namespace kotlin {
+namespace mm {
+
+// Global (de)initialization is undefined in C++. Use single global singleton to define it for simplicity.
+class GlobalData : private Pinned {
+public:
+    static GlobalData& Instance() noexcept { return instance_; }
+
+    ThreadRegistry& threadRegistry() { return threadRegistry_; }
+
+private:
+    GlobalData();
+    ~GlobalData();
+
+    static GlobalData instance_;
+
+    ThreadRegistry threadRegistry_;
+};
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_GLOBAL_DATA_H

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "Memory.h"
+
+#include "ThreadData.hpp"
+#include "ThreadRegistry.hpp"
+#include "Utils.hpp"
+
+using namespace kotlin;
+
+extern "C" struct MemoryState {
+    mm::ThreadRegistry::Node data;
+    // Do not add any other fields: this struct is just a wrapper around ThreadDataNode.
+};
+
+namespace {
+
+ALWAYS_INLINE MemoryState* ToMemoryState(mm::ThreadRegistry::Node* data) {
+    return wrapper_cast(MemoryState, data, data);
+}
+
+ALWAYS_INLINE mm::ThreadRegistry::Node* FromMemoryState(MemoryState* state) {
+    return &state->data;
+}
+
+} // namespace
+
+extern "C" MemoryState* InitMemory(bool firstRuntime) {
+    return ToMemoryState(mm::ThreadRegistry::Instance().RegisterCurrentThread());
+}
+
+extern "C" void DeinitMemory(MemoryState* state, bool destroyRuntime) {
+    mm::ThreadRegistry::Instance().Unregister(FromMemoryState(state));
+}

kotlin-native/runtime/src/mm/cpp/Stubs.cpp

@@ -47,16 +47,8 @@ static void destroyMetaObject(TypeInfo** location) {
 
 extern "C" {
 
-MemoryState* InitMemory(bool firstRuntime) {
+void RestoreMemory(MemoryState*) {
-    RuntimeCheck(false, "Unimplemented");
+    // TODO: Remove this function when legacy MM is gone.
-}
-
-void DeinitMemory(MemoryState*, bool destroyRuntime) {
-    RuntimeCheck(false, "Unimplemented");
-}
-
-void RestoreMemory(MemoryState* memoryState) {
-    RuntimeCheck(false, "Unimplemented");
 }
 
 RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* type_info) {

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_THREAD_DATA_H
+#define RUNTIME_MM_THREAD_DATA_H
+
+#include <pthread.h>
+
+#include "Utils.hpp"
+
+namespace kotlin {
+namespace mm {
+
+// `ThreadData` is supposed to be thread local singleton.
+// Pin it in memory to prevent accidental copying.
+class ThreadData final : private Pinned {
+public:
+    ThreadData(pthread_t threadId) noexcept : threadId_(threadId) {}
+
+    ~ThreadData() = default;
+
+    pthread_t threadId() const noexcept { return threadId_; }
+
+private:
+    const pthread_t threadId_;
+};
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_THREAD_DATA_H

kotlin-native/runtime/src/mm/cpp/ThreadRegistry.cpp

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ThreadRegistry.hpp"
+
+#include "GlobalData.hpp"
+#include "ThreadData.hpp"
+
+using namespace kotlin;
+
+// static
+mm::ThreadRegistry& mm::ThreadRegistry::Instance() noexcept {
+    return mm::GlobalData::Instance().threadRegistry();
+}
+
+mm::ThreadRegistry::Node* mm::ThreadRegistry::RegisterCurrentThread() noexcept {
+    auto* threadDataNode = list_.Emplace(pthread_self());
+    ThreadData*& currentData = currentThreadData_;
+    RuntimeAssert(currentData == nullptr, "This thread already had some data assigned to it.");
+    currentData = threadDataNode->Get();
+    return threadDataNode;
+}
+
+void mm::ThreadRegistry::Unregister(Node* threadDataNode) noexcept {
+    list_.Erase(threadDataNode);
+    // Do not touch `currentThreadData_` as TLS may already have been deallocated.
+}
+
+mm::ThreadRegistry::Iterable mm::ThreadRegistry::Iter() noexcept {
+    return list_.Iter();
+}
+
+mm::ThreadRegistry::ThreadRegistry() = default;
+mm::ThreadRegistry::~ThreadRegistry() = default;
+
+// static
+thread_local mm::ThreadData* mm::ThreadRegistry::currentThreadData_ = nullptr;

kotlin-native/runtime/src/mm/cpp/ThreadRegistry.hpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_THREAD_REGISTRY_H
+#define RUNTIME_MM_THREAD_REGISTRY_H
+
+#include <pthread.h>
+
+#include "SingleLockList.hpp"
+#include "Utils.hpp"
+
+namespace kotlin {
+namespace mm {
+
+class ThreadData;
+
+class ThreadRegistry final : private Pinned {
+public:
+    using Node = SingleLockList<ThreadData>::Node;
+    using Iterable = SingleLockList<ThreadData>::Iterable;
+
+    static ThreadRegistry& Instance() noexcept;
+
+    Node* RegisterCurrentThread() noexcept;
+
+    // `ThreadData` associated with `threadDataNode` cannot be used after this call.
+    void Unregister(Node* threadDataNode) noexcept;
+
+    // Locks `ThreadRegistry` for safe iteration.
+    Iterable Iter() noexcept;
+
+    // Try not to use it very often, as (1) thread local access can be slow on some platforms,
+    // (2) TLS gets deallocated before our thread destruction hooks run.
+    // Using this after `Unregister` for the thread has been called is undefined behaviour.
+    ThreadData* CurrentThreadData() const noexcept { return currentThreadData_; }
+
+private:
+    friend class GlobalData;
+
+    ThreadRegistry();
+    ~ThreadRegistry();
+
+    static thread_local ThreadData* currentThreadData_;
+
+    SingleLockList<ThreadData> list_;
+};
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_THREAD_REGISTRY_H

kotlin-native/runtime/src/mm/cpp/ThreadRegistryTest.cpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ThreadRegistry.hpp"
+
+#include <pthread.h>
+#include <thread>
+
+#include "gtest/gtest.h"
+
+#include "ThreadData.hpp"
+
+using namespace kotlin;
+
+TEST(ThreadRegistryTest, RegisterCurrentThread) {
+    std::thread t([]() {
+        auto* node = mm::ThreadRegistry::Instance().RegisterCurrentThread();
+        auto* threadData = node->Get();
+        EXPECT_EQ(pthread_self(), threadData->threadId());
+        EXPECT_EQ(threadData, mm::ThreadRegistry::Instance().CurrentThreadData());
+    });
+    t.join();
+}