diff --git a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp index 1a514855141..0c933744aec 100644 --- a/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp +++ b/kotlin-native/runtime/src/main/cpp/MultiSourceQueue.hpp @@ -6,6 +6,7 @@ #ifndef RUNTIME_MULTI_SOURCE_QUEUE_H #define RUNTIME_MULTI_SOURCE_QUEUE_H +#include #include #include @@ -17,51 +18,131 @@ namespace kotlin { template class MultiSourceQueue { public: + class Producer; + + // TODO: Consider switching from `std::list` to `SingleLockList` to hide the constructor + // and to not store the iterator. + class Node : private Pinned { + public: + Node(const T& value, Producer* owner) noexcept : value_(value), owner_(owner) {} + + T& operator*() noexcept { return value_; } + + private: + friend class MultiSourceQueue; + + T value_; + std::atomic owner_; // `nullptr` signifies that `MultiSourceQueue` owns it. + typename std::list::iterator position_; + }; + class Producer { public: explicit Producer(MultiSourceQueue& owner) noexcept : owner_(owner) {} ~Producer() { Publish(); } - void Insert(const T& value) noexcept { queue_.push_back(value); } + Node* Insert(const T& value) noexcept { + queue_.emplace_back(value, this); + auto& node = queue_.back(); + node.position_ = std::prev(queue_.end()); + return &node; + } + + void Erase(Node* node) noexcept { + if (node->owner_ == this) { + // If we own it, delete it immediately. + queue_.erase(node->position_); + return; + } + // If it's owned by the global queue or some other `Producer`, queue it. + deletionQueue_.push_back(node); + } // Merge `this` queue with owning `MultiSourceQueue`. `this` will have empty queue after the call. // This call is performed without heap allocations. TODO: Test that no allocations are happening. - void Publish() noexcept { owner_.Collect(*this); } + void Publish() noexcept { + for (auto& node : queue_) { + node.owner_ = nullptr; + } + std::lock_guard guard(owner_.mutex_); + owner_.queue_.splice(owner_.queue_.end(), queue_); + owner_.deletionQueue_.splice(owner_.deletionQueue_.end(), deletionQueue_); + } + + private: + MultiSourceQueue& owner_; // weak + std::list queue_; + std::list deletionQueue_; + }; + + class Iterator { + public: + T& operator*() noexcept { return **position_; } + + Iterator& operator++() noexcept { + ++position_; + return *this; + } + + bool operator==(const Iterator& rhs) const noexcept { return position_ == rhs.position_; } + + bool operator!=(const Iterator& rhs) const noexcept { return position_ != rhs.position_; } private: friend class MultiSourceQueue; - MultiSourceQueue& owner_; // weak - std::list queue_; - }; + explicit Iterator(const typename std::list::iterator& position) noexcept : position_(position) {} - using Iterator = typename std::list::iterator; + typename std::list::iterator position_; + }; class Iterable : MoveOnly { public: - explicit Iterable(MultiSourceQueue& owner) noexcept : owner_(owner), guard_(owner_.mutex_) {} - - Iterator begin() noexcept { return owner_.commonQueue_.begin(); } - Iterator end() noexcept { return owner_.commonQueue_.end(); } + Iterator begin() noexcept { return Iterator(owner_.queue_.begin()); } + Iterator end() noexcept { return Iterator(owner_.queue_.end()); } private: + friend class MultiSourceQueue; + + explicit Iterable(MultiSourceQueue& owner) noexcept : owner_(owner), guard_(owner_.mutex_) {} + MultiSourceQueue& owner_; // weak std::unique_lock guard_; }; - // Lock MultiSourceQueue for safe iteration. + // Lock `MultiSourceQueue` for safe iteration. If element was scheduled for deletion, + // it'll still be iterated. Use `ApplyDeletions` to remove those elements. Iterable Iter() noexcept { return Iterable(*this); } -private: - void Collect(Producer& producer) noexcept { + // Lock `MultiSourceQueue` and apply deletions. Only deletes elements that were published. + void ApplyDeletions() noexcept { std::lock_guard guard(mutex_); - commonQueue_.splice(commonQueue_.end(), producer.queue_); + std::list remainingDeletions; + + auto it = deletionQueue_.begin(); + while (it != deletionQueue_.end()) { + auto next = std::next(it); + Node* node = *it; + + if (node->owner_ != nullptr) { + // If the `Node` is still owned by some `Producer`, skip it. + remainingDeletions.splice(remainingDeletions.end(), deletionQueue_, it); + } else { + queue_.erase(node->position_); + // `node` is invalid after this + } + + it = next; + } + deletionQueue_ = std::move(remainingDeletions); } +private: // Using `std::list` as it allows to implement `Collect` without memory allocations, // which is important for GC mark phase. - std::list commonQueue_; + std::list queue_; + std::list deletionQueue_; SimpleMutex mutex_; }; diff --git a/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp b/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp index 60d8e81fa66..62310bf19ee 100644 --- a/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/MultiSourceQueueTest.cpp @@ -15,16 +15,109 @@ using namespace kotlin; +namespace { + +template +std::vector Collect(MultiSourceQueue& queue) { + std::vector result; + for (const auto& element : queue.Iter()) { + result.push_back(element); + } + return result; +} + +} // namespace + using IntQueue = MultiSourceQueue; +TEST(MultiSourceQueueTest, Insert) { + IntQueue queue; + IntQueue::Producer producer(queue); + + constexpr int kFirst = 1; + constexpr int kSecond = 2; + + auto* node1 = producer.Insert(kFirst); + auto* node2 = producer.Insert(kSecond); + + EXPECT_THAT(**node1, kFirst); + EXPECT_THAT(**node2, kSecond); +} + +TEST(MultiSourceQueueTest, EraseFromTheSameProducer) { + IntQueue queue; + IntQueue::Producer producer(queue); + + constexpr int kFirst = 1; + constexpr int kSecond = 2; + + producer.Insert(kFirst); + auto* node2 = producer.Insert(kSecond); + producer.Erase(node2); + producer.Publish(); + + auto actual = Collect(queue); + EXPECT_THAT(actual, testing::ElementsAre(kFirst)); +} + +TEST(MultiSourceQueueTest, EraseFromGlobal) { + IntQueue queue; + IntQueue::Producer producer(queue); + + constexpr int kFirst = 1; + constexpr int kSecond = 2; + + producer.Insert(kFirst); + auto* node2 = producer.Insert(kSecond); + producer.Publish(); + producer.Erase(node2); + producer.Publish(); + + auto actual1 = Collect(queue); + EXPECT_THAT(actual1, testing::ElementsAre(kFirst, kSecond)); + + queue.ApplyDeletions(); + + auto actual2 = Collect(queue); + EXPECT_THAT(actual2, testing::ElementsAre(kFirst)); +} + +TEST(MultiSourceQueueTest, EraseFromOtherProducer) { + IntQueue queue; + IntQueue::Producer producer1(queue); + IntQueue::Producer producer2(queue); + + constexpr int kFirst = 1; + constexpr int kSecond = 2; + + producer1.Insert(kFirst); + auto* node2 = producer1.Insert(kSecond); + producer2.Erase(node2); + producer1.Publish(); + + auto actual1 = Collect(queue); + EXPECT_THAT(actual1, testing::ElementsAre(kFirst, kSecond)); + + queue.ApplyDeletions(); + + auto actual2 = Collect(queue); + EXPECT_THAT(actual2, testing::ElementsAre(kFirst, kSecond)); + + producer2.Publish(); + + auto actual3 = Collect(queue); + EXPECT_THAT(actual3, testing::ElementsAre(kFirst, kSecond)); + + queue.ApplyDeletions(); + + auto actual4 = Collect(queue); + EXPECT_THAT(actual4, testing::ElementsAre(kFirst)); +} + TEST(MultiSourceQueueTest, Empty) { IntQueue queue; - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::IsEmpty()); } @@ -35,11 +128,7 @@ TEST(MultiSourceQueueTest, DoNotPublish) { producer.Insert(1); producer.Insert(2); - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::IsEmpty()); } @@ -56,11 +145,7 @@ TEST(MultiSourceQueueTest, Publish) { producer1.Publish(); producer2.Publish(); - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::ElementsAre(1, 2, 10, 20)); } @@ -85,11 +170,7 @@ TEST(MultiSourceQueueTest, PublishSeveralTimes) { producer.Insert(5); producer.Publish(); - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::ElementsAre(1, 2, 3, 4, 5)); } @@ -102,11 +183,7 @@ TEST(MultiSourceQueueTest, PublishInDestructor) { producer.Insert(2); } - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::ElementsAre(1, 2)); } @@ -137,11 +214,7 @@ TEST(MultiSourceQueueTest, ConcurrentPublish) { t.join(); } - std::vector actual; - for (int element : queue.Iter()) { - actual.push_back(element); - } - + auto actual = Collect(queue); EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); } @@ -198,10 +271,49 @@ TEST(MultiSourceQueueTest, IterWhileConcurrentPublish) { EXPECT_THAT(actualBefore, testing::ElementsAreArray(expectedBefore)); - std::vector actualAfter; - for (int element : queue.Iter()) { - actualAfter.push_back(element); - } - + auto actualAfter = Collect(queue); EXPECT_THAT(actualAfter, testing::UnorderedElementsAreArray(expectedAfter)); } + +TEST(MultiSourceQueueTest, ConcurrentPublishAndApplyDeletions) { + IntQueue queue; + constexpr int kThreadCount = kDefaultThreadCount; + + std::atomic canStart(false); + std::atomic readyCount(0); + std::atomic startedCount(0); + std::vector threads; + for (int i = 0; i < kThreadCount; ++i) { + threads.emplace_back([&queue, i, &canStart, &readyCount, &startedCount]() { + IntQueue::Producer producer(queue); + auto* node = producer.Insert(i); + producer.Publish(); + producer.Erase(node); + ++readyCount; + while (!canStart) { + } + ++startedCount; + producer.Publish(); + }); + } + + while (readyCount < kThreadCount) { + } + canStart = true; + while (startedCount < kThreadCount) { + } + + queue.ApplyDeletions(); + + for (auto& t : threads) { + t.join(); + } + + // We do not know which elements were deleted at this point. Expecting not to crash by this point. + + // This must make the queue empty. + queue.ApplyDeletions(); + + auto actual = Collect(queue); + EXPECT_THAT(actual, testing::IsEmpty()); +} diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index e063b4e0865..e0b0e0a80e2 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -7,6 +7,7 @@ #define RUNTIME_MM_GLOBAL_DATA_H #include "GlobalsRegistry.hpp" +#include "StableRefRegistry.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" @@ -20,6 +21,7 @@ public: ThreadRegistry& threadRegistry() { return threadRegistry_; } GlobalsRegistry& globalsRegistry() { return globalsRegistry_; } + StableRefRegistry& stableRefRegistry() { return stableRefRegistry_; } private: GlobalData(); @@ -29,6 +31,7 @@ private: ThreadRegistry threadRegistry_; GlobalsRegistry globalsRegistry_; + StableRefRegistry stableRefRegistry_; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalsRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalsRegistry.hpp index 580ddf74ce4..83d3ceac83f 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalsRegistry.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalsRegistry.hpp @@ -46,6 +46,7 @@ private: GlobalsRegistry(); ~GlobalsRegistry(); + // TODO: Add-only MultiSourceQueue can be made more efficient. Measure, if it's a problem. MultiSourceQueue globals_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 9da3c14963f..ba9ab725eeb 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -6,6 +6,7 @@ #include "Memory.h" #include "GlobalsRegistry.hpp" +#include "StableRefRegistry.hpp" #include "ThreadData.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" @@ -19,6 +20,15 @@ extern "C" struct MemoryState : Pinned { ~MemoryState() = delete; }; +// TODO: This name does not make sense anymore. +// Delete all means of creating this type directly as it only serves +// as a typedef for `mm::StableRefRegistry::Node`. +class ForeignRefManager : Pinned { +public: + ForeignRefManager() = delete; + ~ForeignRefManager() = delete; +}; + namespace { // `reinterpret_cast` to it and back to the same type @@ -31,6 +41,14 @@ ALWAYS_INLINE mm::ThreadRegistry::Node* FromMemoryState(MemoryState* state) { return reinterpret_cast(state); } +ALWAYS_INLINE ForeignRefManager* ToForeignRefManager(mm::StableRefRegistry::Node* data) { + return reinterpret_cast(data); +} + +ALWAYS_INLINE mm::StableRefRegistry::Node* FromForeignRefManager(ForeignRefManager* manager) { + return reinterpret_cast(manager); +} + ALWAYS_INLINE mm::ThreadData* GetThreadData(MemoryState* state) { return FromMemoryState(state)->Get(); } @@ -75,3 +93,52 @@ extern "C" RUNTIME_NOTHROW void ClearTLS(MemoryState* memory) { extern "C" RUNTIME_NOTHROW ObjHeader** LookupTLS(void** key, int index) { return mm::ThreadRegistry::Instance().CurrentThreadData()->tls().Lookup(key, index); } + +extern "C" RUNTIME_NOTHROW void* CreateStablePointer(ObjHeader* object) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + return mm::StableRefRegistry::Instance().RegisterStableRef(threadData, object); +} + +extern "C" RUNTIME_NOTHROW void DisposeStablePointer(void* pointer) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* node = static_cast(pointer); + mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node); +} + +extern "C" RUNTIME_NOTHROW OBJ_GETTER(DerefStablePointer, void* pointer) { + auto* node = static_cast(pointer); + ObjHeader* object = **node; + RETURN_OBJ(object); +} + +extern "C" RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void* pointer) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* node = static_cast(pointer); + ObjHeader* object = **node; + UpdateReturnRef(OBJ_RESULT, object); + mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node); + return object; +} + +extern "C" ForeignRefContext InitForeignRef(ObjHeader* object) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* node = mm::StableRefRegistry::Instance().RegisterStableRef(threadData, object); + return ToForeignRefManager(node); +} + +extern "C" void DeinitForeignRef(ObjHeader* object, ForeignRefContext context) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + auto* node = FromForeignRefManager(context); + RuntimeAssert(object == **node, "Must correspond to the same object"); + mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node); +} + +extern "C" bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) { + // TODO: Remove when legacy MM is gone. + return true; +} + +extern "C" void AdoptReferenceFromSharedVariable(ObjHeader* object) { + // TODO: Remove when legacy MM is gone. + // Nothing to do. +} diff --git a/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.cpp b/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.cpp new file mode 100644 index 00000000000..ee4232c7273 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "StableRefRegistry.hpp" + +#include "GlobalData.hpp" +#include "ThreadData.hpp" + +using namespace kotlin; + +// static +mm::StableRefRegistry& mm::StableRefRegistry::Instance() noexcept { + return GlobalData::Instance().stableRefRegistry(); +} + +mm::StableRefRegistry::Node* mm::StableRefRegistry::RegisterStableRef(mm::ThreadData* threadData, ObjHeader* object) noexcept { + return threadData->stableRefThreadQueue().Insert(object); +} + +void mm::StableRefRegistry::UnregisterStableRef(mm::ThreadData* threadData, Node* node) noexcept { + threadData->stableRefThreadQueue().Erase(node); +} + +void mm::StableRefRegistry::ProcessThread(mm::ThreadData* threadData) noexcept { + threadData->stableRefThreadQueue().Publish(); +} + +void mm::StableRefRegistry::ProcessDeletions() noexcept { + stableRefs_.ApplyDeletions(); +} + +mm::StableRefRegistry::StableRefRegistry() = default; +mm::StableRefRegistry::~StableRefRegistry() = default; diff --git a/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.hpp new file mode 100644 index 00000000000..dc2410e4cc6 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/StableRefRegistry.hpp @@ -0,0 +1,73 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_STABLE_REF_REGISTRY_H +#define RUNTIME_MM_STABLE_REF_REGISTRY_H + +#include "Memory.h" +#include "MultiSourceQueue.hpp" +#include "ThreadRegistry.hpp" + +namespace kotlin { +namespace mm { + +// Registry for all objects that have references outside of Kotlin. +class StableRefRegistry : Pinned { +public: + class ThreadQueue : public MultiSourceQueue::Producer { + public: + explicit ThreadQueue(StableRefRegistry& registry) : Producer(registry.stableRefs_) {} + // Do not add fields as this is just a wrapper and Producer does not have virtual destructor. + }; + + using Iterable = MultiSourceQueue::Iterable; + using Iterator = MultiSourceQueue::Iterator; + using Node = MultiSourceQueue::Node; + + static StableRefRegistry& Instance() noexcept; + + Node* RegisterStableRef(mm::ThreadData* threadData, ObjHeader* object) noexcept; + + void UnregisterStableRef(mm::ThreadData* threadData, Node* node) noexcept; + + // Collect stable references from thread corresponding to `threadData`. Must be called by the thread + // when it's asked by GC to stop. + void ProcessThread(mm::ThreadData* threadData) noexcept; + + // Lock registry and apply deletions. Should be called on GC thread after all threads have published, and before `Iter`. + void ProcessDeletions() noexcept; + + // Lock registry for safe iteration. + // TODO: Iteration over `stableRefs_` will be slow, because it's `std::list` collected at different times from + // different threads, and so the nodes are all over the memory. Use metrics to understand how + // much of a problem is it. + Iterable Iter() noexcept { return stableRefs_.Iter(); } + +private: + friend class GlobalData; + + StableRefRegistry(); + ~StableRefRegistry(); + + // Current approach optimizes for creating and disposing of stable refs: + // * creation just enqueues ref, disposing either queues or deletes the ref immediately (if it still resides in the current queue). + // * when thread is stopped, it'll scan through the local queue (to mark that refs no longer reside in it) and push creation and + // deletion queues to the global registry. + // * during marking GC will have to `ProcessDeletions` to actually delete the refs that were enqueued for deletion. + // So, we sacrifice memory (to keep deleted queues) and marking time (to process these queues) to improve creation and disposal times. + // + // Other alternatives: + // * Use a single global collection (e.g. lock free doubly linked list). + // * Sacrifice disposal time to try to delete as early as possible (e.g. post directly into owning producer, so it processes deletions + // before posting queue to the global registry) + // + // TODO: Measure to understand, if this approach is problematic. + MultiSourceQueue stableRefs_; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_STABLE_REF_REGISTRY_H diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index fa8b621acca..5b67ff9f063 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -126,22 +126,6 @@ RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) { RuntimeCheck(false, "Unimplemented"); } -RUNTIME_NOTHROW void* CreateStablePointer(ObjHeader* obj) { - RuntimeCheck(false, "Unimplemented"); -} - -RUNTIME_NOTHROW void DisposeStablePointer(void* pointer) { - RuntimeCheck(false, "Unimplemented"); -} - -RUNTIME_NOTHROW OBJ_GETTER(DerefStablePointer, void*) { - RuntimeCheck(false, "Unimplemented"); -} - -RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void*) { - RuntimeCheck(false, "Unimplemented"); -} - void MutationCheck(ObjHeader* obj) { RuntimeCheck(false, "Unimplemented"); } @@ -194,22 +178,6 @@ ForeignRefContext InitLocalForeignRef(ObjHeader* object) { RuntimeCheck(false, "Unimplemented"); } -ForeignRefContext InitForeignRef(ObjHeader* object) { - RuntimeCheck(false, "Unimplemented"); -} - -void DeinitForeignRef(ObjHeader* object, ForeignRefContext context) { - RuntimeCheck(false, "Unimplemented"); -} - -bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) { - RuntimeCheck(false, "Unimplemented"); -} - -void AdoptReferenceFromSharedVariable(ObjHeader* object) { - RuntimeCheck(false, "Unimplemented"); -} - void CheckGlobalsAccessible() { // Globals are always accessible. } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index cecf7cdb2d6..e7a788e5099 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -9,6 +9,7 @@ #include #include "GlobalsRegistry.hpp" +#include "StableRefRegistry.hpp" #include "ThreadLocalStorage.hpp" #include "Utils.hpp" @@ -19,7 +20,8 @@ namespace mm { // Pin it in memory to prevent accidental copying. class ThreadData final : private Pinned { public: - ThreadData(pthread_t threadId) noexcept : threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()) {} + ThreadData(pthread_t threadId) noexcept : + threadId_(threadId), globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()) {} ~ThreadData() = default; @@ -29,10 +31,13 @@ public: ThreadLocalStorage& tls() noexcept { return tls_; } + StableRefRegistry::ThreadQueue& stableRefThreadQueue() noexcept { return stableRefThreadQueue_; } + private: const pthread_t threadId_; GlobalsRegistry::ThreadQueue globalsThreadQueue_; ThreadLocalStorage tls_; + StableRefRegistry::ThreadQueue stableRefThreadQueue_; }; } // namespace mm