diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index dd5e7067212..fe767630995 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -85,6 +85,8 @@ struct ObjHeader { return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); } + inline bool heap() const { return getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK) == 0; } + static MetaObjHeader* createMetaObject(ObjHeader* object); static void destroyMetaObject(ObjHeader* object); }; diff --git a/kotlin-native/runtime/src/mm/cpp/GC.hpp b/kotlin-native/runtime/src/mm/cpp/GC.hpp new file mode 100644 index 00000000000..83e1c3b2649 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/GC.hpp @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_GC_H +#define RUNTIME_MM_GC_H + +#include "gc/NoOpGC.hpp" + +namespace kotlin { +namespace mm { + +// TODO: GC should be extracted into a separate module, so that we can do different GCs without +// the need to redo the entire MM. For now changing GCs can be done by modifying `using` below. + +using GC = NoOpGC; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_GC_H diff --git a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp index cc06903da16..6cc47c09c69 100644 --- a/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/GlobalData.hpp @@ -8,6 +8,7 @@ #include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" +#include "GC.hpp" #include "StableRefRegistry.hpp" #include "ThreadRegistry.hpp" #include "Utils.hpp" @@ -23,7 +24,8 @@ public: ThreadRegistry& threadRegistry() noexcept { return threadRegistry_; } GlobalsRegistry& globalsRegistry() noexcept { return globalsRegistry_; } StableRefRegistry& stableRefRegistry() noexcept { return stableRefRegistry_; } - ObjectFactory& objectFactory() noexcept { return objectFactory_; } + ObjectFactory& objectFactory() noexcept { return objectFactory_; } + GC& gc() noexcept { return gc_; } private: GlobalData(); @@ -34,7 +36,8 @@ private: ThreadRegistry threadRegistry_; GlobalsRegistry globalsRegistry_; StableRefRegistry stableRefRegistry_; - ObjectFactory objectFactory_; + ObjectFactory objectFactory_; + GC gc_; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index eb9f424fe57..ddab007b465 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -253,6 +253,11 @@ extern "C" RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) { // Nothing to do } +extern "C" void Kotlin_native_internal_GC_collect(ObjHeader*) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().PerformFullGC(); +} + extern "C" void Kotlin_native_internal_GC_collectCyclic(ObjHeader*) { // TODO: Remove when legacy MM is gone. ThrowIllegalArgumentException(); @@ -289,6 +294,10 @@ extern "C" void Kotlin_Any_share(ObjHeader* thiz) { // Nothing to do } +extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) { + GetThreadData(memory)->gc().PerformFullGC(); +} + extern "C" RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) { // TODO: Remove when legacy MM is gone. return true; @@ -366,7 +375,22 @@ extern "C" void AdoptReferenceFromSharedVariable(ObjHeader* object) { // Nothing to do. } -void CheckGlobalsAccessible() { +extern "C" void CheckGlobalsAccessible() { // TODO: Remove when legacy MM is gone. // Always accessible } + +extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().SafePointFunctionEpilogue(); +} + +extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().SafePointLoopBody(); +} + +extern "C" RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + threadData->gc().SafePointExceptionUnwind(); +} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp deleted file mode 100644 index 33b6bb9b497..00000000000 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -#include "ObjectFactory.hpp" - -#include "Alignment.hpp" -#include "Alloc.h" -#include "GlobalData.hpp" -#include "Types.h" - -using namespace kotlin; - -ObjHeader* mm::ObjectFactory::ThreadQueue::CreateObject(const TypeInfo* typeInfo) noexcept { - RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); - size_t allocSize = typeInfo->instanceSize_; - auto& node = producer_.Insert(allocSize); - auto* object = static_cast(node.Data()); - object->typeInfoOrMeta_ = const_cast(typeInfo); - return object; -} - -ArrayHeader* mm::ObjectFactory::ThreadQueue::CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { - RuntimeAssert(typeInfo->IsArray(), "Must be an array"); - uint32_t arraySize = static_cast(-typeInfo->instanceSize_) * count; - // Note: array body is aligned, but for size computation it is enough to align the sum. - size_t allocSize = AlignUp(sizeof(ArrayHeader) + arraySize, kObjectAlignment); - auto& node = producer_.Insert(allocSize); - auto* array = static_cast(node.Data()); - array->typeInfoOrMeta_ = const_cast(typeInfo); - array->count_ = count; - return array; -} - -bool mm::ObjectFactory::Iterator::IsArray() noexcept { - // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can - // be always casted to the other. - auto* object = static_cast((*iterator_).Data()); - return object->type_info()->IsArray(); -} - -ObjHeader* mm::ObjectFactory::Iterator::GetObjHeader() noexcept { - auto* object = static_cast((*iterator_).Data()); - RuntimeAssert(!object->type_info()->IsArray(), "Must not be an array"); - return object; -} - -ArrayHeader* mm::ObjectFactory::Iterator::GetArrayHeader() noexcept { - auto* array = static_cast((*iterator_).Data()); - RuntimeAssert(array->type_info()->IsArray(), "Must be an array"); - return array; -} - -mm::ObjectFactory::ObjectFactory() noexcept = default; -mm::ObjectFactory::~ObjectFactory() = default; - -// static -mm::ObjectFactory& mm::ObjectFactory::Instance() noexcept { - return GlobalData::Instance().objectFactory(); -} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index 9cc78bdc59b..c9fe6e75879 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -25,12 +25,24 @@ namespace internal { // A queue that is constructed by collecting subqueues from several `Producer`s. // This is essentially a heterogeneous `MultiSourceQueue` on top of a singly linked list that -// uses `konanAllocMemory` and `konanFreeMemory` +// uses `Allocator` to allocate and free memory. // TODO: Consider merging with `MultiSourceQueue` somehow. -template +template class ObjectFactoryStorage : private Pinned { static_assert(IsValidAlignment(DataAlignment), "DataAlignment is not a valid alignment"); + template + class Deleter { + public: + void operator()(T* instance) noexcept { + instance->~T(); + Allocator::Free(instance); + } + }; + + template + using unique_ptr = std::unique_ptr>; + public: // This class does not know its size at compile-time. Does not inherit from `KonanAllocatorAware` because // in `KonanAllocatorAware::operator new(size_t size, KonanAllocTag)` `size` would be incorrect. @@ -40,6 +52,13 @@ public: public: ~Node() = default; + static Node& FromData(void* data) noexcept { + constexpr size_t kDataOffset = DataOffset(); + Node* node = reinterpret_cast(reinterpret_cast(data) - kDataOffset); + RuntimeAssert(node->Data() == data, "Node layout has broken"); + return *node; + } + // Note: This can only be trivially destructible data, as nobody can invoke its destructor. void* Data() noexcept { constexpr size_t kDataOffset = DataOffset(); @@ -59,37 +78,36 @@ public: Node() noexcept = default; - static KStdUniquePtr Create(size_t dataSize) noexcept { + static unique_ptr Create(Allocator& allocator, size_t dataSize) noexcept { size_t dataSizeAligned = AlignUp(dataSize, DataAlignment); size_t totalAlignment = std::max(alignof(Node), DataAlignment); size_t totalSize = AlignUp(sizeof(Node) + dataSizeAligned, totalAlignment); RuntimeAssert( DataOffset() + dataSize <= totalSize, "totalSize %zu is not enough to fit data %zu at offset %zu", totalSize, dataSize, DataOffset()); - void* ptr = konanAllocAlignedMemory(totalSize, totalAlignment); + void* ptr = allocator.Alloc(totalSize, totalAlignment); if (!ptr) { - // TODO: Try doing GC first. - konan::consoleErrorf("Out of memory trying to allocate %zu. Aborting.\n", totalSize); + konan::consoleErrorf("Out of memory trying to allocate %zu bytes. Aborting.\n", totalSize); konan::abort(); } RuntimeAssert(IsAligned(ptr, totalAlignment), "Allocator returned unaligned to %zu pointer %p", totalAlignment, ptr); - return KStdUniquePtr(new (ptr) Node()); + return unique_ptr(new (ptr) Node()); } - KStdUniquePtr next_; + unique_ptr next_; // There's some more data of an unknown (at compile-time) size here, but it cannot be represented // with C++ members. }; class Producer : private MoveOnly { public: - explicit Producer(ObjectFactoryStorage& owner) noexcept : owner_(owner) {} + Producer(ObjectFactoryStorage& owner, Allocator allocator) noexcept : owner_(owner), allocator_(std::move(allocator)) {} ~Producer() { Publish(); } Node& Insert(size_t dataSize) noexcept { AssertCorrect(); - auto node = Node::Create(dataSize); + auto node = Node::Create(allocator_, dataSize); auto* nodePtr = node.get(); if (!root_) { root_ = std::move(node); @@ -159,7 +177,8 @@ public: } ObjectFactoryStorage& owner_; // weak - KStdUniquePtr root_; + Allocator allocator_; + unique_ptr root_; Node* last_ = nullptr; }; @@ -245,35 +264,160 @@ private: } } - KStdUniquePtr root_; + unique_ptr root_; Node* last_ = nullptr; SpinLock mutex_; }; +class SimpleAllocator { +public: + void* Alloc(size_t size, size_t alignment) noexcept { return konanAllocAlignedMemory(size, alignment); } + + static void Free(void* instance) noexcept { konanFreeMemory(instance); } +}; + +template +class AllocatorWithGC { +public: + AllocatorWithGC(BaseAllocator base, GC& gc) noexcept : base_(std::move(base)), gc_(gc) {} + + void* Alloc(size_t size, size_t alignment) noexcept { + gc_.SafePointAllocation(size); + if (void* ptr = base_.Alloc(size, alignment)) { + return ptr; + } + // Tell GC that we failed to allocate, and try one more time. + gc_.OnOOM(size); + return base_.Alloc(size, alignment); + } + + static void Free(void* instance) noexcept { BaseAllocator::Free(instance); } + +private: + BaseAllocator base_; + GC& gc_; +}; + } // namespace internal +template class ObjectFactory : private Pinned { + using GCObjectData = typename GC::ObjectData; + using GCThreadData = typename GC::ThreadData; + + using Allocator = internal::AllocatorWithGC; + + struct HeapObjHeader { + GCObjectData gcData; + alignas(kObjectAlignment) ObjHeader object; + }; + + // Needs to be kept compatible with `HeapObjHeader` just like `ArrayHeader` is compatible + // with `ObjHeader`: the former can always be casted to the other. + struct HeapArrayHeader { + GCObjectData gcData; + alignas(kObjectAlignment) ArrayHeader array; + }; + public: - using Storage = internal::ObjectFactoryStorage; + using Storage = internal::ObjectFactoryStorage; + + class NodeRef { + public: + explicit NodeRef(typename Storage::Node& node) noexcept : node_(node) {} + + static NodeRef From(ObjHeader* object) noexcept { + RuntimeAssert(object->heap(), "Must be a heap object"); + auto* heapObject = reinterpret_cast(reinterpret_cast(object) - offsetof(HeapObjHeader, object)); + RuntimeAssert(&heapObject->object == object, "HeapObjHeader layout has broken"); + return NodeRef(Storage::Node::FromData(heapObject)); + } + + static NodeRef From(ArrayHeader* array) noexcept { + // `ArrayHeader` and `ObjHeader` are kept compatible, so the former can + // be always casted to the other. + RuntimeAssert(reinterpret_cast(array)->heap(), "Must be a heap object"); + auto* heapArray = reinterpret_cast(reinterpret_cast(array) - offsetof(HeapArrayHeader, array)); + RuntimeAssert(&heapArray->array == array, "HeapArrayHeader layout has broken"); + return NodeRef(Storage::Node::FromData(heapArray)); + } + + NodeRef* operator->() noexcept { return this; } + + GCObjectData& GCObjectData() noexcept { + // `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can + // be always casted to the other. + return static_cast(node_.Data())->gcData; + } + + bool IsArray() const noexcept { + // `HeapArrayHeader` and `HeapObjHeader` are kept compatible, so the former can + // be always casted to the other. + auto* object = &static_cast(node_.Data())->object; + return object->type_info()->IsArray(); + } + + ObjHeader* GetObjHeader() noexcept { + auto* object = &static_cast(node_.Data())->object; + RuntimeAssert(!object->type_info()->IsArray(), "Must not be an array"); + return object; + } + + ArrayHeader* GetArrayHeader() noexcept { + auto* array = &static_cast(node_.Data())->array; + RuntimeAssert(array->type_info()->IsArray(), "Must be an array"); + return array; + } + + bool operator==(const NodeRef& rhs) const noexcept { return &node_ == &rhs.node_; } + + bool operator!=(const NodeRef& rhs) const noexcept { return !(*this == rhs); } + + private: + typename Storage::Node& node_; + }; class ThreadQueue : private MoveOnly { public: - explicit ThreadQueue(ObjectFactory& owner) noexcept : producer_(owner.storage_) {} + ThreadQueue(ObjectFactory& owner, GCThreadData& gc) noexcept : + producer_(owner.storage_, internal::AllocatorWithGC(internal::SimpleAllocator(), gc)) {} - ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept; - ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept; + ObjHeader* CreateObject(const TypeInfo* typeInfo) noexcept { + RuntimeAssert(!typeInfo->IsArray(), "Must not be an array"); + size_t membersSize = typeInfo->instanceSize_ - sizeof(ObjHeader); + size_t allocSize = AlignUp(sizeof(HeapObjHeader) + membersSize, kObjectAlignment); + auto& node = producer_.Insert(allocSize); + auto* heapObject = new (node.Data()) HeapObjHeader(); + auto* object = &heapObject->object; + object->typeInfoOrMeta_ = const_cast(typeInfo); + return object; + } + + ArrayHeader* CreateArray(const TypeInfo* typeInfo, uint32_t count) noexcept { + RuntimeAssert(typeInfo->IsArray(), "Must be an array"); + uint32_t membersSize = static_cast(-typeInfo->instanceSize_) * count; + // Note: array body is aligned, but for size computation it is enough to align the sum. + size_t allocSize = AlignUp(sizeof(HeapArrayHeader) + membersSize, kObjectAlignment); + auto& node = producer_.Insert(allocSize); + auto* heapArray = new (node.Data()) HeapArrayHeader(); + auto* array = &heapArray->array; + array->typeInfoOrMeta_ = const_cast(typeInfo); + array->count_ = count; + return array; + } void Publish() noexcept { producer_.Publish(); } void ClearForTests() noexcept { producer_.ClearForTests(); } private: - Storage::Producer producer_; + typename Storage::Producer producer_; }; class Iterator { public: - Storage::Node& operator*() noexcept { return *iterator_; } + NodeRef operator*() noexcept { return NodeRef(*iterator_); } + NodeRef operator->() noexcept { return NodeRef(*iterator_); } Iterator& operator++() noexcept { ++iterator_; @@ -284,17 +428,12 @@ public: bool operator!=(const Iterator& rhs) const noexcept { return iterator_ != rhs.iterator_; } - bool IsArray() noexcept; - - ObjHeader* GetObjHeader() noexcept; - ArrayHeader* GetArrayHeader() noexcept; - private: friend class ObjectFactory; - explicit Iterator(Storage::Iterator iterator) noexcept : iterator_(std::move(iterator)) {} + explicit Iterator(typename Storage::Iterator iterator) noexcept : iterator_(std::move(iterator)) {} - Storage::Iterator iterator_; + typename Storage::Iterator iterator_; }; class Iterable { @@ -307,13 +446,11 @@ public: void EraseAndAdvance(Iterator& iterator) noexcept { iter_.EraseAndAdvance(iterator.iterator_); } private: - Storage::Iterable iter_; + typename Storage::Iterable iter_; }; - ObjectFactory() noexcept; - ~ObjectFactory(); - - static ObjectFactory& Instance() noexcept; + ObjectFactory() noexcept = default; + ~ObjectFactory() = default; Iterable Iter() noexcept { return Iterable(*this); } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp index d235e54bb66..ec97e801475 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactoryTest.cpp @@ -12,17 +12,25 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "GC.hpp" #include "TestSupport.hpp" #include "Types.h" using namespace kotlin; +using testing::_; + +namespace { + +using SimpleAllocator = mm::internal::SimpleAllocator; + template -using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; +using ObjectFactoryStorage = mm::internal::ObjectFactoryStorage; using ObjectFactoryStorageRegular = ObjectFactoryStorage; -namespace { +template +using Producer = typename Storage::Producer; template KStdVector Collect(ObjectFactoryStorage& storage) { @@ -76,7 +84,7 @@ TEST(ObjectFactoryStorageTest, Empty) { TEST(ObjectFactoryStorageTest, DoNotPublish) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -88,8 +96,8 @@ TEST(ObjectFactoryStorageTest, DoNotPublish) { TEST(ObjectFactoryStorageTest, Publish) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer1(storage); - ObjectFactoryStorageRegular::Producer producer2(storage); + Producer producer1(storage, SimpleAllocator()); + Producer producer2(storage, SimpleAllocator()); producer1.Insert(1); producer1.Insert(2); @@ -106,7 +114,7 @@ TEST(ObjectFactoryStorageTest, Publish) { TEST(ObjectFactoryStorageTest, PublishDifferentTypes) { ObjectFactoryStorage storage; - ObjectFactoryStorage::Producer producer(storage); + Producer> producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -139,7 +147,7 @@ TEST(ObjectFactoryStorageTest, PublishDifferentTypes) { TEST(ObjectFactoryStorageTest, PublishSeveralTimes) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); // Add 2 elements and publish. producer.Insert(1); @@ -167,7 +175,7 @@ TEST(ObjectFactoryStorageTest, PublishInDestructor) { ObjectFactoryStorageRegular storage; { - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); } @@ -177,9 +185,22 @@ TEST(ObjectFactoryStorageTest, PublishInDestructor) { EXPECT_THAT(actual, testing::ElementsAre(1, 2)); } +TEST(ObjectFactoryStorageTest, FindNode) { + ObjectFactoryStorageRegular storage; + Producer producer(storage, SimpleAllocator()); + + auto& node1 = producer.Insert(1); + auto& node2 = producer.Insert(2); + + producer.Publish(); + + EXPECT_THAT(&ObjectFactoryStorageRegular::Node::FromData(node1.Data()), &node1); + EXPECT_THAT(&ObjectFactoryStorageRegular::Node::FromData(node2.Data()), &node2); +} + TEST(ObjectFactoryStorageTest, EraseFirst) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -205,7 +226,7 @@ TEST(ObjectFactoryStorageTest, EraseFirst) { TEST(ObjectFactoryStorageTest, EraseMiddle) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -231,7 +252,7 @@ TEST(ObjectFactoryStorageTest, EraseMiddle) { TEST(ObjectFactoryStorageTest, EraseLast) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -257,7 +278,7 @@ TEST(ObjectFactoryStorageTest, EraseLast) { TEST(ObjectFactoryStorageTest, EraseAll) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); producer.Insert(2); @@ -279,7 +300,7 @@ TEST(ObjectFactoryStorageTest, EraseAll) { TEST(ObjectFactoryStorageTest, EraseTheOnlyElement) { ObjectFactoryStorageRegular storage; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(1); @@ -307,7 +328,7 @@ TEST(ObjectFactoryStorageTest, ConcurrentPublish) { for (int i = 0; i < kThreadCount; ++i) { expected.push_back(i); threads.emplace_back([i, &storage, &canStart, &readyCount]() { - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(i); ++readyCount; while (!canStart) { @@ -335,7 +356,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) { KStdVector expectedBefore; KStdVector expectedAfter; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); for (int i = 0; i < kStartCount; ++i) { expectedBefore.push_back(i); expectedAfter.push_back(i); @@ -351,7 +372,7 @@ TEST(ObjectFactoryStorageTest, IterWhileConcurrentPublish) { int j = i + kStartCount; expectedAfter.push_back(j); threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() { - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(j); ++readyCount; while (!canStart) { @@ -393,7 +414,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { constexpr int kThreadCount = kDefaultThreadCount; KStdVector expectedAfter; - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); for (int i = 0; i < kStartCount; ++i) { if (i % 2 == 0) { expectedAfter.push_back(i); @@ -410,7 +431,7 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { int j = i + kStartCount; expectedAfter.push_back(j); threads.emplace_back([j, &storage, &canStart, &startedCount, &readyCount]() { - ObjectFactoryStorageRegular::Producer producer(storage); + Producer producer(storage, SimpleAllocator()); producer.Insert(j); ++readyCount; while (!canStart) { @@ -446,10 +467,103 @@ TEST(ObjectFactoryStorageTest, EraseWhileConcurrentPublish) { EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expectedAfter)); } -using mm::ObjectFactory; +using mm::internal::AllocatorWithGC; namespace { +class MockAllocator { +public: + MOCK_METHOD(void*, Alloc, (size_t, size_t)); +}; + +class MockAllocatorWrapper { +public: + MockAllocator& operator*() { return *mock_; } + + void* Alloc(size_t size, size_t alignment) { return mock_->Alloc(size, alignment); } + +private: + KStdUniquePtr> mock_ = make_unique>(); +}; + +class MockGC { +public: + MOCK_METHOD(void, SafePointAllocation, (size_t)); + MOCK_METHOD(void, OnOOM, (size_t)); +}; + +} // namespace + +TEST(AllocatorWithGCTest, AllocateWithoutOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + void* nonNull = reinterpret_cast(1); + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + EXPECT_CALL(gc, OnOOM(_)).Times(0); + } + AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nonNull); +} + +TEST(AllocatorWithGCTest, AllocateWithFixableOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + void* nonNull = reinterpret_cast(1); + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(gc, OnOOM(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nonNull)); + } + AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nonNull); +} + +TEST(AllocatorWithGCTest, AllocateWithUnfixableOOM) { + constexpr size_t size = 256; + constexpr size_t alignment = 8; + MockAllocatorWrapper baseAllocator; + testing::StrictMock gc; + { + testing::InSequence seq; + EXPECT_CALL(gc, SafePointAllocation(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + EXPECT_CALL(gc, OnOOM(size)); + EXPECT_CALL(*baseAllocator, Alloc(size, alignment)).WillOnce(testing::Return(nullptr)); + } + AllocatorWithGC allocator(std::move(baseAllocator), gc); + void* ptr = allocator.Alloc(size, alignment); + EXPECT_THAT(ptr, nullptr); +} + +namespace { + +class GC { +public: + struct ObjectData { + uint32_t flags = 42; + }; + + class ThreadData { + public: + void SafePointAllocation(size_t size) noexcept {} + + void OnOOM(size_t size) noexcept {} + }; +}; + +using ObjectFactory = mm::ObjectFactory; + KStdUniquePtr MakeObjectTypeInfo(int32_t size) { auto typeInfo = make_unique(); typeInfo->typeInfo_ = typeInfo.get(); @@ -468,32 +582,42 @@ KStdUniquePtr MakeArrayTypeInfo(int32_t elementSize) { TEST(ObjectFactoryTest, CreateObject) { auto typeInfo = MakeObjectTypeInfo(24); + GC::ThreadData gc; ObjectFactory objectFactory; - ObjectFactory::ThreadQueue threadQueue(objectFactory); + ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); auto* object = threadQueue.CreateObject(typeInfo.get()); threadQueue.Publish(); + auto node = ObjectFactory::NodeRef::From(object); + EXPECT_FALSE(node.IsArray()); + EXPECT_THAT(node.GetObjHeader(), object); + EXPECT_THAT(node.GCObjectData().flags, 42); + auto iter = objectFactory.Iter(); auto it = iter.begin(); - EXPECT_FALSE(it.IsArray()); - EXPECT_THAT(it.GetObjHeader(), object); + EXPECT_THAT(*it, node); ++it; EXPECT_THAT(it, iter.end()); } TEST(ObjectFactoryTest, CreateArray) { auto typeInfo = MakeArrayTypeInfo(24); + GC::ThreadData gc; ObjectFactory objectFactory; - ObjectFactory::ThreadQueue threadQueue(objectFactory); + ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); auto* array = threadQueue.CreateArray(typeInfo.get(), 3); threadQueue.Publish(); + auto node = ObjectFactory::NodeRef::From(array); + EXPECT_TRUE(node.IsArray()); + EXPECT_THAT(node.GetArrayHeader(), array); + EXPECT_THAT(node.GCObjectData().flags, 42); + auto iter = objectFactory.Iter(); auto it = iter.begin(); - EXPECT_TRUE(it.IsArray()); - EXPECT_THAT(it.GetArrayHeader(), array); + EXPECT_THAT(*it, node); ++it; EXPECT_THAT(it, iter.end()); } @@ -501,8 +625,9 @@ TEST(ObjectFactoryTest, CreateArray) { TEST(ObjectFactoryTest, Erase) { auto objectTypeInfo = MakeObjectTypeInfo(24); auto arrayTypeInfo = MakeArrayTypeInfo(24); + GC::ThreadData gc; ObjectFactory objectFactory; - ObjectFactory::ThreadQueue threadQueue(objectFactory); + ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); for (int i = 0; i < 10; ++i) { threadQueue.CreateObject(objectTypeInfo.get()); @@ -514,7 +639,7 @@ TEST(ObjectFactoryTest, Erase) { { auto iter = objectFactory.Iter(); for (auto it = iter.begin(); it != iter.end();) { - if (it.IsArray()) { + if (it->IsArray()) { iter.EraseAndAdvance(it); } else { ++it; @@ -526,7 +651,7 @@ TEST(ObjectFactoryTest, Erase) { auto iter = objectFactory.Iter(); int count = 0; for (auto it = iter.begin(); it != iter.end(); ++it, ++count) { - EXPECT_FALSE(it.IsArray()); + EXPECT_FALSE(it->IsArray()); } EXPECT_THAT(count, 10); } @@ -544,7 +669,8 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { for (int i = 0; i < kThreadCount; ++i) { threads.emplace_back([&typeInfo, &objectFactory, &canStart, &readyCount, &expected, &expectedMutex]() { - ObjectFactory::ThreadQueue threadQueue(objectFactory); + GC::ThreadData gc; + ObjectFactory::ThreadQueue threadQueue(objectFactory, gc); auto* object = threadQueue.CreateObject(typeInfo.get()); { std::lock_guard guard(expectedMutex); @@ -567,7 +693,7 @@ TEST(ObjectFactoryTest, ConcurrentPublish) { auto iter = objectFactory.Iter(); KStdVector actual; for (auto it = iter.begin(); it != iter.end(); ++it) { - actual.push_back(it.GetObjHeader()); + actual.push_back(it->GetObjHeader()); } EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index 6292c26201f..9bd72a6b2a3 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -26,10 +26,6 @@ void EnsureNeverFrozen(ObjHeader* obj) { TODO(); } -void Kotlin_native_internal_GC_collect(ObjHeader*) { - TODO(); -} - void Kotlin_native_internal_GC_suspend(ObjHeader*) { TODO(); } @@ -78,10 +74,6 @@ bool Kotlin_native_internal_GC_getTuneThreshold(ObjHeader*) { TODO(); } -RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) { - TODO(); -} - bool TryAddHeapRef(const ObjHeader* object) { TODO(); } @@ -98,16 +90,4 @@ ForeignRefContext InitLocalForeignRef(ObjHeader* object) { TODO(); } -RUNTIME_NOTHROW void Kotlin_mm_safePointFunctionEpilogue() { - // TODO: Unimplemented -} - -RUNTIME_NOTHROW void Kotlin_mm_safePointWhileLoopBody() { - // TODO: Unimplemented -} - -RUNTIME_NOTHROW void Kotlin_mm_safePointExceptionUnwind() { - // TODO: Unimplemented -} - } // extern "C" diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 1108c6adcf4..b8bc72dab6f 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -9,7 +9,9 @@ #include #include +#include "GlobalData.hpp" #include "GlobalsRegistry.hpp" +#include "GC.hpp" #include "ObjectFactory.hpp" #include "ShadowStack.hpp" #include "StableRefRegistry.hpp" @@ -32,7 +34,8 @@ public: globalsThreadQueue_(GlobalsRegistry::Instance()), stableRefThreadQueue_(StableRefRegistry::Instance()), state_(ThreadState::kRunnable), - objectFactoryThreadQueue_(ObjectFactory::Instance()) {} + gc_(GlobalData::Instance().gc()), + objectFactoryThreadQueue_(GlobalData::Instance().objectFactory(), gc_) {} ~ThreadData() = default; @@ -48,20 +51,23 @@ public: ThreadState setState(ThreadState state) noexcept { return state_.exchange(state); } - ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } + ObjectFactory::ThreadQueue& objectFactoryThreadQueue() noexcept { return objectFactoryThreadQueue_; } ShadowStack& shadowStack() noexcept { return shadowStack_; } KStdVector>& initializingSingletons() noexcept { return initializingSingletons_; } + GC::ThreadData& gc() noexcept { return gc_; } + private: const pthread_t threadId_; GlobalsRegistry::ThreadQueue globalsThreadQueue_; ThreadLocalStorage tls_; StableRefRegistry::ThreadQueue stableRefThreadQueue_; std::atomic state_; - ObjectFactory::ThreadQueue objectFactoryThreadQueue_; ShadowStack shadowStack_; + GC::ThreadData gc_; + ObjectFactory::ThreadQueue objectFactoryThreadQueue_; KStdVector> initializingSingletons_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/gc/NoOpGC.hpp b/kotlin-native/runtime/src/mm/cpp/gc/NoOpGC.hpp new file mode 100644 index 00000000000..aa7cce129a5 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/gc/NoOpGC.hpp @@ -0,0 +1,50 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_NOOP_GC_H +#define RUNTIME_MM_NOOP_GC_H + +#include + +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +// No-op GC is a GC that does not free memory. +// TODO: It can be made more efficient. +class NoOpGC : private Pinned { +public: + class ObjectData {}; + + class ThreadData : private Pinned { + public: + using ObjectData = NoOpGC::ObjectData; + + explicit ThreadData(NoOpGC& gc) noexcept {} + ~ThreadData() = default; + + void SafePointFunctionEpilogue() noexcept {} + void SafePointLoopBody() noexcept {} + void SafePointExceptionUnwind() noexcept {} + void SafePointAllocation(size_t size) noexcept {} + + void PerformFullGC() noexcept {} + + void OnOOM(size_t size) noexcept {} + + private: + }; + + NoOpGC() noexcept = default; + ~NoOpGC() = default; + +private: +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_NOOP_GC_H