diff --git a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp index 6e046a7323d..b1714bed4c0 100644 --- a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp +++ b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp @@ -173,6 +173,7 @@ void BackRefFromAssociatedObject::releaseRef() { DeinitForeignRef(obj_, context); // From this moment [context] is generally a dangling pointer. // This is handled in [IsForeignRefAccessible] and [addRef]. + // TODO: This probably isn't fine in new MM. Make sure it works. } } diff --git a/kotlin-native/runtime/src/mm/cpp/InitializationScheme.cpp b/kotlin-native/runtime/src/mm/cpp/InitializationScheme.cpp new file mode 100644 index 00000000000..b4e1104ace3 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/InitializationScheme.cpp @@ -0,0 +1,74 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "InitializationScheme.hpp" + +#include "Common.h" +#include "ObjectOps.hpp" +#include "ThreadData.hpp" + +using namespace kotlin; + +OBJ_GETTER(mm::InitThreadLocalSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + if (auto* value = *location) { + // Initialized by someone else. + RETURN_OBJ(value); + } + auto* value = mm::AllocateObject(threadData, typeInfo, OBJ_RESULT); + mm::SetHeapRef(location, value); +#if KONAN_NO_EXCEPTIONS + ctor(value); +#else + try { + ctor(value); + } catch (...) { + mm::SetStackRef(OBJ_RESULT, nullptr); + mm::SetHeapRef(location, nullptr); + throw; + } +#endif + return value; +} + +OBJ_GETTER(mm::InitSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + auto& initializingSingletons = threadData->initializingSingletons(); + + // Search from the top of the stack. + for (auto it = initializingSingletons.rbegin(); it != initializingSingletons.rend(); ++it) { + if (it->first == location) { + RETURN_OBJ(it->second); + } + } + + ObjHeader* initializing = reinterpret_cast(1); + + // Spin lock. + ObjHeader* value = nullptr; + while ((value = __sync_val_compare_and_swap(location, nullptr, initializing)) == initializing) { + } + if (value != nullptr) { + // Initialized by someone else. + RETURN_OBJ(value); + } + auto* object = mm::AllocateObject(threadData, typeInfo, OBJ_RESULT); + initializingSingletons.push_back(std::make_pair(location, object)); + +#if KONAN_NO_EXCEPTIONS + ctor(object); +#else + try { + ctor(object); + } catch (...) { + mm::SetStackRef(OBJ_RESULT, nullptr); + mm::SetHeapRefAtomic(location, nullptr); + initializingSingletons.pop_back(); + throw; + } +#endif + mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location); + mm::SetHeapRefAtomic(location, object); + initializingSingletons.pop_back(); + return object; +} diff --git a/kotlin-native/runtime/src/mm/cpp/InitializationScheme.hpp b/kotlin-native/runtime/src/mm/cpp/InitializationScheme.hpp new file mode 100644 index 00000000000..49420b1df61 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/InitializationScheme.hpp @@ -0,0 +1,22 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_INITIALIZATION_SCHEME_H +#define RUNTIME_MM_INITIALIZATION_SCHEME_H + +#include "Memory.h" + +namespace kotlin { +namespace mm { + +class ThreadData; + +OBJ_GETTER(InitThreadLocalSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); +OBJ_GETTER(InitSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)); + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_INITIALIZATION_SCHEME_H diff --git a/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp b/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp new file mode 100644 index 00000000000..a46c01debf3 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp @@ -0,0 +1,258 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "InitializationScheme.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "TestSupport.hpp" +#include "ThreadData.hpp" +#include "Types.h" + +using namespace kotlin; + +using testing::_; + +namespace { + +class InitSingletonTest : public testing::Test { +public: + InitSingletonTest() { + typeInfo_.typeInfo_ = &typeInfo_; + typeInfo_.instanceSize_ = sizeof(ObjHeader); + + globalConstructor_ = &constructor_; + + for (auto& threadData : threadDatas_) { + threadData = make_unique(pthread_t{}); + } + } + + ~InitSingletonTest() { + globalConstructor_ = nullptr; + // Make sure to clean everything allocated by the tests. + for (auto& threadData : threadDatas_) { + threadData->objectFactoryThreadQueue().ClearForTests(); + } + } + + mm::ThreadData& threadData(size_t threadIndex) { return *threadDatas_[threadIndex]; } + + testing::MockFunction& constructor() { return constructor_; } + + OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, size_t threadIndex) { + RETURN_RESULT_OF(mm::InitThreadLocalSingleton, threadDatas_[threadIndex].get(), location, &typeInfo_, constructorImpl); + } + + OBJ_GETTER(InitSingleton, ObjHeader** location, size_t threadIndex) { + RETURN_RESULT_OF(mm::InitSingleton, threadDatas_[threadIndex].get(), location, &typeInfo_, constructorImpl); + } + +private: + testing::StrictMock> constructor_; + // TODO: It makes sense to somehow abstract `ThreadData` stuff away. Allocation in this case. + std::array, kDefaultThreadCount> threadDatas_; + TypeInfo typeInfo_; // Only used for allocator calls, uninteresting for these tests. + + static testing::MockFunction* globalConstructor_; + + static void constructorImpl(ObjHeader* object) { globalConstructor_->Call(object); } +}; + +// static +testing::MockFunction* InitSingletonTest::globalConstructor_ = nullptr; + +} // namespace + +TEST_F(InitSingletonTest, InitThreadLocalSingleton) { + ObjHeader* location = nullptr; + ObjHeader* stackLocation = nullptr; + + ObjHeader* valueAtConstructor = nullptr; + EXPECT_CALL(constructor(), Call(_)).WillOnce([&location, &stackLocation, &valueAtConstructor](ObjHeader* value) { + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(value, location); + valueAtConstructor = value; + }); + ObjHeader* value = InitThreadLocalSingleton(&location, 0, &stackLocation); + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(value, location); + EXPECT_THAT(valueAtConstructor, location); +} + +TEST_F(InitSingletonTest, InitThreadLocalSingletonTwice) { + ObjHeader previousValue; + ObjHeader* location = &previousValue; + ObjHeader* stackLocation = nullptr; + + EXPECT_CALL(constructor(), Call(_)).Times(0); + ObjHeader* value = InitThreadLocalSingleton(&location, 0, &stackLocation); + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(value, location); + EXPECT_THAT(value, &previousValue); +} + +TEST_F(InitSingletonTest, InitThreadLocalSingletonFail) { + ObjHeader* location = nullptr; + ObjHeader* stackLocation = nullptr; + constexpr int kException = 42; + + EXPECT_CALL(constructor(), Call(_)).WillOnce([]() { throw kException; }); + try { + InitThreadLocalSingleton(&location, 0, &stackLocation); + ASSERT_TRUE(false); // Cannot be reached. + } catch (int exception) { + EXPECT_THAT(exception, kException); + } + EXPECT_THAT(stackLocation, nullptr); + EXPECT_THAT(location, nullptr); +} + +TEST_F(InitSingletonTest, InitSingleton) { + ObjHeader* location = nullptr; + ObjHeader* stackLocation = nullptr; + + ObjHeader* valueAtConstructor = nullptr; + EXPECT_CALL(constructor(), Call(_)).WillOnce([&location, &stackLocation, &valueAtConstructor](ObjHeader* value) { + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(location, reinterpret_cast(1)); + valueAtConstructor = value; + }); + ObjHeader* value = InitSingleton(&location, 0, &stackLocation); + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(value, location); + EXPECT_THAT(valueAtConstructor, location); +} + +TEST_F(InitSingletonTest, InitSingletonTwice) { + ObjHeader previousValue; + ObjHeader* location = &previousValue; + ObjHeader* stackLocation = nullptr; + + EXPECT_CALL(constructor(), Call(_)).Times(0); + ObjHeader* value = InitSingleton(&location, 0, &stackLocation); + EXPECT_THAT(value, stackLocation); + EXPECT_THAT(value, location); + EXPECT_THAT(value, &previousValue); +} + +TEST_F(InitSingletonTest, InitSingletonFail) { + ObjHeader* location = nullptr; + ObjHeader* stackLocation = nullptr; + constexpr int kException = 42; + + EXPECT_CALL(constructor(), Call(_)).WillOnce([]() { throw kException; }); + try { + InitSingleton(&location, 0, &stackLocation); + ASSERT_TRUE(false); // Cannot be reached. + } catch (int exception) { + EXPECT_THAT(exception, kException); + } + EXPECT_THAT(stackLocation, nullptr); + EXPECT_THAT(location, nullptr); +} + +TEST_F(InitSingletonTest, InitSingletonRecursive) { + // The first singleton. Its constructor depends on the second singleton. + ObjHeader* location1 = nullptr; + ObjHeader* stackLocation1 = nullptr; + // The second singleton. Its constructor depends on the first singleton. + ObjHeader* location2 = nullptr; + ObjHeader* stackLocation2 = nullptr; + + EXPECT_CALL(constructor(), Call(_)) + .Times(2) // called only once for each singleton. + .WillRepeatedly([this, &location1, &stackLocation1, &location2, &stackLocation2](ObjHeader* value) { + if (value == stackLocation1) { + ObjHeader* result = InitSingleton(&location2, 0, &stackLocation2); + EXPECT_THAT(result, stackLocation2); + EXPECT_THAT(result, location2); + EXPECT_THAT(result, testing::Ne(reinterpret_cast(1))); + } else { + ObjHeader* result = InitSingleton(&location1, 0, &stackLocation1); + EXPECT_THAT(result, stackLocation1); + EXPECT_THAT(result, testing::Ne(location1)); + EXPECT_THAT(location1, reinterpret_cast(1)); + } + }); + ObjHeader* value = InitSingleton(&location1, 0, &stackLocation1); + EXPECT_THAT(value, stackLocation1); + EXPECT_THAT(value, location1); +} + +TEST_F(InitSingletonTest, InitSingletonConcurrent) { + constexpr size_t kThreadCount = kDefaultThreadCount; + std::atomic canStart(false); + std::atomic readyCount(0); + KStdVector threads; + ObjHeader* location = nullptr; + KStdVector stackLocations(kThreadCount, nullptr); + KStdVector actual(kThreadCount, nullptr); + + for (size_t i = 0; i < kThreadCount; ++i) { + threads.emplace_back([this, i, &location, &stackLocations, &actual, &readyCount, &canStart]() { + ++readyCount; + while (!canStart) { + } + actual[i] = InitSingleton(&location, i, &stackLocations[i]); + }); + } + + while (readyCount < kThreadCount) { + } + // Constructor is called exactly once. + EXPECT_CALL(constructor(), Call(_)); + canStart = true; + for (auto& t : threads) { + t.join(); + } + testing::Mock::VerifyAndClearExpectations(&constructor()); + + EXPECT_THAT(location, testing::Ne(nullptr)); + EXPECT_THAT(location, testing::Ne(reinterpret_cast(1))); + EXPECT_THAT(stackLocations, testing::Each(location)); + EXPECT_THAT(actual, testing::Each(location)); +} + +TEST_F(InitSingletonTest, InitSingletonConcurrentFailing) { + constexpr size_t kThreadCount = kDefaultThreadCount; + std::atomic canStart(false); + std::atomic readyCount(0); + KStdVector threads; + constexpr int kException = 42; + ObjHeader* location = nullptr; + KStdVector stackLocations(kThreadCount, nullptr); + + for (size_t i = 0; i < kThreadCount; ++i) { + threads.emplace_back([this, i, &location, &stackLocations, &readyCount, &canStart]() { + ++readyCount; + while (!canStart) { + } + try { + InitSingleton(&location, i, &stackLocations[i]); + ASSERT_TRUE(false); // Cannot be reached. + } catch (int exception) { + EXPECT_THAT(exception, kException); + } + }); + } + + while (readyCount < kThreadCount) { + } + // Constructor is called exactly `kThreadCount` times. + EXPECT_CALL(constructor(), Call(_)).Times(kThreadCount).WillRepeatedly([]() { throw kException; }); + canStart = true; + for (auto& t : threads) { + t.join(); + } + testing::Mock::VerifyAndClearExpectations(&constructor()); + + EXPECT_THAT(location, nullptr); + EXPECT_THAT(stackLocations, testing::Each(nullptr)); +} diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 0ea354c650f..f07c9bead4c 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -8,8 +8,11 @@ #include "Exceptions.h" #include "ExtraObjectData.hpp" #include "GlobalsRegistry.hpp" +#include "InitializationScheme.hpp" #include "KAssert.h" +#include "Natives.h" #include "Porting.h" +#include "ObjectOps.hpp" #include "StableRefRegistry.hpp" #include "ThreadData.hpp" #include "ThreadRegistry.hpp" @@ -111,8 +114,7 @@ extern "C" void RestoreMemory(MemoryState*) { extern "C" RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo); - RETURN_OBJ(object); + RETURN_RESULT_OF(mm::AllocateObject, threadData, typeInfo); } extern "C" OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elements) { @@ -120,28 +122,84 @@ extern "C" OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elem ThrowIllegalArgumentException(); } auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast(elements)); - // `ArrayHeader` and `ObjHeader` are expected to be compatible. - RETURN_OBJ(reinterpret_cast(array)); + RETURN_RESULT_OF(mm::AllocateArray, threadData, typeInfo, static_cast(elements)); } -extern "C" OBJ_GETTER(InitSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { +extern "C" ALWAYS_INLINE OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); - // TODO: This should only be called if singleton is actually created here. It's possible that the - // singleton will be created on a different thread and here we should check that, instead of creating - // another one (and registering `location` twice). - mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location); - TODO(); + + RETURN_RESULT_OF(mm::InitThreadLocalSingleton, threadData, location, typeInfo, ctor); +} + +extern "C" ALWAYS_INLINE OBJ_GETTER(InitSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { + auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); + + RETURN_RESULT_OF(mm::InitSingleton, threadData, location, typeInfo, ctor); } extern "C" RUNTIME_NOTHROW void InitAndRegisterGlobal(ObjHeader** location, const ObjHeader* initialValue) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location); - TODO(); + mm::SetHeapRef(location, const_cast(initialValue)); } extern "C" const MemoryModel CurrentMemoryModel = MemoryModel::kExperimental; +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) { + mm::SetStackRef(location, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) { + mm::SetHeapRef(location, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) { + mm::SetHeapRef(location, nullptr); +} + +extern "C" RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) { + for (uint32_t index = 0; index < array->count_; ++index) { + ObjHeader** location = ArrayAddressOfElementAt(array, index); + mm::SetHeapRef(location, nullptr); + } +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) { + mm::SetStackRef(location, nullptr); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { + mm::SetStackRef(location, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { + mm::SetHeapRef(location, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { + if (object == nullptr) return; + ObjHeader* result = nullptr; // No need to store this value in a rootset. + mm::CompareAndSwapHeapRef(location, nullptr, const_cast(object), &result); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { + mm::SetStackRef(returnSlot, const_cast(object)); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER( + SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { + RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRefLocked( + ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { + mm::SetHeapRefAtomic(location, newValue); +} + +extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) { + RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location); +} + extern "C" OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index) { // TODO: Remove when legacy MM is gone. ThrowNotImplementedError(); @@ -250,12 +308,15 @@ extern "C" RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void* pointer) { auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); auto* node = static_cast(pointer); ObjHeader* object = **node; - UpdateReturnRef(OBJ_RESULT, object); + // Make sure `object` stays in the rootset: put it on the stack before removing it from `StableRefRegistry`. + mm::SetStackRef(OBJ_RESULT, object); mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node); return object; } extern "C" RUNTIME_NOTHROW void CheckLifetimesConstraint(ObjHeader* obj, ObjHeader* pointee) { + // TODO: Consider making it a `RuntimeCheck`. Probably all `RuntimeCheck`s and `RuntimeAssert`s should specify + // that their firing is a compiler bug and should be reported. if (!obj->local() && pointee != nullptr && pointee->local()) { konan::consolePrintf("Attempt to store a stack object %p into a heap object %p\n", pointee, obj); konan::consolePrintf("This is a compiler bug, please report it to https://kotl.in/issue\n"); diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp index b28d093f149..9cc78bdc59b 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp @@ -140,6 +140,12 @@ public: owner_.AssertCorrectUnsafe(); } + void ClearForTests() noexcept { + // Since it's only for tests, no need to worry about stack overflows. + root_.reset(); + last_ = nullptr; + } + private: friend class ObjectFactoryStorage; @@ -259,6 +265,8 @@ public: void Publish() noexcept { producer_.Publish(); } + void ClearForTests() noexcept { producer_.ClearForTests(); } + private: Storage::Producer producer_; }; diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp new file mode 100644 index 00000000000..539ef456138 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -0,0 +1,63 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ObjectOps.hpp" + +#include "Common.h" +#include "ThreadData.hpp" + +using namespace kotlin; + +// TODO: Memory barriers. + +ALWAYS_INLINE void mm::SetStackRef(ObjHeader** location, ObjHeader* value) noexcept { + *location = value; +} + +ALWAYS_INLINE void mm::SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept { + *location = value; +} + +#pragma clang diagnostic push +// On 32-bit android arm clang warns of significant performance penalty because of large +// atomic operations. TODO: Consider using alternative ways of ordering memory operations if they +// turn out to be more efficient on these platforms. +#pragma clang diagnostic ignored "-Watomic-alignment" + +ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept { + __atomic_store_n(location, value, __ATOMIC_RELEASE); +} + +ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept { + // TODO: Make this work with GCs that can stop thread at any point. + auto result = __atomic_load_n(location, __ATOMIC_ACQUIRE); + RETURN_OBJ(result); +} + +ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept { + // TODO: Make this work with GCs that can stop thread at any point. + ObjHeader* actual = expected; + // TODO: Do we need this strong memory model? Do we need to use strong CAS? + // This intrinsic modifies `actual` non-atomically. + __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); + // On success, we already have old value (== `expected`) in `actual`. + // On failure, we have the old value written into `actual`. + RETURN_OBJ(actual); +} + +#pragma clang diagnostic pop + +OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept { + // TODO: Make this work with GCs that can stop thread at any point. + auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo); + RETURN_OBJ(object); +} + +OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept { + // TODO: Make this work with GCs that can stop thread at any point. + auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast(elements)); + // `ArrayHeader` and `ObjHeader` are expected to be compatible. + RETURN_OBJ(reinterpret_cast(array)); +} diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp new file mode 100644 index 00000000000..2c20468ae97 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_OBJECT_OPS_H +#define RUNTIME_MM_OBJECT_OPS_H + +#include "Memory.h" + +namespace kotlin { +namespace mm { + +class ThreadData; + +// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals. + +// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which +// will have these operations for a friendlier API. + +// TODO: `OBJ_GETTER` is used because the returned objects needs to be accessible via the rootset before the function +// returns. If we had a different way to efficiently keep the object in the roots, `OBJ_GETTER` can be removed. + +void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept; +void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept; +void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept; +OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept; +OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; +OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; +OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_OBJECT_OPS_H diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index cf53f4f8b73..31214be2e4f 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -17,59 +17,6 @@ ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) { extern "C" { -OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) { - TODO(); -} - -RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) { - TODO(); -} - -RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) { - TODO(); -} - -RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) { - TODO(); -} - -RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { - TODO(); -} - -RUNTIME_NOTHROW OBJ_GETTER( - SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { - TODO(); -} - -RUNTIME_NOTHROW void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { - TODO(); -} - -RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) { - TODO(); -} - void MutationCheck(ObjHeader* obj) { TODO(); } diff --git a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp index 07292b26237..1108c6adcf4 100644 --- a/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ThreadData.hpp @@ -9,13 +9,16 @@ #include #include -#include "ObjectFactory.hpp" #include "GlobalsRegistry.hpp" +#include "ObjectFactory.hpp" #include "ShadowStack.hpp" #include "StableRefRegistry.hpp" #include "ThreadLocalStorage.hpp" -#include "Utils.hpp" #include "ThreadState.hpp" +#include "Types.h" +#include "Utils.hpp" + +struct ObjHeader; namespace kotlin { namespace mm { @@ -49,6 +52,8 @@ public: ShadowStack& shadowStack() noexcept { return shadowStack_; } + KStdVector>& initializingSingletons() noexcept { return initializingSingletons_; } + private: const pthread_t threadId_; GlobalsRegistry::ThreadQueue globalsThreadQueue_; @@ -57,6 +62,7 @@ private: std::atomic state_; ObjectFactory::ThreadQueue objectFactoryThreadQueue_; ShadowStack shadowStack_; + KStdVector> initializingSingletons_; }; } // namespace mm