Implement ref updates in new MM (#4653)

kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp

@@ -173,6 +173,7 @@ void BackRefFromAssociatedObject::releaseRef() {
     DeinitForeignRef(obj_, context);
     // From this moment [context] is generally a dangling pointer.
     // This is handled in [IsForeignRefAccessible] and [addRef].
+    // TODO: This probably isn't fine in new MM. Make sure it works.
   }
 }
 

kotlin-native/runtime/src/mm/cpp/InitializationScheme.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "InitializationScheme.hpp"
+
+#include "Common.h"
+#include "ObjectOps.hpp"
+#include "ThreadData.hpp"
+
+using namespace kotlin;
+
+OBJ_GETTER(mm::InitThreadLocalSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+    if (auto* value = *location) {
+        // Initialized by someone else.
+        RETURN_OBJ(value);
+    }
+    auto* value = mm::AllocateObject(threadData, typeInfo, OBJ_RESULT);
+    mm::SetHeapRef(location, value);
+#if KONAN_NO_EXCEPTIONS
+    ctor(value);
+#else
+    try {
+        ctor(value);
+    } catch (...) {
+        mm::SetStackRef(OBJ_RESULT, nullptr);
+        mm::SetHeapRef(location, nullptr);
+        throw;
+    }
+#endif
+    return value;
+}
+
+OBJ_GETTER(mm::InitSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+    auto& initializingSingletons = threadData->initializingSingletons();
+
+    // Search from the top of the stack.
+    for (auto it = initializingSingletons.rbegin(); it != initializingSingletons.rend(); ++it) {
+        if (it->first == location) {
+            RETURN_OBJ(it->second);
+        }
+    }
+
+    ObjHeader* initializing = reinterpret_cast<ObjHeader*>(1);
+
+    // Spin lock.
+    ObjHeader* value = nullptr;
+    while ((value = __sync_val_compare_and_swap(location, nullptr, initializing)) == initializing) {
+    }
+    if (value != nullptr) {
+        // Initialized by someone else.
+        RETURN_OBJ(value);
+    }
+    auto* object = mm::AllocateObject(threadData, typeInfo, OBJ_RESULT);
+    initializingSingletons.push_back(std::make_pair(location, object));
+
+#if KONAN_NO_EXCEPTIONS
+    ctor(object);
+#else
+    try {
+        ctor(object);
+    } catch (...) {
+        mm::SetStackRef(OBJ_RESULT, nullptr);
+        mm::SetHeapRefAtomic(location, nullptr);
+        initializingSingletons.pop_back();
+        throw;
+    }
+#endif
+    mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location);
+    mm::SetHeapRefAtomic(location, object);
+    initializingSingletons.pop_back();
+    return object;
+}

kotlin-native/runtime/src/mm/cpp/InitializationScheme.hpp

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_INITIALIZATION_SCHEME_H
+#define RUNTIME_MM_INITIALIZATION_SCHEME_H
+
+#include "Memory.h"
+
+namespace kotlin {
+namespace mm {
+
+class ThreadData;
+
+OBJ_GETTER(InitThreadLocalSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+OBJ_GETTER(InitSingleton, ThreadData* threadData, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_INITIALIZATION_SCHEME_H

kotlin-native/runtime/src/mm/cpp/InitializationSchemeTest.cpp

@@ -0,0 +1,258 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "InitializationScheme.hpp"
+
+#include <atomic>
+#include <thread>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "TestSupport.hpp"
+#include "ThreadData.hpp"
+#include "Types.h"
+
+using namespace kotlin;
+
+using testing::_;
+
+namespace {
+
+class InitSingletonTest : public testing::Test {
+public:
+    InitSingletonTest() {
+        typeInfo_.typeInfo_ = &typeInfo_;
+        typeInfo_.instanceSize_ = sizeof(ObjHeader);
+
+        globalConstructor_ = &constructor_;
+
+        for (auto& threadData : threadDatas_) {
+            threadData = make_unique<mm::ThreadData>(pthread_t{});
+        }
+    }
+
+    ~InitSingletonTest() {
+        globalConstructor_ = nullptr;
+        // Make sure to clean everything allocated by the tests.
+        for (auto& threadData : threadDatas_) {
+            threadData->objectFactoryThreadQueue().ClearForTests();
+        }
+    }
+
+    mm::ThreadData& threadData(size_t threadIndex) { return *threadDatas_[threadIndex]; }
+
+    testing::MockFunction<void(ObjHeader*)>& constructor() { return constructor_; }
+
+    OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, size_t threadIndex) {
+        RETURN_RESULT_OF(mm::InitThreadLocalSingleton, threadDatas_[threadIndex].get(), location, &typeInfo_, constructorImpl);
+    }
+
+    OBJ_GETTER(InitSingleton, ObjHeader** location, size_t threadIndex) {
+        RETURN_RESULT_OF(mm::InitSingleton, threadDatas_[threadIndex].get(), location, &typeInfo_, constructorImpl);
+    }
+
+private:
+    testing::StrictMock<testing::MockFunction<void(ObjHeader*)>> constructor_;
+    // TODO: It makes sense to somehow abstract `ThreadData` stuff away. Allocation in this case.
+    std::array<KStdUniquePtr<mm::ThreadData>, kDefaultThreadCount> threadDatas_;
+    TypeInfo typeInfo_; // Only used for allocator calls, uninteresting for these tests.
+
+    static testing::MockFunction<void(ObjHeader*)>* globalConstructor_;
+
+    static void constructorImpl(ObjHeader* object) { globalConstructor_->Call(object); }
+};
+
+// static
+testing::MockFunction<void(ObjHeader*)>* InitSingletonTest::globalConstructor_ = nullptr;
+
+} // namespace
+
+TEST_F(InitSingletonTest, InitThreadLocalSingleton) {
+    ObjHeader* location = nullptr;
+    ObjHeader* stackLocation = nullptr;
+
+    ObjHeader* valueAtConstructor = nullptr;
+    EXPECT_CALL(constructor(), Call(_)).WillOnce([&location, &stackLocation, &valueAtConstructor](ObjHeader* value) {
+        EXPECT_THAT(value, stackLocation);
+        EXPECT_THAT(value, location);
+        valueAtConstructor = value;
+    });
+    ObjHeader* value = InitThreadLocalSingleton(&location, 0, &stackLocation);
+    EXPECT_THAT(value, stackLocation);
+    EXPECT_THAT(value, location);
+    EXPECT_THAT(valueAtConstructor, location);
+}
+
+TEST_F(InitSingletonTest, InitThreadLocalSingletonTwice) {
+    ObjHeader previousValue;
+    ObjHeader* location = &previousValue;
+    ObjHeader* stackLocation = nullptr;
+
+    EXPECT_CALL(constructor(), Call(_)).Times(0);
+    ObjHeader* value = InitThreadLocalSingleton(&location, 0, &stackLocation);
+    EXPECT_THAT(value, stackLocation);
+    EXPECT_THAT(value, location);
+    EXPECT_THAT(value, &previousValue);
+}
+
+TEST_F(InitSingletonTest, InitThreadLocalSingletonFail) {
+    ObjHeader* location = nullptr;
+    ObjHeader* stackLocation = nullptr;
+    constexpr int kException = 42;
+
+    EXPECT_CALL(constructor(), Call(_)).WillOnce([]() { throw kException; });
+    try {
+        InitThreadLocalSingleton(&location, 0, &stackLocation);
+        ASSERT_TRUE(false); // Cannot be reached.
+    } catch (int exception) {
+        EXPECT_THAT(exception, kException);
+    }
+    EXPECT_THAT(stackLocation, nullptr);
+    EXPECT_THAT(location, nullptr);
+}
+
+TEST_F(InitSingletonTest, InitSingleton) {
+    ObjHeader* location = nullptr;
+    ObjHeader* stackLocation = nullptr;
+
+    ObjHeader* valueAtConstructor = nullptr;
+    EXPECT_CALL(constructor(), Call(_)).WillOnce([&location, &stackLocation, &valueAtConstructor](ObjHeader* value) {
+        EXPECT_THAT(value, stackLocation);
+        EXPECT_THAT(location, reinterpret_cast<ObjHeader*>(1));
+        valueAtConstructor = value;
+    });
+    ObjHeader* value = InitSingleton(&location, 0, &stackLocation);
+    EXPECT_THAT(value, stackLocation);
+    EXPECT_THAT(value, location);
+    EXPECT_THAT(valueAtConstructor, location);
+}
+
+TEST_F(InitSingletonTest, InitSingletonTwice) {
+    ObjHeader previousValue;
+    ObjHeader* location = &previousValue;
+    ObjHeader* stackLocation = nullptr;
+
+    EXPECT_CALL(constructor(), Call(_)).Times(0);
+    ObjHeader* value = InitSingleton(&location, 0, &stackLocation);
+    EXPECT_THAT(value, stackLocation);
+    EXPECT_THAT(value, location);
+    EXPECT_THAT(value, &previousValue);
+}
+
+TEST_F(InitSingletonTest, InitSingletonFail) {
+    ObjHeader* location = nullptr;
+    ObjHeader* stackLocation = nullptr;
+    constexpr int kException = 42;
+
+    EXPECT_CALL(constructor(), Call(_)).WillOnce([]() { throw kException; });
+    try {
+        InitSingleton(&location, 0, &stackLocation);
+        ASSERT_TRUE(false); // Cannot be reached.
+    } catch (int exception) {
+        EXPECT_THAT(exception, kException);
+    }
+    EXPECT_THAT(stackLocation, nullptr);
+    EXPECT_THAT(location, nullptr);
+}
+
+TEST_F(InitSingletonTest, InitSingletonRecursive) {
+    // The first singleton. Its constructor depends on the second singleton.
+    ObjHeader* location1 = nullptr;
+    ObjHeader* stackLocation1 = nullptr;
+    // The second singleton. Its constructor depends on the first singleton.
+    ObjHeader* location2 = nullptr;
+    ObjHeader* stackLocation2 = nullptr;
+
+    EXPECT_CALL(constructor(), Call(_))
+            .Times(2) // called only once for each singleton.
+            .WillRepeatedly([this, &location1, &stackLocation1, &location2, &stackLocation2](ObjHeader* value) {
+                if (value == stackLocation1) {
+                    ObjHeader* result = InitSingleton(&location2, 0, &stackLocation2);
+                    EXPECT_THAT(result, stackLocation2);
+                    EXPECT_THAT(result, location2);
+                    EXPECT_THAT(result, testing::Ne(reinterpret_cast<ObjHeader*>(1)));
+                } else {
+                    ObjHeader* result = InitSingleton(&location1, 0, &stackLocation1);
+                    EXPECT_THAT(result, stackLocation1);
+                    EXPECT_THAT(result, testing::Ne(location1));
+                    EXPECT_THAT(location1, reinterpret_cast<ObjHeader*>(1));
+                }
+            });
+    ObjHeader* value = InitSingleton(&location1, 0, &stackLocation1);
+    EXPECT_THAT(value, stackLocation1);
+    EXPECT_THAT(value, location1);
+}
+
+TEST_F(InitSingletonTest, InitSingletonConcurrent) {
+    constexpr size_t kThreadCount = kDefaultThreadCount;
+    std::atomic<bool> canStart(false);
+    std::atomic<size_t> readyCount(0);
+    KStdVector<std::thread> threads;
+    ObjHeader* location = nullptr;
+    KStdVector<ObjHeader*> stackLocations(kThreadCount, nullptr);
+    KStdVector<ObjHeader*> actual(kThreadCount, nullptr);
+
+    for (size_t i = 0; i < kThreadCount; ++i) {
+        threads.emplace_back([this, i, &location, &stackLocations, &actual, &readyCount, &canStart]() {
+            ++readyCount;
+            while (!canStart) {
+            }
+            actual[i] = InitSingleton(&location, i, &stackLocations[i]);
+        });
+    }
+
+    while (readyCount < kThreadCount) {
+    }
+    // Constructor is called exactly once.
+    EXPECT_CALL(constructor(), Call(_));
+    canStart = true;
+    for (auto& t : threads) {
+        t.join();
+    }
+    testing::Mock::VerifyAndClearExpectations(&constructor());
+
+    EXPECT_THAT(location, testing::Ne(nullptr));
+    EXPECT_THAT(location, testing::Ne(reinterpret_cast<ObjHeader*>(1)));
+    EXPECT_THAT(stackLocations, testing::Each(location));
+    EXPECT_THAT(actual, testing::Each(location));
+}
+
+TEST_F(InitSingletonTest, InitSingletonConcurrentFailing) {
+    constexpr size_t kThreadCount = kDefaultThreadCount;
+    std::atomic<bool> canStart(false);
+    std::atomic<size_t> readyCount(0);
+    KStdVector<std::thread> threads;
+    constexpr int kException = 42;
+    ObjHeader* location = nullptr;
+    KStdVector<ObjHeader*> stackLocations(kThreadCount, nullptr);
+
+    for (size_t i = 0; i < kThreadCount; ++i) {
+        threads.emplace_back([this, i, &location, &stackLocations, &readyCount, &canStart]() {
+            ++readyCount;
+            while (!canStart) {
+            }
+            try {
+                InitSingleton(&location, i, &stackLocations[i]);
+                ASSERT_TRUE(false); // Cannot be reached.
+            } catch (int exception) {
+                EXPECT_THAT(exception, kException);
+            }
+        });
+    }
+
+    while (readyCount < kThreadCount) {
+    }
+    // Constructor is called exactly `kThreadCount` times.
+    EXPECT_CALL(constructor(), Call(_)).Times(kThreadCount).WillRepeatedly([]() { throw kException; });
+    canStart = true;
+    for (auto& t : threads) {
+        t.join();
+    }
+    testing::Mock::VerifyAndClearExpectations(&constructor());
+
+    EXPECT_THAT(location, nullptr);
+    EXPECT_THAT(stackLocations, testing::Each(nullptr));
+}

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -8,8 +8,11 @@
 #include "Exceptions.h"
 #include "ExtraObjectData.hpp"
 #include "GlobalsRegistry.hpp"
+#include "InitializationScheme.hpp"
 #include "KAssert.h"
+#include "Natives.h"
 #include "Porting.h"
+#include "ObjectOps.hpp"
 #include "StableRefRegistry.hpp"
 #include "ThreadData.hpp"
 #include "ThreadRegistry.hpp"
@@ -111,8 +114,7 @@ extern "C" void RestoreMemory(MemoryState*) {
 
 extern "C" RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* typeInfo) {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo);
-    RETURN_OBJ(object);
+    RETURN_RESULT_OF(mm::AllocateObject, threadData, typeInfo);
 }
 
 extern "C" OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elements) {
@@ -120,28 +122,84 @@ extern "C" OBJ_GETTER(AllocArrayInstance, const TypeInfo* typeInfo, int32_t elem
         ThrowIllegalArgumentException();
     }
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast<uint32_t>(elements));
-    // `ArrayHeader` and `ObjHeader` are expected to be compatible.
-    RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
+    RETURN_RESULT_OF(mm::AllocateArray, threadData, typeInfo, static_cast<uint32_t>(elements));
 }
 
-extern "C" OBJ_GETTER(InitSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+extern "C" ALWAYS_INLINE OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
-    // TODO: This should only be called if singleton is actually created here. It's possible that the
-    // singleton will be created on a different thread and here we should check that, instead of creating
-    // another one (and registering `location` twice).
-    mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location);
-    TODO();
+
+    RETURN_RESULT_OF(mm::InitThreadLocalSingleton, threadData, location, typeInfo, ctor);
+}
+
+extern "C" ALWAYS_INLINE OBJ_GETTER(InitSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+    auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
+
+    RETURN_RESULT_OF(mm::InitSingleton, threadData, location, typeInfo, ctor);
 }
 
 extern "C" RUNTIME_NOTHROW void InitAndRegisterGlobal(ObjHeader** location, const ObjHeader* initialValue) {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location);
-    TODO();
+    mm::SetHeapRef(location, const_cast<ObjHeader*>(initialValue));
 }
 
 extern "C" const MemoryModel CurrentMemoryModel = MemoryModel::kExperimental;
 
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) {
+    mm::SetStackRef(location, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) {
+    mm::SetHeapRef(location, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) {
+    mm::SetHeapRef(location, nullptr);
+}
+
+extern "C" RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) {
+    for (uint32_t index = 0; index < array->count_; ++index) {
+        ObjHeader** location = ArrayAddressOfElementAt(array, index);
+        mm::SetHeapRef(location, nullptr);
+    }
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) {
+    mm::SetStackRef(location, nullptr);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
+    mm::SetStackRef(location, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) {
+    mm::SetHeapRef(location, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
+    if (object == nullptr) return;
+    ObjHeader* result = nullptr; // No need to store this value in a rootset.
+    mm::CompareAndSwapHeapRef(location, nullptr, const_cast<ObjHeader*>(object), &result);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
+    mm::SetStackRef(returnSlot, const_cast<ObjHeader*>(object));
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(
+        SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
+    RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRefLocked(
+        ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
+    mm::SetHeapRefAtomic(location, newValue);
+}
+
+extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) {
+    RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location);
+}
+
 extern "C" OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index) {
     // TODO: Remove when legacy MM is gone.
     ThrowNotImplementedError();
@@ -250,12 +308,15 @@ extern "C" RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void* pointer) {
     auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
     auto* node = static_cast<mm::StableRefRegistry::Node*>(pointer);
     ObjHeader* object = **node;
-    UpdateReturnRef(OBJ_RESULT, object);
+    // Make sure `object` stays in the rootset: put it on the stack before removing it from `StableRefRegistry`.
+    mm::SetStackRef(OBJ_RESULT, object);
     mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node);
     return object;
 }
 
 extern "C" RUNTIME_NOTHROW void CheckLifetimesConstraint(ObjHeader* obj, ObjHeader* pointee) {
+    // TODO: Consider making it a `RuntimeCheck`. Probably all `RuntimeCheck`s and `RuntimeAssert`s should specify
+    //       that their firing is a compiler bug and should be reported.
     if (!obj->local() && pointee != nullptr && pointee->local()) {
         konan::consolePrintf("Attempt to store a stack object %p into a heap object %p\n", pointee, obj);
         konan::consolePrintf("This is a compiler bug, please report it to https://kotl.in/issue\n");

kotlin-native/runtime/src/mm/cpp/ObjectFactory.hpp

@@ -140,6 +140,12 @@ public:
             owner_.AssertCorrectUnsafe();
         }
 
+        void ClearForTests() noexcept {
+            // Since it's only for tests, no need to worry about stack overflows.
+            root_.reset();
+            last_ = nullptr;
+        }
+
     private:
         friend class ObjectFactoryStorage;
 
@@ -259,6 +265,8 @@ public:
 
         void Publish() noexcept { producer_.Publish(); }
 
+        void ClearForTests() noexcept { producer_.ClearForTests(); }
+
     private:
         Storage::Producer producer_;
     };

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ObjectOps.hpp"
+
+#include "Common.h"
+#include "ThreadData.hpp"
+
+using namespace kotlin;
+
+// TODO: Memory barriers.
+
+ALWAYS_INLINE void mm::SetStackRef(ObjHeader** location, ObjHeader* value) noexcept {
+    *location = value;
+}
+
+ALWAYS_INLINE void mm::SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept {
+    *location = value;
+}
+
+#pragma clang diagnostic push
+// On 32-bit android arm clang warns of significant performance penalty because of large
+// atomic operations. TODO: Consider using alternative ways of ordering memory operations if they
+// turn out to be more efficient on these platforms.
+#pragma clang diagnostic ignored "-Watomic-alignment"
+
+ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept {
+    __atomic_store_n(location, value, __ATOMIC_RELEASE);
+}
+
+ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept {
+    // TODO: Make this work with GCs that can stop thread at any point.
+    auto result = __atomic_load_n(location, __ATOMIC_ACQUIRE);
+    RETURN_OBJ(result);
+}
+
+ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept {
+    // TODO: Make this work with GCs that can stop thread at any point.
+    ObjHeader* actual = expected;
+    // TODO: Do we need this strong memory model? Do we need to use strong CAS?
+    // This intrinsic modifies `actual` non-atomically.
+    __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+    // On success, we already have old value (== `expected`) in `actual`.
+    // On failure, we have the old value written into `actual`.
+    RETURN_OBJ(actual);
+}
+
+#pragma clang diagnostic pop
+
+OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept {
+    // TODO: Make this work with GCs that can stop thread at any point.
+    auto* object = threadData->objectFactoryThreadQueue().CreateObject(typeInfo);
+    RETURN_OBJ(object);
+}
+
+OBJ_GETTER(mm::AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept {
+    // TODO: Make this work with GCs that can stop thread at any point.
+    auto* array = threadData->objectFactoryThreadQueue().CreateArray(typeInfo, static_cast<uint32_t>(elements));
+    // `ArrayHeader` and `ObjHeader` are expected to be compatible.
+    RETURN_OBJ(reinterpret_cast<ObjHeader*>(array));
+}

kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_OBJECT_OPS_H
+#define RUNTIME_MM_OBJECT_OPS_H
+
+#include "Memory.h"
+
+namespace kotlin {
+namespace mm {
+
+class ThreadData;
+
+// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals.
+
+// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which
+//       will have these operations for a friendlier API.
+
+// TODO: `OBJ_GETTER` is used because the returned objects needs to be accessible via the rootset before the function
+//       returns. If we had a different way to efficiently keep the object in the roots, `OBJ_GETTER` can be removed.
+
+void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept;
+void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept;
+void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept;
+OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept;
+OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept;
+OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept;
+OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_OBJECT_OPS_H

kotlin-native/runtime/src/mm/cpp/Stubs.cpp

@@ -17,59 +17,6 @@ ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {
 
 extern "C" {
 
-OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
-    TODO();
-}
-
-RUNTIME_NOTHROW OBJ_GETTER(
-        SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
-    TODO();
-}
-
-RUNTIME_NOTHROW void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
-    TODO();
-}
-
-RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) {
-    TODO();
-}
-
 void MutationCheck(ObjHeader* obj) {
     TODO();
 }

kotlin-native/runtime/src/mm/cpp/ThreadData.hpp

@@ -9,13 +9,16 @@
 #include <atomic>
 #include <pthread.h>
 
-#include "ObjectFactory.hpp"
 #include "GlobalsRegistry.hpp"
+#include "ObjectFactory.hpp"
 #include "ShadowStack.hpp"
 #include "StableRefRegistry.hpp"
 #include "ThreadLocalStorage.hpp"
-#include "Utils.hpp"
 #include "ThreadState.hpp"
+#include "Types.h"
+#include "Utils.hpp"
+
+struct ObjHeader;
 
 namespace kotlin {
 namespace mm {
@@ -49,6 +52,8 @@ public:
 
     ShadowStack& shadowStack() noexcept { return shadowStack_; }
 
+    KStdVector<std::pair<ObjHeader**, ObjHeader*>>& initializingSingletons() noexcept { return initializingSingletons_; }
+
 private:
     const pthread_t threadId_;
     GlobalsRegistry::ThreadQueue globalsThreadQueue_;
@@ -57,6 +62,7 @@ private:
     std::atomic<ThreadState> state_;
     ObjectFactory::ThreadQueue objectFactoryThreadQueue_;
     ShadowStack shadowStack_;
+    KStdVector<std::pair<ObjHeader**, ObjHeader*>> initializingSingletons_;
 };
 
 } // namespace mm