[K/N] Runtime reference accessors

Encapsulate reference-access-related logic,
    such as GC barriers and tsan workarounds
    inside (Direct)RefAccessor classes.

kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp

@@ -22,9 +22,9 @@ using namespace kotlin;
 namespace {
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields = {
             &Payload::field1,

kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp

@@ -851,8 +851,8 @@ public:
 using ObjectFactory = alloc::ObjectFactory<ObjectFactoryTraits>;
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
 
     static constexpr std::array kFields{
             &Payload::field1,

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp

@@ -35,8 +35,8 @@ void processFieldInMark(void* state, ObjHeader* field) noexcept {
 
 template <typename Traits>
 void processObjectInMark(void* state, ObjHeader* object) noexcept {
-    traverseClassObjectFields(object, [state] (ObjHeader** fieldLocation) noexcept {
+    traverseClassObjectFields(object, [state] (auto fieldAccessor) noexcept {
-        if (auto field = *fieldLocation) {
+        if (ObjHeader* field = fieldAccessor.direct()) {
             processFieldInMark<Traits>(state, field);
         }
     });
@@ -44,8 +44,8 @@ void processObjectInMark(void* state, ObjHeader* object) noexcept {
 
 template <typename Traits>
 void processArrayInMark(void* state, ArrayHeader* array) noexcept {
-    traverseArrayOfObjectsElements(array, [state] (ObjHeader** elemLocation) noexcept {
+    traverseArrayOfObjectsElements(array, [state] (auto elemAccessor) noexcept {
-        if (auto elem = *elemLocation) {
+        if (ObjHeader* elem = elemAccessor.direct()) {
             processFieldInMark<Traits>(state, elem);
         }
     });

kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp

@@ -22,9 +22,9 @@ using namespace kotlin;
 namespace {
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields = {
             &Payload::field1,

kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp

@@ -30,9 +30,9 @@
 using namespace kotlin;
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields = {
             &Payload::field1,
@@ -97,7 +97,7 @@ public:
     test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); }
     test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); }
 
-    ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; }
+    mm::RefField& operator[](size_t index) noexcept { return (**this).elements()[index]; }
 
 private:
     ObjHeader* location_ = nullptr;
@@ -144,7 +144,7 @@ public:
     test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); }
     test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); }
 
-    ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; }
+    mm::RefField& operator[](size_t index) noexcept { return (**this).elements()[index]; }
 
 private:
     ObjHolder holder_;
@@ -343,7 +343,7 @@ TYPED_TEST_P(TracingGCTest, FreeObjectWithHoldedWeak) {
     RunInNewThread([](mm::ThreadData& threadData) {
         auto& object1 = AllocateObject(threadData);
         StackObjectHolder stack{threadData};
-        auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
+        auto& weak1 = InstallWeakReference(threadData, object1.header(), stack->field1.ptr());
 
         ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
         ASSERT_THAT(gc::isMarked(object1.header()), false);

kotlin-native/runtime/src/main/cpp/ExceptionsTest.cpp

@@ -31,8 +31,7 @@ namespace {
 struct Payload {
     int value = 0;
 
-    using Field = ObjHeader* Payload::*;
+    static constexpr test_support::NoRefFields<Payload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 using Object = test_support::Object<Payload>;

kotlin-native/runtime/src/main/cpp/FinalizerHooksTest.cpp

@@ -20,8 +20,7 @@ using ::testing::_;
 namespace {
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 class FinalizerHooksTest : public testing::Test {

kotlin-native/runtime/src/main/cpp/FreezeHooksTest.cpp

@@ -19,8 +19,7 @@ using ::testing::_;
 namespace {
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 class FreezeHooksTest : public testing::Test {

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -191,6 +191,7 @@ struct type_layout::descriptor<ArrayBody> {
     };
 };
 
+
 } // namespace kotlin
 
 ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj);
@@ -292,10 +293,6 @@ enum class MemoryModel {
 // Controls the current memory model, is compile-time constant.
 extern const MemoryModel CurrentMemoryModel;
 
-// Sets stack location.
-void SetStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
-// Sets heap location.
-void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Zeroes heap location.
 void ZeroHeapRef(ObjHeader** location) RUNTIME_NOTHROW;
 // Zeroes an array.
@@ -315,18 +312,8 @@ OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue)
 // Updates heap/static data in one array.
 void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW;
 // Updates location if it is null, atomically.
-void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates reference in return slot.
 void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW;
-// Compares and swaps reference with taken lock.
-OBJ_GETTER(SwapHeapRefLocked,
-    ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock,
-    int32_t* cookie) RUNTIME_NOTHROW;
-// Sets reference with taken lock.
-void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock,
-    int32_t* cookie) RUNTIME_NOTHROW;
-// Reads reference with taken lock.
-OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) RUNTIME_NOTHROW;
 OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index);
 // Called on frame enter, if it has object slots.
 void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
@@ -421,6 +408,7 @@ struct FrameOverlay {
 };
 
 // Class holding reference to an object, holding object during C++ scope.
+// TODO adopt ref accessors
 class ObjHolder {
  public:
    ObjHolder() : obj_(nullptr) {
@@ -429,7 +417,7 @@ class ObjHolder {
 
    explicit ObjHolder(const ObjHeader* obj) {
      EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
-     ::SetStackRef(slot(), obj);
+     ::UpdateStackRef(slot(), obj);
    }
 
    ~ObjHolder() {

kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp

@@ -11,6 +11,7 @@
 
 #include "KAssert.h"
 #include "Memory.h"
+#include "ReferenceOps.hpp"
 #include "TypeInfo.h"
 #include "Types.h"
 #include "Utils.hpp"
@@ -20,6 +21,12 @@ namespace test_support {
 
 // TODO: Some concepts from here can be used in production code.
 
+template<typename Host>
+using RefFieldPtr = mm::RefField Host::*;
+
+template<typename Host>
+using NoRefFields = std::array<RefFieldPtr<Host>, 0>;
+
 class TypeInfoHolder : private Pinned {
 private:
     class Builder {
@@ -96,6 +103,9 @@ public:
         typeInfo_.instanceAlignment_ = builder.instanceAlignment_;
     }
 
+    template<>
+    class ArrayBuilder<mm::RefField> : public ArrayBuilder<ObjHeader*> {};
+
     TypeInfo* typeInfo() noexcept { return &typeInfo_; }
 
 private:
@@ -123,9 +133,9 @@ public:
     public:
         FieldIterator(Object& owner, size_t index) noexcept : owner_(owner), index_(index) {}
 
-        ObjHeader*& operator*() noexcept {
+        mm::RefField& operator*() noexcept {
             auto* header = &owner_.header_;
-            return *reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(header) + header->type_info()->objOffsets_[index_]);
+            return *reinterpret_cast<mm::RefField*>(reinterpret_cast<uintptr_t>(header) + header->type_info()->objOffsets_[index_]);
         }
 
         FieldIterator& operator++() noexcept {
@@ -148,7 +158,7 @@ public:
 
         size_t size() const noexcept { return owner_.header_.type_info()->objOffsetsCount_; }
 
-        ObjHeader*& operator[](size_t index) noexcept { return *FieldIterator(owner_, index); }
+        mm::RefField& operator[](size_t index) noexcept { return *FieldIterator(owner_, index); }
 
         FieldIterator begin() noexcept { return FieldIterator(owner_, 0); }
         FieldIterator end() noexcept { return FieldIterator(owner_, size()); }
@@ -190,8 +200,7 @@ TypeInfoHolder::ObjectBuilder<Payload>::ObjectBuilder() noexcept {
     char c;
     Object<Payload>& object = *reinterpret_cast<Object<Payload>*>(&c);
     auto& payload = *object;
-    using Field = ObjHeader* Payload::*;
+    for (RefFieldPtr<Payload> field : Payload::kFields) {
-    for (Field field : Payload::kFields) {
         auto& actualField = payload.*field;
         objOffsets_.push_back(reinterpret_cast<uintptr_t>(&actualField) - reinterpret_cast<uintptr_t>(object.header()));
     }
@@ -237,13 +246,13 @@ private:
 } // namespace internal
 
 template <size_t ElementCount>
-class ObjectArray : public internal::Array<ObjHeader*, ElementCount> {
+class ObjectArray : public internal::Array<mm::RefField, ElementCount> {
 public:
     static ObjectArray<ElementCount>& FromArrayHeader(ArrayHeader* arr) noexcept {
-        return static_cast<ObjectArray<ElementCount>&>(internal::Array<ObjHeader*, ElementCount>::FromArrayHeader(arr));
+        return static_cast<ObjectArray<ElementCount>&>(internal::Array<mm::RefField, ElementCount>::FromArrayHeader(arr));
     }
 
-    ObjectArray() noexcept : internal::Array<ObjHeader*, ElementCount>(theArrayTypeInfo) {}
+    ObjectArray() noexcept : internal::Array<mm::RefField, ElementCount>(theArrayTypeInfo) {}
 };
 
 template <size_t ElementCount>
@@ -350,8 +359,7 @@ struct RegularWeakReferenceImplPayload {
     void* weakRef;
     void* referred;
 
-    using Field = ObjHeader* RegularWeakReferenceImplPayload::*;
+    static constexpr test_support::NoRefFields<RegularWeakReferenceImplPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader*);

kotlin-native/runtime/src/main/cpp/ObjectTestSupportTest.cpp

@@ -18,9 +18,9 @@ using namespace kotlin;
 namespace {
 
 struct RegularPayload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields{
             &RegularPayload::field1,
@@ -31,11 +31,11 @@ struct RegularPayload {
 
 struct IrregularPayload {
     int skipBefore;
-    ObjHeader* field1;
+    mm::RefField field1;
     int skip;
-    ObjHeader* field2;
+    mm::RefField field2;
     std::array<int, 10> skipALot;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields{
             &IrregularPayload::field1,
@@ -70,8 +70,8 @@ using ObjectTestCases = testing::Types<RegularObjectTestCase, IrregularObjectTes
 TYPED_TEST_SUITE(ObjectTestSupportObjectTest, ObjectTestCases, ObjectTestCaseNames);
 
 template <typename Payload>
-std::vector<ObjHeader**> Collect(test_support::Object<Payload>& object) {
+std::vector<mm::RefField*> Collect(test_support::Object<Payload>& object) {
-    std::vector<ObjHeader**> result;
+    std::vector<mm::RefField*> result;
     for (auto& field : object.fields()) {
         result.push_back(&field);
     }
@@ -108,9 +108,9 @@ TYPED_TEST(ObjectTestSupportObjectTest, Local) {
 
     EXPECT_THAT(Collect(object), testing::ElementsAre(&object->field1, &object->field2, &object->field3));
 
-    EXPECT_THAT(object.fields()[0], nullptr);
+    EXPECT_THAT(object.fields()[0].direct().load(), nullptr);
-    EXPECT_THAT(object.fields()[1], nullptr);
+    EXPECT_THAT(object.fields()[1].direct().load(), nullptr);
-    EXPECT_THAT(object.fields()[2], nullptr);
+    EXPECT_THAT(object.fields()[2].direct().load(), nullptr);
 
     auto& recoveredObject = test_support::Object<Payload>::FromObjHeader(object.header());
     EXPECT_THAT(&recoveredObject, &object);
@@ -149,9 +149,9 @@ TYPED_TEST(ObjectTestSupportObjectTest, Heap) {
 
         EXPECT_THAT(Collect(object), testing::ElementsAre(&object->field1, &object->field2, &object->field3));
 
-        EXPECT_THAT(object.fields()[0], nullptr);
+        EXPECT_THAT(object.fields()[0].direct().load(), nullptr);
-        EXPECT_THAT(object.fields()[1], nullptr);
+        EXPECT_THAT(object.fields()[1].direct().load(), nullptr);
-        EXPECT_THAT(object.fields()[2], nullptr);
+        EXPECT_THAT(object.fields()[2].direct().load(), nullptr);
     });
 }
 
@@ -161,7 +161,7 @@ template <typename Payload>
 struct PayloadTraits;
 
 template <>
-struct PayloadTraits<ObjHeader*> {
+struct PayloadTraits<mm::RefField> {
     template <size_t Size>
     using Array = test_support::ObjectArray<Size>;
     static const TypeInfo* GetTypeInfo() { return theArrayTypeInfo; }
@@ -284,8 +284,8 @@ public:
 template <typename TestCase>
 class ObjectTestSupportArrayTest : public testing::Test {};
 using ArrayTestCases = testing::Types<
-        ArrayTestCase<ObjHeader*, 0>,
+        ArrayTestCase<mm::RefField, 0>,
-        ArrayTestCase<ObjHeader*, 3>,
+        ArrayTestCase<mm::RefField, 3>,
         ArrayTestCase<KBoolean, 0>,
         ArrayTestCase<KBoolean, 3>,
         ArrayTestCase<KByte, 0>,
@@ -335,7 +335,7 @@ TYPED_TEST(ObjectTestSupportArrayTest, Local) {
     for (size_t i = 0; i < size; ++i) {
         auto* element = AddressOfElementAt<Payload>(array.arrayHeader(), i);
         EXPECT_THAT(&array.elements()[i], element);
-        EXPECT_THAT(array.elements()[i], Payload{});
+        EXPECT_TRUE(array.elements()[i] == Payload{});
         expected.push_back(element);
     }
 
@@ -366,7 +366,7 @@ TYPED_TEST(ObjectTestSupportArrayTest, Heap) {
         for (size_t i = 0; i < size; ++i) {
             auto* element = AddressOfElementAt<Payload>(array.arrayHeader(), i);
             EXPECT_THAT(&array.elements()[i], element);
-            EXPECT_THAT(array.elements()[i], Payload{});
+            EXPECT_TRUE(array.elements()[i] == Payload{});
             expected.push_back(element);
         }
 

kotlin-native/runtime/src/main/cpp/ReferenceOps.hpp

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#pragma once
+
+#include "Memory.h"
+#include "std_support/AtomicRef.hpp"
+
+// Concurrent GC may cause conflicting unordered accesses to references in heap.
+// C++ memory model declares that accesses have data race unless they are atomic.
+// Thus TSAN would report such non-atomic accesses.
+// TODO find out if compiler may take advantage of the theoretical UB here.
+//
+// However, in practice all the ptr-sized loads and stores are atomic on CPU-level
+// even if they are not std::atomic.
+// And as far as we aware,
+// std::atomic operations are not optimized by LLVM even if they have relaxed memory order.
+// So we don't want to compile every heap reference access into std::atomic access.
+#define ALWAYS_ATOMIC_REFS __has_feature(thread_sanitizer)
+
+namespace kotlin::mm {
+
+// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals.
+
+// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which
+//       will have these operations for a friendlier API.
+
+/**
+ * Represents direct low-level operations on Koltin references.
+ * No GC barriers are inserted. Should be used with care!
+ */
+class DirectRefAccessor {
+public:
+    DirectRefAccessor() = delete;
+    DirectRefAccessor& operator=(const DirectRefAccessor&) = delete;
+
+    explicit DirectRefAccessor(ObjHeader*& fieldRef) noexcept : ref_(fieldRef) {}
+    explicit DirectRefAccessor(ObjHeader** fieldPtr) noexcept : DirectRefAccessor(*fieldPtr) {}
+    DirectRefAccessor(const DirectRefAccessor& other) noexcept : DirectRefAccessor(other.ref_) {}
+
+    ObjHeader** location() const noexcept { return &ref_; }
+
+    ALWAYS_INLINE operator ObjHeader*() const noexcept { return load(); }
+    ALWAYS_INLINE ObjHeader* operator=(ObjHeader* desired) noexcept { store(desired); return desired; }
+
+    ALWAYS_INLINE ObjHeader* load() const noexcept {
+#if ALWAYS_ATOMIC_REFS
+        return loadAtomic(std::memory_order_relaxed);
+#else
+        return ref_;
+#endif
+    }
+
+    ALWAYS_INLINE void store(ObjHeader* desired) noexcept {
+#if ALWAYS_ATOMIC_REFS
+        storeAtomic(desired, std::memory_order_relaxed);
+#else
+        ref_ = desired;
+#endif
+    }
+
+    ALWAYS_INLINE auto atomic() noexcept {
+        return std_support::atomic_ref<ObjHeader*>{ref_};
+    }
+    ALWAYS_INLINE auto atomic() const noexcept {
+        return std_support::atomic_ref<ObjHeader*>{ref_};
+    }
+
+    ALWAYS_INLINE ObjHeader* loadAtomic(std::memory_order order) const noexcept {
+        return atomic().load(order);
+    }
+    ALWAYS_INLINE void storeAtomic(ObjHeader* desired, std::memory_order order) noexcept {
+        atomic().store(desired, order);
+    }
+    ALWAYS_INLINE ObjHeader* exchange(ObjHeader* desired, std::memory_order order) noexcept {
+        return atomic().exchange(desired, order);
+    }
+    ALWAYS_INLINE bool compareAndExchange(ObjHeader*& expected, ObjHeader* desired, std::memory_order order) noexcept {
+        return atomic().compare_exchange_strong(expected, desired, order);
+    }
+
+private:
+    ObjHeader*& ref_;
+};
+
+/**
+ * Represents Koltin-level operations on Koltin references.
+ * With all the necessary GC barriers etc.
+ * Prefer using aliases below.
+ */
+template<bool kOnStack>
+class RefAccessor {
+public:
+    RefAccessor() = delete;
+    RefAccessor& operator=(const RefAccessor&) = delete;
+
+    explicit RefAccessor(ObjHeader*& fieldRef) noexcept : direct_(fieldRef) {}
+    explicit RefAccessor(ObjHeader** fieldPtr) noexcept : RefAccessor(*fieldPtr) {}
+    RefAccessor(const RefAccessor& other) noexcept : direct_(other.direct_) {}
+
+    DirectRefAccessor direct() const noexcept { return direct_; }
+
+    void beforeLoad() noexcept;
+    void afterLoad() noexcept;
+    void beforeStore(ObjHeader* value) noexcept;
+    void afterStore(ObjHeader* value) noexcept;
+
+    ALWAYS_INLINE operator ObjHeader*() noexcept { return load(); }
+
+    ALWAYS_INLINE ObjHeader* load() noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeLoad();
+        auto result = direct_.load();
+        afterLoad();
+        return result;
+    }
+
+    ALWAYS_INLINE ObjHeader* loadAtomic(std::memory_order order) noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeLoad();
+        auto result = direct_.loadAtomic(order);
+        afterLoad();
+        return result;
+    }
+
+    ALWAYS_INLINE ObjHeader* operator=(ObjHeader* desired) noexcept { store(desired); return desired; }
+
+    ALWAYS_INLINE void store(ObjHeader* desired) noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeStore(desired);
+        direct_.store(desired);
+        afterStore(desired);
+    }
+
+    ALWAYS_INLINE void storeAtomic(ObjHeader* desired, std::memory_order order) noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeStore(desired);
+        direct_.storeAtomic(desired, order);
+        afterStore(desired);
+    }
+
+    ALWAYS_INLINE ObjHeader* exchange(ObjHeader* desired, std::memory_order order) noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeLoad();
+        beforeStore(desired);
+        auto result = direct_.exchange(desired, order);
+        afterStore(desired);
+        afterLoad();
+        return result;
+    }
+
+    ALWAYS_INLINE bool compareAndExchange(ObjHeader*& expected, ObjHeader* desired, std::memory_order order) noexcept {
+        AssertThreadState(ThreadState::kRunnable);
+        beforeLoad();
+        beforeStore(desired);
+        bool result = direct_.compareAndExchange(expected, desired, order);
+        afterStore(desired);
+        afterLoad();
+        return result;
+    }
+
+private:
+    DirectRefAccessor direct_;
+};
+
+using RefFieldAccessor = RefAccessor<false>;
+using GlobalRefAccessor = RefAccessor<false>;
+using StackRefAccessor = RefAccessor<true>;
+
+class RefField : private Pinned {
+public:
+    auto accessor() noexcept {
+        return mm::RefFieldAccessor(value_);
+    }
+    auto direct() noexcept {
+        return accessor().direct();
+    }
+    // FIXME probably most of the uses should instead use accessor
+    auto ptr() noexcept {
+        return direct().location();
+    }
+
+    // TODO consider adding other operations
+    ObjHeader* operator=(ObjHeader* value) noexcept {
+        accessor() = value;
+        return value_;
+    }
+
+    bool operator==(const RefField& other) const noexcept {
+        return value_ == other.value_;
+    }
+
+    bool operator!=(const RefField& other) const noexcept {
+        return !operator==(other);
+    }
+
+private:
+    ObjHeader* value_ = nullptr;
+};
+
+OBJ_GETTER(weakRefReadBarrier, std::atomic<ObjHeader*>& referee) noexcept;
+
+}

kotlin-native/runtime/src/main/cpp/TypeInfoTest.cpp

@@ -16,13 +16,12 @@ using namespace kotlin;
 namespace {
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 struct Payload1 {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
 
     static constexpr std::array kFields{
             &Payload1::field1,
@@ -31,8 +30,8 @@ struct Payload1 {
 };
 
 struct Payload2 {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
 
     static constexpr std::array kFields{
             &Payload2::field1,

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -19,8 +19,7 @@ using namespace kotlin;
 namespace {
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 class ExtraObjectDataTest : public testing::Test {

kotlin-native/runtime/src/mm/cpp/FreezingTest.cpp

@@ -29,14 +29,13 @@ struct NoFreezeHook {
 };
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields{
             &Payload::field1,
@@ -66,7 +65,7 @@ public:
 
     ObjHeader* header() { return object_.header(); }
 
-    ObjHeader*& operator[](size_t field) { return object_.fields()[field]; }
+    mm::RefField& operator[](size_t field) { return object_.fields()[field]; }
 
     void MakePermanent() { header()->typeInfoOrMeta_ = setPointerBits(header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); }
 
@@ -89,7 +88,7 @@ public:
 
     ObjHeader* header() { return array_.header(); }
 
-    ObjHeader*& operator[](size_t index) { return array_.elements()[index]; }
+    mm::RefField& operator[](size_t index) { return array_.elements()[index]; }
 
     void MakePermanent() { header()->typeInfoOrMeta_ = setPointerBits(header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); }
 

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -16,6 +16,7 @@
 #include "Natives.h"
 #include "ObjectOps.hpp"
 #include "Porting.h"
+#include "ReferenceOps.hpp"
 #include "Runtime.h"
 #include "SafePoint.hpp"
 #include "StableRef.hpp"
@@ -144,63 +145,51 @@ extern "C" RUNTIME_NOTHROW void InitAndRegisterGlobal(ObjHeader** location, cons
     mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location);
     // Null `initialValue` means that the appropriate value was already set by static initialization.
     if (initialValue != nullptr) {
-        mm::SetHeapRef(location, const_cast<ObjHeader*>(initialValue));
+        UpdateHeapRef(location, const_cast<ObjHeader*>(initialValue));
     }
 }
 
 extern "C" const MemoryModel CurrentMemoryModel = MemoryModel::kExperimental;
 
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) {
-    mm::SetStackRef(location, const_cast<ObjHeader*>(object));
-}
-
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) {
-    mm::SetHeapRef(location, const_cast<ObjHeader*>(object));
-}
-
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) {
-    mm::SetHeapRef(location, nullptr);
+    mm::RefAccessor<false>{location} = nullptr;
 }
 
 extern "C" RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) {
     for (uint32_t index = 0; index < array->count_; ++index) {
         ObjHeader** location = ArrayAddressOfElementAt(array, index);
-        mm::SetHeapRef(location, nullptr);
+        mm::RefFieldAccessor{location} = nullptr;
     }
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) {
-    mm::SetStackRef(location, nullptr);
+    mm::StackRefAccessor{location} = nullptr;
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
-    mm::SetStackRef(location, const_cast<ObjHeader*>(object));
+    mm::StackRefAccessor{location} = const_cast<ObjHeader*>(object);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) {
-    mm::SetHeapRef(location, const_cast<ObjHeader*>(object));
+    mm::RefAccessor<false>{location} = const_cast<ObjHeader*>(object);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) {
-    mm::SetHeapRefAtomicSeqCst(location, const_cast<ObjHeader*>(object));
+    mm::RefAccessor<false>{location}.storeAtomic(const_cast<ObjHeader*>(object), std::memory_order_seq_cst);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
-    RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue);
+    ObjHeader* actual = expectedValue;
+    mm::RefAccessor<false>{location}.compareAndExchange(actual, newValue, std::memory_order_seq_cst);
+    RETURN_OBJ(actual);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) {
-    return mm::CompareAndSetHeapRef(location, expectedValue, newValue);
+    return mm::RefAccessor<false>{location}.compareAndExchange(expectedValue, newValue, std::memory_order_seq_cst);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) {
-    RETURN_RESULT_OF(mm::GetAndSetHeapRef, location, newValue);
+    RETURN_OBJ(mm::RefAccessor<false>{location}.exchange(newValue, std::memory_order_seq_cst));
-}
-
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
-    if (object == nullptr) return;
-    ObjHeader* result = nullptr; // No need to store this value in a rootset.
-    mm::CompareAndSwapHeapRef(location, nullptr, const_cast<ObjHeader*>(object), &result);
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex,
@@ -209,22 +198,10 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const
 }
 
 extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
-    mm::SetStackRef(returnSlot, const_cast<ObjHeader*>(object));
+    UpdateStackRef(returnSlot, object);
 }
 
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(
-        SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
-    RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue);
-}
 
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRefLocked(
-        ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
-    mm::SetHeapRefAtomic(location, newValue);
-}
-
-extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) {
-    RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location);
-}
 
 extern "C" OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index) {
     // TODO: Remove when legacy MM is gone.
@@ -513,7 +490,7 @@ extern "C" RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void* pointer) {
     AssertThreadState(ThreadState::kRunnable);
     mm::StableRef stableRef(static_cast<mm::RawSpecialRef*>(pointer));
     auto* obj = *stableRef;
-    mm::SetStackRef(OBJ_RESULT, obj);
+    UpdateStackRef(OBJ_RESULT, obj);
     std::move(stableRef).dispose();
     return obj;
 }

kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp

@@ -11,66 +11,6 @@
 
 using namespace kotlin;
 
-// TODO: Memory barriers.
-
-ALWAYS_INLINE void mm::SetStackRef(ObjHeader** location, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    *location = value;
-}
-
-ALWAYS_INLINE void mm::SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    *location = value;
-}
-
-#pragma clang diagnostic push
-// On 32-bit android arm clang warns of significant performance penalty because of large
-// atomic operations. TODO: Consider using alternative ways of ordering memory operations if they
-// turn out to be more efficient on these platforms.
-#pragma clang diagnostic ignored "-Watomic-alignment"
-
-ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    __atomic_store_n(location, value, __ATOMIC_RELEASE);
-}
-
-ALWAYS_INLINE void mm::SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    __atomic_store_n(location, value, __ATOMIC_SEQ_CST);
-}
-
-
-ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    // TODO: Make this work with GCs that can stop thread at any point.
-    auto result = __atomic_load_n(location, __ATOMIC_ACQUIRE);
-    RETURN_OBJ(result);
-}
-
-ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    // TODO: Make this work with GCs that can stop thread at any point.
-    ObjHeader* actual = expected;
-    __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
-    RETURN_OBJ(actual);
-}
-
-ALWAYS_INLINE bool mm::CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);
-    // TODO: Make this work with GCs that can stop thread at any point.
-    ObjHeader* actual = expected;
-    return __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
-}
-
-ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept {
-    AssertThreadState(ThreadState::kRunnable);;
-    auto *actual = __atomic_exchange_n(location, value,  __ATOMIC_SEQ_CST);
-    RETURN_OBJ(actual);
-}
-
-
-#pragma clang diagnostic pop
-
 OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept {
     AssertThreadState(threadData, ThreadState::kRunnable);
     // TODO: Make this work with GCs that can stop thread at any point.

kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
  * that can be found in the LICENSE file.
  */
 
@@ -13,22 +13,9 @@ namespace mm {
 
 class ThreadData;
 
-// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals.
-
-// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which
-//       will have these operations for a friendlier API.
-
 // TODO: `OBJ_GETTER` is used because the returned objects needs to be accessible via the rootset before the function
 //       returns. If we had a different way to efficiently keep the object in the roots, `OBJ_GETTER` can be removed.
 
-void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept;
-void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept;
-void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept;
-void SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept;
-OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept;
-OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept;
-bool CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept;
-OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept;
 OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept;
 OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept;
 

kotlin-native/runtime/src/mm/cpp/ObjectTraversal.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
  * that can be found in the LICENSE file.
  */
 
@@ -10,49 +10,54 @@
 
 #include "Memory.h"
 #include "Natives.h"
+#include "ReferenceOps.hpp"
 #include "Types.h"
+#include "ObjectOps.hpp"
 
 namespace kotlin {
 
 // TODO: Consider an iterator/ranges based approaches for traversals.
 
 template <typename F>
-ALWAYS_INLINE void traverseClassObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval<ObjHeader**>()))) {
+ALWAYS_INLINE void traverseClassObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval<mm::RefFieldAccessor>()))) {
     const TypeInfo* typeInfo = object->type_info();
     RuntimeAssert(typeInfo != theArrayTypeInfo, "Must not be an array of objects");
     for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
-        process(reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(object) + typeInfo->objOffsets_[index]));
+        auto fieldPtr = reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(object) + typeInfo->objOffsets_[index]);
+        process(mm::RefFieldAccessor(fieldPtr));
     }
 }
 
 template <typename F>
-ALWAYS_INLINE void traverseArrayOfObjectsElements(ArrayHeader* array, F process) noexcept(noexcept(process(std::declval<ObjHeader**>()))) {
+ALWAYS_INLINE void traverseArrayOfObjectsElements(ArrayHeader* array, F process) noexcept(noexcept(process(std::declval<mm::RefFieldAccessor>()))) {
     RuntimeAssert(array->type_info() == theArrayTypeInfo, "Must be an array of objects");
     for (uint32_t index = 0; index < array->count_; index++) {
-        process(ArrayAddressOfElementAt(array, index));
+        process(mm::RefFieldAccessor(ArrayAddressOfElementAt(array, index)));
     }
 }
 
 template <typename F>
-void traverseObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval<ObjHeader**>()))) {
+void traverseObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval<mm::RefFieldAccessor>()))) {
     const TypeInfo* typeInfo = object->type_info();
     // Only consider arrays of objects, not arrays of primitives.
     if (typeInfo != theArrayTypeInfo) {
         for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
-            process(reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(object) + typeInfo->objOffsets_[index]));
+            auto fieldPtr = reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(object) + typeInfo->objOffsets_[index]);
+            process(mm::RefFieldAccessor(fieldPtr));
         }
     } else {
         ArrayHeader* array = object->array();
         for (uint32_t index = 0; index < array->count_; index++) {
-            process(ArrayAddressOfElementAt(array, index));
+            process(mm::RefFieldAccessor(ArrayAddressOfElementAt(array, index)));
         }
     }
 }
 
+// FIXME explicitly mention no barriers
 template <typename F>
 void traverseReferredObjects(ObjHeader* object, F process) noexcept(noexcept(process(std::declval<ObjHeader*>()))) {
-    traverseObjectFields(object, [&process](ObjHeader** location) noexcept(noexcept(process(std::declval<ObjHeader*>()))) {
+    traverseObjectFields(object, [&process](auto accessor) noexcept(noexcept(process(std::declval<ObjHeader*>()))) {
-        if (ObjHeader* ref = *location) {
+        if (ObjHeader* ref = accessor.direct()) {
             process(ref);
         }
     });

kotlin-native/runtime/src/mm/cpp/ObjectTraversalTest.cpp

@@ -9,8 +9,10 @@
 #include "gtest/gtest.h"
 
 #include "ObjectTestSupport.hpp"
+#include "ReferenceOps.hpp"
 #include "Types.h"
 #include "Utils.hpp"
+#include "ObjectOps.hpp"
 
 using namespace kotlin;
 
@@ -18,25 +20,14 @@ using ::testing::_;
 
 namespace {
 
-struct CallableWithExceptions {
-    void operator()(ObjHeader*) noexcept(false) {}
-    void operator()(ObjHeader**) noexcept(false) {}
-};
-
-struct CallableWithoutExceptions {
-    void operator()(ObjHeader*) noexcept {}
-    void operator()(ObjHeader**) noexcept {}
-};
-
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 struct Payload {
-    ObjHeader* field1;
+    mm::RefField field1;
-    ObjHeader* field2;
+    mm::RefField field2;
-    ObjHeader* field3;
+    mm::RefField field3;
 
     static constexpr std::array kFields{
             &Payload::field1,
@@ -45,9 +36,22 @@ struct Payload {
     };
 };
 
+using ProcessFunMock = testing::StrictMock<testing::MockFunction<void(mm::RefFieldAccessor)>>;
+
+MATCHER_P(SameAccessor, accessor, "") {
+    return arg.direct().location() == accessor.direct().location();
+}
+
 } // namespace
 
 TEST(ObjectTraversalTest, TraverseFieldsExceptions) {
+    struct CallableWithExceptions {
+        void operator()(mm::RefFieldAccessor) noexcept(false) {}
+    };
+    struct CallableWithoutExceptions {
+        void operator()(mm::RefFieldAccessor) noexcept {}
+    };
+
     static_assert(
             noexcept(traverseObjectFields(std::declval<ObjHeader*>(), std::declval<CallableWithoutExceptions>())),
             "Callable is noexcept, so traverse is noexcept");
@@ -59,10 +63,10 @@ TEST(ObjectTraversalTest, TraverseFieldsExceptions) {
 TEST(ObjectTraversalTest, TraverseEmptyObjectFields) {
     test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
     test_support::Object<EmptyPayload> object(type.typeInfo());
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
     EXPECT_CALL(process, Call(_)).Times(0);
-    traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); });
+    traverseObjectFields(object.header(), process.AsStdFunction());
 }
 
 TEST(ObjectTraversalTest, TraverseObjectFields) {
@@ -70,14 +74,14 @@ TEST(ObjectTraversalTest, TraverseObjectFields) {
     ObjHeader field1;
     ObjHeader field3;
     test_support::Object<Payload> object(type.typeInfo());
-    object->field1 = &field1;
+    object->field1.direct() = &field1;
-    object->field3 = &field3;
+    object->field3.direct() = &field3;
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
-    EXPECT_CALL(process, Call(&object->field1));
+    EXPECT_CALL(process, Call(SameAccessor(object->field1.accessor())));
-    EXPECT_CALL(process, Call(&object->field2));
+    EXPECT_CALL(process, Call(SameAccessor(object->field2.accessor())));
-    EXPECT_CALL(process, Call(&object->field3));
+    EXPECT_CALL(process, Call(SameAccessor(object->field3.accessor())));
-    traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); });
+    traverseObjectFields(object.header(), process.AsStdFunction());
 }
 
 TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) {
@@ -88,16 +92,16 @@ TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) {
     ObjHeader field2;
     ObjHeader field3;
     test_support::Object<Payload> object(type.typeInfo());
-    object->field1 = &field1;
+    object->field1.direct() = &field1;
-    object->field2 = &field2;
+    object->field2.direct() = &field2;
-    object->field3 = &field3;
+    object->field3.direct() = &field3;
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
-    EXPECT_CALL(process, Call(&object->field1));
+    EXPECT_CALL(process, Call(SameAccessor(object->field1.accessor())));
-    EXPECT_CALL(process, Call(&object->field2)).WillOnce([]() { throw kException; });
+    EXPECT_CALL(process, Call(SameAccessor(object->field2.accessor()))).WillOnce([]() { throw kException; });
-    EXPECT_CALL(process, Call(&object->field3)).Times(0);
+    EXPECT_CALL(process, Call(SameAccessor(object->field3.accessor()))).Times(0);
     try {
-        traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); });
+        traverseObjectFields(object.header(), process.AsStdFunction());
     } catch (int exception) {
         EXPECT_THAT(exception, kException);
     } catch (...) {
@@ -107,24 +111,24 @@ TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) {
 
 TEST(ObjectTraversalTest, TraverseEmptyArrayFields) {
     test_support::ObjectArray<0> array;
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
     EXPECT_CALL(process, Call(_)).Times(0);
-    traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); });
+    traverseObjectFields(array.header(), process.AsStdFunction());
 }
 
 TEST(ObjectTraversalTest, TraverseArrayFields) {
     ObjHeader element1;
     ObjHeader element3;
     test_support::ObjectArray<3> array;
-    array.elements()[0] = &element1;
+    array.elements()[0].direct() = &element1;
-    array.elements()[2] = &element3;
+    array.elements()[2].direct() = &element3;
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
-    EXPECT_CALL(process, Call(&array.elements()[0]));
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[0].accessor())));
-    EXPECT_CALL(process, Call(&array.elements()[1]));
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[1].accessor())));
-    EXPECT_CALL(process, Call(&array.elements()[2]));
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[2].accessor())));
-    traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); });
+    traverseObjectFields(array.header(), process.AsStdFunction());
 }
 
 TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) {
@@ -134,16 +138,16 @@ TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) {
     ObjHeader element2;
     ObjHeader element3;
     test_support::ObjectArray<3> array;
-    array.elements()[0] = &element1;
+    array.elements()[0].direct() = &element1;
-    array.elements()[1] = &element2;
+    array.elements()[1].direct() = &element2;
-    array.elements()[2] = &element3;
+    array.elements()[2].direct() = &element3;
-    testing::StrictMock<testing::MockFunction<void(ObjHeader**)>> process;
+    ProcessFunMock process;
 
-    EXPECT_CALL(process, Call(&array.elements()[0]));
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[0].accessor())));
-    EXPECT_CALL(process, Call(&array.elements()[1])).WillOnce([]() { throw kException; });
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[1].accessor()))).WillOnce([]() { throw kException; });
-    EXPECT_CALL(process, Call(&array.elements()[2])).Times(0);
+    EXPECT_CALL(process, Call(SameAccessor(array.elements()[2].accessor()))).Times(0);
     try {
-        traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); });
+        traverseObjectFields(array.header(), process.AsStdFunction());
     } catch (int exception) {
         EXPECT_THAT(exception, kException);
     } catch (...) {
@@ -152,6 +156,13 @@ TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) {
 }
 
 TEST(ObjectTraversalTest, TraverseRefsExceptions) {
+    struct CallableWithExceptions {
+        void operator()(ObjHeader*) noexcept(false) {}
+    };
+    struct CallableWithoutExceptions {
+        void operator()(ObjHeader*) noexcept {}
+    };
+
     static_assert(
             noexcept(traverseReferredObjects(std::declval<ObjHeader*>(), std::declval<CallableWithoutExceptions>())),
             "Callable is noexcept, so traverse is noexcept");
@@ -174,8 +185,8 @@ TEST(ObjectTraversalTest, TraverseObjectRefs) {
     ObjHeader field1;
     ObjHeader field3;
     test_support::Object<Payload> object(type.typeInfo());
-    object->field1 = &field1;
+    object->field1.direct() = &field1;
-    object->field3 = &field3;
+    object->field3.direct() = &field3;
     testing::StrictMock<testing::MockFunction<void(ObjHeader*)>> process;
 
     EXPECT_CALL(process, Call(&field1));
@@ -191,9 +202,9 @@ TEST(ObjectTraversalTest, TraverseObjectRefsWithException) {
     ObjHeader field2;
     ObjHeader field3;
     test_support::Object<Payload> object(type.typeInfo());
-    object->field1 = &field1;
+    object->field1.direct() = &field1;
-    object->field2 = &field2;
+    object->field2.direct() = &field2;
-    object->field3 = &field3;
+    object->field3.direct() = &field3;
     testing::StrictMock<testing::MockFunction<void(ObjHeader*)>> process;
 
     EXPECT_CALL(process, Call(&field1));
@@ -220,8 +231,8 @@ TEST(ObjectTraversalTest, TraverseArrayRefs) {
     ObjHeader element1;
     ObjHeader element3;
     test_support::ObjectArray<3> array;
-    array.elements()[0] = &element1;
+    array.elements()[0].direct() = &element1;
-    array.elements()[2] = &element3;
+    array.elements()[2].direct() = &element3;
     testing::StrictMock<testing::MockFunction<void(ObjHeader*)>> process;
 
     EXPECT_CALL(process, Call(&element1));
@@ -236,9 +247,9 @@ TEST(ObjectTraversalTest, TraverseArrayRefsWithException) {
     ObjHeader element2;
     ObjHeader element3;
     test_support::ObjectArray<3> array;
-    array.elements()[0] = &element1;
+    array.elements()[0].direct() = &element1;
-    array.elements()[1] = &element2;
+    array.elements()[1].direct() = &element2;
-    array.elements()[2] = &element3;
+    array.elements()[2].direct() = &element3;
     testing::StrictMock<testing::MockFunction<void(ObjHeader*)>> process;
 
     EXPECT_CALL(process, Call(&element1));

kotlin-native/runtime/src/mm/cpp/ReferenceOps.cpp

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ReferenceOps.hpp"
+#include "ThreadData.hpp"
+#include "ThreadRegistry.hpp"
+
+using namespace kotlin;
+
+// on stack
+template<> void mm::RefAccessor<true>::beforeStore(ObjHeader*) noexcept {}
+template<> void mm::RefAccessor<true>::afterStore(ObjHeader*) noexcept {}
+template<> void mm::RefAccessor<true>::beforeLoad() noexcept {}
+template<> void mm::RefAccessor<true>::afterLoad() noexcept {}
+
+// on heap
+template<> void mm::RefAccessor<false>::beforeStore(ObjHeader*) noexcept {}
+template<> void mm::RefAccessor<false>::afterStore(ObjHeader*) noexcept {}
+template<> void mm::RefAccessor<false>::beforeLoad() noexcept {}
+template<> void mm::RefAccessor<false>::afterLoad() noexcept {}
+
+ALWAYS_INLINE OBJ_GETTER(mm::weakRefReadBarrier, std::atomic<ObjHeader*>& referee) noexcept {
+    RETURN_RESULT_OF(kotlin::gc::tryRef, referee);
+}

kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp

@@ -11,6 +11,7 @@
 #include "GC.hpp"
 #include "Memory.h"
 #include "RawPtr.hpp"
+#include "ReferenceOps.hpp"
 #include "ThreadRegistry.hpp"
 
 namespace kotlin::mm {
@@ -102,7 +103,7 @@ class SpecialRefRegistry : private Pinned {
 
         OBJ_GETTER0(tryRef) noexcept {
             AssertThreadState(ThreadState::kRunnable);
-            RETURN_RESULT_OF(kotlin::gc::tryRef, obj_);
+            RETURN_RESULT_OF(mm::weakRefReadBarrier, obj_);
         }
 
         void retainRef() noexcept {

kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp

@@ -22,12 +22,11 @@ testing::MockFunction<void(KRef)>* kotlin::test_support::internal::Kotlin_runUnh
 namespace {
 
 struct EmptyPayload {
-    using Field = ObjHeader* EmptyPayload::*;
+    static constexpr kotlin::test_support::NoRefFields<EmptyPayload> kFields{};
-    static constexpr std::array<Field, 0> kFields{};
 };
 
 kotlin::test_support::TypeInfoHolder theAnyTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
-kotlin::test_support::TypeInfoHolder theArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<ObjHeader*>()};
+kotlin::test_support::TypeInfoHolder theArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<kotlin::mm::RefField>()};
 kotlin::test_support::TypeInfoHolder theBooleanArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<KBoolean>()};
 kotlin::test_support::TypeInfoHolder theByteArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<KByte>()};
 kotlin::test_support::TypeInfoHolder theCharArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder<KChar>()};