diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp index 77515dedc61..87b4c064049 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactorySweepTest.cpp @@ -22,9 +22,9 @@ using namespace kotlin; namespace { struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields = { &Payload::field1, diff --git a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp index 3cf1e3b6d15..7127420ab87 100644 --- a/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp +++ b/kotlin-native/runtime/src/alloc/legacy/cpp/ObjectFactoryTest.cpp @@ -851,8 +851,8 @@ public: using ObjectFactory = alloc::ObjectFactory; struct Payload { - ObjHeader* field1; - ObjHeader* field2; + mm::RefField field1; + mm::RefField field2; static constexpr std::array kFields{ &Payload::field1, diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index f95b1d94878..4538a841f55 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -35,8 +35,8 @@ void processFieldInMark(void* state, ObjHeader* field) noexcept { template void processObjectInMark(void* state, ObjHeader* object) noexcept { - traverseClassObjectFields(object, [state] (ObjHeader** fieldLocation) noexcept { - if (auto field = *fieldLocation) { + traverseClassObjectFields(object, [state] (auto fieldAccessor) noexcept { + if (ObjHeader* field = fieldAccessor.direct()) { processFieldInMark(state, field); } }); @@ -44,8 +44,8 @@ void processObjectInMark(void* state, ObjHeader* object) noexcept { template void processArrayInMark(void* state, ArrayHeader* array) noexcept { - traverseArrayOfObjectsElements(array, [state] (ObjHeader** elemLocation) noexcept { - if (auto elem = *elemLocation) { + traverseArrayOfObjectsElements(array, [state] (auto elemAccessor) noexcept { + if (ObjHeader* elem = elemAccessor.direct()) { processFieldInMark(state, elem); } }); diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index ec5fd90cb23..d922c418d5b 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -22,9 +22,9 @@ using namespace kotlin; namespace { struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields = { &Payload::field1, diff --git a/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp index 7c45030cd8e..dfb1225bf68 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/TracingGCTest.hpp @@ -30,9 +30,9 @@ using namespace kotlin; struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields = { &Payload::field1, @@ -97,7 +97,7 @@ public: test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(location_->array()); } - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } + mm::RefField& operator[](size_t index) noexcept { return (**this).elements()[index]; } private: ObjHeader* location_ = nullptr; @@ -144,7 +144,7 @@ public: test_support::ObjectArray<3>& operator*() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } test_support::ObjectArray<3>& operator->() { return test_support::ObjectArray<3>::FromArrayHeader(holder_.obj()->array()); } - ObjHeader*& operator[](size_t index) noexcept { return (**this).elements()[index]; } + mm::RefField& operator[](size_t index) noexcept { return (**this).elements()[index]; } private: ObjHolder holder_; @@ -343,7 +343,7 @@ TYPED_TEST_P(TracingGCTest, FreeObjectWithHoldedWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); StackObjectHolder stack{threadData}; - auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); + auto& weak1 = InstallWeakReference(threadData, object1.header(), stack->field1.ptr()); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); ASSERT_THAT(gc::isMarked(object1.header()), false); diff --git a/kotlin-native/runtime/src/main/cpp/ExceptionsTest.cpp b/kotlin-native/runtime/src/main/cpp/ExceptionsTest.cpp index 127c828008a..fb25e036c2c 100644 --- a/kotlin-native/runtime/src/main/cpp/ExceptionsTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/ExceptionsTest.cpp @@ -31,8 +31,7 @@ namespace { struct Payload { int value = 0; - using Field = ObjHeader* Payload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; using Object = test_support::Object; diff --git a/kotlin-native/runtime/src/main/cpp/FinalizerHooksTest.cpp b/kotlin-native/runtime/src/main/cpp/FinalizerHooksTest.cpp index c2fdcc6fbff..e409185c0f8 100644 --- a/kotlin-native/runtime/src/main/cpp/FinalizerHooksTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/FinalizerHooksTest.cpp @@ -20,8 +20,7 @@ using ::testing::_; namespace { struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; class FinalizerHooksTest : public testing::Test { diff --git a/kotlin-native/runtime/src/main/cpp/FreezeHooksTest.cpp b/kotlin-native/runtime/src/main/cpp/FreezeHooksTest.cpp index 11193ae2b70..fa341700371 100644 --- a/kotlin-native/runtime/src/main/cpp/FreezeHooksTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/FreezeHooksTest.cpp @@ -19,8 +19,7 @@ using ::testing::_; namespace { struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; class FreezeHooksTest : public testing::Test { diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index ee2c1132277..32d5c8e45bd 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -191,6 +191,7 @@ struct type_layout::descriptor { }; }; + } // namespace kotlin ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj); @@ -292,10 +293,6 @@ enum class MemoryModel { // Controls the current memory model, is compile-time constant. extern const MemoryModel CurrentMemoryModel; -// Sets stack location. -void SetStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; -// Sets heap location. -void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Zeroes heap location. void ZeroHeapRef(ObjHeader** location) RUNTIME_NOTHROW; // Zeroes an array. @@ -315,18 +312,8 @@ OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) // Updates heap/static data in one array. void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, int toIndex, int count) RUNTIME_NOTHROW; // Updates location if it is null, atomically. -void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Updates reference in return slot. void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW; -// Compares and swaps reference with taken lock. -OBJ_GETTER(SwapHeapRefLocked, - ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, - int32_t* cookie) RUNTIME_NOTHROW; -// Sets reference with taken lock. -void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, - int32_t* cookie) RUNTIME_NOTHROW; -// Reads reference with taken lock. -OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) RUNTIME_NOTHROW; OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index); // Called on frame enter, if it has object slots. void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW; @@ -421,6 +408,7 @@ struct FrameOverlay { }; // Class holding reference to an object, holding object during C++ scope. +// TODO adopt ref accessors class ObjHolder { public: ObjHolder() : obj_(nullptr) { @@ -429,7 +417,7 @@ class ObjHolder { explicit ObjHolder(const ObjHeader* obj) { EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*)); - ::SetStackRef(slot(), obj); + ::UpdateStackRef(slot(), obj); } ~ObjHolder() { diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp index 4230aedc019..0d92d2294a9 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp @@ -11,6 +11,7 @@ #include "KAssert.h" #include "Memory.h" +#include "ReferenceOps.hpp" #include "TypeInfo.h" #include "Types.h" #include "Utils.hpp" @@ -20,6 +21,12 @@ namespace test_support { // TODO: Some concepts from here can be used in production code. +template +using RefFieldPtr = mm::RefField Host::*; + +template +using NoRefFields = std::array, 0>; + class TypeInfoHolder : private Pinned { private: class Builder { @@ -96,6 +103,9 @@ public: typeInfo_.instanceAlignment_ = builder.instanceAlignment_; } + template<> + class ArrayBuilder : public ArrayBuilder {}; + TypeInfo* typeInfo() noexcept { return &typeInfo_; } private: @@ -123,9 +133,9 @@ public: public: FieldIterator(Object& owner, size_t index) noexcept : owner_(owner), index_(index) {} - ObjHeader*& operator*() noexcept { + mm::RefField& operator*() noexcept { auto* header = &owner_.header_; - return *reinterpret_cast(reinterpret_cast(header) + header->type_info()->objOffsets_[index_]); + return *reinterpret_cast(reinterpret_cast(header) + header->type_info()->objOffsets_[index_]); } FieldIterator& operator++() noexcept { @@ -148,7 +158,7 @@ public: size_t size() const noexcept { return owner_.header_.type_info()->objOffsetsCount_; } - ObjHeader*& operator[](size_t index) noexcept { return *FieldIterator(owner_, index); } + mm::RefField& operator[](size_t index) noexcept { return *FieldIterator(owner_, index); } FieldIterator begin() noexcept { return FieldIterator(owner_, 0); } FieldIterator end() noexcept { return FieldIterator(owner_, size()); } @@ -190,8 +200,7 @@ TypeInfoHolder::ObjectBuilder::ObjectBuilder() noexcept { char c; Object& object = *reinterpret_cast*>(&c); auto& payload = *object; - using Field = ObjHeader* Payload::*; - for (Field field : Payload::kFields) { + for (RefFieldPtr field : Payload::kFields) { auto& actualField = payload.*field; objOffsets_.push_back(reinterpret_cast(&actualField) - reinterpret_cast(object.header())); } @@ -237,13 +246,13 @@ private: } // namespace internal template -class ObjectArray : public internal::Array { +class ObjectArray : public internal::Array { public: static ObjectArray& FromArrayHeader(ArrayHeader* arr) noexcept { - return static_cast&>(internal::Array::FromArrayHeader(arr)); + return static_cast&>(internal::Array::FromArrayHeader(arr)); } - ObjectArray() noexcept : internal::Array(theArrayTypeInfo) {} + ObjectArray() noexcept : internal::Array(theArrayTypeInfo) {} }; template @@ -350,8 +359,7 @@ struct RegularWeakReferenceImplPayload { void* weakRef; void* referred; - using Field = ObjHeader* RegularWeakReferenceImplPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader*); diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupportTest.cpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupportTest.cpp index 498dc6ccebd..1c7f957cae8 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupportTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupportTest.cpp @@ -18,9 +18,9 @@ using namespace kotlin; namespace { struct RegularPayload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields{ &RegularPayload::field1, @@ -31,11 +31,11 @@ struct RegularPayload { struct IrregularPayload { int skipBefore; - ObjHeader* field1; + mm::RefField field1; int skip; - ObjHeader* field2; + mm::RefField field2; std::array skipALot; - ObjHeader* field3; + mm::RefField field3; static constexpr std::array kFields{ &IrregularPayload::field1, @@ -70,8 +70,8 @@ using ObjectTestCases = testing::Types -std::vector Collect(test_support::Object& object) { - std::vector result; +std::vector Collect(test_support::Object& object) { + std::vector result; for (auto& field : object.fields()) { result.push_back(&field); } @@ -108,9 +108,9 @@ TYPED_TEST(ObjectTestSupportObjectTest, Local) { EXPECT_THAT(Collect(object), testing::ElementsAre(&object->field1, &object->field2, &object->field3)); - EXPECT_THAT(object.fields()[0], nullptr); - EXPECT_THAT(object.fields()[1], nullptr); - EXPECT_THAT(object.fields()[2], nullptr); + EXPECT_THAT(object.fields()[0].direct().load(), nullptr); + EXPECT_THAT(object.fields()[1].direct().load(), nullptr); + EXPECT_THAT(object.fields()[2].direct().load(), nullptr); auto& recoveredObject = test_support::Object::FromObjHeader(object.header()); EXPECT_THAT(&recoveredObject, &object); @@ -149,9 +149,9 @@ TYPED_TEST(ObjectTestSupportObjectTest, Heap) { EXPECT_THAT(Collect(object), testing::ElementsAre(&object->field1, &object->field2, &object->field3)); - EXPECT_THAT(object.fields()[0], nullptr); - EXPECT_THAT(object.fields()[1], nullptr); - EXPECT_THAT(object.fields()[2], nullptr); + EXPECT_THAT(object.fields()[0].direct().load(), nullptr); + EXPECT_THAT(object.fields()[1].direct().load(), nullptr); + EXPECT_THAT(object.fields()[2].direct().load(), nullptr); }); } @@ -161,7 +161,7 @@ template struct PayloadTraits; template <> -struct PayloadTraits { +struct PayloadTraits { template using Array = test_support::ObjectArray; static const TypeInfo* GetTypeInfo() { return theArrayTypeInfo; } @@ -284,8 +284,8 @@ public: template class ObjectTestSupportArrayTest : public testing::Test {}; using ArrayTestCases = testing::Types< - ArrayTestCase, - ArrayTestCase, + ArrayTestCase, + ArrayTestCase, ArrayTestCase, ArrayTestCase, ArrayTestCase, @@ -335,7 +335,7 @@ TYPED_TEST(ObjectTestSupportArrayTest, Local) { for (size_t i = 0; i < size; ++i) { auto* element = AddressOfElementAt(array.arrayHeader(), i); EXPECT_THAT(&array.elements()[i], element); - EXPECT_THAT(array.elements()[i], Payload{}); + EXPECT_TRUE(array.elements()[i] == Payload{}); expected.push_back(element); } @@ -366,7 +366,7 @@ TYPED_TEST(ObjectTestSupportArrayTest, Heap) { for (size_t i = 0; i < size; ++i) { auto* element = AddressOfElementAt(array.arrayHeader(), i); EXPECT_THAT(&array.elements()[i], element); - EXPECT_THAT(array.elements()[i], Payload{}); + EXPECT_TRUE(array.elements()[i] == Payload{}); expected.push_back(element); } diff --git a/kotlin-native/runtime/src/main/cpp/ReferenceOps.hpp b/kotlin-native/runtime/src/main/cpp/ReferenceOps.hpp new file mode 100644 index 00000000000..bd1731beb48 --- /dev/null +++ b/kotlin-native/runtime/src/main/cpp/ReferenceOps.hpp @@ -0,0 +1,205 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Memory.h" +#include "std_support/AtomicRef.hpp" + +// Concurrent GC may cause conflicting unordered accesses to references in heap. +// C++ memory model declares that accesses have data race unless they are atomic. +// Thus TSAN would report such non-atomic accesses. +// TODO find out if compiler may take advantage of the theoretical UB here. +// +// However, in practice all the ptr-sized loads and stores are atomic on CPU-level +// even if they are not std::atomic. +// And as far as we aware, +// std::atomic operations are not optimized by LLVM even if they have relaxed memory order. +// So we don't want to compile every heap reference access into std::atomic access. +#define ALWAYS_ATOMIC_REFS __has_feature(thread_sanitizer) + +namespace kotlin::mm { + +// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals. + +// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which +// will have these operations for a friendlier API. + +/** + * Represents direct low-level operations on Koltin references. + * No GC barriers are inserted. Should be used with care! + */ +class DirectRefAccessor { +public: + DirectRefAccessor() = delete; + DirectRefAccessor& operator=(const DirectRefAccessor&) = delete; + + explicit DirectRefAccessor(ObjHeader*& fieldRef) noexcept : ref_(fieldRef) {} + explicit DirectRefAccessor(ObjHeader** fieldPtr) noexcept : DirectRefAccessor(*fieldPtr) {} + DirectRefAccessor(const DirectRefAccessor& other) noexcept : DirectRefAccessor(other.ref_) {} + + ObjHeader** location() const noexcept { return &ref_; } + + ALWAYS_INLINE operator ObjHeader*() const noexcept { return load(); } + ALWAYS_INLINE ObjHeader* operator=(ObjHeader* desired) noexcept { store(desired); return desired; } + + ALWAYS_INLINE ObjHeader* load() const noexcept { +#if ALWAYS_ATOMIC_REFS + return loadAtomic(std::memory_order_relaxed); +#else + return ref_; +#endif + } + + ALWAYS_INLINE void store(ObjHeader* desired) noexcept { +#if ALWAYS_ATOMIC_REFS + storeAtomic(desired, std::memory_order_relaxed); +#else + ref_ = desired; +#endif + } + + ALWAYS_INLINE auto atomic() noexcept { + return std_support::atomic_ref{ref_}; + } + ALWAYS_INLINE auto atomic() const noexcept { + return std_support::atomic_ref{ref_}; + } + + ALWAYS_INLINE ObjHeader* loadAtomic(std::memory_order order) const noexcept { + return atomic().load(order); + } + ALWAYS_INLINE void storeAtomic(ObjHeader* desired, std::memory_order order) noexcept { + atomic().store(desired, order); + } + ALWAYS_INLINE ObjHeader* exchange(ObjHeader* desired, std::memory_order order) noexcept { + return atomic().exchange(desired, order); + } + ALWAYS_INLINE bool compareAndExchange(ObjHeader*& expected, ObjHeader* desired, std::memory_order order) noexcept { + return atomic().compare_exchange_strong(expected, desired, order); + } + +private: + ObjHeader*& ref_; +}; + +/** + * Represents Koltin-level operations on Koltin references. + * With all the necessary GC barriers etc. + * Prefer using aliases below. + */ +template +class RefAccessor { +public: + RefAccessor() = delete; + RefAccessor& operator=(const RefAccessor&) = delete; + + explicit RefAccessor(ObjHeader*& fieldRef) noexcept : direct_(fieldRef) {} + explicit RefAccessor(ObjHeader** fieldPtr) noexcept : RefAccessor(*fieldPtr) {} + RefAccessor(const RefAccessor& other) noexcept : direct_(other.direct_) {} + + DirectRefAccessor direct() const noexcept { return direct_; } + + void beforeLoad() noexcept; + void afterLoad() noexcept; + void beforeStore(ObjHeader* value) noexcept; + void afterStore(ObjHeader* value) noexcept; + + ALWAYS_INLINE operator ObjHeader*() noexcept { return load(); } + + ALWAYS_INLINE ObjHeader* load() noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeLoad(); + auto result = direct_.load(); + afterLoad(); + return result; + } + + ALWAYS_INLINE ObjHeader* loadAtomic(std::memory_order order) noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeLoad(); + auto result = direct_.loadAtomic(order); + afterLoad(); + return result; + } + + ALWAYS_INLINE ObjHeader* operator=(ObjHeader* desired) noexcept { store(desired); return desired; } + + ALWAYS_INLINE void store(ObjHeader* desired) noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeStore(desired); + direct_.store(desired); + afterStore(desired); + } + + ALWAYS_INLINE void storeAtomic(ObjHeader* desired, std::memory_order order) noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeStore(desired); + direct_.storeAtomic(desired, order); + afterStore(desired); + } + + ALWAYS_INLINE ObjHeader* exchange(ObjHeader* desired, std::memory_order order) noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeLoad(); + beforeStore(desired); + auto result = direct_.exchange(desired, order); + afterStore(desired); + afterLoad(); + return result; + } + + ALWAYS_INLINE bool compareAndExchange(ObjHeader*& expected, ObjHeader* desired, std::memory_order order) noexcept { + AssertThreadState(ThreadState::kRunnable); + beforeLoad(); + beforeStore(desired); + bool result = direct_.compareAndExchange(expected, desired, order); + afterStore(desired); + afterLoad(); + return result; + } + +private: + DirectRefAccessor direct_; +}; + +using RefFieldAccessor = RefAccessor; +using GlobalRefAccessor = RefAccessor; +using StackRefAccessor = RefAccessor; + +class RefField : private Pinned { +public: + auto accessor() noexcept { + return mm::RefFieldAccessor(value_); + } + auto direct() noexcept { + return accessor().direct(); + } + // FIXME probably most of the uses should instead use accessor + auto ptr() noexcept { + return direct().location(); + } + + // TODO consider adding other operations + ObjHeader* operator=(ObjHeader* value) noexcept { + accessor() = value; + return value_; + } + + bool operator==(const RefField& other) const noexcept { + return value_ == other.value_; + } + + bool operator!=(const RefField& other) const noexcept { + return !operator==(other); + } + +private: + ObjHeader* value_ = nullptr; +}; + +OBJ_GETTER(weakRefReadBarrier, std::atomic& referee) noexcept; + +} diff --git a/kotlin-native/runtime/src/main/cpp/TypeInfoTest.cpp b/kotlin-native/runtime/src/main/cpp/TypeInfoTest.cpp index da376b6c0fa..937b9b10670 100644 --- a/kotlin-native/runtime/src/main/cpp/TypeInfoTest.cpp +++ b/kotlin-native/runtime/src/main/cpp/TypeInfoTest.cpp @@ -16,13 +16,12 @@ using namespace kotlin; namespace { struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; struct Payload1 { - ObjHeader* field1; - ObjHeader* field2; + mm::RefField field1; + mm::RefField field2; static constexpr std::array kFields{ &Payload1::field1, @@ -31,8 +30,8 @@ struct Payload1 { }; struct Payload2 { - ObjHeader* field1; - ObjHeader* field2; + mm::RefField field1; + mm::RefField field2; static constexpr std::array kFields{ &Payload2::field1, diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index ce9aac794d6..e638c89c025 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -19,8 +19,7 @@ using namespace kotlin; namespace { struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; class ExtraObjectDataTest : public testing::Test { diff --git a/kotlin-native/runtime/src/mm/cpp/FreezingTest.cpp b/kotlin-native/runtime/src/mm/cpp/FreezingTest.cpp index 9bb776222bc..95db5ee9268 100644 --- a/kotlin-native/runtime/src/mm/cpp/FreezingTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/FreezingTest.cpp @@ -29,14 +29,13 @@ struct NoFreezeHook { }; struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields{ &Payload::field1, @@ -66,7 +65,7 @@ public: ObjHeader* header() { return object_.header(); } - ObjHeader*& operator[](size_t field) { return object_.fields()[field]; } + mm::RefField& operator[](size_t field) { return object_.fields()[field]; } void MakePermanent() { header()->typeInfoOrMeta_ = setPointerBits(header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); } @@ -89,7 +88,7 @@ public: ObjHeader* header() { return array_.header(); } - ObjHeader*& operator[](size_t index) { return array_.elements()[index]; } + mm::RefField& operator[](size_t index) { return array_.elements()[index]; } void MakePermanent() { header()->typeInfoOrMeta_ = setPointerBits(header()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); } diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index cc5be1f1d07..d08b33d89c3 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -16,6 +16,7 @@ #include "Natives.h" #include "ObjectOps.hpp" #include "Porting.h" +#include "ReferenceOps.hpp" #include "Runtime.h" #include "SafePoint.hpp" #include "StableRef.hpp" @@ -144,63 +145,51 @@ extern "C" RUNTIME_NOTHROW void InitAndRegisterGlobal(ObjHeader** location, cons mm::GlobalsRegistry::Instance().RegisterStorageForGlobal(threadData, location); // Null `initialValue` means that the appropriate value was already set by static initialization. if (initialValue != nullptr) { - mm::SetHeapRef(location, const_cast(initialValue)); + UpdateHeapRef(location, const_cast(initialValue)); } } extern "C" const MemoryModel CurrentMemoryModel = MemoryModel::kExperimental; -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) { - mm::SetStackRef(location, const_cast(object)); -} - -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) { - mm::SetHeapRef(location, const_cast(object)); -} - extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) { - mm::SetHeapRef(location, nullptr); + mm::RefAccessor{location} = nullptr; } extern "C" RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) { for (uint32_t index = 0; index < array->count_; ++index) { ObjHeader** location = ArrayAddressOfElementAt(array, index); - mm::SetHeapRef(location, nullptr); + mm::RefFieldAccessor{location} = nullptr; } } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) { - mm::SetStackRef(location, nullptr); + mm::StackRefAccessor{location} = nullptr; } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) { - mm::SetStackRef(location, const_cast(object)); + mm::StackRefAccessor{location} = const_cast(object); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) { - mm::SetHeapRef(location, const_cast(object)); + mm::RefAccessor{location} = const_cast(object); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateVolatileHeapRef(ObjHeader** location, const ObjHeader* object) { - mm::SetHeapRefAtomicSeqCst(location, const_cast(object)); + mm::RefAccessor{location}.storeAtomic(const_cast(object), std::memory_order_seq_cst); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(CompareAndSwapVolatileHeapRef, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { - RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue); + ObjHeader* actual = expectedValue; + mm::RefAccessor{location}.compareAndExchange(actual, newValue, std::memory_order_seq_cst); + RETURN_OBJ(actual); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW bool CompareAndSetVolatileHeapRef(ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue) { - return mm::CompareAndSetHeapRef(location, expectedValue, newValue); + return mm::RefAccessor{location}.compareAndExchange(expectedValue, newValue, std::memory_order_seq_cst); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(GetAndSetVolatileHeapRef, ObjHeader** location, ObjHeader* newValue) { - RETURN_RESULT_OF(mm::GetAndSetHeapRef, location, newValue); -} - -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) { - if (object == nullptr) return; - ObjHeader* result = nullptr; // No need to store this value in a rootset. - mm::CompareAndSwapHeapRef(location, nullptr, const_cast(object), &result); + RETURN_OBJ(mm::RefAccessor{location}.exchange(newValue, std::memory_order_seq_cst)); } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const ArrayHeader* array, int fromIndex, @@ -209,22 +198,10 @@ extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateHeapRefsInsideOneArray(const } extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { - mm::SetStackRef(returnSlot, const_cast(object)); + UpdateStackRef(returnSlot, object); } -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER( - SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { - RETURN_RESULT_OF(mm::CompareAndSwapHeapRef, location, expectedValue, newValue); -} -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW void SetHeapRefLocked( - ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) { - mm::SetHeapRefAtomic(location, newValue); -} - -extern "C" ALWAYS_INLINE RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) { - RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location); -} extern "C" OBJ_GETTER(ReadHeapRefNoLock, ObjHeader* object, int32_t index) { // TODO: Remove when legacy MM is gone. @@ -513,7 +490,7 @@ extern "C" RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void* pointer) { AssertThreadState(ThreadState::kRunnable); mm::StableRef stableRef(static_cast(pointer)); auto* obj = *stableRef; - mm::SetStackRef(OBJ_RESULT, obj); + UpdateStackRef(OBJ_RESULT, obj); std::move(stableRef).dispose(); return obj; } diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp index 6e6307ab463..9520f1c3b33 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.cpp @@ -11,66 +11,6 @@ using namespace kotlin; -// TODO: Memory barriers. - -ALWAYS_INLINE void mm::SetStackRef(ObjHeader** location, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - *location = value; -} - -ALWAYS_INLINE void mm::SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - *location = value; -} - -#pragma clang diagnostic push -// On 32-bit android arm clang warns of significant performance penalty because of large -// atomic operations. TODO: Consider using alternative ways of ordering memory operations if they -// turn out to be more efficient on these platforms. -#pragma clang diagnostic ignored "-Watomic-alignment" - -ALWAYS_INLINE void mm::SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - __atomic_store_n(location, value, __ATOMIC_RELEASE); -} - -ALWAYS_INLINE void mm::SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - __atomic_store_n(location, value, __ATOMIC_SEQ_CST); -} - - -ALWAYS_INLINE OBJ_GETTER(mm::ReadHeapRefAtomic, ObjHeader** location) noexcept { - AssertThreadState(ThreadState::kRunnable); - // TODO: Make this work with GCs that can stop thread at any point. - auto result = __atomic_load_n(location, __ATOMIC_ACQUIRE); - RETURN_OBJ(result); -} - -ALWAYS_INLINE OBJ_GETTER(mm::CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - // TODO: Make this work with GCs that can stop thread at any point. - ObjHeader* actual = expected; - __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); - RETURN_OBJ(actual); -} - -ALWAYS_INLINE bool mm::CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable); - // TODO: Make this work with GCs that can stop thread at any point. - ObjHeader* actual = expected; - return __atomic_compare_exchange_n(location, &actual, value, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} - -ALWAYS_INLINE OBJ_GETTER(mm::GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept { - AssertThreadState(ThreadState::kRunnable);; - auto *actual = __atomic_exchange_n(location, value, __ATOMIC_SEQ_CST); - RETURN_OBJ(actual); -} - - -#pragma clang diagnostic pop - OBJ_GETTER(mm::AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept { AssertThreadState(threadData, ThreadState::kRunnable); // TODO: Make this work with GCs that can stop thread at any point. diff --git a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp index e26fc66220e..0dd8e80fcf2 100644 --- a/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectOps.hpp @@ -1,5 +1,5 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ @@ -13,22 +13,9 @@ namespace mm { class ThreadData; -// TODO: Make sure these operations work with any kind of thread stopping: safepoints and signals. - -// TODO: Consider adding some kind of an `Object` type (that wraps `ObjHeader*`) which -// will have these operations for a friendlier API. - // TODO: `OBJ_GETTER` is used because the returned objects needs to be accessible via the rootset before the function // returns. If we had a different way to efficiently keep the object in the roots, `OBJ_GETTER` can be removed. -void SetStackRef(ObjHeader** location, ObjHeader* value) noexcept; -void SetHeapRef(ObjHeader** location, ObjHeader* value) noexcept; -void SetHeapRefAtomic(ObjHeader** location, ObjHeader* value) noexcept; -void SetHeapRefAtomicSeqCst(ObjHeader** location, ObjHeader* value) noexcept; -OBJ_GETTER(ReadHeapRefAtomic, ObjHeader** location) noexcept; -OBJ_GETTER(CompareAndSwapHeapRef, ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; -bool CompareAndSetHeapRef(ObjHeader** location, ObjHeader* expected, ObjHeader* value) noexcept; -OBJ_GETTER(GetAndSetHeapRef, ObjHeader** location, ObjHeader* value) noexcept; OBJ_GETTER(AllocateObject, ThreadData* threadData, const TypeInfo* typeInfo) noexcept; OBJ_GETTER(AllocateArray, ThreadData* threadData, const TypeInfo* typeInfo, uint32_t elements) noexcept; diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTraversal.hpp b/kotlin-native/runtime/src/mm/cpp/ObjectTraversal.hpp similarity index 60% rename from kotlin-native/runtime/src/main/cpp/ObjectTraversal.hpp rename to kotlin-native/runtime/src/mm/cpp/ObjectTraversal.hpp index 5fb85c11b67..5d69e00f944 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTraversal.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectTraversal.hpp @@ -1,5 +1,5 @@ /* - * Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ @@ -10,49 +10,54 @@ #include "Memory.h" #include "Natives.h" +#include "ReferenceOps.hpp" #include "Types.h" +#include "ObjectOps.hpp" namespace kotlin { // TODO: Consider an iterator/ranges based approaches for traversals. template -ALWAYS_INLINE void traverseClassObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval()))) { +ALWAYS_INLINE void traverseClassObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval()))) { const TypeInfo* typeInfo = object->type_info(); RuntimeAssert(typeInfo != theArrayTypeInfo, "Must not be an array of objects"); for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { - process(reinterpret_cast(reinterpret_cast(object) + typeInfo->objOffsets_[index])); + auto fieldPtr = reinterpret_cast(reinterpret_cast(object) + typeInfo->objOffsets_[index]); + process(mm::RefFieldAccessor(fieldPtr)); } } template -ALWAYS_INLINE void traverseArrayOfObjectsElements(ArrayHeader* array, F process) noexcept(noexcept(process(std::declval()))) { +ALWAYS_INLINE void traverseArrayOfObjectsElements(ArrayHeader* array, F process) noexcept(noexcept(process(std::declval()))) { RuntimeAssert(array->type_info() == theArrayTypeInfo, "Must be an array of objects"); for (uint32_t index = 0; index < array->count_; index++) { - process(ArrayAddressOfElementAt(array, index)); + process(mm::RefFieldAccessor(ArrayAddressOfElementAt(array, index))); } } template -void traverseObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval()))) { +void traverseObjectFields(ObjHeader* object, F process) noexcept(noexcept(process(std::declval()))) { const TypeInfo* typeInfo = object->type_info(); // Only consider arrays of objects, not arrays of primitives. if (typeInfo != theArrayTypeInfo) { for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { - process(reinterpret_cast(reinterpret_cast(object) + typeInfo->objOffsets_[index])); + auto fieldPtr = reinterpret_cast(reinterpret_cast(object) + typeInfo->objOffsets_[index]); + process(mm::RefFieldAccessor(fieldPtr)); } } else { ArrayHeader* array = object->array(); for (uint32_t index = 0; index < array->count_; index++) { - process(ArrayAddressOfElementAt(array, index)); + process(mm::RefFieldAccessor(ArrayAddressOfElementAt(array, index))); } } } +// FIXME explicitly mention no barriers template void traverseReferredObjects(ObjHeader* object, F process) noexcept(noexcept(process(std::declval()))) { - traverseObjectFields(object, [&process](ObjHeader** location) noexcept(noexcept(process(std::declval()))) { - if (ObjHeader* ref = *location) { + traverseObjectFields(object, [&process](auto accessor) noexcept(noexcept(process(std::declval()))) { + if (ObjHeader* ref = accessor.direct()) { process(ref); } }); diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTraversalTest.cpp b/kotlin-native/runtime/src/mm/cpp/ObjectTraversalTest.cpp similarity index 65% rename from kotlin-native/runtime/src/main/cpp/ObjectTraversalTest.cpp rename to kotlin-native/runtime/src/mm/cpp/ObjectTraversalTest.cpp index 903d776f3cb..78c33942641 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTraversalTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ObjectTraversalTest.cpp @@ -9,8 +9,10 @@ #include "gtest/gtest.h" #include "ObjectTestSupport.hpp" +#include "ReferenceOps.hpp" #include "Types.h" #include "Utils.hpp" +#include "ObjectOps.hpp" using namespace kotlin; @@ -18,25 +20,14 @@ using ::testing::_; namespace { -struct CallableWithExceptions { - void operator()(ObjHeader*) noexcept(false) {} - void operator()(ObjHeader**) noexcept(false) {} -}; - -struct CallableWithoutExceptions { - void operator()(ObjHeader*) noexcept {} - void operator()(ObjHeader**) noexcept {} -}; - struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr test_support::NoRefFields kFields{}; }; struct Payload { - ObjHeader* field1; - ObjHeader* field2; - ObjHeader* field3; + mm::RefField field1; + mm::RefField field2; + mm::RefField field3; static constexpr std::array kFields{ &Payload::field1, @@ -45,9 +36,22 @@ struct Payload { }; }; +using ProcessFunMock = testing::StrictMock>; + +MATCHER_P(SameAccessor, accessor, "") { + return arg.direct().location() == accessor.direct().location(); +} + } // namespace TEST(ObjectTraversalTest, TraverseFieldsExceptions) { + struct CallableWithExceptions { + void operator()(mm::RefFieldAccessor) noexcept(false) {} + }; + struct CallableWithoutExceptions { + void operator()(mm::RefFieldAccessor) noexcept {} + }; + static_assert( noexcept(traverseObjectFields(std::declval(), std::declval())), "Callable is noexcept, so traverse is noexcept"); @@ -59,10 +63,10 @@ TEST(ObjectTraversalTest, TraverseFieldsExceptions) { TEST(ObjectTraversalTest, TraverseEmptyObjectFields) { test_support::TypeInfoHolder type{test_support::TypeInfoHolder::ObjectBuilder()}; test_support::Object object(type.typeInfo()); - testing::StrictMock> process; + ProcessFunMock process; EXPECT_CALL(process, Call(_)).Times(0); - traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); }); + traverseObjectFields(object.header(), process.AsStdFunction()); } TEST(ObjectTraversalTest, TraverseObjectFields) { @@ -70,14 +74,14 @@ TEST(ObjectTraversalTest, TraverseObjectFields) { ObjHeader field1; ObjHeader field3; test_support::Object object(type.typeInfo()); - object->field1 = &field1; - object->field3 = &field3; - testing::StrictMock> process; + object->field1.direct() = &field1; + object->field3.direct() = &field3; + ProcessFunMock process; - EXPECT_CALL(process, Call(&object->field1)); - EXPECT_CALL(process, Call(&object->field2)); - EXPECT_CALL(process, Call(&object->field3)); - traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); }); + EXPECT_CALL(process, Call(SameAccessor(object->field1.accessor()))); + EXPECT_CALL(process, Call(SameAccessor(object->field2.accessor()))); + EXPECT_CALL(process, Call(SameAccessor(object->field3.accessor()))); + traverseObjectFields(object.header(), process.AsStdFunction()); } TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) { @@ -88,16 +92,16 @@ TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) { ObjHeader field2; ObjHeader field3; test_support::Object object(type.typeInfo()); - object->field1 = &field1; - object->field2 = &field2; - object->field3 = &field3; - testing::StrictMock> process; + object->field1.direct() = &field1; + object->field2.direct() = &field2; + object->field3.direct() = &field3; + ProcessFunMock process; - EXPECT_CALL(process, Call(&object->field1)); - EXPECT_CALL(process, Call(&object->field2)).WillOnce([]() { throw kException; }); - EXPECT_CALL(process, Call(&object->field3)).Times(0); + EXPECT_CALL(process, Call(SameAccessor(object->field1.accessor()))); + EXPECT_CALL(process, Call(SameAccessor(object->field2.accessor()))).WillOnce([]() { throw kException; }); + EXPECT_CALL(process, Call(SameAccessor(object->field3.accessor()))).Times(0); try { - traverseObjectFields(object.header(), [&process](ObjHeader** field) { process.Call(field); }); + traverseObjectFields(object.header(), process.AsStdFunction()); } catch (int exception) { EXPECT_THAT(exception, kException); } catch (...) { @@ -107,24 +111,24 @@ TEST(ObjectTraversalTest, TraverseObjectFieldsWithException) { TEST(ObjectTraversalTest, TraverseEmptyArrayFields) { test_support::ObjectArray<0> array; - testing::StrictMock> process; + ProcessFunMock process; EXPECT_CALL(process, Call(_)).Times(0); - traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); }); + traverseObjectFields(array.header(), process.AsStdFunction()); } TEST(ObjectTraversalTest, TraverseArrayFields) { ObjHeader element1; ObjHeader element3; test_support::ObjectArray<3> array; - array.elements()[0] = &element1; - array.elements()[2] = &element3; - testing::StrictMock> process; + array.elements()[0].direct() = &element1; + array.elements()[2].direct() = &element3; + ProcessFunMock process; - EXPECT_CALL(process, Call(&array.elements()[0])); - EXPECT_CALL(process, Call(&array.elements()[1])); - EXPECT_CALL(process, Call(&array.elements()[2])); - traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); }); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[0].accessor()))); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[1].accessor()))); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[2].accessor()))); + traverseObjectFields(array.header(), process.AsStdFunction()); } TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) { @@ -134,16 +138,16 @@ TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) { ObjHeader element2; ObjHeader element3; test_support::ObjectArray<3> array; - array.elements()[0] = &element1; - array.elements()[1] = &element2; - array.elements()[2] = &element3; - testing::StrictMock> process; + array.elements()[0].direct() = &element1; + array.elements()[1].direct() = &element2; + array.elements()[2].direct() = &element3; + ProcessFunMock process; - EXPECT_CALL(process, Call(&array.elements()[0])); - EXPECT_CALL(process, Call(&array.elements()[1])).WillOnce([]() { throw kException; }); - EXPECT_CALL(process, Call(&array.elements()[2])).Times(0); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[0].accessor()))); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[1].accessor()))).WillOnce([]() { throw kException; }); + EXPECT_CALL(process, Call(SameAccessor(array.elements()[2].accessor()))).Times(0); try { - traverseObjectFields(array.header(), [&process](ObjHeader** field) { process.Call(field); }); + traverseObjectFields(array.header(), process.AsStdFunction()); } catch (int exception) { EXPECT_THAT(exception, kException); } catch (...) { @@ -152,6 +156,13 @@ TEST(ObjectTraversalTest, TraverseArrayFieldsWithException) { } TEST(ObjectTraversalTest, TraverseRefsExceptions) { + struct CallableWithExceptions { + void operator()(ObjHeader*) noexcept(false) {} + }; + struct CallableWithoutExceptions { + void operator()(ObjHeader*) noexcept {} + }; + static_assert( noexcept(traverseReferredObjects(std::declval(), std::declval())), "Callable is noexcept, so traverse is noexcept"); @@ -174,8 +185,8 @@ TEST(ObjectTraversalTest, TraverseObjectRefs) { ObjHeader field1; ObjHeader field3; test_support::Object object(type.typeInfo()); - object->field1 = &field1; - object->field3 = &field3; + object->field1.direct() = &field1; + object->field3.direct() = &field3; testing::StrictMock> process; EXPECT_CALL(process, Call(&field1)); @@ -191,9 +202,9 @@ TEST(ObjectTraversalTest, TraverseObjectRefsWithException) { ObjHeader field2; ObjHeader field3; test_support::Object object(type.typeInfo()); - object->field1 = &field1; - object->field2 = &field2; - object->field3 = &field3; + object->field1.direct() = &field1; + object->field2.direct() = &field2; + object->field3.direct() = &field3; testing::StrictMock> process; EXPECT_CALL(process, Call(&field1)); @@ -220,8 +231,8 @@ TEST(ObjectTraversalTest, TraverseArrayRefs) { ObjHeader element1; ObjHeader element3; test_support::ObjectArray<3> array; - array.elements()[0] = &element1; - array.elements()[2] = &element3; + array.elements()[0].direct() = &element1; + array.elements()[2].direct() = &element3; testing::StrictMock> process; EXPECT_CALL(process, Call(&element1)); @@ -236,9 +247,9 @@ TEST(ObjectTraversalTest, TraverseArrayRefsWithException) { ObjHeader element2; ObjHeader element3; test_support::ObjectArray<3> array; - array.elements()[0] = &element1; - array.elements()[1] = &element2; - array.elements()[2] = &element3; + array.elements()[0].direct() = &element1; + array.elements()[1].direct() = &element2; + array.elements()[2].direct() = &element3; testing::StrictMock> process; EXPECT_CALL(process, Call(&element1)); diff --git a/kotlin-native/runtime/src/mm/cpp/ReferenceOps.cpp b/kotlin-native/runtime/src/mm/cpp/ReferenceOps.cpp new file mode 100644 index 00000000000..7dd3317ebe9 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ReferenceOps.cpp @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ReferenceOps.hpp" +#include "ThreadData.hpp" +#include "ThreadRegistry.hpp" + +using namespace kotlin; + +// on stack +template<> void mm::RefAccessor::beforeStore(ObjHeader*) noexcept {} +template<> void mm::RefAccessor::afterStore(ObjHeader*) noexcept {} +template<> void mm::RefAccessor::beforeLoad() noexcept {} +template<> void mm::RefAccessor::afterLoad() noexcept {} + +// on heap +template<> void mm::RefAccessor::beforeStore(ObjHeader*) noexcept {} +template<> void mm::RefAccessor::afterStore(ObjHeader*) noexcept {} +template<> void mm::RefAccessor::beforeLoad() noexcept {} +template<> void mm::RefAccessor::afterLoad() noexcept {} + +ALWAYS_INLINE OBJ_GETTER(mm::weakRefReadBarrier, std::atomic& referee) noexcept { + RETURN_RESULT_OF(kotlin::gc::tryRef, referee); +} diff --git a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp index d42712a54f4..f095765a281 100644 --- a/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp +++ b/kotlin-native/runtime/src/mm/cpp/SpecialRefRegistry.hpp @@ -11,6 +11,7 @@ #include "GC.hpp" #include "Memory.h" #include "RawPtr.hpp" +#include "ReferenceOps.hpp" #include "ThreadRegistry.hpp" namespace kotlin::mm { @@ -102,7 +103,7 @@ class SpecialRefRegistry : private Pinned { OBJ_GETTER0(tryRef) noexcept { AssertThreadState(ThreadState::kRunnable); - RETURN_RESULT_OF(kotlin::gc::tryRef, obj_); + RETURN_RESULT_OF(mm::weakRefReadBarrier, obj_); } void retainRef() noexcept { diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index d008820b8d7..f7b9e2d8389 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -22,12 +22,11 @@ testing::MockFunction* kotlin::test_support::internal::Kotlin_runUnh namespace { struct EmptyPayload { - using Field = ObjHeader* EmptyPayload::*; - static constexpr std::array kFields{}; + static constexpr kotlin::test_support::NoRefFields kFields{}; }; kotlin::test_support::TypeInfoHolder theAnyTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; -kotlin::test_support::TypeInfoHolder theArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; +kotlin::test_support::TypeInfoHolder theArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; kotlin::test_support::TypeInfoHolder theBooleanArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; kotlin::test_support::TypeInfoHolder theByteArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()}; kotlin::test_support::TypeInfoHolder theCharArrayTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ArrayBuilder()};