diff --git a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp index 9b037760c7b..0e2b288e235 100644 --- a/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp +++ b/kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp @@ -56,7 +56,7 @@ bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStackClearWeakReferenceCounter(); + extraObject->ClearRegularWeakReferenceImpl(); if (extraObject->HasAssociatedObject()) { extraObject->DetachAssociatedObject(); extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); diff --git a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp index 1786c624691..281607ed4e3 100644 --- a/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp @@ -42,20 +42,8 @@ struct Payload { }; }; -// TODO: This should go into test support for weak references. -struct WeakCounterPayload { - void* referred; - KInt lock; - KInt cookie; - - static constexpr std::array kFields{}; -}; - -using WeakCounter = test_support::Object; - test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; -test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder()}; // TODO: Clean GlobalObjectHolder after it's gone. class GlobalObjectHolder : private Pinned { @@ -204,14 +192,14 @@ bool IsMarked(ObjHeader* objHeader) { return nodeRef.ObjectData().marked(); } -WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { - mm::AllocateObject(&threadData, typeHolderWeakCounter.typeInfo(), location); - auto& weakCounter = WeakCounter::FromObjHeader(*location); +test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { + mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); + auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); - auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header()); - EXPECT_EQ(setCounter, weakCounter.header()); - weakCounter->referred = objHeader; - return weakCounter; + weakReference->referred = objHeader; + auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header()); + EXPECT_EQ(setWeakRef, weakReference.header()); + return weakReference; } class ConcurrentMarkAndSweepTest : public testing::TestWithParam { @@ -351,15 +339,15 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) { TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); - auto& weak1 = ([&threadData, &object1]() -> WeakCounter& { + auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& { ObjHolder holder; - return InstallWeakCounter(threadData, object1.header(), holder.slot()); + return InstallWeakReference(threadData, object1.header(), holder.slot()); })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(weak1.header()), false); - ASSERT_THAT(weak1->referred, object1.header()); + ASSERT_THAT(weak1.get(), object1.header()); threadData.gc().ScheduleAndWaitFullGCWithFinalizers(); @@ -371,18 +359,18 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); StackObjectHolder stack{threadData}; - auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1); + auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(weak1.header()), false); - ASSERT_THAT(weak1->referred, object1.header()); + ASSERT_THAT(weak1.get(), object1.header()); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(IsMarked(weak1.header()), false); - EXPECT_THAT(weak1->referred, nullptr); + EXPECT_THAT(weak1.get(), nullptr); }); } @@ -971,16 +959,16 @@ TEST_P(ConcurrentMarkAndSweepTest, CrossThreadReference) { TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { std_support::vector mutators(kDefaultThreadCount); ObjHeader* globalRoot = nullptr; - WeakCounter* weak = nullptr; + test_support::RegularWeakReferenceImpl* weak = nullptr; mutators[0] .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { auto& global = mutator.AddGlobalRoot(); auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> WeakCounter& { + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { ObjHolder holder; - return InstallWeakCounter(threadData, object.header(), holder.slot()); + return InstallWeakReference(threadData, object.header(), holder.slot()); })(); global->field1 = objectWeak.header(); weak = &objectWeak; @@ -997,7 +985,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGC(); - EXPECT_THAT((*weak)->referred, nullptr); + EXPECT_THAT(weak->get(), nullptr); }); // Spin until thread suspension is requested. @@ -1007,7 +995,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) { for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); - EXPECT_THAT((*weak)->referred, nullptr); + EXPECT_THAT(weak->get(), nullptr); }); } @@ -1105,7 +1093,7 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) { TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { std_support::vector mutators(2); std::atomic*> object1 = nullptr; - std::atomic weak = nullptr; + std::atomic weak = nullptr; std::atomic done = false; auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { GlobalObjectHolder global1{threadData}; @@ -1120,7 +1108,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(IsMarked(object1_local.header()), false); ASSERT_THAT(IsMarked(weak.load()->header()), false); - ASSERT_THAT((*weak.load())->referred, object1_local.header()); + ASSERT_THAT(weak.load()->get(), object1_local.header()); global1->field1 = nullptr; @@ -1133,7 +1121,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { while (object1.load() == nullptr) {} ObjHolder holder; - auto &weak_local = InstallWeakCounter(threadData, object1.load()->header(), holder.slot()); + auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot()); weak = &weak_local; *holder.slot() = nullptr; while (!done) threadData.gc().SafePointLoopBody(); diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp index 333f7304359..caf0af2322c 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp @@ -74,16 +74,16 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc // TODO: Consider making it noinline to keep loop in `Mark` small. template void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQueue, mm::ExtraObjectData& extraObjectData, ObjHeader* object) noexcept { - if (auto weakCounter = extraObjectData.GetWeakReferenceCounter()) { + if (auto weakReference = extraObjectData.GetRegularWeakReferenceImpl()) { RuntimeAssert( - weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", object, weakCounter, - weakCounter->permanent(), weakCounter->local()); - // Do not schedule WeakReferenceCounter but process it right away. + weakReference->heap(), "Weak reference must be a heap object. object=%p weak=%p permanent=%d local=%d", object, + weakReference, weakReference->permanent(), weakReference->local()); + // Do not schedule RegularWeakReferenceImpl but process it right away. // This will skip markQueue interaction. - if (Traits::tryMark(weakCounter)) { - markHandle.addObject(mm::GetAllocatedHeapSize(weakCounter)); - // WeakReferenceCounter is empty, but keeping this just in case. - Traits::processInMark(markQueue, weakCounter); + if (Traits::tryMark(weakReference)) { + markHandle.addObject(mm::GetAllocatedHeapSize(weakReference)); + // RegularWeakReferenceImpl is empty, but keeping this just in case. + Traits::processInMark(markQueue, weakReference); } } } @@ -114,7 +114,7 @@ void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::It for (auto it = factoryIter.begin(); it != factoryIter.end();) { auto &extraObject = *it; if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) { - extraObject.ClearWeakReferenceCounter(); + extraObject.ClearRegularWeakReferenceImpl(); if (extraObject.HasAssociatedObject()) { extraObject.DetachAssociatedObject(); extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE); diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp index 689f823c696..3cfeeb49f3b 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp @@ -35,77 +35,40 @@ struct Payload { test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; -// TODO: This base might belong in `test_support` -class BaseObject { +void InstallWeakReference(test_support::Any& object, test_support::RegularWeakReferenceImpl& weakRef) noexcept { + auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(object.header()); + weakRef->referred = object.header(); + auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(object.header(), weakRef.header()); + EXPECT_EQ(setWeakRef, weakRef.header()); + EXPECT_EQ(extraObjectData.GetBaseObject(), object.header()); +} + +void Finalize(test_support::Any& object) noexcept { + if (auto* extraObjectData = mm::ExtraObjectData::Get(object.header())) { + extraObjectData->ClearRegularWeakReferenceImpl(); + } + RunFinalizers(object.header()); +} + +class Object : public test_support::Object { public: - enum class Kind { - kPermanent, - kStackLocal, - kHeapLike // Treated as heap object for the purposes of the test. - }; + Object() : test_support::Object(typeHolder.typeInfo()) {} - virtual ObjHeader* GetObjHeader() = 0; - - void InstallExtraData() { mm::ExtraObjectData::Install(GetObjHeader()); } - - void InstallWeakCounter(BaseObject& counter) { - auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(GetObjHeader()); - auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(GetObjHeader(), counter.GetObjHeader()); - EXPECT_EQ(setCounter, counter.GetObjHeader()); - EXPECT_EQ(extraObjectData.GetBaseObject(), GetObjHeader()); - } - -protected: - void SetKind(Kind kind) { - switch (kind) { - case Kind::kPermanent: - GetObjHeader()->typeInfoOrMeta_ = setPointerBits(GetObjHeader()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); - RuntimeAssert(GetObjHeader()->permanent(), "Must be permanent"); - break; - case Kind::kHeapLike: - RuntimeAssert(GetObjHeader()->heap(), "Must be heap"); - break; - case Kind::kStackLocal: - GetObjHeader()->typeInfoOrMeta_ = setPointerBits(GetObjHeader()->typeInfoOrMeta_, - OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER); - RuntimeAssert(GetObjHeader()->local(), "Must be stack local"); - break; - } - } - - void Finalize() { - if (auto* extraObjectData = mm::ExtraObjectData::Get(GetObjHeader())) { - extraObjectData->ClearWeakReferenceCounter(); - } - RunFinalizers(GetObjHeader()); - } + ~Object() { Finalize(*this); } }; -class Object : public BaseObject, public test_support::Object { +class ObjectArray : public test_support::ObjectArray<3> { public: - explicit Object(Kind kind = Kind::kHeapLike) : test_support::Object(typeHolder.typeInfo()) { SetKind(kind); } + ObjectArray() : test_support::ObjectArray<3>() {} - ~Object() { Finalize(); } - - ObjHeader* GetObjHeader() override { return header(); } + ~ObjectArray() { Finalize(*this); } }; -class ObjectArray : public BaseObject, public test_support::ObjectArray<3> { +class CharArray : public test_support::CharArray<3> { public: - explicit ObjectArray(Kind kind = Kind::kHeapLike) : test_support::ObjectArray<3>() { SetKind(kind); } + CharArray() : test_support::CharArray<3>() {} - ~ObjectArray() { Finalize(); } - - ObjHeader* GetObjHeader() override { return header(); } -}; - -class CharArray : public BaseObject, public test_support::CharArray<3> { -public: - explicit CharArray(Kind kind = Kind::kHeapLike) : test_support::CharArray<3>() { SetKind(kind); } - - ~CharArray() { Finalize(); } - - ObjHeader* GetObjHeader() override { return header(); } + ~CharArray() { Finalize(*this); } }; class ScopedMarkTraits : private Pinned { @@ -169,17 +132,17 @@ class MarkAndSweepUtilsMarkTest : public ::testing::Test { public: const std_support::unordered_set& marked() const { return markTraits_.marked(); } - auto MarkedMatcher(std::initializer_list> expected) { + auto MarkedMatcher(std::initializer_list> expected) { std_support::vector objects; for (auto& object : expected) { - objects.push_back(object.get().GetObjHeader()); + objects.push_back(object.get().header()); } return testing::UnorderedElementsAreArray(objects); } - gc::MemoryUsage Mark(std::initializer_list> graySet) { + gc::MemoryUsage Mark(std::initializer_list> graySet) { std_support::vector objects; - for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().GetObjHeader()); + for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().header()); auto handle = gc::GCHandle::create(epoch_++); gc::Mark(handle, objects); handle.finished(); @@ -194,17 +157,17 @@ private: ScopedMarkTraits markTraits_; }; -size_t GetObjectsSize(std::initializer_list> objects) { +size_t GetObjectsSize(std::initializer_list> objects) { size_t size = 0; for (auto& object : objects) { - size += mm::GetAllocatedHeapSize(object.get().GetObjHeader()); + size += mm::GetAllocatedHeapSize(object.get().header()); } return size; } #define EXPECT_MARKED(stats, ...) \ do { \ - std::initializer_list> objects = {__VA_ARGS__}; \ + std::initializer_list> objects = {__VA_ARGS__}; \ EXPECT_THAT(stats.objectsCount, objects.size()); \ EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \ EXPECT_THAT(marked(), MarkedMatcher(objects)); \ @@ -273,7 +236,7 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) { TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) { Object object; - object.InstallExtraData(); + object.installMetaObject(); auto stats = Mark({object}); @@ -282,7 +245,7 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) { TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) { ObjectArray array; - array.InstallExtraData(); + array.installMetaObject(); auto stats = Mark({array}); @@ -291,82 +254,76 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) { TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) { CharArray array; - array.InstallExtraData(); + array.installMetaObject(); auto stats = Mark({array}); EXPECT_MARKED(stats, array); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; Object object; - weakCounter->field1 = object.header(); - object.InstallWeakCounter(weakCounter); + InstallWeakReference(object, weakReference); auto stats = Mark({object}); - EXPECT_MARKED(stats, object, weakCounter); + EXPECT_MARKED(stats, object, weakReference); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; ObjectArray array; - weakCounter->field1 = array.header(); - array.InstallWeakCounter(weakCounter); + InstallWeakReference(array, weakReference); auto stats = Mark({array}); - EXPECT_MARKED(stats, array, weakCounter); + EXPECT_MARKED(stats, array, weakReference); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; CharArray array; - weakCounter->field1 = array.header(); - array.InstallWeakCounter(weakCounter); + InstallWeakReference(array, weakReference); auto stats = Mark({array}); - EXPECT_MARKED(stats, array, weakCounter); + EXPECT_MARKED(stats, array, weakReference); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; Object object; object->field1 = nullptr; - weakCounter->field1 = object.header(); - object.InstallWeakCounter(weakCounter); + InstallWeakReference(object, weakReference); auto stats = Mark({object}); - EXPECT_MARKED(stats, object, weakCounter); + EXPECT_MARKED(stats, object, weakReference); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; ObjectArray array; array.elements()[0] = nullptr; - weakCounter->field1 = array.header(); - array.InstallWeakCounter(weakCounter); + InstallWeakReference(array, weakReference); auto stats = Mark({array}); - EXPECT_MARKED(stats, array, weakCounter); + EXPECT_MARKED(stats, array, weakReference); } -TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) { - Object weakCounter; +TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakReference) { + test_support::RegularWeakReferenceImpl weakReference; CharArray array; array.elements()[0] = 'a'; array.elements()[1] = 'b'; array.elements()[2] = 'c'; - weakCounter->field1 = array.header(); - array.InstallWeakCounter(weakCounter); + InstallWeakReference(array, weakReference); auto stats = Mark({array}); - EXPECT_MARKED(stats, array, weakCounter); + EXPECT_MARKED(stats, array, weakReference); } TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) { diff --git a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp index f815e248a5f..de2700261fc 100644 --- a/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp @@ -31,18 +31,8 @@ struct Payload { }; }; -// TODO: This should go into test support for weak references. -struct WeakCounterPayload { - void* referred; - KInt lock; - KInt cookie; - - static constexpr std::array kFields{}; -}; - test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; -test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder()}; struct ObjectFactoryTraits { struct ObjectData { @@ -68,10 +58,9 @@ public: static Object& FromObjHeader(ObjHeader* obj) { return static_cast(test_support::Object::FromObjHeader(obj)); } - - bool HasWeakCounter() { + bool HasWeakReference() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { - return extraObjectData->HasWeakReferenceCounter(); + return extraObjectData->HasRegularWeakReferenceImpl(); } return false; } @@ -94,9 +83,9 @@ public: return static_cast(test_support::ObjectArray<3>::FromArrayHeader(array)); } - bool HasWeakCounter() { + bool HasWeakReference() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { - return extraObjectData->HasWeakReferenceCounter(); + return extraObjectData->HasRegularWeakReferenceImpl(); } return false; } @@ -119,9 +108,9 @@ public: return static_cast(test_support::CharArray<3>::FromArrayHeader(array)); } - bool HasWeakCounter() { + bool HasWeakReference() { if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) { - return extraObjectData->HasWeakReferenceCounter(); + return extraObjectData->HasRegularWeakReferenceImpl(); } return false; } @@ -134,14 +123,12 @@ private: ObjectFactoryTraits::ObjectData& objectData() { return ObjectFactory::NodeRef::From(header()).ObjectData(); } }; -using WeakCounter = test_support::Object; - -void MarkWeakCounter(WeakCounter& counter) { - ObjectFactory::NodeRef::From(counter.header()).ObjectData().state = ObjectFactoryTraits::ObjectData::State::kMarked; +void MarkWeakReference(test_support::RegularWeakReferenceImpl& weakRef) { + ObjectFactory::NodeRef::From(weakRef.header()).ObjectData().state = ObjectFactoryTraits::ObjectData::State::kMarked; } -ObjectFactoryTraits::ObjectData::State GetWeakCounterState(WeakCounter& counter) { - return ObjectFactory::NodeRef::From(counter.header()).ObjectData().state; +ObjectFactoryTraits::ObjectData::State GetWeakReferenceState(test_support::RegularWeakReferenceImpl& weakRef) { + return ObjectFactory::NodeRef::From(weakRef.header()).ObjectData().state; } struct SweepTraits { @@ -191,7 +178,7 @@ public: for (auto node : objectFactory_.LockForIter()) { auto* obj = node->GetObjHeader(); if (auto* extraObject = mm::ExtraObjectData::Get(obj)) { - extraObject->ClearWeakReferenceCounter(); + extraObject->ClearRegularWeakReferenceImpl(); deallocExtraObject(obj); } RunFinalizers(obj); @@ -250,15 +237,15 @@ public: return *mm::ExtraObjectData::Get(objHeader); } - WeakCounter& InstallWeakCounter(ObjHeader *objHeader) { - auto* weakCounterHeader = objectFactoryThreadQueue_.CreateObject(typeHolderWeakCounter.typeInfo()); + test_support::RegularWeakReferenceImpl& InstallWeakReference(ObjHeader* objHeader) { + auto* weakReferenceHeader = objectFactoryThreadQueue_.CreateObject(theRegularWeakReferenceImplTypeInfo); objectFactoryThreadQueue_.Publish(); - auto& weakCounter = WeakCounter::FromObjHeader(weakCounterHeader); + auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(weakReferenceHeader); auto& extraObjectData = InstallExtraData(objHeader); - auto *setHeader = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header()); - EXPECT_EQ(setHeader, weakCounter.header()); - weakCounter->referred = objHeader; - return weakCounter; + auto* setHeader = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header()); + EXPECT_EQ(setHeader, weakReference.header()); + weakReference->referred = objHeader; + return weakReference; } testing::MockFunction& finalizerHook() { return finalizerHooks_.finalizerHook(); } @@ -451,10 +438,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithFinalizerHook) { EXPECT_THAT(object.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakReference) { auto& object = AllocateObject(); - auto& weakCounter = InstallWeakCounter(object.header()); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header())); + auto& weakReference = InstallWeakReference(object.header()); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header())); auto finalizers = Sweep(); @@ -462,10 +449,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakCounter) { EXPECT_THAT(Alive(), testing::UnorderedElementsAre()); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakReference) { auto& array = AllocateObjectArray(); - auto& weakCounter = InstallWeakCounter(array.header()); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + auto& weakReference = InstallWeakReference(array.header()); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); auto finalizers = Sweep(); @@ -473,10 +460,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakCounter) { EXPECT_THAT(Alive(), testing::UnorderedElementsAre()); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakReference) { auto& array = AllocateCharArray(); - auto& weakCounter = InstallWeakCounter(array.header()); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + auto& weakReference = InstallWeakReference(array.header()); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); auto finalizers = Sweep(); @@ -484,55 +471,55 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakCounter) { EXPECT_THAT(Alive(), testing::UnorderedElementsAre()); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakReference) { auto& object = AllocateObject(); - auto& weakCounter = InstallWeakCounter(object.header()); + auto& weakReference = InstallWeakReference(object.header()); object.Mark(); - MarkWeakCounter(weakCounter); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header())); + MarkWeakReference(weakReference); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header())); auto finalizers = Sweep(); EXPECT_THAT(finalizers, testing::UnorderedElementsAre()); - EXPECT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header())); + EXPECT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header())); EXPECT_THAT(object.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_TRUE(object.HasWeakCounter()); - EXPECT_NE(weakCounter->referred, nullptr); + EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset); + EXPECT_TRUE(object.HasWeakReference()); + EXPECT_NE(weakReference.get(), nullptr); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakReference) { auto& array = AllocateObjectArray(); - auto& weakCounter = InstallWeakCounter(array.header()); + auto& weakReference = InstallWeakReference(array.header()); array.Mark(); - MarkWeakCounter(weakCounter); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + MarkWeakReference(weakReference); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); auto finalizers = Sweep(); EXPECT_THAT(finalizers, testing::UnorderedElementsAre()); - EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); EXPECT_THAT(array.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_TRUE(array.HasWeakCounter()); - EXPECT_NE(weakCounter->referred, nullptr); + EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset); + EXPECT_TRUE(array.HasWeakReference()); + EXPECT_NE(weakReference.get(), nullptr); } -TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) { +TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakReference) { auto& array = AllocateCharArray(); - auto& weakCounter = InstallWeakCounter(array.header()); + auto& weakReference = InstallWeakReference(array.header()); array.Mark(); - MarkWeakCounter(weakCounter); - ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + MarkWeakReference(weakReference); + ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); auto finalizers = Sweep(); EXPECT_THAT(finalizers, testing::UnorderedElementsAre()); - EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header())); + EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header())); EXPECT_THAT(array.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset); - EXPECT_TRUE(array.HasWeakCounter()); - EXPECT_NE(weakCounter->referred, nullptr); + EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset); + EXPECT_TRUE(array.HasWeakReference()); + EXPECT_NE(weakReference.get(), nullptr); } TEST_F(MarkAndSweepUtilsSweepTest, SweepObjects) { diff --git a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp index 0b90eb431aa..df492de9c2e 100644 --- a/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp +++ b/kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp @@ -43,20 +43,8 @@ struct Payload { }; }; -// TODO: This should go into test support for weak references. -struct WeakCounterPayload { - void* referred; - KInt lock; - KInt cookie; - - static constexpr std::array kFields{}; -}; - -using WeakCounter = test_support::Object; - test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder()}; test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder().addFlag(TF_HAS_FINALIZER)}; -test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder()}; // TODO: Clean GlobalObjectHolder after it's gone. class GlobalObjectHolder : private Pinned { @@ -205,14 +193,14 @@ bool IsMarked(ObjHeader* objHeader) { return nodeRef.ObjectData().marked(); } -WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { - mm::AllocateObject(&threadData, typeHolderWeakCounter.typeInfo(), location); - auto& weakCounter = WeakCounter::FromObjHeader(*location); +test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) { + mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location); + auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location); auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader); - auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header()); - EXPECT_EQ(setCounter, weakCounter.header()); - weakCounter->referred = objHeader; - return weakCounter; + weakReference->referred = objHeader; + auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header()); + EXPECT_EQ(setWeakRef, weakReference.header()); + return weakReference; } class SameThreadMarkAndSweepTest : public testing::Test { @@ -347,15 +335,15 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) { TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); - auto& weak1 = ([&threadData, &object1]() -> WeakCounter& { + auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& { ObjHolder holder; - return InstallWeakCounter(threadData, object1.header(), holder.slot()); + return InstallWeakReference(threadData, object1.header(), holder.slot()); })(); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header())); ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(weak1.header()), false); - ASSERT_THAT(weak1->referred, object1.header()); + ASSERT_THAT(weak1.get(), object1.header()); threadData.gc().ScheduleAndWaitFullGC(); @@ -367,18 +355,18 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) { RunInNewThread([](mm::ThreadData& threadData) { auto& object1 = AllocateObject(threadData); StackObjectHolder stack{threadData}; - auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1); + auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1); ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header())); ASSERT_THAT(IsMarked(object1.header()), false); ASSERT_THAT(IsMarked(weak1.header()), false); - ASSERT_THAT(weak1->referred, object1.header()); + ASSERT_THAT(weak1.get(), object1.header()); threadData.gc().ScheduleAndWaitFullGC(); EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header())); EXPECT_THAT(IsMarked(weak1.header()), false); - EXPECT_THAT(weak1->referred, nullptr); + EXPECT_THAT(weak1.get(), nullptr); }); } @@ -955,16 +943,16 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) { TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { std_support::vector mutators(kDefaultThreadCount); ObjHeader* globalRoot = nullptr; - WeakCounter* weak = nullptr; + test_support::RegularWeakReferenceImpl* weak = nullptr; mutators[0] .Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) { auto& global = mutator.AddGlobalRoot(); auto& object = AllocateObject(threadData); - auto& objectWeak = ([&threadData, &object]() -> WeakCounter& { + auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& { ObjHolder holder; - return InstallWeakCounter(threadData, object.header(), holder.slot()); + return InstallWeakReference(threadData, object.header(), holder.slot()); })(); global->field1 = objectWeak.header(); weak = &objectWeak; @@ -981,7 +969,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().ScheduleAndWaitFullGC(); - EXPECT_THAT((*weak)->referred, nullptr); + EXPECT_THAT(weak->get(), nullptr); }); // Spin until thread suspension is requested. @@ -991,7 +979,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) { for (int i = 1; i < kDefaultThreadCount; ++i) { gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) { threadData.gc().SafePointFunctionPrologue(); - EXPECT_THAT((*weak)->referred, nullptr); + EXPECT_THAT(weak->get(), nullptr); }); } @@ -1089,7 +1077,7 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) { TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { std_support::vector mutators(2); std::atomic*> object1 = nullptr; - std::atomic weak = nullptr; + std::atomic weak = nullptr; std::atomic done = false; auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) { GlobalObjectHolder global1{threadData}; @@ -1104,7 +1092,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { ASSERT_THAT(IsMarked(global1.header()), false); ASSERT_THAT(IsMarked(object1_local.header()), false); ASSERT_THAT(IsMarked(weak.load()->header()), false); - ASSERT_THAT((*weak.load())->referred, object1_local.header()); + ASSERT_THAT(weak.load()->get(), object1_local.header()); global1->field1 = nullptr; @@ -1117,7 +1105,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) { auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) { while (object1.load() == nullptr) {} ObjHolder holder; - auto &weak_local = InstallWeakCounter(threadData, object1.load()->header(), holder.slot()); + auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot()); weak = &weak_local; *holder.slot() = nullptr; while (!done) threadData.gc().SafePointLoopBody(); diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index dfc3600e5cc..0d647ded955 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -3242,10 +3242,6 @@ RUNTIME_NOTHROW void ReleaseHeapRefNoCollectRelaxed(const ObjHeader* object) { releaseHeapRef(const_cast(object)); } -RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object) { - RuntimeFail("Only for experimental MM"); -} - ForeignRefContext InitLocalForeignRef(ObjHeader* object) { return initLocalForeignRef(object); } @@ -3784,5 +3780,3 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {} bool kotlin::FinalizersThreadIsRunning() noexcept { return false; } - - diff --git a/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp b/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp index 9a0aa4b13ff..1fa04b78806 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp +++ b/kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp @@ -327,6 +327,18 @@ void ReleaseHeapRef(const ObjHeader* object) RUNTIME_NOTHROW; MODEL_VARIANTS(void, ReleaseHeapRef, const ObjHeader* object); MODEL_VARIANTS(void, ReleaseHeapRefNoCollect, const ObjHeader* object); +bool TryAddHeapRef(const ObjHeader* object); +void ReleaseHeapRefNoCollect(const ObjHeader* object) RUNTIME_NOTHROW; + +ForeignRefContext InitLocalForeignRef(ObjHeader* object); +ForeignRefContext InitForeignRef(ObjHeader* object); +void DeinitForeignRef(ObjHeader* object, ForeignRefContext context); +bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context); + +// Should be used when reference is read from a possibly shared variable, +// and there's nothing else keeping the object alive. +void AdoptReferenceFromSharedVariable(ObjHeader* object); + } // extern "C" #endif // RUNTIME_MEMORYPRIVATE_HPP diff --git a/kotlin-native/runtime/src/legacymm/cpp/MemorySharedRefs.cpp b/kotlin-native/runtime/src/legacymm/cpp/MemorySharedRefs.cpp new file mode 100644 index 00000000000..1d6efa0fa37 --- /dev/null +++ b/kotlin-native/runtime/src/legacymm/cpp/MemorySharedRefs.cpp @@ -0,0 +1,200 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "MemorySharedRefs.hpp" + +#include "Exceptions.h" +#include "MemoryPrivate.hpp" +#include "Runtime.h" + +namespace { + +inline bool isForeignRefAccessible(ObjHeader* object, ForeignRefContext context) { + // If runtime has not been initialized on this thread, then the object is either unowned or shared. + // In the former case initialized runtime is required to throw exceptions + // in the latter case -- to provide proper execution context for caller. + // TODO: this probably can't be called in uninitialized state in the new MM. + Kotlin_initRuntimeIfNeeded(); + + return IsForeignRefAccessible(object, context); +} + +RUNTIME_NORETURN inline void throwIllegalSharingException(ObjHeader* object) { + // TODO: add some info about the context. + // Note: retrieving 'type_info()' is supposed to be correct even for unowned object. + ThrowIllegalObjectSharingException(object->type_info(), object); +} + +RUNTIME_NORETURN inline void terminateWithIllegalSharingException(ObjHeader* object) { +#if KONAN_NO_EXCEPTIONS + // This will terminate. + throwIllegalSharingException(object); +#else + try { + throwIllegalSharingException(object); + } catch (...) { + // A trick to terminate with unhandled exception. This will print a stack trace + // and write to iOS crash log. + std::terminate(); + } +#endif +} + +template +bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) { + static_assert(errorPolicy != ErrorPolicy::kIgnore, "Must've been handled by specialization"); + + if (isForeignRefAccessible(object, context)) { + return true; + } + + switch (errorPolicy) { + case ErrorPolicy::kDefaultValue: + return false; + case ErrorPolicy::kThrow: + throwIllegalSharingException(object); + case ErrorPolicy::kTerminate: + terminateWithIllegalSharingException(object); + } +} + +template <> +bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) { + return true; +} + +} // namespace + +void KRefSharedHolder::initLocal(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + + context_ = InitLocalForeignRef(obj); + obj_ = obj; +} + +void KRefSharedHolder::init(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + + context_ = InitForeignRef(obj); + obj_ = obj; +} + +template +ObjHeader* KRefSharedHolder::ref() const { + if (!ensureRefAccessible(obj_, context_)) { + return nullptr; + } + + AdoptReferenceFromSharedVariable(obj_); + return obj_; +} + +template ObjHeader* KRefSharedHolder::ref() const; +template ObjHeader* KRefSharedHolder::ref() const; +template ObjHeader* KRefSharedHolder::ref() const; + +void KRefSharedHolder::dispose() { + if (obj_ == nullptr) { + // To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc]. + return; + } + + DeinitForeignRef(obj_, context_); +} + +void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + obj_ = obj; + + // Generally a specialized addRef below: + context_ = InitForeignRef(obj); + refCount = 1; +} + +template +void BackRefFromAssociatedObject::addRef() { + static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); + + // Can be called both from Native state (if ObjC or Swift code adds RC) + // and from Runnable state (Kotlin_ObjCExport_refToObjC). + + if (atomicAdd(&refCount, 1) == 1) { + if (obj_ == nullptr) return; // E.g. after [detach]. + + // There are no references to the associated object itself, so Kotlin object is being passed from Kotlin, + // and it is owned therefore. + ensureRefAccessible(obj_, context_); // TODO: consider removing explicit verification. + + // Foreign reference has already been deinitialized (see [releaseRef]). + // Create a new one: + context_ = InitForeignRef(obj_); + } +} + +template void BackRefFromAssociatedObject::addRef(); +template void BackRefFromAssociatedObject::addRef(); + +template +bool BackRefFromAssociatedObject::tryAddRef() { + static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); + + if (obj_ == nullptr) return false; // E.g. after [detach]. + + // Suboptimal but simple: + ensureRefAccessible(obj_, context_); + + ObjHeader* obj = obj_; + + if (!TryAddHeapRef(obj)) return false; + RuntimeAssert(isForeignRefAccessible(obj_, context_), "Cannot be inaccessible because of the check above"); + // TODO: This is a very weird way to ask for "unsafe" addRef. + addRef(); + ReleaseHeapRefNoCollect(obj); // Balance TryAddHeapRef. + // TODO: consider optimizing for non-shared objects. + + return true; +} + +template bool BackRefFromAssociatedObject::tryAddRef(); +template bool BackRefFromAssociatedObject::tryAddRef(); + +void BackRefFromAssociatedObject::releaseRef() { + ForeignRefContext context = context_; + if (atomicAdd(&refCount, -1) == 0) { + if (obj_ == nullptr) return; // E.g. after [detach]. + + // Note: by this moment "subsequent" addRef may have already happened and patched context_. + // So use the value loaded before refCount update: + DeinitForeignRef(obj_, context); + // From this moment [context] is generally a dangling pointer. + // This is handled in [IsForeignRefAccessible] and [addRef]. + // TODO: This probably isn't fine in new MM. Make sure it works. + } +} + +void BackRefFromAssociatedObject::detach() { + RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount"); + obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref. +} + +ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() { + RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_); +} + +template +ObjHeader* BackRefFromAssociatedObject::ref() const { + RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found"); + + if (!ensureRefAccessible(obj_, context_)) { + return nullptr; + } + + AdoptReferenceFromSharedVariable(obj_); + return obj_; +} + +template ObjHeader* BackRefFromAssociatedObject::ref() const; +template ObjHeader* BackRefFromAssociatedObject::ref() const; +template ObjHeader* BackRefFromAssociatedObject::ref() const; diff --git a/kotlin-native/runtime/src/main/cpp/Weak.cpp b/kotlin-native/runtime/src/legacymm/cpp/Weak.cpp similarity index 89% rename from kotlin-native/runtime/src/main/cpp/Weak.cpp rename to kotlin-native/runtime/src/legacymm/cpp/Weak.cpp index 5ad173a840e..0b538f1e752 100644 --- a/kotlin-native/runtime/src/main/cpp/Weak.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Weak.cpp @@ -51,7 +51,7 @@ inline void unlock(int32_t* address) { extern "C" { -OBJ_GETTER(makeWeakReferenceCounter, void*); +OBJ_GETTER(makeWeakReferenceCounterLegacyMM, void*); OBJ_GETTER(makeObjCWeakReferenceImpl, void*); OBJ_GETTER(makePermanentWeakReferenceImpl, ObjHeader*); @@ -72,14 +72,18 @@ OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) { if (weakCounter == nullptr) { ObjHolder counterHolder; // Cast unneeded, just to emphasize we store an object reference as void*. - ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast(referred), counterHolder.slot()); + ObjHeader* counter = makeWeakReferenceCounterLegacyMM(reinterpret_cast(referred), counterHolder.slot()); weakCounter = referred->GetOrSetWeakCounter(counter); } RETURN_OBJ(weakCounter); } +OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader* counter) { + RuntimeFail("New MM only"); +} + // Materialize a weak reference to either null or the real reference. -OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) { +OBJ_GETTER(Konan_WeakReferenceCounterLegacyMM_get, ObjHeader* counter) { ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred; #if KONAN_NO_THREADS RETURN_OBJ(*referredAddress); @@ -89,10 +93,6 @@ OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) { #endif } -ALWAYS_INLINE ObjHeader* UnsafeWeakReferenceCounterGet(ObjHeader* counter) { - return asWeakReferenceCounter(counter)->referred; -} - void WeakReferenceCounterClear(ObjHeader* counter) { ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred; // Note, that we don't do UpdateRef here, as reference is weak. diff --git a/kotlin-native/runtime/src/main/cpp/Weak.h b/kotlin-native/runtime/src/legacymm/cpp/Weak.h similarity index 85% rename from kotlin-native/runtime/src/main/cpp/Weak.h rename to kotlin-native/runtime/src/legacymm/cpp/Weak.h index eeec78f0016..00054883cbf 100644 --- a/kotlin-native/runtime/src/main/cpp/Weak.h +++ b/kotlin-native/runtime/src/legacymm/cpp/Weak.h @@ -12,7 +12,6 @@ extern "C" { // Atomically clears counter object reference. void WeakReferenceCounterClear(ObjHeader* counter); -ObjHeader* UnsafeWeakReferenceCounterGet(ObjHeader* counter); } // extern "C" diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index cb45afce7eb..18614d4f0f6 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -149,10 +149,17 @@ ALWAYS_INLINE inline bool isNullOrMarker(const ObjHeader* obj) noexcept { return reinterpret_cast(obj) <= 1; } -class ForeignRefManager; struct FrameOverlay; + +// Legacy MM only: +class ForeignRefManager; typedef ForeignRefManager* ForeignRefContext; +namespace kotlin::mm { +// New MM only: +struct RawSpecialRef; +} // namespace kotlin::mm + #ifdef __cplusplus extern "C" { #endif @@ -337,24 +344,6 @@ bool Kotlin_Any_isShareable(ObjHeader* thiz); void Kotlin_Any_share(ObjHeader* thiz); void PerformFullGC(MemoryState* memory) RUNTIME_NOTHROW; -// Only for legacy -bool TryAddHeapRef(const ObjHeader* object); -void ReleaseHeapRefNoCollect(const ObjHeader* object) RUNTIME_NOTHROW; - -// Only for experimental -OBJ_GETTER(TryRef, ObjHeader* object) RUNTIME_NOTHROW; - -ForeignRefContext InitLocalForeignRef(ObjHeader* object); - -ForeignRefContext InitForeignRef(ObjHeader* object); -void DeinitForeignRef(ObjHeader* object, ForeignRefContext context); - -bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context); - -// Should be used when reference is read from a possibly shared variable, -// and there's nothing else keeping the object alive. -void AdoptReferenceFromSharedVariable(ObjHeader* object); - void CheckGlobalsAccessible(); // Sets state of the current thread to NATIVE (used by the new MM). diff --git a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp index ac88e6a0b82..23bd6b9bcbf 100644 --- a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp +++ b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp @@ -3,9 +3,8 @@ * that can be found in the LICENSE file. */ -#include "Exceptions.h" #include "MemorySharedRefs.hpp" -#include "Runtime.h" + #include "Types.h" extern "C" { @@ -13,217 +12,11 @@ extern "C" { OBJ_GETTER(DescribeObjectForDebugging, KConstNativePtr typeInfo, KConstNativePtr address); } // extern "C" -namespace { - -inline bool isForeignRefAccessible(ObjHeader* object, ForeignRefContext context) { - // If runtime has not been initialized on this thread, then the object is either unowned or shared. - // In the former case initialized runtime is required to throw exceptions - // in the latter case -- to provide proper execution context for caller. - // TODO: this probably can't be called in uninitialized state in the new MM. - Kotlin_initRuntimeIfNeeded(); - - return IsForeignRefAccessible(object, context); -} - -RUNTIME_NORETURN inline void throwIllegalSharingException(ObjHeader* object) { - // TODO: add some info about the context. - // Note: retrieving 'type_info()' is supposed to be correct even for unowned object. - ThrowIllegalObjectSharingException(object->type_info(), object); -} - -RUNTIME_NORETURN inline void terminateWithIllegalSharingException(ObjHeader* object) { -#if KONAN_NO_EXCEPTIONS - // This will terminate. - throwIllegalSharingException(object); -#else - try { - throwIllegalSharingException(object); - } catch (...) { - // A trick to terminate with unhandled exception. This will print a stack trace - // and write to iOS crash log. - std::terminate(); - } -#endif -} - -template -bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) { - static_assert(errorPolicy != ErrorPolicy::kIgnore, "Must've been handled by specialization"); - - if (isForeignRefAccessible(object, context)) { - return true; - } - - switch (errorPolicy) { - case ErrorPolicy::kDefaultValue: - return false; - case ErrorPolicy::kThrow: - throwIllegalSharingException(object); - case ErrorPolicy::kTerminate: - terminateWithIllegalSharingException(object); - } -} - -template <> -bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) { - return true; -} - -} // namespace - -void KRefSharedHolder::initLocal(ObjHeader* obj) { - RuntimeAssert(obj != nullptr, "must not be null"); - context_ = InitLocalForeignRef(obj); - obj_ = obj; -} - -void KRefSharedHolder::init(ObjHeader* obj) { - RuntimeAssert(obj != nullptr, "must not be null"); - context_ = InitForeignRef(obj); - obj_ = obj; -} - -template -ObjHeader* KRefSharedHolder::ref() const { - kotlin::AssertThreadState(kotlin::ThreadState::kRunnable); - if (!ensureRefAccessible(obj_, context_)) { - return nullptr; - } - - AdoptReferenceFromSharedVariable(obj_); - return obj_; -} - -template ObjHeader* KRefSharedHolder::ref() const; -template ObjHeader* KRefSharedHolder::ref() const; -template ObjHeader* KRefSharedHolder::ref() const; - -void KRefSharedHolder::dispose() const { - if (obj_ == nullptr) { - // To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc]. - return; - } - - DeinitForeignRef(obj_, context_); -} - OBJ_GETTER0(KRefSharedHolder::describe) const { // Note: retrieving 'type_info()' is supposed to be correct even for unowned object. RETURN_RESULT_OF(DescribeObjectForDebugging, obj_->type_info(), obj_); } -void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) { - RuntimeAssert(obj != nullptr, "must not be null"); - obj_ = obj; - - // Generally a specialized addRef below: - context_ = InitForeignRef(obj); - refCount = 1; -} - -template -void BackRefFromAssociatedObject::addRef() { - static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); - - // Can be called both from Native state (if ObjC or Swift code adds RC) - // and from Runnable state (Kotlin_ObjCExport_refToObjC). - - if (atomicAdd(&refCount, 1) == 1) { - if (obj_ == nullptr) return; // E.g. after [detach]. - - kotlin::CalledFromNativeGuard guard(/* reentrant */ true); - - // There are no references to the associated object itself, so Kotlin object is being passed from Kotlin, - // and it is owned therefore. - ensureRefAccessible(obj_, context_); // TODO: consider removing explicit verification. - - // Foreign reference has already been deinitialized (see [releaseRef]). - // Create a new one: - context_ = InitForeignRef(obj_); - } -} - -template void BackRefFromAssociatedObject::addRef(); -template void BackRefFromAssociatedObject::addRef(); - -template -bool BackRefFromAssociatedObject::tryAddRef() { - static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); - kotlin::CalledFromNativeGuard guard; - - if (obj_ == nullptr) return false; // E.g. after [detach]. - - if (CurrentMemoryModel == MemoryModel::kExperimental) { - ObjHolder holder; - ObjHeader* obj = TryRef(obj_, holder.slot()); - // Failed to lock weak reference. - if (obj == nullptr) return false; - RuntimeAssert(obj == obj_, "Mismatched locked weak. obj=%p obj_=%p", obj, obj_); - // TODO: This is a very weird way to ask for "unsafe" addRef. - addRef(); - return true; - } else { - // Suboptimal but simple: - ensureRefAccessible(obj_, context_); - - ObjHeader* obj = obj_; - - if (!TryAddHeapRef(obj)) return false; - RuntimeAssert(isForeignRefAccessible(obj_, context_), "Cannot be inaccessible because of the check above"); - // TODO: This is a very weird way to ask for "unsafe" addRef. - addRef(); - ReleaseHeapRefNoCollect(obj); // Balance TryAddHeapRef. - // TODO: consider optimizing for non-shared objects. - - return true; - } -} - -template bool BackRefFromAssociatedObject::tryAddRef(); -template bool BackRefFromAssociatedObject::tryAddRef(); - -void BackRefFromAssociatedObject::releaseRef() { - ForeignRefContext context = context_; - if (atomicAdd(&refCount, -1) == 0) { - if (obj_ == nullptr) return; // E.g. after [detach]. - - kotlin::CalledFromNativeGuard guard; - - // Note: by this moment "subsequent" addRef may have already happened and patched context_. - // So use the value loaded before refCount update: - DeinitForeignRef(obj_, context); - // From this moment [context] is generally a dangling pointer. - // This is handled in [IsForeignRefAccessible] and [addRef]. - // TODO: This probably isn't fine in new MM. Make sure it works. - } -} - -void BackRefFromAssociatedObject::detach() { - RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount"); - obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref. -} - -ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() { - RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_); -} - -template -ObjHeader* BackRefFromAssociatedObject::ref() const { - kotlin::AssertThreadState(kotlin::ThreadState::kRunnable); - RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found"); - - if (!ensureRefAccessible(obj_, context_)) { - return nullptr; - } - - AdoptReferenceFromSharedVariable(obj_); - return obj_; -} - -template ObjHeader* BackRefFromAssociatedObject::ref() const; -template ObjHeader* BackRefFromAssociatedObject::ref() const; -template ObjHeader* BackRefFromAssociatedObject::ref() const; - extern "C" { RUNTIME_NOTHROW void KRefSharedHolder_initLocal(KRefSharedHolder* holder, ObjHeader* obj) { holder->initLocal(obj); @@ -233,8 +26,8 @@ RUNTIME_NOTHROW void KRefSharedHolder_init(KRefSharedHolder* holder, ObjHeader* holder->init(obj); } -RUNTIME_NOTHROW void KRefSharedHolder_dispose(const KRefSharedHolder* holder) { - holder->dispose(); +RUNTIME_NOTHROW void KRefSharedHolder_dispose(KRefSharedHolder* holder) { + holder->dispose(); } RUNTIME_NOTHROW ObjHeader* KRefSharedHolder_ref(const KRefSharedHolder* holder) { diff --git a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.hpp b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.hpp index af0a86bc9de..ea3ccc2beea 100644 --- a/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.hpp +++ b/kotlin-native/runtime/src/main/cpp/MemorySharedRefs.hpp @@ -28,9 +28,9 @@ class KRefSharedHolder { template ObjHeader* ref() const; - void dispose() const; + void dispose(); - void disposeFromNative() const { + void disposeFromNative() { kotlin::CalledFromNativeGuard guard; dispose(); } @@ -78,7 +78,7 @@ static_assert( extern "C" { RUNTIME_NOTHROW void KRefSharedHolder_initLocal(KRefSharedHolder* holder, ObjHeader* obj); RUNTIME_NOTHROW void KRefSharedHolder_init(KRefSharedHolder* holder, ObjHeader* obj); -RUNTIME_NOTHROW void KRefSharedHolder_dispose(const KRefSharedHolder* holder); +RUNTIME_NOTHROW void KRefSharedHolder_dispose(KRefSharedHolder* holder); RUNTIME_NOTHROW ObjHeader* KRefSharedHolder_ref(const KRefSharedHolder* holder); } // extern "C" diff --git a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp index eb3f5b39d6c..489094e3171 100644 --- a/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp +++ b/kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp @@ -101,8 +101,21 @@ private: std_support::vector objOffsets_; }; +class Any : private Pinned { +public: + ObjHeader* header() noexcept { return &header_; } + + void installMetaObject() noexcept { (void)header()->meta_object(); } + +protected: + Any() noexcept = default; + ~Any() = default; + + ObjHeader header_; +}; + template -class Object : private Pinned { +class Object : public Any { public: class FieldIterator { public: @@ -160,15 +173,12 @@ public: header_.typeInfoOrMeta_ = const_cast(typeInfo); } - ObjHeader* header() noexcept { return &header_; } - Payload& operator*() noexcept { return payload_; } Payload* operator->() noexcept { return &payload_; } FieldIterable fields() noexcept { return FieldIterable(*this); } private: - ObjHeader header_; Payload payload_{}; }; @@ -191,7 +201,7 @@ namespace internal { // Array types are predetermined, use one of the subclasses below. template -class Array : private Pinned { +class Array : public Any { public: static Array& FromArrayHeader(ArrayHeader* arr) noexcept { static_assert(std::is_trivially_destructible_v, "Array destructor is not guaranteed to be called."); @@ -210,17 +220,16 @@ public: TypeInfoHolder{TypeInfoHolder::ArrayBuilder()}.typeInfo()->IsLayoutCompatible(typeInfo), "constructing array from incompatible type info"); header_.typeInfoOrMeta_ = const_cast(typeInfo); - header_.count_ = ElementCount; + count_ = ElementCount; } - ObjHeader* header() noexcept { return header_.obj(); } - ArrayHeader* arrayHeader() noexcept { return &header_; } + ArrayHeader* arrayHeader() noexcept { return header()->array(); } std::array& elements() noexcept { return elements_; } private: - ArrayHeader header_; - std::array elements_{}; + uint32_t count_; + alignas(ArrayHeader) std::array elements_{}; }; } // namespace internal @@ -335,5 +344,31 @@ public: String() noexcept : internal::Array(theStringTypeInfo) {} }; +struct RegularWeakReferenceImplPayload { + void* referred; + + using Field = ObjHeader* RegularWeakReferenceImplPayload::*; + static constexpr std::array kFields{}; +}; + +extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader*); + +class RegularWeakReferenceImpl : public Object { +public: + static RegularWeakReferenceImpl& FromObjHeader(ObjHeader* obj) noexcept { + RuntimeAssert(obj->type_info() == theRegularWeakReferenceImplTypeInfo, "Invalid type"); + return static_cast(Object::FromObjHeader(obj)); + } + + RegularWeakReferenceImpl() noexcept : Object(theRegularWeakReferenceImplTypeInfo) {} + + OBJ_GETTER0(get) noexcept { RETURN_RESULT_OF(Konan_RegularWeakReferenceImpl_get, header()); } + + ObjHeader* get() noexcept { + ObjHeader* result; + return get(&result); + } +}; + } // namespace test_support } // namespace kotlin diff --git a/kotlin-native/runtime/src/main/cpp/Types.h b/kotlin-native/runtime/src/main/cpp/Types.h index 5188f099742..28a9e78ea23 100644 --- a/kotlin-native/runtime/src/main/cpp/Types.h +++ b/kotlin-native/runtime/src/main/cpp/Types.h @@ -70,6 +70,7 @@ extern const TypeInfo* theThrowableTypeInfo; extern const TypeInfo* theUnitTypeInfo; extern const TypeInfo* theWorkerBoundReferenceTypeInfo; extern const TypeInfo* theCleanerImplTypeInfo; +extern const TypeInfo* theRegularWeakReferenceImplTypeInfo; KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info) RUNTIME_PURE; KBoolean IsInstanceOfClassFast(const ObjHeader* obj, int32_t lo, int32_t hi) RUNTIME_PURE; diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/ref/WeakPrivate.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/ref/WeakPrivate.kt index 67298d14585..df8f4370b7b 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/ref/WeakPrivate.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/ref/WeakPrivate.kt @@ -6,46 +6,54 @@ package kotlin.native.ref import kotlinx.cinterop.COpaquePointer -import kotlin.native.internal.ExportForCppRuntime -import kotlin.native.internal.Frozen -import kotlin.native.internal.GCUnsafeCall -import kotlin.native.internal.NoReorderFields -import kotlin.native.internal.Escapes +import kotlin.native.internal.* /** * Theory of operations: * - * Weak references in Kotlin/Native are implemented in the following way. Whenever weak reference to an - * object is created, we atomically modify type info pointer in the object to point into a metaobject. - * This metaobject contains a strong reference to the counter object (instance of WeakReferenceCounter class). - * Every other weak reference contains a strong reference to the counter object. + * Weak references in Kotlin/Native are implemented in the following way. Whenever weak reference to a + * object is created, we try to create one of `WeakReferenceImpl`: + * `PermanentWeakReferenceImpl` for permanent objects, + * `ObjCWeakReferenceImpl` for objects coming from ObjC, + * or `RegularWeakReferenceImpl` for all other Kotlin objects. + * The latter are stored in object's extra data, so only one per object is created. + * Every other weak reference contains a strong reference to the impl object. * - * [weak1] [weak2] - * \ / - * V V - * .......[Counter] <---- - * . | - * . | - * ->[Object] -> [Meta]- + * [weak1] [weak2] + * \ / + * V V + * ...... [WeakReferenceImpl] <------ + * . | + * . | + * ->[Object] -> [ExtraObjectData]- * - * References from weak reference objects to the counter and from the metaobject to the counter are strong, - * and from the counter to the object is nullably weak. So whenever an object dies, if it has a metaobject, - * it is traversed to find a counter object, and atomically nullify reference to the object. Afterward, all attempts - * to get the object would yield null. + * References from weak reference objects to the `RegularWeakReferenceImpl` and from the + * extra data to the `RegularWeakReferenceImpl` are strong, and from the + * `RegularWeakReferenceImpl` to the object is nullably weak. During GC if the object is + * considered dead, the reference inside `RegularWeakReferenceImpl` is nulled out. */ // Clear holding the counter object, which refers to the actual object. @NoReorderFields @Frozen @OptIn(FreezingIsDeprecated::class) -internal class WeakReferenceCounter(var referred: COpaquePointer?) : WeakReferenceImpl() { +internal class WeakReferenceCounterLegacyMM(var referred: COpaquePointer?) : WeakReferenceImpl() { // Spinlock, potentially taken when materializing or removing 'referred' object. var lock: Int = 0 // Optimization for concurrent access. var cookie: Int = 0 - @GCUnsafeCall("Konan_WeakReferenceCounter_get") + @GCUnsafeCall("Konan_WeakReferenceCounterLegacyMM_get") + external override fun get(): Any? +} + +@NoReorderFields +@ExportTypeInfo("theRegularWeakReferenceImplTypeInfo") +internal class RegularWeakReferenceImpl( + val referred: COpaquePointer, +) : WeakReferenceImpl() { + @GCUnsafeCall("Konan_RegularWeakReferenceImpl_get") external override fun get(): Any? } @@ -59,9 +67,13 @@ internal abstract class WeakReferenceImpl { @Escapes(0b01) // referent escapes. external internal fun getWeakReferenceImpl(referent: Any): WeakReferenceImpl +// Create a counter object for legacy MM. +@ExportForCppRuntime +internal fun makeWeakReferenceCounterLegacyMM(referred: COpaquePointer) = WeakReferenceCounterLegacyMM(referred) + // Create a counter object. @ExportForCppRuntime -internal fun makeWeakReferenceCounter(referred: COpaquePointer) = WeakReferenceCounter(referred) +internal fun makeRegularWeakReferenceImpl(referred: COpaquePointer) = RegularWeakReferenceImpl(referred) internal class PermanentWeakReferenceImpl(val referred: Any): kotlin.native.ref.WeakReferenceImpl() { override fun get(): Any? = referred diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp index facce78fd0f..c8e3fb1b41e 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -7,7 +7,6 @@ #include "PointerBits.h" #include "ThreadData.hpp" -#include "Weak.h" #ifdef KONAN_OBJC_INTEROP #include "ObjCMMAPI.h" @@ -77,20 +76,19 @@ bool mm::ExtraObjectData::HasAssociatedObject() noexcept { #endif } - -void mm::ExtraObjectData::ClearWeakReferenceCounter() noexcept { - if (!HasWeakReferenceCounter()) return; +void mm::ExtraObjectData::ClearRegularWeakReferenceImpl() noexcept { + if (!HasRegularWeakReferenceImpl()) return; auto *object = GetBaseObject(); - WeakReferenceCounterClear(GetWeakReferenceCounter()); + disposeRegularWeakReferenceImpl(GetRegularWeakReferenceImpl()); // Not using `mm::SetHeapRef here`, because this code is called during sweep phase by the GC thread, // and so cannot affect marking. // TODO: Asserts on the above? - weakReferenceCounterOrBaseObject_ = object; + weakReferenceOrBaseObject_ = object; } mm::ExtraObjectData::~ExtraObjectData() { - RuntimeAssert(!HasWeakReferenceCounter(), "Object must have cleared weak references"); + RuntimeAssert(!HasRegularWeakReferenceImpl(), "Object must have cleared weak references"); #ifdef KONAN_OBJC_INTEROP RuntimeAssert(associatedObject_ == nullptr, "Object must have cleared associated object"); diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp index 1ce2b7019ab..122d2a8050a 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp @@ -13,7 +13,7 @@ #include "Memory.h" #include "TypeInfo.h" #include "Utils.hpp" -#include "Weak.h" +#include "Weak.hpp" namespace kotlin { namespace mm { @@ -52,33 +52,32 @@ public: bool getFlag(Flags value) noexcept { return (flags_.load() & (1u << static_cast(value))) != 0; } void setFlag(Flags value) noexcept { flags_.fetch_or(1u << static_cast(value)); } - - bool HasWeakReferenceCounter() noexcept { return hasPointerBits(weakReferenceCounterOrBaseObject_.load(), WEAK_REF_TAG); } - void ClearWeakReferenceCounter() noexcept; - ObjHeader* GetWeakReferenceCounter() noexcept { - auto *pointer = weakReferenceCounterOrBaseObject_.load(); + bool HasRegularWeakReferenceImpl() noexcept { return hasPointerBits(weakReferenceOrBaseObject_.load(), WEAK_REF_TAG); } + void ClearRegularWeakReferenceImpl() noexcept; + ObjHeader* GetRegularWeakReferenceImpl() noexcept { + auto* pointer = weakReferenceOrBaseObject_.load(); if (hasPointerBits(pointer, WEAK_REF_TAG)) return clearPointerBits(pointer, WEAK_REF_TAG); return nullptr; } - ObjHeader* GetOrSetWeakReferenceCounter(ObjHeader* object, ObjHeader* counter) noexcept { - if (weakReferenceCounterOrBaseObject_.compare_exchange_strong(object, setPointerBits(counter, WEAK_REF_TAG))) { - return counter; + ObjHeader* GetOrSetRegularWeakReferenceImpl(ObjHeader* object, ObjHeader* weakRef) noexcept { + if (weakReferenceOrBaseObject_.compare_exchange_strong(object, setPointerBits(weakRef, WEAK_REF_TAG))) { + return weakRef; } else { - return clearPointerBits(object, WEAK_REF_TAG); // on fail current value of counter is stored to object + return clearPointerBits(object, WEAK_REF_TAG); // on fail current value of weakRef is stored to object } } ObjHeader* GetBaseObject() noexcept { - auto *header = weakReferenceCounterOrBaseObject_.load(); + auto* header = weakReferenceOrBaseObject_.load(); if (hasPointerBits(header, WEAK_REF_TAG)) { - return UnsafeWeakReferenceCounterGet(clearPointerBits(header, WEAK_REF_TAG)); + return regularWeakReferenceImplBaseObjectUnsafe(clearPointerBits(header, WEAK_REF_TAG)); } else { return header; } } // info must be equal to objHeader->type_info(), but it needs to be loaded in advance to avoid data races - explicit ExtraObjectData(ObjHeader* objHeader, const TypeInfo *info) noexcept : - typeInfo_(nullptr), weakReferenceCounterOrBaseObject_(objHeader) { + explicit ExtraObjectData(ObjHeader* objHeader, const TypeInfo* info) noexcept : + typeInfo_(nullptr), weakReferenceOrBaseObject_(objHeader) { atomicSetRelease(&typeInfo_, info); } ~ExtraObjectData(); @@ -92,7 +91,7 @@ private: std::atomic associatedObject_ = nullptr; #endif - std::atomic weakReferenceCounterOrBaseObject_; + std::atomic weakReferenceOrBaseObject_; }; } // namespace mm diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp index 8f51bdf871f..806cd4cb2fd 100644 --- a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -49,7 +49,7 @@ TEST_F(ExtraObjectDataTest, Install) { EXPECT_TRUE(object.header()->has_meta_object()); EXPECT_THAT(object.header()->meta_object(), extraData.AsMetaObjHeader()); EXPECT_THAT(object.header()->type_info(), typeInfo); - EXPECT_FALSE(extraData.HasWeakReferenceCounter()); + EXPECT_FALSE(extraData.HasRegularWeakReferenceImpl()); EXPECT_THAT(extraData.GetBaseObject(), object.header()); extraData.Uninstall(); diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 430826ab70a..a0bbaa71e57 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -48,11 +48,11 @@ ALWAYS_INLINE mm::StableRefRegistry::Node* FromForeignRefManager(ForeignRefManag } // namespace ObjHeader* ObjHeader::GetWeakCounter() { - return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetWeakReferenceCounter(); + RuntimeFail("Only for legacy MM"); } ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) { - return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetOrSetWeakReferenceCounter(this, counter); + RuntimeFail("Only for legacy MM"); } #ifdef KONAN_OBJC_INTEROP @@ -464,14 +464,6 @@ extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) { threadData->gc().ScheduleAndWaitFullGCWithFinalizers(); } -extern "C" bool TryAddHeapRef(const ObjHeader* object) { - RuntimeFail("Only for legacy MM"); -} - -extern "C" RUNTIME_NOTHROW void ReleaseHeapRefNoCollect(const ObjHeader* object) { - RuntimeFail("Only for legacy MM"); -} - extern "C" RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object) { // TODO: With CMS this needs: // * during marking phase if `object` is unmarked: barrier (might be automatic because of the stack write) @@ -562,13 +554,6 @@ extern "C" void EnsureNeverFrozen(ObjHeader* obj) { } } -extern "C" ForeignRefContext InitLocalForeignRef(ObjHeader* object) { - AssertThreadState(ThreadState::kRunnable); - // TODO: Remove when legacy MM is gone. - // Nothing to do. - return nullptr; -} - extern "C" ForeignRefContext InitForeignRef(ObjHeader* object) { AssertThreadState(ThreadState::kRunnable); auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData(); @@ -585,16 +570,6 @@ extern "C" void DeinitForeignRef(ObjHeader* object, ForeignRefContext context) { mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node); } -extern "C" bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) { - // TODO: Remove when legacy MM is gone. - return true; -} - -extern "C" void AdoptReferenceFromSharedVariable(ObjHeader* object) { - // TODO: Remove when legacy MM is gone. - // Nothing to do. -} - extern "C" void CheckGlobalsAccessible() { // TODO: Remove when legacy MM is gone. // Always accessible @@ -661,3 +636,26 @@ RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(vo // Empty object. Nothing to do. // TODO: Try to generate it in the code generator. } + +extern "C" OBJ_GETTER(makePermanentWeakReferenceImpl, ObjHeader*); +extern "C" OBJ_GETTER(makeObjCWeakReferenceImpl, void*); + +RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) { + if (referred->permanent()) { + RETURN_RESULT_OF(makePermanentWeakReferenceImpl, referred); + } +#if KONAN_OBJC_INTEROP + if (IsInstance(referred, theObjCObjectWrapperTypeInfo)) { + RETURN_RESULT_OF(makeObjCWeakReferenceImpl, referred->GetAssociatedObject()); + } +#endif // KONAN_OBJC_INTEROP + RETURN_RESULT_OF(mm::createRegularWeakReferenceImpl, referred); +} + +RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_WeakReferenceCounterLegacyMM_get, ObjHeader* counter) { + RuntimeFail("Legacy MM only"); +} + +RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader* weakRef) { + RETURN_RESULT_OF(mm::derefRegularWeakReferenceImpl, weakRef); +} diff --git a/kotlin-native/runtime/src/mm/cpp/MemoryPrivate.hpp b/kotlin-native/runtime/src/mm/cpp/MemoryPrivate.hpp index 1efeb01e4dc..be51d75684c 100644 --- a/kotlin-native/runtime/src/mm/cpp/MemoryPrivate.hpp +++ b/kotlin-native/runtime/src/mm/cpp/MemoryPrivate.hpp @@ -39,4 +39,10 @@ extern "C" struct MemoryState : kotlin::Pinned { } }; +extern "C" { +ForeignRefContext InitForeignRef(ObjHeader* object); +void DeinitForeignRef(ObjHeader* object, ForeignRefContext context); +RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object); +} + #endif //RUNTIME_MEMORYPRIVATE_HPP diff --git a/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp b/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp new file mode 100644 index 00000000000..01d5e81c950 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp @@ -0,0 +1,128 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "MemorySharedRefs.hpp" + +#include "MemoryPrivate.hpp" + +using namespace kotlin; + +void KRefSharedHolder::initLocal(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + context_ = nullptr; + obj_ = obj; +} + +void KRefSharedHolder::init(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + context_ = InitForeignRef(obj); + obj_ = obj; +} + +template +ObjHeader* KRefSharedHolder::ref() const { + AssertThreadState(ThreadState::kRunnable); + // ref_ may be null if created with initLocal. + return obj_; +} + +template ObjHeader* KRefSharedHolder::ref() const; +template ObjHeader* KRefSharedHolder::ref() const; +template ObjHeader* KRefSharedHolder::ref() const; + +void KRefSharedHolder::dispose() { + if (obj_ == nullptr) { + // To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc]. + return; + } + + DeinitForeignRef(obj_, context_); +} + +void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) { + RuntimeAssert(obj != nullptr, "must not be null"); + obj_ = obj; + + // Generally a specialized addRef below: + context_ = InitForeignRef(obj); + refCount = 1; +} + +template +void BackRefFromAssociatedObject::addRef() { + static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); + + // Can be called both from Native state (if ObjC or Swift code adds RC) + // and from Runnable state (Kotlin_ObjCExport_refToObjC). + + if (atomicAdd(&refCount, 1) == 1) { + if (obj_ == nullptr) return; // E.g. after [detach]. + + kotlin::CalledFromNativeGuard guard(/* reentrant */ true); + + // Foreign reference has already been deinitialized (see [releaseRef]). + // Create a new one: + context_ = InitForeignRef(obj_); + } +} + +template void BackRefFromAssociatedObject::addRef(); +template void BackRefFromAssociatedObject::addRef(); + +template +bool BackRefFromAssociatedObject::tryAddRef() { + static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here"); + kotlin::CalledFromNativeGuard guard; + + if (obj_ == nullptr) return false; // E.g. after [detach]. + + ObjHolder holder; + ObjHeader* obj = TryRef(obj_, holder.slot()); + // Failed to lock weak reference. + if (obj == nullptr) return false; + RuntimeAssert(obj == obj_, "Mismatched locked weak. obj=%p obj_=%p", obj, obj_); + // TODO: This is a very weird way to ask for "unsafe" addRef. + addRef(); + return true; +} + +template bool BackRefFromAssociatedObject::tryAddRef(); +template bool BackRefFromAssociatedObject::tryAddRef(); + +void BackRefFromAssociatedObject::releaseRef() { + ForeignRefContext context = context_; + if (atomicAdd(&refCount, -1) == 0) { + if (obj_ == nullptr) return; // E.g. after [detach]. + + kotlin::CalledFromNativeGuard guard; + + // Note: by this moment "subsequent" addRef may have already happened and patched context_. + // So use the value loaded before refCount update: + DeinitForeignRef(obj_, context); + // From this moment [context] is generally a dangling pointer. + // This is handled in [IsForeignRefAccessible] and [addRef]. + // TODO: This probably isn't fine in new MM. Make sure it works. + } +} + +void BackRefFromAssociatedObject::detach() { + RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount"); + obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref. +} + +ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() { + RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_); +} + +template +ObjHeader* BackRefFromAssociatedObject::ref() const { + kotlin::AssertThreadState(kotlin::ThreadState::kRunnable); + RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found"); + return obj_; +} + +template ObjHeader* BackRefFromAssociatedObject::ref() const; +template ObjHeader* BackRefFromAssociatedObject::ref() const; +template ObjHeader* BackRefFromAssociatedObject::ref() const; diff --git a/kotlin-native/runtime/src/mm/cpp/Weak.cpp b/kotlin-native/runtime/src/mm/cpp/Weak.cpp new file mode 100644 index 00000000000..0838794704c --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/Weak.cpp @@ -0,0 +1,57 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "Weak.hpp" + +#include "ExtraObjectData.hpp" +#include "ObjectOps.hpp" +#include "ThreadState.hpp" +#include "Types.h" + +using namespace kotlin; + +extern "C" { +OBJ_GETTER(makeRegularWeakReferenceImpl, void*); +} + +namespace { + +struct RegularWeakReferenceImpl { + ObjHeader header; + void* referred; +}; + +RegularWeakReferenceImpl* asRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept { + return reinterpret_cast(weakRef); +} + +} // namespace + +OBJ_GETTER(mm::createRegularWeakReferenceImpl, ObjHeader* object) noexcept { + auto* thread = mm::ThreadRegistry::Instance().CurrentThreadData(); + AssertThreadState(thread, ThreadState::kRunnable); + + auto& extraObject = mm::ExtraObjectData::GetOrInstall(object); + if (auto* weakRef = extraObject.GetRegularWeakReferenceImpl()) { + RETURN_OBJ(weakRef); + } + ObjHolder holder; + auto* weakRef = makeRegularWeakReferenceImpl(object, holder.slot()); + auto* setWeakRef = extraObject.GetOrSetRegularWeakReferenceImpl(object, weakRef); + RETURN_OBJ(setWeakRef); +} + +void mm::disposeRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept { + asRegularWeakReferenceImpl(weakRef)->referred = nullptr; +} + +OBJ_GETTER(mm::derefRegularWeakReferenceImpl, ObjHeader* weakRef) noexcept { + ObjHeader** location = reinterpret_cast(&asRegularWeakReferenceImpl(weakRef)->referred); + RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location); +} + +ObjHeader* mm::regularWeakReferenceImplBaseObjectUnsafe(ObjHeader* weakRef) noexcept { + return static_cast(asRegularWeakReferenceImpl(weakRef)->referred); +} diff --git a/kotlin-native/runtime/src/mm/cpp/Weak.hpp b/kotlin-native/runtime/src/mm/cpp/Weak.hpp new file mode 100644 index 00000000000..136f19f25db --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/Weak.hpp @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#pragma once + +#include "Memory.h" +#include "Utils.hpp" + +namespace kotlin::mm { + +OBJ_GETTER(createRegularWeakReferenceImpl, ObjHeader* object) noexcept; +void disposeRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept; + +OBJ_GETTER(derefRegularWeakReferenceImpl, ObjHeader* weakRef) noexcept; +ObjHeader* regularWeakReferenceImplBaseObjectUnsafe(ObjHeader* weakRef) noexcept; + +} // namespace kotlin::mm diff --git a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp index 5e6f8b65398..a4175202c45 100644 --- a/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp +++ b/kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp @@ -50,6 +50,8 @@ kotlin::test_support::TypeInfoHolder theUnitTypeInfoHolder{kotlin::test_support: kotlin::test_support::TypeInfoHolder theWorkerBoundReferenceTypeInfoHolder{ kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; kotlin::test_support::TypeInfoHolder theCleanerImplTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; +kotlin::test_support::TypeInfoHolder theRegularWeakReferenceImplTypeInfoHolder{ + kotlin::test_support::TypeInfoHolder::ObjectBuilder()}; ArrayHeader theEmptyStringImpl = {theStringTypeInfoHolder.typeInfo(), /* element count */ 0}; @@ -90,6 +92,7 @@ extern const TypeInfo* theThrowableTypeInfo = theThrowableTypeInfoHolder.typeInf extern const TypeInfo* theUnitTypeInfo = theUnitTypeInfoHolder.typeInfo(); extern const TypeInfo* theWorkerBoundReferenceTypeInfo = theWorkerBoundReferenceTypeInfoHolder.typeInfo(); extern const TypeInfo* theCleanerImplTypeInfo = theCleanerImplTypeInfoHolder.typeInfo(); +extern const TypeInfo* theRegularWeakReferenceImplTypeInfo = theRegularWeakReferenceImplTypeInfoHolder.typeInfo(); extern const ArrayHeader theEmptyArray = {theArrayTypeInfoHolder.typeInfo(), /* element count */ 0}; @@ -97,7 +100,11 @@ OBJ_GETTER0(TheEmptyString) { RETURN_OBJ(theEmptyStringImpl.obj()); } -RUNTIME_NORETURN OBJ_GETTER(makeWeakReferenceCounter, void*) { +RUNTIME_NORETURN OBJ_GETTER(makeRegularWeakReferenceImpl, void*) { + throw std::runtime_error("Not implemented for tests"); +} + +RUNTIME_NORETURN OBJ_GETTER(makeWeakReferenceCounterLegacyMM, void*) { throw std::runtime_error("Not implemented for tests"); }