azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Split Weak and MemorySharedRefs between mm implementations ^KT-56233

Browse Source 
* A separate RegularWeakReferenceImpl for the new mm. Name is chosen for
  better consistency with other WeakReferenceImpl implementations.
* KRefSharedHolder and BackRefFromAssociatedObject implementations are
  independent between the legacy and the new mm.
* Better support for WeakReference in runtime unit tests.
This commit is contained in:
Alexander Shabalin
2023-03-31 14:39:00 +02:00
committed by Space Team
parent 87da670319
commit 1f1b26f2a5
26 changed files with 738 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/custom_alloc/cpp/GCApi.cpp
+1 -1
View File
@@ -56,7 +56,7 @@ bool SweepExtraObject(ExtraObjectCell* extraObjectCell, AtomicStack<ExtraObjectC
CustomAllocDebug("SweepIsCollectable(%p): base object (%p) is alive", extraObject, baseObject);
return false;
}
extraObject->ClearWeakReferenceCounter();
extraObject->ClearRegularWeakReferenceImpl();
if (extraObject->HasAssociatedObject()) {
extraObject->DetachAssociatedObject();
extraObject->setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweepTest.cpp
+21 -33
View File
@@ -42,20 +42,8 @@ struct Payload {
};
};
// TODO: This should go into test support for weak references.
struct WeakCounterPayload {
void* referred;
KInt lock;
KInt cookie;
static constexpr std::array<ObjHeader * WeakCounterPayload::*, 0> kFields{};
};
using WeakCounter = test_support::Object<WeakCounterPayload>;
test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder<Payload>().addFlag(TF_HAS_FINALIZER)};
test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder<WeakCounterPayload>()};
// TODO: Clean GlobalObjectHolder after it's gone.
class GlobalObjectHolder : private Pinned {
@@ -204,14 +192,14 @@ bool IsMarked(ObjHeader* objHeader) {
return nodeRef.ObjectData().marked();
}
WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, typeHolderWeakCounter.typeInfo(), location);
auto& weakCounter = WeakCounter::FromObjHeader(*location);
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader);
auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header());
EXPECT_EQ(setCounter, weakCounter.header());
weakCounter->referred = objHeader;
return weakCounter;
weakReference->referred = objHeader;
auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header());
EXPECT_EQ(setWeakRef, weakReference.header());
return weakReference;
}
class ConcurrentMarkAndSweepTest : public testing::TestWithParam<gc::ConcurrentMarkAndSweep::MarkingBehavior> {
@@ -351,15 +339,15 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectsWithFinalizers) {
TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeak) {
RunInNewThread([](mm::ThreadData& threadData) {
auto& object1 = AllocateObject(threadData);
auto& weak1 = ([&threadData, &object1]() -> WeakCounter& {
auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& {
ObjHolder holder;
return InstallWeakCounter(threadData, object1.header(), holder.slot());
return InstallWeakReference(threadData, object1.header(), holder.slot());
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGCWithFinalizers();
@@ -371,18 +359,18 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithHoldedWeak) {
RunInNewThread([](mm::ThreadData& threadData) {
auto& object1 = AllocateObject(threadData);
StackObjectHolder stack{threadData};
auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(weak1->referred, nullptr);
EXPECT_THAT(weak1.get(), nullptr);
});
}
@@ -971,16 +959,16 @@ TEST_P(ConcurrentMarkAndSweepTest, CrossThreadReference) {
TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) {
std_support::vector<Mutator> mutators(kDefaultThreadCount);
ObjHeader* globalRoot = nullptr;
WeakCounter* weak = nullptr;
test_support::RegularWeakReferenceImpl* weak = nullptr;
mutators[0]
.Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) {
auto& global = mutator.AddGlobalRoot();
auto& object = AllocateObject(threadData);
auto& objectWeak = ([&threadData, &object]() -> WeakCounter& {
auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& {
ObjHolder holder;
return InstallWeakCounter(threadData, object.header(), holder.slot());
return InstallWeakReference(threadData, object.header(), holder.slot());
})();
global->field1 = objectWeak.header();
weak = &objectWeak;
@@ -997,7 +985,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) {
gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT((*weak)->referred, nullptr);
EXPECT_THAT(weak->get(), nullptr);
});
// Spin until thread suspension is requested.
@@ -1007,7 +995,7 @@ TEST_P(ConcurrentMarkAndSweepTest, MultipleMutatorsWeaks) {
for (int i = 1; i < kDefaultThreadCount; ++i) {
gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().SafePointFunctionPrologue();
EXPECT_THAT((*weak)->referred, nullptr);
EXPECT_THAT(weak->get(), nullptr);
});
}
@@ -1105,7 +1093,7 @@ TEST_P(ConcurrentMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
std_support::vector<Mutator> mutators(2);
std::atomic<test_support::Object<Payload>*> object1 = nullptr;
std::atomic<WeakCounter*> weak = nullptr;
std::atomic<test_support::RegularWeakReferenceImpl*> weak = nullptr;
std::atomic<bool> done = false;
auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) {
GlobalObjectHolder global1{threadData};
@@ -1120,7 +1108,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT((*weak.load())->referred, object1_local.header());
ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr;
@@ -1133,7 +1121,7 @@ TEST_P(ConcurrentMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) {
while (object1.load() == nullptr) {}
ObjHolder holder;
auto &weak_local = InstallWeakCounter(threadData, object1.load()->header(), holder.slot());
auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot());
weak = &weak_local;
*holder.slot() = nullptr;
while (!done) threadData.gc().SafePointLoopBody();
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtils.hpp
+9 -9
View File
@@ -74,16 +74,16 @@ bool collectRoot(typename Traits::MarkQueue& markQueue, ObjHeader* object) noexc
// TODO: Consider making it noinline to keep loop in `Mark` small.
template <typename Traits>
void processExtraObjectData(GCHandle::GCMarkScope& markHandle, typename Traits::MarkQueue& markQueue, mm::ExtraObjectData& extraObjectData, ObjHeader* object) noexcept {
if (auto weakCounter = extraObjectData.GetWeakReferenceCounter()) {
if (auto weakReference = extraObjectData.GetRegularWeakReferenceImpl()) {
RuntimeAssert(
weakCounter->heap(), "Weak counter must be a heap object. object=%p counter=%p permanent=%d local=%d", object, weakCounter,
weakCounter->permanent(), weakCounter->local());
// Do not schedule WeakReferenceCounter but process it right away.
weakReference->heap(), "Weak reference must be a heap object. object=%p weak=%p permanent=%d local=%d", object,
weakReference, weakReference->permanent(), weakReference->local());
// Do not schedule RegularWeakReferenceImpl but process it right away.
// This will skip markQueue interaction.
if (Traits::tryMark(weakCounter)) {
markHandle.addObject(mm::GetAllocatedHeapSize(weakCounter));
// WeakReferenceCounter is empty, but keeping this just in case.
Traits::processInMark(markQueue, weakCounter);
if (Traits::tryMark(weakReference)) {
markHandle.addObject(mm::GetAllocatedHeapSize(weakReference));
// RegularWeakReferenceImpl is empty, but keeping this just in case.
Traits::processInMark(markQueue, weakReference);
}
}
}
@@ -114,7 +114,7 @@ void SweepExtraObjects(GCHandle handle, typename Traits::ExtraObjectsFactory::It
for (auto it = factoryIter.begin(); it != factoryIter.end();) {
auto &extraObject = *it;
if (!extraObject.getFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE) && !Traits::IsMarkedByExtraObject(extraObject)) {
extraObject.ClearWeakReferenceCounter();
extraObject.ClearRegularWeakReferenceImpl();
if (extraObject.HasAssociatedObject()) {
extraObject.DetachAssociatedObject();
extraObject.setFlag(mm::ExtraObjectData::FLAGS_IN_FINALIZER_QUEUE);
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsMarkTest.cpp
+58 -101
View File
@@ -35,77 +35,40 @@ struct Payload {
test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
// TODO: This base might belong in `test_support`
class BaseObject {
void InstallWeakReference(test_support::Any& object, test_support::RegularWeakReferenceImpl& weakRef) noexcept {
auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(object.header());
weakRef->referred = object.header();
auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(object.header(), weakRef.header());
EXPECT_EQ(setWeakRef, weakRef.header());
EXPECT_EQ(extraObjectData.GetBaseObject(), object.header());
}
void Finalize(test_support::Any& object) noexcept {
if (auto* extraObjectData = mm::ExtraObjectData::Get(object.header())) {
extraObjectData->ClearRegularWeakReferenceImpl();
}
RunFinalizers(object.header());
}
class Object : public test_support::Object<Payload> {
public:
enum class Kind {
kPermanent,
kStackLocal,
kHeapLike // Treated as heap object for the purposes of the test.
};
Object() : test_support::Object<Payload>(typeHolder.typeInfo()) {}
virtual ObjHeader* GetObjHeader() = 0;
void InstallExtraData() { mm::ExtraObjectData::Install(GetObjHeader()); }
void InstallWeakCounter(BaseObject& counter) {
auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(GetObjHeader());
auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(GetObjHeader(), counter.GetObjHeader());
EXPECT_EQ(setCounter, counter.GetObjHeader());
EXPECT_EQ(extraObjectData.GetBaseObject(), GetObjHeader());
}
protected:
void SetKind(Kind kind) {
switch (kind) {
case Kind::kPermanent:
GetObjHeader()->typeInfoOrMeta_ = setPointerBits(GetObjHeader()->typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
RuntimeAssert(GetObjHeader()->permanent(), "Must be permanent");
break;
case Kind::kHeapLike:
RuntimeAssert(GetObjHeader()->heap(), "Must be heap");
break;
case Kind::kStackLocal:
GetObjHeader()->typeInfoOrMeta_ = setPointerBits(GetObjHeader()->typeInfoOrMeta_,
OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER);
RuntimeAssert(GetObjHeader()->local(), "Must be stack local");
break;
}
}
void Finalize() {
if (auto* extraObjectData = mm::ExtraObjectData::Get(GetObjHeader())) {
extraObjectData->ClearWeakReferenceCounter();
}
RunFinalizers(GetObjHeader());
}
~Object() { Finalize(*this); }
};
class Object : public BaseObject, public test_support::Object<Payload> {
class ObjectArray : public test_support::ObjectArray<3> {
public:
explicit Object(Kind kind = Kind::kHeapLike) : test_support::Object<Payload>(typeHolder.typeInfo()) { SetKind(kind); }
ObjectArray() : test_support::ObjectArray<3>() {}
~Object() { Finalize(); }
ObjHeader* GetObjHeader() override { return header(); }
~ObjectArray() { Finalize(*this); }
};
class ObjectArray : public BaseObject, public test_support::ObjectArray<3> {
class CharArray : public test_support::CharArray<3> {
public:
explicit ObjectArray(Kind kind = Kind::kHeapLike) : test_support::ObjectArray<3>() { SetKind(kind); }
CharArray() : test_support::CharArray<3>() {}
~ObjectArray() { Finalize(); }
ObjHeader* GetObjHeader() override { return header(); }
};
class CharArray : public BaseObject, public test_support::CharArray<3> {
public:
explicit CharArray(Kind kind = Kind::kHeapLike) : test_support::CharArray<3>() { SetKind(kind); }
~CharArray() { Finalize(); }
ObjHeader* GetObjHeader() override { return header(); }
~CharArray() { Finalize(*this); }
};
class ScopedMarkTraits : private Pinned {
@@ -169,17 +132,17 @@ class MarkAndSweepUtilsMarkTest : public ::testing::Test {
public:
const std_support::unordered_set<ObjHeader*>& marked() const { return markTraits_.marked(); }
auto MarkedMatcher(std::initializer_list<std::reference_wrapper<BaseObject>> expected) {
auto MarkedMatcher(std::initializer_list<std::reference_wrapper<test_support::Any>> expected) {
std_support::vector<ObjHeader*> objects;
for (auto& object : expected) {
objects.push_back(object.get().GetObjHeader());
objects.push_back(object.get().header());
}
return testing::UnorderedElementsAreArray(objects);
}
gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<BaseObject>> graySet) {
gc::MemoryUsage Mark(std::initializer_list<std::reference_wrapper<test_support::Any>> graySet) {
std_support::vector<ObjHeader*> objects;
for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().GetObjHeader());
for (auto& object : graySet) ScopedMarkTraits::tryEnqueue(objects, object.get().header());
auto handle = gc::GCHandle::create(epoch_++);
gc::Mark<ScopedMarkTraits>(handle, objects);
handle.finished();
@@ -194,17 +157,17 @@ private:
ScopedMarkTraits markTraits_;
};
size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<BaseObject>> objects) {
size_t GetObjectsSize(std::initializer_list<std::reference_wrapper<test_support::Any>> objects) {
size_t size = 0;
for (auto& object : objects) {
size += mm::GetAllocatedHeapSize(object.get().GetObjHeader());
size += mm::GetAllocatedHeapSize(object.get().header());
}
return size;
}
#define EXPECT_MARKED(stats, ...) \
do { \
std::initializer_list<std::reference_wrapper<BaseObject>> objects = {__VA_ARGS__}; \
std::initializer_list<std::reference_wrapper<test_support::Any>> objects = {__VA_ARGS__}; \
EXPECT_THAT(stats.objectsCount, objects.size()); \
EXPECT_THAT(stats.totalObjectsSize, GetObjectsSize(objects)); \
EXPECT_THAT(marked(), MarkedMatcher(objects)); \
@@ -273,7 +236,7 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeData) {
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) {
Object object;
object.InstallExtraData();
object.installMetaObject();
auto stats = Mark({object});
@@ -282,7 +245,7 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithExtraData) {
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) {
ObjectArray array;
array.InstallExtraData();
array.installMetaObject();
auto stats = Mark({array});
@@ -291,82 +254,76 @@ TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithExtraData) {
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithExtraData) {
CharArray array;
array.InstallExtraData();
array.installMetaObject();
auto stats = Mark({array});
EXPECT_MARKED(stats, array);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
Object object;
weakCounter->field1 = object.header();
object.InstallWeakCounter(weakCounter);
InstallWeakReference(object, weakReference);
auto stats = Mark({object});
EXPECT_MARKED(stats, object, weakCounter);
EXPECT_MARKED(stats, object, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
ObjectArray array;
weakCounter->field1 = array.header();
array.InstallWeakCounter(weakCounter);
InstallWeakReference(array, weakReference);
auto stats = Mark({array});
EXPECT_MARKED(stats, array, weakCounter);
EXPECT_MARKED(stats, array, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
CharArray array;
weakCounter->field1 = array.header();
array.InstallWeakCounter(weakCounter);
InstallWeakReference(array, weakReference);
auto stats = Mark({array});
EXPECT_MARKED(stats, array, weakCounter);
EXPECT_MARKED(stats, array, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectWithInvalidFieldsWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
Object object;
object->field1 = nullptr;
weakCounter->field1 = object.header();
object.InstallWeakCounter(weakCounter);
InstallWeakReference(object, weakReference);
auto stats = Mark({object});
EXPECT_MARKED(stats, object, weakCounter);
EXPECT_MARKED(stats, object, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleObjectArrayWithInvalidFieldsWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
ObjectArray array;
array.elements()[0] = nullptr;
weakCounter->field1 = array.header();
array.InstallWeakCounter(weakCounter);
InstallWeakReference(array, weakReference);
auto stats = Mark({array});
EXPECT_MARKED(stats, array, weakCounter);
EXPECT_MARKED(stats, array, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakCounter) {
Object weakCounter;
TEST_F(MarkAndSweepUtilsMarkTest, MarkSingleCharArrayWithSomeDataWithWeakReference) {
test_support::RegularWeakReferenceImpl weakReference;
CharArray array;
array.elements()[0] = 'a';
array.elements()[1] = 'b';
array.elements()[2] = 'c';
weakCounter->field1 = array.header();
array.InstallWeakCounter(weakCounter);
InstallWeakReference(array, weakReference);
auto stats = Mark({array});
EXPECT_MARKED(stats, array, weakCounter);
EXPECT_MARKED(stats, array, weakReference);
}
TEST_F(MarkAndSweepUtilsMarkTest, MarkTree) {
kotlin-native/runtime/src/gc/common/cpp/MarkAndSweepUtilsSweepTest.cpp
+51 -64
View File
@@ -31,18 +31,8 @@ struct Payload {
};
};
// TODO: This should go into test support for weak references.
struct WeakCounterPayload {
void* referred;
KInt lock;
KInt cookie;
static constexpr std::array<ObjHeader * WeakCounterPayload::*, 0> kFields{};
};
test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder<Payload>().addFlag(TF_HAS_FINALIZER)};
test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder<WeakCounterPayload>()};
struct ObjectFactoryTraits {
struct ObjectData {
@@ -68,10 +58,9 @@ public:
static Object& FromObjHeader(ObjHeader* obj) { return static_cast<Object&>(test_support::Object<Payload>::FromObjHeader(obj)); }
bool HasWeakCounter() {
bool HasWeakReference() {
if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
return extraObjectData->HasWeakReferenceCounter();
return extraObjectData->HasRegularWeakReferenceImpl();
}
return false;
}
@@ -94,9 +83,9 @@ public:
return static_cast<ObjectArray&>(test_support::ObjectArray<3>::FromArrayHeader(array));
}
bool HasWeakCounter() {
bool HasWeakReference() {
if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
return extraObjectData->HasWeakReferenceCounter();
return extraObjectData->HasRegularWeakReferenceImpl();
}
return false;
}
@@ -119,9 +108,9 @@ public:
return static_cast<CharArray&>(test_support::CharArray<3>::FromArrayHeader(array));
}
bool HasWeakCounter() {
bool HasWeakReference() {
if (auto* extraObjectData = mm::ExtraObjectData::Get(header())) {
return extraObjectData->HasWeakReferenceCounter();
return extraObjectData->HasRegularWeakReferenceImpl();
}
return false;
}
@@ -134,14 +123,12 @@ private:
ObjectFactoryTraits::ObjectData& objectData() { return ObjectFactory::NodeRef::From(header()).ObjectData(); }
};
using WeakCounter = test_support::Object<WeakCounterPayload>;
void MarkWeakCounter(WeakCounter& counter) {
ObjectFactory::NodeRef::From(counter.header()).ObjectData().state = ObjectFactoryTraits::ObjectData::State::kMarked;
void MarkWeakReference(test_support::RegularWeakReferenceImpl& weakRef) {
ObjectFactory::NodeRef::From(weakRef.header()).ObjectData().state = ObjectFactoryTraits::ObjectData::State::kMarked;
}
ObjectFactoryTraits::ObjectData::State GetWeakCounterState(WeakCounter& counter) {
return ObjectFactory::NodeRef::From(counter.header()).ObjectData().state;
ObjectFactoryTraits::ObjectData::State GetWeakReferenceState(test_support::RegularWeakReferenceImpl& weakRef) {
return ObjectFactory::NodeRef::From(weakRef.header()).ObjectData().state;
}
struct SweepTraits {
@@ -191,7 +178,7 @@ public:
for (auto node : objectFactory_.LockForIter()) {
auto* obj = node->GetObjHeader();
if (auto* extraObject = mm::ExtraObjectData::Get(obj)) {
extraObject->ClearWeakReferenceCounter();
extraObject->ClearRegularWeakReferenceImpl();
deallocExtraObject(obj);
}
RunFinalizers(obj);
@@ -250,15 +237,15 @@ public:
return *mm::ExtraObjectData::Get(objHeader);
}
WeakCounter& InstallWeakCounter(ObjHeader *objHeader) {
auto* weakCounterHeader = objectFactoryThreadQueue_.CreateObject(typeHolderWeakCounter.typeInfo());
test_support::RegularWeakReferenceImpl& InstallWeakReference(ObjHeader* objHeader) {
auto* weakReferenceHeader = objectFactoryThreadQueue_.CreateObject(theRegularWeakReferenceImplTypeInfo);
objectFactoryThreadQueue_.Publish();
auto& weakCounter = WeakCounter::FromObjHeader(weakCounterHeader);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(weakReferenceHeader);
auto& extraObjectData = InstallExtraData(objHeader);
auto *setHeader = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header());
EXPECT_EQ(setHeader, weakCounter.header());
weakCounter->referred = objHeader;
return weakCounter;
auto* setHeader = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header());
EXPECT_EQ(setHeader, weakReference.header());
weakReference->referred = objHeader;
return weakReference;
}
testing::MockFunction<void(ObjHeader*)>& finalizerHook() { return finalizerHooks_.finalizerHook(); }
@@ -451,10 +438,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithFinalizerHook) {
EXPECT_THAT(object.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset);
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakReference) {
auto& object = AllocateObject();
auto& weakCounter = InstallWeakCounter(object.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header()));
auto& weakReference = InstallWeakReference(object.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header()));
auto finalizers = Sweep();
@@ -462,10 +449,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectWithWeakCounter) {
EXPECT_THAT(Alive(), testing::UnorderedElementsAre());
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakReference) {
auto& array = AllocateObjectArray();
auto& weakCounter = InstallWeakCounter(array.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
auto& weakReference = InstallWeakReference(array.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
auto finalizers = Sweep();
@@ -473,10 +460,10 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleObjectArrayWithWeakCounter) {
EXPECT_THAT(Alive(), testing::UnorderedElementsAre());
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakReference) {
auto& array = AllocateCharArray();
auto& weakCounter = InstallWeakCounter(array.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
auto& weakReference = InstallWeakReference(array.header());
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
auto finalizers = Sweep();
@@ -484,55 +471,55 @@ TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleCharArrayWithWeakCounter) {
EXPECT_THAT(Alive(), testing::UnorderedElementsAre());
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectWithWeakReference) {
auto& object = AllocateObject();
auto& weakCounter = InstallWeakCounter(object.header());
auto& weakReference = InstallWeakReference(object.header());
object.Mark();
MarkWeakCounter(weakCounter);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header()));
MarkWeakReference(weakReference);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header()));
auto finalizers = Sweep();
EXPECT_THAT(finalizers, testing::UnorderedElementsAre());
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakCounter.header()));
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(object.header(), weakReference.header()));
EXPECT_THAT(object.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(object.HasWeakCounter());
EXPECT_NE(weakCounter->referred, nullptr);
EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(object.HasWeakReference());
EXPECT_NE(weakReference.get(), nullptr);
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedObjectArrayWithWeakReference) {
auto& array = AllocateObjectArray();
auto& weakCounter = InstallWeakCounter(array.header());
auto& weakReference = InstallWeakReference(array.header());
array.Mark();
MarkWeakCounter(weakCounter);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
MarkWeakReference(weakReference);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
auto finalizers = Sweep();
EXPECT_THAT(finalizers, testing::UnorderedElementsAre());
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
EXPECT_THAT(array.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(array.HasWeakCounter());
EXPECT_NE(weakCounter->referred, nullptr);
EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(array.HasWeakReference());
EXPECT_NE(weakReference.get(), nullptr);
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakCounter) {
TEST_F(MarkAndSweepUtilsSweepTest, SweepSingleMarkedCharArrayWithWeakReference) {
auto& array = AllocateCharArray();
auto& weakCounter = InstallWeakCounter(array.header());
auto& weakReference = InstallWeakReference(array.header());
array.Mark();
MarkWeakCounter(weakCounter);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
MarkWeakReference(weakReference);
ASSERT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
auto finalizers = Sweep();
EXPECT_THAT(finalizers, testing::UnorderedElementsAre());
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakCounter.header()));
EXPECT_THAT(Alive(), testing::UnorderedElementsAre(array.header(), weakReference.header()));
EXPECT_THAT(array.state(), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_THAT(GetWeakCounterState(weakCounter), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(array.HasWeakCounter());
EXPECT_NE(weakCounter->referred, nullptr);
EXPECT_THAT(GetWeakReferenceState(weakReference), ObjectFactoryTraits::ObjectData::State::kMarkReset);
EXPECT_TRUE(array.HasWeakReference());
EXPECT_NE(weakReference.get(), nullptr);
}
TEST_F(MarkAndSweepUtilsSweepTest, SweepObjects) {
kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweepTest.cpp
+21 -33
View File
@@ -43,20 +43,8 @@ struct Payload {
};
};
// TODO: This should go into test support for weak references.
struct WeakCounterPayload {
void* referred;
KInt lock;
KInt cookie;
static constexpr std::array<ObjHeader * WeakCounterPayload::*, 0> kFields{};
};
using WeakCounter = test_support::Object<WeakCounterPayload>;
test_support::TypeInfoHolder typeHolder{test_support::TypeInfoHolder::ObjectBuilder<Payload>()};
test_support::TypeInfoHolder typeHolderWithFinalizer{test_support::TypeInfoHolder::ObjectBuilder<Payload>().addFlag(TF_HAS_FINALIZER)};
test_support::TypeInfoHolder typeHolderWeakCounter{test_support::TypeInfoHolder::ObjectBuilder<WeakCounterPayload>()};
// TODO: Clean GlobalObjectHolder after it's gone.
class GlobalObjectHolder : private Pinned {
@@ -205,14 +193,14 @@ bool IsMarked(ObjHeader* objHeader) {
return nodeRef.ObjectData().marked();
}
WeakCounter& InstallWeakCounter(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, typeHolderWeakCounter.typeInfo(), location);
auto& weakCounter = WeakCounter::FromObjHeader(*location);
test_support::RegularWeakReferenceImpl& InstallWeakReference(mm::ThreadData& threadData, ObjHeader* objHeader, ObjHeader** location) {
mm::AllocateObject(&threadData, theRegularWeakReferenceImplTypeInfo, location);
auto& weakReference = test_support::RegularWeakReferenceImpl::FromObjHeader(*location);
auto& extraObjectData = mm::ExtraObjectData::GetOrInstall(objHeader);
auto *setCounter = extraObjectData.GetOrSetWeakReferenceCounter(objHeader, weakCounter.header());
EXPECT_EQ(setCounter, weakCounter.header());
weakCounter->referred = objHeader;
return weakCounter;
weakReference->referred = objHeader;
auto* setWeakRef = extraObjectData.GetOrSetRegularWeakReferenceImpl(objHeader, weakReference.header());
EXPECT_EQ(setWeakRef, weakReference.header());
return weakReference;
}
class SameThreadMarkAndSweepTest : public testing::Test {
@@ -347,15 +335,15 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectsWithFinalizers) {
TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeak) {
RunInNewThread([](mm::ThreadData& threadData) {
auto& object1 = AllocateObject(threadData);
auto& weak1 = ([&threadData, &object1]() -> WeakCounter& {
auto& weak1 = ([&threadData, &object1]() -> test_support::RegularWeakReferenceImpl& {
ObjHolder holder;
return InstallWeakCounter(threadData, object1.header(), holder.slot());
return InstallWeakReference(threadData, object1.header(), holder.slot());
})();
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGC();
@@ -367,18 +355,18 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithHoldedWeak) {
RunInNewThread([](mm::ThreadData& threadData) {
auto& object1 = AllocateObject(threadData);
StackObjectHolder stack{threadData};
auto& weak1 = InstallWeakCounter(threadData, object1.header(), &stack->field1);
auto& weak1 = InstallWeakReference(threadData, object1.header(), &stack->field1);
ASSERT_THAT(Alive(threadData), testing::UnorderedElementsAre(object1.header(), weak1.header(), stack.header()));
ASSERT_THAT(IsMarked(object1.header()), false);
ASSERT_THAT(IsMarked(weak1.header()), false);
ASSERT_THAT(weak1->referred, object1.header());
ASSERT_THAT(weak1.get(), object1.header());
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT(Alive(threadData), testing::UnorderedElementsAre(weak1.header(), stack.header()));
EXPECT_THAT(IsMarked(weak1.header()), false);
EXPECT_THAT(weak1->referred, nullptr);
EXPECT_THAT(weak1.get(), nullptr);
});
}
@@ -955,16 +943,16 @@ TEST_F(SameThreadMarkAndSweepTest, CrossThreadReference) {
TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) {
std_support::vector<Mutator> mutators(kDefaultThreadCount);
ObjHeader* globalRoot = nullptr;
WeakCounter* weak = nullptr;
test_support::RegularWeakReferenceImpl* weak = nullptr;
mutators[0]
.Execute([&weak, &globalRoot](mm::ThreadData& threadData, Mutator& mutator) {
auto& global = mutator.AddGlobalRoot();
auto& object = AllocateObject(threadData);
auto& objectWeak = ([&threadData, &object]() -> WeakCounter& {
auto& objectWeak = ([&threadData, &object]() -> test_support::RegularWeakReferenceImpl& {
ObjHolder holder;
return InstallWeakCounter(threadData, object.header(), holder.slot());
return InstallWeakReference(threadData, object.header(), holder.slot());
})();
global->field1 = objectWeak.header();
weak = &objectWeak;
@@ -981,7 +969,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) {
gcFutures[0] = mutators[0].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().ScheduleAndWaitFullGC();
EXPECT_THAT((*weak)->referred, nullptr);
EXPECT_THAT(weak->get(), nullptr);
});
// Spin until thread suspension is requested.
@@ -991,7 +979,7 @@ TEST_F(SameThreadMarkAndSweepTest, MultipleMutatorsWeaks) {
for (int i = 1; i < kDefaultThreadCount; ++i) {
gcFutures[i] = mutators[i].Execute([weak](mm::ThreadData& threadData, Mutator& mutator) {
threadData.gc().SafePointFunctionPrologue();
EXPECT_THAT((*weak)->referred, nullptr);
EXPECT_THAT(weak->get(), nullptr);
});
}
@@ -1089,7 +1077,7 @@ TEST_F(SameThreadMarkAndSweepTest, NewThreadsWhileRequestingCollection) {
TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
std_support::vector<Mutator> mutators(2);
std::atomic<test_support::Object<Payload>*> object1 = nullptr;
std::atomic<WeakCounter*> weak = nullptr;
std::atomic<test_support::RegularWeakReferenceImpl*> weak = nullptr;
std::atomic<bool> done = false;
auto f0 = mutators[0].Execute([&](mm::ThreadData& threadData, Mutator &) {
GlobalObjectHolder global1{threadData};
@@ -1104,7 +1092,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
ASSERT_THAT(IsMarked(global1.header()), false);
ASSERT_THAT(IsMarked(object1_local.header()), false);
ASSERT_THAT(IsMarked(weak.load()->header()), false);
ASSERT_THAT((*weak.load())->referred, object1_local.header());
ASSERT_THAT(weak.load()->get(), object1_local.header());
global1->field1 = nullptr;
@@ -1117,7 +1105,7 @@ TEST_F(SameThreadMarkAndSweepTest, FreeObjectWithFreeWeakReversedOrder) {
auto f1 = mutators[1].Execute([&](mm::ThreadData& threadData, Mutator &) {
while (object1.load() == nullptr) {}
ObjHolder holder;
auto &weak_local = InstallWeakCounter(threadData, object1.load()->header(), holder.slot());
auto& weak_local = InstallWeakReference(threadData, object1.load()->header(), holder.slot());
weak = &weak_local;
*holder.slot() = nullptr;
while (!done) threadData.gc().SafePointLoopBody();
kotlin-native/runtime/src/legacymm/cpp/Memory.cpp
-6
View File
@@ -3242,10 +3242,6 @@ RUNTIME_NOTHROW void ReleaseHeapRefNoCollectRelaxed(const ObjHeader* object) {
releaseHeapRef<false, /* CanCollect = */ false>(const_cast<ObjHeader*>(object));
}
RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object) {
RuntimeFail("Only for experimental MM");
}
ForeignRefContext InitLocalForeignRef(ObjHeader* object) {
return initLocalForeignRef(object);
}
@@ -3784,5 +3780,3 @@ void kotlin::StartFinalizerThreadIfNeeded() noexcept {}
bool kotlin::FinalizersThreadIsRunning() noexcept {
return false;
}
kotlin-native/runtime/src/legacymm/cpp/MemoryPrivate.hpp
+12
View File
@@ -327,6 +327,18 @@ void ReleaseHeapRef(const ObjHeader* object) RUNTIME_NOTHROW;
MODEL_VARIANTS(void, ReleaseHeapRef, const ObjHeader* object);
MODEL_VARIANTS(void, ReleaseHeapRefNoCollect, const ObjHeader* object);
bool TryAddHeapRef(const ObjHeader* object);
void ReleaseHeapRefNoCollect(const ObjHeader* object) RUNTIME_NOTHROW;
ForeignRefContext InitLocalForeignRef(ObjHeader* object);
ForeignRefContext InitForeignRef(ObjHeader* object);
void DeinitForeignRef(ObjHeader* object, ForeignRefContext context);
bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context);
// Should be used when reference is read from a possibly shared variable,
// and there's nothing else keeping the object alive.
void AdoptReferenceFromSharedVariable(ObjHeader* object);
} // extern "C"
#endif // RUNTIME_MEMORYPRIVATE_HPP
kotlin-native/runtime/src/legacymm/cpp/MemorySharedRefs.cpp
+200
View File
@@ -0,0 +1,200 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "MemorySharedRefs.hpp"
#include "Exceptions.h"
#include "MemoryPrivate.hpp"
#include "Runtime.h"
namespace {
inline bool isForeignRefAccessible(ObjHeader* object, ForeignRefContext context) {
// If runtime has not been initialized on this thread, then the object is either unowned or shared.
// In the former case initialized runtime is required to throw exceptions
// in the latter case -- to provide proper execution context for caller.
// TODO: this probably can't be called in uninitialized state in the new MM.
Kotlin_initRuntimeIfNeeded();
return IsForeignRefAccessible(object, context);
}
RUNTIME_NORETURN inline void throwIllegalSharingException(ObjHeader* object) {
// TODO: add some info about the context.
// Note: retrieving 'type_info()' is supposed to be correct even for unowned object.
ThrowIllegalObjectSharingException(object->type_info(), object);
}
RUNTIME_NORETURN inline void terminateWithIllegalSharingException(ObjHeader* object) {
#if KONAN_NO_EXCEPTIONS
// This will terminate.
throwIllegalSharingException(object);
#else
try {
throwIllegalSharingException(object);
} catch (...) {
// A trick to terminate with unhandled exception. This will print a stack trace
// and write to iOS crash log.
std::terminate();
}
#endif
}
template <ErrorPolicy errorPolicy>
bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) {
static_assert(errorPolicy != ErrorPolicy::kIgnore, "Must've been handled by specialization");
if (isForeignRefAccessible(object, context)) {
return true;
}
switch (errorPolicy) {
case ErrorPolicy::kDefaultValue:
return false;
case ErrorPolicy::kThrow:
throwIllegalSharingException(object);
case ErrorPolicy::kTerminate:
terminateWithIllegalSharingException(object);
}
}
template <>
bool ensureRefAccessible<ErrorPolicy::kIgnore>(ObjHeader* object, ForeignRefContext context) {
return true;
}
} // namespace
void KRefSharedHolder::initLocal(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = InitLocalForeignRef(obj);
obj_ = obj;
}
void KRefSharedHolder::init(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = InitForeignRef(obj);
obj_ = obj;
}
template <ErrorPolicy errorPolicy>
ObjHeader* KRefSharedHolder::ref() const {
if (!ensureRefAccessible<errorPolicy>(obj_, context_)) {
return nullptr;
}
AdoptReferenceFromSharedVariable(obj_);
return obj_;
}
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kTerminate>() const;
void KRefSharedHolder::dispose() {
if (obj_ == nullptr) {
// To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc].
return;
}
DeinitForeignRef(obj_, context_);
}
void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
obj_ = obj;
// Generally a specialized addRef below:
context_ = InitForeignRef(obj);
refCount = 1;
}
template <ErrorPolicy errorPolicy>
void BackRefFromAssociatedObject::addRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
// Can be called both from Native state (if ObjC or Swift code adds RC)
// and from Runnable state (Kotlin_ObjCExport_refToObjC).
if (atomicAdd(&refCount, 1) == 1) {
if (obj_ == nullptr) return; // E.g. after [detach].
// There are no references to the associated object itself, so Kotlin object is being passed from Kotlin,
// and it is owned therefore.
ensureRefAccessible<errorPolicy>(obj_, context_); // TODO: consider removing explicit verification.
// Foreign reference has already been deinitialized (see [releaseRef]).
// Create a new one:
context_ = InitForeignRef(obj_);
}
}
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kThrow>();
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kTerminate>();
template <ErrorPolicy errorPolicy>
bool BackRefFromAssociatedObject::tryAddRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
if (obj_ == nullptr) return false; // E.g. after [detach].
// Suboptimal but simple:
ensureRefAccessible<errorPolicy>(obj_, context_);
ObjHeader* obj = obj_;
if (!TryAddHeapRef(obj)) return false;
RuntimeAssert(isForeignRefAccessible(obj_, context_), "Cannot be inaccessible because of the check above");
// TODO: This is a very weird way to ask for "unsafe" addRef.
addRef<ErrorPolicy::kIgnore>();
ReleaseHeapRefNoCollect(obj); // Balance TryAddHeapRef.
// TODO: consider optimizing for non-shared objects.
return true;
}
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kThrow>();
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kTerminate>();
void BackRefFromAssociatedObject::releaseRef() {
ForeignRefContext context = context_;
if (atomicAdd(&refCount, -1) == 0) {
if (obj_ == nullptr) return; // E.g. after [detach].
// Note: by this moment "subsequent" addRef may have already happened and patched context_.
// So use the value loaded before refCount update:
DeinitForeignRef(obj_, context);
// From this moment [context] is generally a dangling pointer.
// This is handled in [IsForeignRefAccessible] and [addRef].
// TODO: This probably isn't fine in new MM. Make sure it works.
}
}
void BackRefFromAssociatedObject::detach() {
RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount");
obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref.
}
ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() {
RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_);
}
template <ErrorPolicy errorPolicy>
ObjHeader* BackRefFromAssociatedObject::ref() const {
RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found");
if (!ensureRefAccessible<errorPolicy>(obj_, context_)) {
return nullptr;
}
AdoptReferenceFromSharedVariable(obj_);
return obj_;
}
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kTerminate>() const;
kotlin-native/runtime/src/main/cpp/Weak.cpp → kotlin-native/runtime/src/legacymm/cpp/Weak.cpp
+7 -7
View File
@@ -51,7 +51,7 @@ inline void unlock(int32_t* address) {
extern "C" {
OBJ_GETTER(makeWeakReferenceCounter, void*);
OBJ_GETTER(makeWeakReferenceCounterLegacyMM, void*);
OBJ_GETTER(makeObjCWeakReferenceImpl, void*);
OBJ_GETTER(makePermanentWeakReferenceImpl, ObjHeader*);
@@ -72,14 +72,18 @@ OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) {
if (weakCounter == nullptr) {
ObjHolder counterHolder;
// Cast unneeded, just to emphasize we store an object reference as void*.
ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast<void*>(referred), counterHolder.slot());
ObjHeader* counter = makeWeakReferenceCounterLegacyMM(reinterpret_cast<void*>(referred), counterHolder.slot());
weakCounter = referred->GetOrSetWeakCounter(counter);
}
RETURN_OBJ(weakCounter);
}
OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader* counter) {
RuntimeFail("New MM only");
}
// Materialize a weak reference to either null or the real reference.
OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) {
OBJ_GETTER(Konan_WeakReferenceCounterLegacyMM_get, ObjHeader* counter) {
ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred;
#if KONAN_NO_THREADS
RETURN_OBJ(*referredAddress);
@@ -89,10 +93,6 @@ OBJ_GETTER(Konan_WeakReferenceCounter_get, ObjHeader* counter) {
#endif
}
ALWAYS_INLINE ObjHeader* UnsafeWeakReferenceCounterGet(ObjHeader* counter) {
return asWeakReferenceCounter(counter)->referred;
}
void WeakReferenceCounterClear(ObjHeader* counter) {
ObjHeader** referredAddress = &asWeakReferenceCounter(counter)->referred;
// Note, that we don't do UpdateRef here, as reference is weak.
kotlin-native/runtime/src/main/cpp/Weak.h → kotlin-native/runtime/src/legacymm/cpp/Weak.h
-1
View File
@@ -12,7 +12,6 @@ extern "C" {
// Atomically clears counter object reference.
void WeakReferenceCounterClear(ObjHeader* counter);
ObjHeader* UnsafeWeakReferenceCounterGet(ObjHeader* counter);
} // extern "C"
kotlin-native/runtime/src/main/cpp/Memory.h
+8 -19
View File
@@ -149,10 +149,17 @@ ALWAYS_INLINE inline bool isNullOrMarker(const ObjHeader* obj) noexcept {
return reinterpret_cast<uintptr_t>(obj) <= 1;
}
class ForeignRefManager;
struct FrameOverlay;
// Legacy MM only:
class ForeignRefManager;
typedef ForeignRefManager* ForeignRefContext;
namespace kotlin::mm {
// New MM only:
struct RawSpecialRef;
} // namespace kotlin::mm
#ifdef __cplusplus
extern "C" {
#endif
@@ -337,24 +344,6 @@ bool Kotlin_Any_isShareable(ObjHeader* thiz);
void Kotlin_Any_share(ObjHeader* thiz);
void PerformFullGC(MemoryState* memory) RUNTIME_NOTHROW;
// Only for legacy
bool TryAddHeapRef(const ObjHeader* object);
void ReleaseHeapRefNoCollect(const ObjHeader* object) RUNTIME_NOTHROW;
// Only for experimental
OBJ_GETTER(TryRef, ObjHeader* object) RUNTIME_NOTHROW;
ForeignRefContext InitLocalForeignRef(ObjHeader* object);
ForeignRefContext InitForeignRef(ObjHeader* object);
void DeinitForeignRef(ObjHeader* object, ForeignRefContext context);
bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context);
// Should be used when reference is read from a possibly shared variable,
// and there's nothing else keeping the object alive.
void AdoptReferenceFromSharedVariable(ObjHeader* object);
void CheckGlobalsAccessible();
// Sets state of the current thread to NATIVE (used by the new MM).
kotlin-native/runtime/src/main/cpp/MemorySharedRefs.cpp
+3 -210
View File
@@ -3,9 +3,8 @@
* that can be found in the LICENSE file.
*/
#include "Exceptions.h"
#include "MemorySharedRefs.hpp"
#include "Runtime.h"
#include "Types.h"
extern "C" {
@@ -13,217 +12,11 @@ extern "C" {
OBJ_GETTER(DescribeObjectForDebugging, KConstNativePtr typeInfo, KConstNativePtr address);
} // extern "C"
namespace {
inline bool isForeignRefAccessible(ObjHeader* object, ForeignRefContext context) {
// If runtime has not been initialized on this thread, then the object is either unowned or shared.
// In the former case initialized runtime is required to throw exceptions
// in the latter case -- to provide proper execution context for caller.
// TODO: this probably can't be called in uninitialized state in the new MM.
Kotlin_initRuntimeIfNeeded();
return IsForeignRefAccessible(object, context);
}
RUNTIME_NORETURN inline void throwIllegalSharingException(ObjHeader* object) {
// TODO: add some info about the context.
// Note: retrieving 'type_info()' is supposed to be correct even for unowned object.
ThrowIllegalObjectSharingException(object->type_info(), object);
}
RUNTIME_NORETURN inline void terminateWithIllegalSharingException(ObjHeader* object) {
#if KONAN_NO_EXCEPTIONS
// This will terminate.
throwIllegalSharingException(object);
#else
try {
throwIllegalSharingException(object);
} catch (...) {
// A trick to terminate with unhandled exception. This will print a stack trace
// and write to iOS crash log.
std::terminate();
}
#endif
}
template <ErrorPolicy errorPolicy>
bool ensureRefAccessible(ObjHeader* object, ForeignRefContext context) {
static_assert(errorPolicy != ErrorPolicy::kIgnore, "Must've been handled by specialization");
if (isForeignRefAccessible(object, context)) {
return true;
}
switch (errorPolicy) {
case ErrorPolicy::kDefaultValue:
return false;
case ErrorPolicy::kThrow:
throwIllegalSharingException(object);
case ErrorPolicy::kTerminate:
terminateWithIllegalSharingException(object);
}
}
template <>
bool ensureRefAccessible<ErrorPolicy::kIgnore>(ObjHeader* object, ForeignRefContext context) {
return true;
}
} // namespace
void KRefSharedHolder::initLocal(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = InitLocalForeignRef(obj);
obj_ = obj;
}
void KRefSharedHolder::init(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = InitForeignRef(obj);
obj_ = obj;
}
template <ErrorPolicy errorPolicy>
ObjHeader* KRefSharedHolder::ref() const {
kotlin::AssertThreadState(kotlin::ThreadState::kRunnable);
if (!ensureRefAccessible<errorPolicy>(obj_, context_)) {
return nullptr;
}
AdoptReferenceFromSharedVariable(obj_);
return obj_;
}
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kTerminate>() const;
void KRefSharedHolder::dispose() const {
if (obj_ == nullptr) {
// To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc].
return;
}
DeinitForeignRef(obj_, context_);
}
OBJ_GETTER0(KRefSharedHolder::describe) const {
// Note: retrieving 'type_info()' is supposed to be correct even for unowned object.
RETURN_RESULT_OF(DescribeObjectForDebugging, obj_->type_info(), obj_);
}
void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
obj_ = obj;
// Generally a specialized addRef below:
context_ = InitForeignRef(obj);
refCount = 1;
}
template <ErrorPolicy errorPolicy>
void BackRefFromAssociatedObject::addRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
// Can be called both from Native state (if ObjC or Swift code adds RC)
// and from Runnable state (Kotlin_ObjCExport_refToObjC).
if (atomicAdd(&refCount, 1) == 1) {
if (obj_ == nullptr) return; // E.g. after [detach].
kotlin::CalledFromNativeGuard guard(/* reentrant */ true);
// There are no references to the associated object itself, so Kotlin object is being passed from Kotlin,
// and it is owned therefore.
ensureRefAccessible<errorPolicy>(obj_, context_); // TODO: consider removing explicit verification.
// Foreign reference has already been deinitialized (see [releaseRef]).
// Create a new one:
context_ = InitForeignRef(obj_);
}
}
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kThrow>();
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kTerminate>();
template <ErrorPolicy errorPolicy>
bool BackRefFromAssociatedObject::tryAddRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
kotlin::CalledFromNativeGuard guard;
if (obj_ == nullptr) return false; // E.g. after [detach].
if (CurrentMemoryModel == MemoryModel::kExperimental) {
ObjHolder holder;
ObjHeader* obj = TryRef(obj_, holder.slot());
// Failed to lock weak reference.
if (obj == nullptr) return false;
RuntimeAssert(obj == obj_, "Mismatched locked weak. obj=%p obj_=%p", obj, obj_);
// TODO: This is a very weird way to ask for "unsafe" addRef.
addRef<ErrorPolicy::kIgnore>();
return true;
} else {
// Suboptimal but simple:
ensureRefAccessible<errorPolicy>(obj_, context_);
ObjHeader* obj = obj_;
if (!TryAddHeapRef(obj)) return false;
RuntimeAssert(isForeignRefAccessible(obj_, context_), "Cannot be inaccessible because of the check above");
// TODO: This is a very weird way to ask for "unsafe" addRef.
addRef<ErrorPolicy::kIgnore>();
ReleaseHeapRefNoCollect(obj); // Balance TryAddHeapRef.
// TODO: consider optimizing for non-shared objects.
return true;
}
}
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kThrow>();
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kTerminate>();
void BackRefFromAssociatedObject::releaseRef() {
ForeignRefContext context = context_;
if (atomicAdd(&refCount, -1) == 0) {
if (obj_ == nullptr) return; // E.g. after [detach].
kotlin::CalledFromNativeGuard guard;
// Note: by this moment "subsequent" addRef may have already happened and patched context_.
// So use the value loaded before refCount update:
DeinitForeignRef(obj_, context);
// From this moment [context] is generally a dangling pointer.
// This is handled in [IsForeignRefAccessible] and [addRef].
// TODO: This probably isn't fine in new MM. Make sure it works.
}
}
void BackRefFromAssociatedObject::detach() {
RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount");
obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref.
}
ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() {
RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_);
}
template <ErrorPolicy errorPolicy>
ObjHeader* BackRefFromAssociatedObject::ref() const {
kotlin::AssertThreadState(kotlin::ThreadState::kRunnable);
RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found");
if (!ensureRefAccessible<errorPolicy>(obj_, context_)) {
return nullptr;
}
AdoptReferenceFromSharedVariable(obj_);
return obj_;
}
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kTerminate>() const;
extern "C" {
RUNTIME_NOTHROW void KRefSharedHolder_initLocal(KRefSharedHolder* holder, ObjHeader* obj) {
holder->initLocal(obj);
@@ -233,8 +26,8 @@ RUNTIME_NOTHROW void KRefSharedHolder_init(KRefSharedHolder* holder, ObjHeader*
holder->init(obj);
}
RUNTIME_NOTHROW void KRefSharedHolder_dispose(const KRefSharedHolder* holder) {
holder->dispose();
RUNTIME_NOTHROW void KRefSharedHolder_dispose(KRefSharedHolder* holder) {
holder->dispose();
}
RUNTIME_NOTHROW ObjHeader* KRefSharedHolder_ref(const KRefSharedHolder* holder) {
kotlin-native/runtime/src/main/cpp/MemorySharedRefs.hpp
+3 -3
View File
@@ -28,9 +28,9 @@ class KRefSharedHolder {
template <ErrorPolicy errorPolicy>
ObjHeader* ref() const;
void dispose() const;
void dispose();
void disposeFromNative() const {
void disposeFromNative() {
kotlin::CalledFromNativeGuard guard;
dispose();
}
@@ -78,7 +78,7 @@ static_assert(
extern "C" {
RUNTIME_NOTHROW void KRefSharedHolder_initLocal(KRefSharedHolder* holder, ObjHeader* obj);
RUNTIME_NOTHROW void KRefSharedHolder_init(KRefSharedHolder* holder, ObjHeader* obj);
RUNTIME_NOTHROW void KRefSharedHolder_dispose(const KRefSharedHolder* holder);
RUNTIME_NOTHROW void KRefSharedHolder_dispose(KRefSharedHolder* holder);
RUNTIME_NOTHROW ObjHeader* KRefSharedHolder_ref(const KRefSharedHolder* holder);
} // extern "C"
kotlin-native/runtime/src/main/cpp/ObjectTestSupport.hpp
+45 -10
View File
@@ -101,8 +101,21 @@ private:
std_support::vector<int32_t> objOffsets_;
};
class Any : private Pinned {
public:
ObjHeader* header() noexcept { return &header_; }
void installMetaObject() noexcept { (void)header()->meta_object(); }
protected:
Any() noexcept = default;
~Any() = default;
ObjHeader header_;
};
template <typename Payload>
class Object : private Pinned {
class Object : public Any {
public:
class FieldIterator {
public:
@@ -160,15 +173,12 @@ public:
header_.typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
}
ObjHeader* header() noexcept { return &header_; }
Payload& operator*() noexcept { return payload_; }
Payload* operator->() noexcept { return &payload_; }
FieldIterable fields() noexcept { return FieldIterable(*this); }
private:
ObjHeader header_;
Payload payload_{};
};
@@ -191,7 +201,7 @@ namespace internal {
// Array types are predetermined, use one of the subclasses below.
template <typename Payload, size_t ElementCount>
class Array : private Pinned {
class Array : public Any {
public:
static Array<Payload, ElementCount>& FromArrayHeader(ArrayHeader* arr) noexcept {
static_assert(std::is_trivially_destructible_v<Array>, "Array destructor is not guaranteed to be called.");
@@ -210,17 +220,16 @@ public:
TypeInfoHolder{TypeInfoHolder::ArrayBuilder<Payload>()}.typeInfo()->IsLayoutCompatible(typeInfo),
"constructing array from incompatible type info");
header_.typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
header_.count_ = ElementCount;
count_ = ElementCount;
}
ObjHeader* header() noexcept { return header_.obj(); }
ArrayHeader* arrayHeader() noexcept { return &header_; }
ArrayHeader* arrayHeader() noexcept { return header()->array(); }
std::array<Payload, ElementCount>& elements() noexcept { return elements_; }
private:
ArrayHeader header_;
std::array<Payload, ElementCount> elements_{};
uint32_t count_;
alignas(ArrayHeader) std::array<Payload, ElementCount> elements_{};
};
} // namespace internal
@@ -335,5 +344,31 @@ public:
String() noexcept : internal::Array<KChar, ElementCount>(theStringTypeInfo) {}
};
struct RegularWeakReferenceImplPayload {
void* referred;
using Field = ObjHeader* RegularWeakReferenceImplPayload::*;
static constexpr std::array<Field, 0> kFields{};
};
extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader*);
class RegularWeakReferenceImpl : public Object<RegularWeakReferenceImplPayload> {
public:
static RegularWeakReferenceImpl& FromObjHeader(ObjHeader* obj) noexcept {
RuntimeAssert(obj->type_info() == theRegularWeakReferenceImplTypeInfo, "Invalid type");
return static_cast<RegularWeakReferenceImpl&>(Object::FromObjHeader(obj));
}
RegularWeakReferenceImpl() noexcept : Object(theRegularWeakReferenceImplTypeInfo) {}
OBJ_GETTER0(get) noexcept { RETURN_RESULT_OF(Konan_RegularWeakReferenceImpl_get, header()); }
ObjHeader* get() noexcept {
ObjHeader* result;
return get(&result);
}
};
} // namespace test_support
} // namespace kotlin
kotlin-native/runtime/src/main/cpp/Types.h
+1
View File
@@ -70,6 +70,7 @@ extern const TypeInfo* theThrowableTypeInfo;
extern const TypeInfo* theUnitTypeInfo;
extern const TypeInfo* theWorkerBoundReferenceTypeInfo;
extern const TypeInfo* theCleanerImplTypeInfo;
extern const TypeInfo* theRegularWeakReferenceImplTypeInfo;
KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info) RUNTIME_PURE;
KBoolean IsInstanceOfClassFast(const ObjHeader* obj, int32_t lo, int32_t hi) RUNTIME_PURE;
kotlin-native/runtime/src/main/kotlin/kotlin/native/ref/WeakPrivate.kt
+35 -23
View File
@@ -6,46 +6,54 @@
package kotlin.native.ref
import kotlinx.cinterop.COpaquePointer
import kotlin.native.internal.ExportForCppRuntime
import kotlin.native.internal.Frozen
import kotlin.native.internal.GCUnsafeCall
import kotlin.native.internal.NoReorderFields
import kotlin.native.internal.Escapes
import kotlin.native.internal.*
/**
* Theory of operations:
*
* Weak references in Kotlin/Native are implemented in the following way. Whenever weak reference to an
* object is created, we atomically modify type info pointer in the object to point into a metaobject.
* This metaobject contains a strong reference to the counter object (instance of WeakReferenceCounter class).
* Every other weak reference contains a strong reference to the counter object.
* Weak references in Kotlin/Native are implemented in the following way. Whenever weak reference to a
* object is created, we try to create one of `WeakReferenceImpl`:
* `PermanentWeakReferenceImpl` for permanent objects,
* `ObjCWeakReferenceImpl` for objects coming from ObjC,
* or `RegularWeakReferenceImpl` for all other Kotlin objects.
* The latter are stored in object's extra data, so only one per object is created.
* Every other weak reference contains a strong reference to the impl object.
*
* [weak1] [weak2]
* \ /
* V V
* .......[Counter] <----
* . |
* . |
* ->[Object] -> [Meta]-
* [weak1] [weak2]
* \ /
* V V
* ...... [WeakReferenceImpl] <------
* . |
* . |
* ->[Object] -> [ExtraObjectData]-
*
* References from weak reference objects to the counter and from the metaobject to the counter are strong,
* and from the counter to the object is nullably weak. So whenever an object dies, if it has a metaobject,
* it is traversed to find a counter object, and atomically nullify reference to the object. Afterward, all attempts
* to get the object would yield null.
* References from weak reference objects to the `RegularWeakReferenceImpl` and from the
* extra data to the `RegularWeakReferenceImpl` are strong, and from the
* `RegularWeakReferenceImpl` to the object is nullably weak. During GC if the object is
* considered dead, the reference inside `RegularWeakReferenceImpl` is nulled out.
*/
// Clear holding the counter object, which refers to the actual object.
@NoReorderFields
@Frozen
@OptIn(FreezingIsDeprecated::class)
internal class WeakReferenceCounter(var referred: COpaquePointer?) : WeakReferenceImpl() {
internal class WeakReferenceCounterLegacyMM(var referred: COpaquePointer?) : WeakReferenceImpl() {
// Spinlock, potentially taken when materializing or removing 'referred' object.
var lock: Int = 0
// Optimization for concurrent access.
var cookie: Int = 0
@GCUnsafeCall("Konan_WeakReferenceCounter_get")
@GCUnsafeCall("Konan_WeakReferenceCounterLegacyMM_get")
external override fun get(): Any?
}
@NoReorderFields
@ExportTypeInfo("theRegularWeakReferenceImplTypeInfo")
internal class RegularWeakReferenceImpl(
val referred: COpaquePointer,
) : WeakReferenceImpl() {
@GCUnsafeCall("Konan_RegularWeakReferenceImpl_get")
external override fun get(): Any?
}
@@ -59,9 +67,13 @@ internal abstract class WeakReferenceImpl {
@Escapes(0b01) // referent escapes.
external internal fun getWeakReferenceImpl(referent: Any): WeakReferenceImpl
// Create a counter object for legacy MM.
@ExportForCppRuntime
internal fun makeWeakReferenceCounterLegacyMM(referred: COpaquePointer) = WeakReferenceCounterLegacyMM(referred)
// Create a counter object.
@ExportForCppRuntime
internal fun makeWeakReferenceCounter(referred: COpaquePointer) = WeakReferenceCounter(referred)
internal fun makeRegularWeakReferenceImpl(referred: COpaquePointer) = RegularWeakReferenceImpl(referred)
internal class PermanentWeakReferenceImpl(val referred: Any): kotlin.native.ref.WeakReferenceImpl() {
override fun get(): Any? = referred
kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp
+5 -7
View File
@@ -7,7 +7,6 @@
#include "PointerBits.h"
#include "ThreadData.hpp"
#include "Weak.h"
#ifdef KONAN_OBJC_INTEROP
#include "ObjCMMAPI.h"
@@ -77,20 +76,19 @@ bool mm::ExtraObjectData::HasAssociatedObject() noexcept {
#endif
}
void mm::ExtraObjectData::ClearWeakReferenceCounter() noexcept {
if (!HasWeakReferenceCounter()) return;
void mm::ExtraObjectData::ClearRegularWeakReferenceImpl() noexcept {
if (!HasRegularWeakReferenceImpl()) return;
auto *object = GetBaseObject();
WeakReferenceCounterClear(GetWeakReferenceCounter());
disposeRegularWeakReferenceImpl(GetRegularWeakReferenceImpl());
// Not using `mm::SetHeapRef here`, because this code is called during sweep phase by the GC thread,
// and so cannot affect marking.
// TODO: Asserts on the above?
weakReferenceCounterOrBaseObject_ = object;
weakReferenceOrBaseObject_ = object;
}
mm::ExtraObjectData::~ExtraObjectData() {
RuntimeAssert(!HasWeakReferenceCounter(), "Object must have cleared weak references");
RuntimeAssert(!HasRegularWeakReferenceImpl(), "Object must have cleared weak references");
#ifdef KONAN_OBJC_INTEROP
RuntimeAssert(associatedObject_ == nullptr, "Object must have cleared associated object");
kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp
+14 -15
View File
@@ -13,7 +13,7 @@
#include "Memory.h"
#include "TypeInfo.h"
#include "Utils.hpp"
#include "Weak.h"
#include "Weak.hpp"
namespace kotlin {
namespace mm {
@@ -52,33 +52,32 @@ public:
bool getFlag(Flags value) noexcept { return (flags_.load() & (1u << static_cast<uint32_t>(value))) != 0; }
void setFlag(Flags value) noexcept { flags_.fetch_or(1u << static_cast<uint32_t>(value)); }
bool HasWeakReferenceCounter() noexcept { return hasPointerBits(weakReferenceCounterOrBaseObject_.load(), WEAK_REF_TAG); }
void ClearWeakReferenceCounter() noexcept;
ObjHeader* GetWeakReferenceCounter() noexcept {
auto *pointer = weakReferenceCounterOrBaseObject_.load();
bool HasRegularWeakReferenceImpl() noexcept { return hasPointerBits(weakReferenceOrBaseObject_.load(), WEAK_REF_TAG); }
void ClearRegularWeakReferenceImpl() noexcept;
ObjHeader* GetRegularWeakReferenceImpl() noexcept {
auto* pointer = weakReferenceOrBaseObject_.load();
if (hasPointerBits(pointer, WEAK_REF_TAG)) return clearPointerBits(pointer, WEAK_REF_TAG);
return nullptr;
}
ObjHeader* GetOrSetWeakReferenceCounter(ObjHeader* object, ObjHeader* counter) noexcept {
if (weakReferenceCounterOrBaseObject_.compare_exchange_strong(object, setPointerBits(counter, WEAK_REF_TAG))) {
return counter;
ObjHeader* GetOrSetRegularWeakReferenceImpl(ObjHeader* object, ObjHeader* weakRef) noexcept {
if (weakReferenceOrBaseObject_.compare_exchange_strong(object, setPointerBits(weakRef, WEAK_REF_TAG))) {
return weakRef;
} else {
return clearPointerBits(object, WEAK_REF_TAG); // on fail current value of counter is stored to object
return clearPointerBits(object, WEAK_REF_TAG); // on fail current value of weakRef is stored to object
}
}
ObjHeader* GetBaseObject() noexcept {
auto *header = weakReferenceCounterOrBaseObject_.load();
auto* header = weakReferenceOrBaseObject_.load();
if (hasPointerBits(header, WEAK_REF_TAG)) {
return UnsafeWeakReferenceCounterGet(clearPointerBits(header, WEAK_REF_TAG));
return regularWeakReferenceImplBaseObjectUnsafe(clearPointerBits(header, WEAK_REF_TAG));
} else {
return header;
}
}
// info must be equal to objHeader->type_info(), but it needs to be loaded in advance to avoid data races
explicit ExtraObjectData(ObjHeader* objHeader, const TypeInfo *info) noexcept :
typeInfo_(nullptr), weakReferenceCounterOrBaseObject_(objHeader) {
explicit ExtraObjectData(ObjHeader* objHeader, const TypeInfo* info) noexcept :
typeInfo_(nullptr), weakReferenceOrBaseObject_(objHeader) {
atomicSetRelease(&typeInfo_, info);
}
~ExtraObjectData();
@@ -92,7 +91,7 @@ private:
std::atomic<void*> associatedObject_ = nullptr;
#endif
std::atomic<ObjHeader*> weakReferenceCounterOrBaseObject_;
std::atomic<ObjHeader*> weakReferenceOrBaseObject_;
};
} // namespace mm
kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp
+1 -1
View File
@@ -49,7 +49,7 @@ TEST_F(ExtraObjectDataTest, Install) {
EXPECT_TRUE(object.header()->has_meta_object());
EXPECT_THAT(object.header()->meta_object(), extraData.AsMetaObjHeader());
EXPECT_THAT(object.header()->type_info(), typeInfo);
EXPECT_FALSE(extraData.HasWeakReferenceCounter());
EXPECT_FALSE(extraData.HasRegularWeakReferenceImpl());
EXPECT_THAT(extraData.GetBaseObject(), object.header());
extraData.Uninstall();
kotlin-native/runtime/src/mm/cpp/Memory.cpp
+25 -27
View File
@@ -48,11 +48,11 @@ ALWAYS_INLINE mm::StableRefRegistry::Node* FromForeignRefManager(ForeignRefManag
} // namespace
ObjHeader* ObjHeader::GetWeakCounter() {
return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetWeakReferenceCounter();
RuntimeFail("Only for legacy MM");
}
ObjHeader* ObjHeader::GetOrSetWeakCounter(ObjHeader* counter) {
return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetOrSetWeakReferenceCounter(this, counter);
RuntimeFail("Only for legacy MM");
}
#ifdef KONAN_OBJC_INTEROP
@@ -464,14 +464,6 @@ extern "C" RUNTIME_NOTHROW void PerformFullGC(MemoryState* memory) {
threadData->gc().ScheduleAndWaitFullGCWithFinalizers();
}
extern "C" bool TryAddHeapRef(const ObjHeader* object) {
RuntimeFail("Only for legacy MM");
}
extern "C" RUNTIME_NOTHROW void ReleaseHeapRefNoCollect(const ObjHeader* object) {
RuntimeFail("Only for legacy MM");
}
extern "C" RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object) {
// TODO: With CMS this needs:
// * during marking phase if `object` is unmarked: barrier (might be automatic because of the stack write)
@@ -562,13 +554,6 @@ extern "C" void EnsureNeverFrozen(ObjHeader* obj) {
}
}
extern "C" ForeignRefContext InitLocalForeignRef(ObjHeader* object) {
AssertThreadState(ThreadState::kRunnable);
// TODO: Remove when legacy MM is gone.
// Nothing to do.
return nullptr;
}
extern "C" ForeignRefContext InitForeignRef(ObjHeader* object) {
AssertThreadState(ThreadState::kRunnable);
auto* threadData = mm::ThreadRegistry::Instance().CurrentThreadData();
@@ -585,16 +570,6 @@ extern "C" void DeinitForeignRef(ObjHeader* object, ForeignRefContext context) {
mm::StableRefRegistry::Instance().UnregisterStableRef(threadData, node);
}
extern "C" bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) {
// TODO: Remove when legacy MM is gone.
return true;
}
extern "C" void AdoptReferenceFromSharedVariable(ObjHeader* object) {
// TODO: Remove when legacy MM is gone.
// Nothing to do.
}
extern "C" void CheckGlobalsAccessible() {
// TODO: Remove when legacy MM is gone.
// Always accessible
@@ -661,3 +636,26 @@ RUNTIME_NOTHROW ALWAYS_INLINE extern "C" void Kotlin_processEmptyObjectInMark(vo
// Empty object. Nothing to do.
// TODO: Try to generate it in the code generator.
}
extern "C" OBJ_GETTER(makePermanentWeakReferenceImpl, ObjHeader*);
extern "C" OBJ_GETTER(makeObjCWeakReferenceImpl, void*);
RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) {
if (referred->permanent()) {
RETURN_RESULT_OF(makePermanentWeakReferenceImpl, referred);
}
#if KONAN_OBJC_INTEROP
if (IsInstance(referred, theObjCObjectWrapperTypeInfo)) {
RETURN_RESULT_OF(makeObjCWeakReferenceImpl, referred->GetAssociatedObject());
}
#endif // KONAN_OBJC_INTEROP
RETURN_RESULT_OF(mm::createRegularWeakReferenceImpl, referred);
}
RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_WeakReferenceCounterLegacyMM_get, ObjHeader* counter) {
RuntimeFail("Legacy MM only");
}
RUNTIME_NOTHROW extern "C" OBJ_GETTER(Konan_RegularWeakReferenceImpl_get, ObjHeader* weakRef) {
RETURN_RESULT_OF(mm::derefRegularWeakReferenceImpl, weakRef);
}
kotlin-native/runtime/src/mm/cpp/MemoryPrivate.hpp
+6
View File
@@ -39,4 +39,10 @@ extern "C" struct MemoryState : kotlin::Pinned {
}
};
extern "C" {
ForeignRefContext InitForeignRef(ObjHeader* object);
void DeinitForeignRef(ObjHeader* object, ForeignRefContext context);
RUNTIME_NOTHROW OBJ_GETTER(TryRef, ObjHeader* object);
}
#endif //RUNTIME_MEMORYPRIVATE_HPP
kotlin-native/runtime/src/mm/cpp/MemorySharedRefs.cpp
+128
View File
@@ -0,0 +1,128 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "MemorySharedRefs.hpp"
#include "MemoryPrivate.hpp"
using namespace kotlin;
void KRefSharedHolder::initLocal(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = nullptr;
obj_ = obj;
}
void KRefSharedHolder::init(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
context_ = InitForeignRef(obj);
obj_ = obj;
}
template <ErrorPolicy errorPolicy>
ObjHeader* KRefSharedHolder::ref() const {
AssertThreadState(ThreadState::kRunnable);
// ref_ may be null if created with initLocal.
return obj_;
}
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* KRefSharedHolder::ref<ErrorPolicy::kTerminate>() const;
void KRefSharedHolder::dispose() {
if (obj_ == nullptr) {
// To handle the case when it is not initialized. See [KotlinMutableSet/Dictionary dealloc].
return;
}
DeinitForeignRef(obj_, context_);
}
void BackRefFromAssociatedObject::initAndAddRef(ObjHeader* obj) {
RuntimeAssert(obj != nullptr, "must not be null");
obj_ = obj;
// Generally a specialized addRef below:
context_ = InitForeignRef(obj);
refCount = 1;
}
template <ErrorPolicy errorPolicy>
void BackRefFromAssociatedObject::addRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
// Can be called both from Native state (if ObjC or Swift code adds RC)
// and from Runnable state (Kotlin_ObjCExport_refToObjC).
if (atomicAdd(&refCount, 1) == 1) {
if (obj_ == nullptr) return; // E.g. after [detach].
kotlin::CalledFromNativeGuard guard(/* reentrant */ true);
// Foreign reference has already been deinitialized (see [releaseRef]).
// Create a new one:
context_ = InitForeignRef(obj_);
}
}
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kThrow>();
template void BackRefFromAssociatedObject::addRef<ErrorPolicy::kTerminate>();
template <ErrorPolicy errorPolicy>
bool BackRefFromAssociatedObject::tryAddRef() {
static_assert(errorPolicy != ErrorPolicy::kDefaultValue, "Cannot use default return value here");
kotlin::CalledFromNativeGuard guard;
if (obj_ == nullptr) return false; // E.g. after [detach].
ObjHolder holder;
ObjHeader* obj = TryRef(obj_, holder.slot());
// Failed to lock weak reference.
if (obj == nullptr) return false;
RuntimeAssert(obj == obj_, "Mismatched locked weak. obj=%p obj_=%p", obj, obj_);
// TODO: This is a very weird way to ask for "unsafe" addRef.
addRef<ErrorPolicy::kIgnore>();
return true;
}
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kThrow>();
template bool BackRefFromAssociatedObject::tryAddRef<ErrorPolicy::kTerminate>();
void BackRefFromAssociatedObject::releaseRef() {
ForeignRefContext context = context_;
if (atomicAdd(&refCount, -1) == 0) {
if (obj_ == nullptr) return; // E.g. after [detach].
kotlin::CalledFromNativeGuard guard;
// Note: by this moment "subsequent" addRef may have already happened and patched context_.
// So use the value loaded before refCount update:
DeinitForeignRef(obj_, context);
// From this moment [context] is generally a dangling pointer.
// This is handled in [IsForeignRefAccessible] and [addRef].
// TODO: This probably isn't fine in new MM. Make sure it works.
}
}
void BackRefFromAssociatedObject::detach() {
RuntimeAssert(atomicGet(&refCount) == 0, "unexpected refCount");
obj_ = nullptr; // Handled in addRef/tryAddRef/releaseRef/ref.
}
ALWAYS_INLINE void BackRefFromAssociatedObject::assertDetached() {
RuntimeAssert(obj_ == nullptr, "Expecting this=%p to be detached, but found obj_=%p", this, obj_);
}
template <ErrorPolicy errorPolicy>
ObjHeader* BackRefFromAssociatedObject::ref() const {
kotlin::AssertThreadState(kotlin::ThreadState::kRunnable);
RuntimeAssert(obj_ != nullptr, "no valid Kotlin object found");
return obj_;
}
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kDefaultValue>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kThrow>() const;
template ObjHeader* BackRefFromAssociatedObject::ref<ErrorPolicy::kTerminate>() const;
kotlin-native/runtime/src/mm/cpp/Weak.cpp
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#include "Weak.hpp"
#include "ExtraObjectData.hpp"
#include "ObjectOps.hpp"
#include "ThreadState.hpp"
#include "Types.h"
using namespace kotlin;
extern "C" {
OBJ_GETTER(makeRegularWeakReferenceImpl, void*);
}
namespace {
struct RegularWeakReferenceImpl {
ObjHeader header;
void* referred;
};
RegularWeakReferenceImpl* asRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept {
return reinterpret_cast<RegularWeakReferenceImpl*>(weakRef);
}
} // namespace
OBJ_GETTER(mm::createRegularWeakReferenceImpl, ObjHeader* object) noexcept {
auto* thread = mm::ThreadRegistry::Instance().CurrentThreadData();
AssertThreadState(thread, ThreadState::kRunnable);
auto& extraObject = mm::ExtraObjectData::GetOrInstall(object);
if (auto* weakRef = extraObject.GetRegularWeakReferenceImpl()) {
RETURN_OBJ(weakRef);
}
ObjHolder holder;
auto* weakRef = makeRegularWeakReferenceImpl(object, holder.slot());
auto* setWeakRef = extraObject.GetOrSetRegularWeakReferenceImpl(object, weakRef);
RETURN_OBJ(setWeakRef);
}
void mm::disposeRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept {
asRegularWeakReferenceImpl(weakRef)->referred = nullptr;
}
OBJ_GETTER(mm::derefRegularWeakReferenceImpl, ObjHeader* weakRef) noexcept {
ObjHeader** location = reinterpret_cast<ObjHeader**>(&asRegularWeakReferenceImpl(weakRef)->referred);
RETURN_RESULT_OF(mm::ReadHeapRefAtomic, location);
}
ObjHeader* mm::regularWeakReferenceImplBaseObjectUnsafe(ObjHeader* weakRef) noexcept {
return static_cast<ObjHeader*>(asRegularWeakReferenceImpl(weakRef)->referred);
}
kotlin-native/runtime/src/mm/cpp/Weak.hpp
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
#pragma once
#include "Memory.h"
#include "Utils.hpp"
namespace kotlin::mm {
OBJ_GETTER(createRegularWeakReferenceImpl, ObjHeader* object) noexcept;
void disposeRegularWeakReferenceImpl(ObjHeader* weakRef) noexcept;
OBJ_GETTER(derefRegularWeakReferenceImpl, ObjHeader* weakRef) noexcept;
ObjHeader* regularWeakReferenceImplBaseObjectUnsafe(ObjHeader* weakRef) noexcept;
} // namespace kotlin::mm
kotlin-native/runtime/src/test_support/cpp/CompilerGenerated.cpp
+8 -1
View File
@@ -50,6 +50,8 @@ kotlin::test_support::TypeInfoHolder theUnitTypeInfoHolder{kotlin::test_support:
kotlin::test_support::TypeInfoHolder theWorkerBoundReferenceTypeInfoHolder{
kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
kotlin::test_support::TypeInfoHolder theCleanerImplTypeInfoHolder{kotlin::test_support::TypeInfoHolder::ObjectBuilder<EmptyPayload>()};
kotlin::test_support::TypeInfoHolder theRegularWeakReferenceImplTypeInfoHolder{
kotlin::test_support::TypeInfoHolder::ObjectBuilder<kotlin::test_support::RegularWeakReferenceImplPayload>()};
ArrayHeader theEmptyStringImpl = {theStringTypeInfoHolder.typeInfo(), /* element count */ 0};
@@ -90,6 +92,7 @@ extern const TypeInfo* theThrowableTypeInfo = theThrowableTypeInfoHolder.typeInf
extern const TypeInfo* theUnitTypeInfo = theUnitTypeInfoHolder.typeInfo();
extern const TypeInfo* theWorkerBoundReferenceTypeInfo = theWorkerBoundReferenceTypeInfoHolder.typeInfo();
extern const TypeInfo* theCleanerImplTypeInfo = theCleanerImplTypeInfoHolder.typeInfo();
extern const TypeInfo* theRegularWeakReferenceImplTypeInfo = theRegularWeakReferenceImplTypeInfoHolder.typeInfo();
extern const ArrayHeader theEmptyArray = {theArrayTypeInfoHolder.typeInfo(), /* element count */ 0};
@@ -97,7 +100,11 @@ OBJ_GETTER0(TheEmptyString) {
RETURN_OBJ(theEmptyStringImpl.obj());
}
RUNTIME_NORETURN OBJ_GETTER(makeWeakReferenceCounter, void*) {
RUNTIME_NORETURN OBJ_GETTER(makeRegularWeakReferenceImpl, void*) {
throw std::runtime_error("Not implemented for tests");
}
RUNTIME_NORETURN OBJ_GETTER(makeWeakReferenceCounterLegacyMM, void*) {
throw std::runtime_error("Not implemented for tests");
}