diff --git a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp index d826cd0218c..5fd2ab38e73 100644 --- a/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/legacymm/cpp/Memory.cpp @@ -1218,7 +1218,7 @@ ALWAYS_INLINE void runDeallocationHooks(ContainerHeader* container) { CycleDetector::removeCandidateIfNeeded(obj); #endif // USE_CYCLE_DETECTOR if (obj->has_meta_object()) { - ObjHeader::destroyMetaObject(&obj->typeInfoOrMeta_); + ObjHeader::destroyMetaObject(obj); } obj = reinterpret_cast(reinterpret_cast(obj) + objectSize(obj)); } @@ -3102,7 +3102,8 @@ OBJ_GETTER(findCycle, KRef root) { } // namespace -MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { +MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { + TypeInfo** location = &object->typeInfoOrMeta_; TypeInfo* typeInfo = *location; RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged"); @@ -3128,7 +3129,8 @@ MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { return meta; } -void ObjHeader::destroyMetaObject(TypeInfo** location) { +void ObjHeader::destroyMetaObject(ObjHeader* object) { + TypeInfo** location = &object->typeInfoOrMeta_; MetaObjHeader* meta = clearPointerBits(*(reinterpret_cast(location)), OBJECT_TAG_MASK); *const_cast(location) = meta->typeInfo_; if (meta->WeakReference.counter_ != nullptr) { diff --git a/kotlin-native/runtime/src/main/cpp/Memory.h b/kotlin-native/runtime/src/main/cpp/Memory.h index 5f0b0e1eb85..647eb8b35b2 100644 --- a/kotlin-native/runtime/src/main/cpp/Memory.h +++ b/kotlin-native/runtime/src/main/cpp/Memory.h @@ -38,19 +38,29 @@ struct MetaObjHeader; struct ObjHeader { TypeInfo* typeInfoOrMeta_; + // Returns `nullptr` if it's not a meta object. + static MetaObjHeader* AsMetaObject(TypeInfo* typeInfo) noexcept { + auto* typeInfoOrMeta = clearPointerBits(typeInfo, OBJECT_TAG_MASK); + if (typeInfoOrMeta != typeInfoOrMeta->typeInfo_) { + return reinterpret_cast(typeInfoOrMeta); + } else { + return nullptr; + } + } + const TypeInfo* type_info() const { return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_; } bool has_meta_object() const { - auto* typeInfoOrMeta = clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK); - return (typeInfoOrMeta != typeInfoOrMeta->typeInfo_); + return AsMetaObject(typeInfoOrMeta_) != nullptr; } MetaObjHeader* meta_object() { - return has_meta_object() ? - reinterpret_cast(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)) : - createMetaObject(&typeInfoOrMeta_); + if (auto* metaObject = AsMetaObject(typeInfoOrMeta_)) { + return metaObject; + } + return createMetaObject(this); } ALWAYS_INLINE ObjHeader** GetWeakCounterLocation(); @@ -75,8 +85,8 @@ struct ObjHeader { return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); } - static MetaObjHeader* createMetaObject(TypeInfo** location); - static void destroyMetaObject(TypeInfo** location); + static MetaObjHeader* createMetaObject(ObjHeader* object); + static void destroyMetaObject(ObjHeader* object); }; // Header of value type array objects. Keep layout in sync with that of object header. diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp new file mode 100644 index 00000000000..4fc9b1a4914 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ExtraObjectData.hpp" + +#include "PointerBits.h" +#include "Weak.h" + +#ifdef KONAN_OBJC_INTEROP +#include "ObjCMMAPI.h" +#endif + +using namespace kotlin; + +// static +mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept { + TypeInfo* typeInfo = object->typeInfoOrMeta_; + if (auto* metaObject = ObjHeader::AsMetaObject(typeInfo)) { + return mm::ExtraObjectData::FromMetaObjHeader(metaObject); + } + + RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged"); + + auto* data = new ExtraObjectData(typeInfo); + + TypeInfo* old = __sync_val_compare_and_swap(&object->typeInfoOrMeta_, typeInfo, reinterpret_cast(data)); + if (old != typeInfo) { + // Somebody else created `mm::ExtraObjectData` for this object + delete data; + return *reinterpret_cast(old); + } + + return *data; +} + +// static +void mm::ExtraObjectData::Uninstall(ObjHeader* object) noexcept { + RuntimeAssert(object->has_meta_object(), "Object must have a meta object set"); + + auto& data = ExtraObjectData::FromMetaObjHeader(object->meta_object()); + + *const_cast(&object->typeInfoOrMeta_) = data.typeInfo_; + + delete &data; +} + +mm::ExtraObjectData::~ExtraObjectData() { + if (weakReferenceCounter_) { + WeakReferenceCounterClear(weakReferenceCounter_); + ZeroHeapRef(&weakReferenceCounter_); + } + +#ifdef KONAN_OBJC_INTEROP + Kotlin_ObjCExport_releaseAssociatedObject(associatedObject_); +#endif +} diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp new file mode 100644 index 00000000000..f88524af2ee --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_H +#define RUNTIME_MM_EXTRA_OBJECT_DATA_H + +#include +#include + +#include "Memory.h" +#include "TypeInfo.h" +#include "Utils.hpp" + +namespace kotlin { +namespace mm { + +// Optional data that's lazily allocated only for objects that need it. +class ExtraObjectData : private Pinned { +public: + MetaObjHeader* AsMetaObjHeader() noexcept { return reinterpret_cast(this); } + static ExtraObjectData& FromMetaObjHeader(MetaObjHeader* header) noexcept { return *reinterpret_cast(header); } + + static ExtraObjectData& Install(ObjHeader* object) noexcept; + static void Uninstall(ObjHeader* object) noexcept; + +#ifdef KONAN_OBJC_INTEROP + void** GetAssociatedObjectLocation() noexcept { return &associatedObject_; } +#endif + + ObjHeader** GetWeakCounterLocation() noexcept { return &weakReferenceCounter_; } + +private: + explicit ExtraObjectData(const TypeInfo* typeInfo) noexcept : typeInfo_(typeInfo) {} + ~ExtraObjectData(); + + // Must be first to match `TypeInfo` layout. + const TypeInfo* typeInfo_; + +#ifdef KONAN_OBJC_INTEROP + void* associatedObject_ = nullptr; +#endif + + // TODO: Need to respect when marking. + ObjHeader* weakReferenceCounter_ = nullptr; +}; + +} // namespace mm +} // namespace kotlin + +#endif // RUNTIME_MM_EXTRA_OBJECT_DATA_H diff --git a/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp new file mode 100644 index 00000000000..594c47e0305 --- /dev/null +++ b/kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp @@ -0,0 +1,74 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +#include "ExtraObjectData.hpp" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include "TestSupport.hpp" + +using namespace kotlin; + +TEST(ExtraObjectDataTest, Install) { + TypeInfo typeInfo; + typeInfo.typeInfo_ = &typeInfo; + ObjHeader object; + object.typeInfoOrMeta_ = &typeInfo; + + ASSERT_FALSE(object.has_meta_object()); + + auto& extraData = mm::ExtraObjectData::Install(&object); + + EXPECT_TRUE(object.has_meta_object()); + EXPECT_THAT(object.meta_object(), extraData.AsMetaObjHeader()); + EXPECT_THAT(object.type_info(), &typeInfo); + + mm::ExtraObjectData::Uninstall(&object); + + EXPECT_FALSE(object.has_meta_object()); + EXPECT_THAT(object.type_info(), &typeInfo); +} + +TEST(ExtraObjectDataTest, ConcurrentInstall) { + TypeInfo typeInfo; + typeInfo.typeInfo_ = &typeInfo; + ObjHeader object; + object.typeInfoOrMeta_ = &typeInfo; + + constexpr int kThreadCount = kDefaultThreadCount; + + std::atomic canStart(false); + std::atomic readyCount(0); + std::vector threads; + std::vector actual(kThreadCount, nullptr); + + for (int i = 0; i < kThreadCount; ++i) { + threads.emplace_back([i, &actual, &object, &canStart, &readyCount]() { + ++readyCount; + while (!canStart) { + } + auto& extraData = mm::ExtraObjectData::Install(&object); + actual[i] = &extraData; + }); + } + + while (readyCount < kThreadCount) { + } + canStart = true; + + for (auto& t : threads) { + t.join(); + } + + std::vector expected(kThreadCount, actual[0]); + + EXPECT_THAT(actual, testing::ElementsAreArray(expected)); + + mm::ExtraObjectData::Uninstall(&object); +} diff --git a/kotlin-native/runtime/src/mm/cpp/Memory.cpp b/kotlin-native/runtime/src/mm/cpp/Memory.cpp index 8eb8ef9e4fd..84a47c0512f 100644 --- a/kotlin-native/runtime/src/mm/cpp/Memory.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Memory.cpp @@ -6,6 +6,7 @@ #include "Memory.h" #include "Exceptions.h" +#include "ExtraObjectData.hpp" #include "GlobalsRegistry.hpp" #include "KAssert.h" #include "Porting.h" @@ -58,6 +59,39 @@ ALWAYS_INLINE mm::ThreadData* GetThreadData(MemoryState* state) { } // namespace +ObjHeader** ObjHeader::GetWeakCounterLocation() { + return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetWeakCounterLocation(); +} + +#ifdef KONAN_OBJC_INTEROP + +void* ObjHeader::GetAssociatedObject() { + if (!has_meta_object()) { + return nullptr; + } + return *GetAssociatedObjectLocation(); +} + +void** ObjHeader::GetAssociatedObjectLocation() { + return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetAssociatedObjectLocation(); +} + +void ObjHeader::SetAssociatedObject(void* obj) { + *GetAssociatedObjectLocation() = obj; +} + +#endif // KONAN_OBJC_INTEROP + +// static +MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) { + return mm::ExtraObjectData::Install(object).AsMetaObjHeader(); +} + +// static +void ObjHeader::destroyMetaObject(ObjHeader* object) { + mm::ExtraObjectData::Uninstall(object); +} + ALWAYS_INLINE bool isShareable(const ObjHeader* obj) { // TODO: Remove when legacy MM is gone. return true; diff --git a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp index dd3cc74c1c6..869760b745a 100644 --- a/kotlin-native/runtime/src/mm/cpp/Stubs.cpp +++ b/kotlin-native/runtime/src/mm/cpp/Stubs.cpp @@ -15,34 +15,6 @@ ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) { TODO(); } -ObjHeader** ObjHeader::GetWeakCounterLocation() { - TODO(); -} - -#ifdef KONAN_OBJC_INTEROP - -void* ObjHeader::GetAssociatedObject() { - TODO(); -} - -void** ObjHeader::GetAssociatedObjectLocation() { - TODO(); -} - -void ObjHeader::SetAssociatedObject(void* obj) { - TODO(); -} - -#endif // KONAN_OBJC_INTEROP - -static MetaObjHeader* createMetaObject(TypeInfo** location) { - TODO(); -} - -static void destroyMetaObject(TypeInfo** location) { - TODO(); -} - extern "C" { OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {