Add ExtraObjectData (#4602)

kotlin-native/runtime/src/legacymm/cpp/Memory.cpp

@@ -1218,7 +1218,7 @@ ALWAYS_INLINE void runDeallocationHooks(ContainerHeader* container) {
     CycleDetector::removeCandidateIfNeeded(obj);
 #endif  // USE_CYCLE_DETECTOR
     if (obj->has_meta_object()) {
-      ObjHeader::destroyMetaObject(&obj->typeInfoOrMeta_);
+      ObjHeader::destroyMetaObject(obj);
     }
     obj = reinterpret_cast<ObjHeader*>(reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
   }
@@ -3102,7 +3102,8 @@ OBJ_GETTER(findCycle, KRef root) {
 
 }  // namespace
 
-MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
+MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) {
+  TypeInfo** location = &object->typeInfoOrMeta_;
   TypeInfo* typeInfo = *location;
   RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
 
@@ -3128,7 +3129,8 @@ MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
   return meta;
 }
 
-void ObjHeader::destroyMetaObject(TypeInfo** location) {
+void ObjHeader::destroyMetaObject(ObjHeader* object) {
+  TypeInfo** location = &object->typeInfoOrMeta_;
   MetaObjHeader* meta = clearPointerBits(*(reinterpret_cast<MetaObjHeader**>(location)), OBJECT_TAG_MASK);
   *const_cast<const TypeInfo**>(location) = meta->typeInfo_;
   if (meta->WeakReference.counter_ != nullptr) {

kotlin-native/runtime/src/main/cpp/Memory.h

@@ -38,19 +38,29 @@ struct MetaObjHeader;
 struct ObjHeader {
   TypeInfo* typeInfoOrMeta_;
 
+  // Returns `nullptr` if it's not a meta object.
+  static MetaObjHeader* AsMetaObject(TypeInfo* typeInfo) noexcept {
+      auto* typeInfoOrMeta = clearPointerBits(typeInfo, OBJECT_TAG_MASK);
+      if (typeInfoOrMeta != typeInfoOrMeta->typeInfo_) {
+          return reinterpret_cast<MetaObjHeader*>(typeInfoOrMeta);
+      } else {
+          return nullptr;
+      }
+  }
+
   const TypeInfo* type_info() const {
     return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
   }
 
   bool has_meta_object() const {
-    auto* typeInfoOrMeta = clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
-    return (typeInfoOrMeta != typeInfoOrMeta->typeInfo_);
+      return AsMetaObject(typeInfoOrMeta_) != nullptr;
   }
 
   MetaObjHeader* meta_object() {
-     return has_meta_object() ?
-        reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)) :
-        createMetaObject(&typeInfoOrMeta_);
+      if (auto* metaObject = AsMetaObject(typeInfoOrMeta_)) {
+          return metaObject;
+      }
+      return createMetaObject(this);
   }
 
   ALWAYS_INLINE ObjHeader** GetWeakCounterLocation();
@@ -75,8 +85,8 @@ struct ObjHeader {
     return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
   }
 
-  static MetaObjHeader* createMetaObject(TypeInfo** location);
-  static void destroyMetaObject(TypeInfo** location);
+  static MetaObjHeader* createMetaObject(ObjHeader* object);
+  static void destroyMetaObject(ObjHeader* object);
 };
 
 // Header of value type array objects. Keep layout in sync with that of object header.

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.cpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ExtraObjectData.hpp"
+
+#include "PointerBits.h"
+#include "Weak.h"
+
+#ifdef KONAN_OBJC_INTEROP
+#include "ObjCMMAPI.h"
+#endif
+
+using namespace kotlin;
+
+// static
+mm::ExtraObjectData& mm::ExtraObjectData::Install(ObjHeader* object) noexcept {
+    TypeInfo* typeInfo = object->typeInfoOrMeta_;
+    if (auto* metaObject = ObjHeader::AsMetaObject(typeInfo)) {
+        return mm::ExtraObjectData::FromMetaObjHeader(metaObject);
+    }
+
+    RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
+
+    auto* data = new ExtraObjectData(typeInfo);
+
+    TypeInfo* old = __sync_val_compare_and_swap(&object->typeInfoOrMeta_, typeInfo, reinterpret_cast<TypeInfo*>(data));
+    if (old != typeInfo) {
+        // Somebody else created `mm::ExtraObjectData` for this object
+        delete data;
+        return *reinterpret_cast<mm::ExtraObjectData*>(old);
+    }
+
+    return *data;
+}
+
+// static
+void mm::ExtraObjectData::Uninstall(ObjHeader* object) noexcept {
+    RuntimeAssert(object->has_meta_object(), "Object must have a meta object set");
+
+    auto& data = ExtraObjectData::FromMetaObjHeader(object->meta_object());
+
+    *const_cast<const TypeInfo**>(&object->typeInfoOrMeta_) = data.typeInfo_;
+
+    delete &data;
+}
+
+mm::ExtraObjectData::~ExtraObjectData() {
+    if (weakReferenceCounter_) {
+        WeakReferenceCounterClear(weakReferenceCounter_);
+        ZeroHeapRef(&weakReferenceCounter_);
+    }
+
+#ifdef KONAN_OBJC_INTEROP
+    Kotlin_ObjCExport_releaseAssociatedObject(associatedObject_);
+#endif
+}

kotlin-native/runtime/src/mm/cpp/ExtraObjectData.hpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_MM_EXTRA_OBJECT_DATA_H
+#define RUNTIME_MM_EXTRA_OBJECT_DATA_H
+
+#include <cstddef>
+#include <cstdint>
+
+#include "Memory.h"
+#include "TypeInfo.h"
+#include "Utils.hpp"
+
+namespace kotlin {
+namespace mm {
+
+// Optional data that's lazily allocated only for objects that need it.
+class ExtraObjectData : private Pinned {
+public:
+    MetaObjHeader* AsMetaObjHeader() noexcept { return reinterpret_cast<MetaObjHeader*>(this); }
+    static ExtraObjectData& FromMetaObjHeader(MetaObjHeader* header) noexcept { return *reinterpret_cast<ExtraObjectData*>(header); }
+
+    static ExtraObjectData& Install(ObjHeader* object) noexcept;
+    static void Uninstall(ObjHeader* object) noexcept;
+
+#ifdef KONAN_OBJC_INTEROP
+    void** GetAssociatedObjectLocation() noexcept { return &associatedObject_; }
+#endif
+
+    ObjHeader** GetWeakCounterLocation() noexcept { return &weakReferenceCounter_; }
+
+private:
+    explicit ExtraObjectData(const TypeInfo* typeInfo) noexcept : typeInfo_(typeInfo) {}
+    ~ExtraObjectData();
+
+    // Must be first to match `TypeInfo` layout.
+    const TypeInfo* typeInfo_;
+
+#ifdef KONAN_OBJC_INTEROP
+    void* associatedObject_ = nullptr;
+#endif
+
+    // TODO: Need to respect when marking.
+    ObjHeader* weakReferenceCounter_ = nullptr;
+};
+
+} // namespace mm
+} // namespace kotlin
+
+#endif // RUNTIME_MM_EXTRA_OBJECT_DATA_H

kotlin-native/runtime/src/mm/cpp/ExtraObjectDataTest.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "ExtraObjectData.hpp"
+
+#include <atomic>
+#include <thread>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "TestSupport.hpp"
+
+using namespace kotlin;
+
+TEST(ExtraObjectDataTest, Install) {
+    TypeInfo typeInfo;
+    typeInfo.typeInfo_ = &typeInfo;
+    ObjHeader object;
+    object.typeInfoOrMeta_ = &typeInfo;
+
+    ASSERT_FALSE(object.has_meta_object());
+
+    auto& extraData = mm::ExtraObjectData::Install(&object);
+
+    EXPECT_TRUE(object.has_meta_object());
+    EXPECT_THAT(object.meta_object(), extraData.AsMetaObjHeader());
+    EXPECT_THAT(object.type_info(), &typeInfo);
+
+    mm::ExtraObjectData::Uninstall(&object);
+
+    EXPECT_FALSE(object.has_meta_object());
+    EXPECT_THAT(object.type_info(), &typeInfo);
+}
+
+TEST(ExtraObjectDataTest, ConcurrentInstall) {
+    TypeInfo typeInfo;
+    typeInfo.typeInfo_ = &typeInfo;
+    ObjHeader object;
+    object.typeInfoOrMeta_ = &typeInfo;
+
+    constexpr int kThreadCount = kDefaultThreadCount;
+
+    std::atomic<bool> canStart(false);
+    std::atomic<int> readyCount(0);
+    std::vector<std::thread> threads;
+    std::vector<mm::ExtraObjectData*> actual(kThreadCount, nullptr);
+
+    for (int i = 0; i < kThreadCount; ++i) {
+        threads.emplace_back([i, &actual, &object, &canStart, &readyCount]() {
+            ++readyCount;
+            while (!canStart) {
+            }
+            auto& extraData = mm::ExtraObjectData::Install(&object);
+            actual[i] = &extraData;
+        });
+    }
+
+    while (readyCount < kThreadCount) {
+    }
+    canStart = true;
+
+    for (auto& t : threads) {
+        t.join();
+    }
+
+    std::vector<mm::ExtraObjectData*> expected(kThreadCount, actual[0]);
+
+    EXPECT_THAT(actual, testing::ElementsAreArray(expected));
+
+    mm::ExtraObjectData::Uninstall(&object);
+}

kotlin-native/runtime/src/mm/cpp/Memory.cpp

@@ -6,6 +6,7 @@
 #include "Memory.h"
 
 #include "Exceptions.h"
+#include "ExtraObjectData.hpp"
 #include "GlobalsRegistry.hpp"
 #include "KAssert.h"
 #include "Porting.h"
@@ -58,6 +59,39 @@ ALWAYS_INLINE mm::ThreadData* GetThreadData(MemoryState* state) {
 
 } // namespace
 
+ObjHeader** ObjHeader::GetWeakCounterLocation() {
+    return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetWeakCounterLocation();
+}
+
+#ifdef KONAN_OBJC_INTEROP
+
+void* ObjHeader::GetAssociatedObject() {
+    if (!has_meta_object()) {
+        return nullptr;
+    }
+    return *GetAssociatedObjectLocation();
+}
+
+void** ObjHeader::GetAssociatedObjectLocation() {
+    return mm::ExtraObjectData::FromMetaObjHeader(this->meta_object()).GetAssociatedObjectLocation();
+}
+
+void ObjHeader::SetAssociatedObject(void* obj) {
+    *GetAssociatedObjectLocation() = obj;
+}
+
+#endif // KONAN_OBJC_INTEROP
+
+// static
+MetaObjHeader* ObjHeader::createMetaObject(ObjHeader* object) {
+    return mm::ExtraObjectData::Install(object).AsMetaObjHeader();
+}
+
+// static
+void ObjHeader::destroyMetaObject(ObjHeader* object) {
+    mm::ExtraObjectData::Uninstall(object);
+}
+
 ALWAYS_INLINE bool isShareable(const ObjHeader* obj) {
     // TODO: Remove when legacy MM is gone.
     return true;

kotlin-native/runtime/src/mm/cpp/Stubs.cpp

@@ -15,34 +15,6 @@ ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {
     TODO();
 }
 
-ObjHeader** ObjHeader::GetWeakCounterLocation() {
-    TODO();
-}
-
-#ifdef KONAN_OBJC_INTEROP
-
-void* ObjHeader::GetAssociatedObject() {
-    TODO();
-}
-
-void** ObjHeader::GetAssociatedObjectLocation() {
-    TODO();
-}
-
-void ObjHeader::SetAssociatedObject(void* obj) {
-    TODO();
-}
-
-#endif // KONAN_OBJC_INTEROP
-
-static MetaObjHeader* createMetaObject(TypeInfo** location) {
-    TODO();
-}
-
-static void destroyMetaObject(TypeInfo** location) {
-    TODO();
-}
-
 extern "C" {
 
 OBJ_GETTER(InitThreadLocalSingleton, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {