Meta-object support. (#1451)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -616,7 +616,9 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         val typeInfoPtr: LLVMValueRef = if (owner.isObjCClass()) {
             call(context.llvm.getObjCKotlinTypeInfo, listOf(receiver))
         } else {
-            val typeInfoPtrPtr = LLVMBuildStructGEP(builder, receiver, 0 /* type_info */, "")!!
+            val typeInfoOrMetaPtr = structGep(receiver, 0  /* typeInfoOrMeta_ */)
+            val typeInfoOrMeta = load(typeInfoOrMetaPtr)
+            val typeInfoPtrPtr = structGep(typeInfoOrMeta, 0 /* typeInfo */)
             load(typeInfoPtrPtr)
         }
 
@@ -655,10 +657,10 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
 
         val objectPtr = codegen.getObjectInstanceStorage(descriptor, shared)
         val bbCurrent = currentBlock
-        val bbInit    = basicBlock("label_init", locationInfo)
+        val bbInit= basicBlock("label_init", locationInfo)
-        val bbExit    = basicBlock("label_continue", locationInfo)
+        val bbExit= basicBlock("label_continue", locationInfo)
         val objectVal = loadSlot(objectPtr, false)
-        val objectInitialized = icmpUGt(ptrToInt(objectVal, codegen.intPtrType), codegen.immOneIntPtrType)
+        val objectInitialized= icmpUGt(ptrToInt(objectVal, codegen.intPtrType), codegen.immOneIntPtrType)
         condBr(objectInitialized, bbExit, bbInit)
 
         positionAtEnd(bbInit)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -27,7 +27,6 @@ internal val LLVMValueRef.type: LLVMTypeRef
  * Represents the value which can be emitted as bitcode const value
  */
 internal interface ConstValue {
-
     val llvm: LLVMValueRef
 }
 
@@ -54,9 +53,8 @@ private class ConstGetElementPtr(val pointer: ConstPointer, val index: Int) : Co
 
 internal fun ConstPointer.bitcast(toType: LLVMTypeRef) = constPointer(LLVMConstBitCast(this.llvm, toType)!!)
 
-internal class ConstArray(val elemType: LLVMTypeRef?, val elements: List<ConstValue>) : ConstValue {
+internal class ConstArray(elementType: LLVMTypeRef?, val elements: List<ConstValue>) : ConstValue {
-
+    override val llvm = LLVMConstArray(elementType, elements.map { it.llvm }.toCValues(), elements.size)!!
-    override val llvm = LLVMConstArray(elemType, elements.map { it.llvm }.toCValues(), elements.size)!!
 }
 
 internal open class Struct(val type: LLVMTypeRef?, val elements: List<ConstValue?>) : ConstValue {
@@ -108,7 +106,7 @@ internal class Zero(val type: LLVMTypeRef) : ConstValue {
     override val llvm = LLVMConstNull(type)!!
 }
 
-internal class NullPointer(val pointeeType: LLVMTypeRef): ConstPointer {
+internal class NullPointer(pointeeType: LLVMTypeRef): ConstPointer {
     override val llvm = LLVMConstNull(pointerType(pointeeType))!!
 }
 
@@ -149,7 +147,6 @@ internal val kInt8Ptr      = pointerType(int8Type)
 internal val kInt8PtrPtr   = pointerType(kInt8Ptr)
 internal val kNullInt8Ptr  = LLVMConstNull(kInt8Ptr)!!
 internal val kImmInt32One  = Int32(1).llvm
-internal val kImmInt64One  = Int64(1).llvm
 internal val ContextUtils.kNullObjHeaderPtr: LLVMValueRef
     get() = LLVMConstNull(this.kObjHeaderPtr)!!
 internal val ContextUtils.kNullObjHeaderPtrPtr: LLVMValueRef

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt

@@ -35,22 +35,28 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
     inner class MethodTableRecord(val nameSignature: LocalHash, val methodEntryPoint: ConstPointer?) :
             Struct(runtime.methodTableRecordType, nameSignature, methodEntryPoint)
 
-    private inner class TypeInfo(val name: ConstValue, val size: Int,
+    private inner class TypeInfo(
-                                 val superType: ConstValue,
+            selfPtr: ConstPointer,
-                                 val objOffsets: ConstValue,
+            name: ConstValue,
-                                 val objOffsetsCount: Int,
+            size: Int,
-                                 val interfaces: ConstValue,
+            superType: ConstValue,
-                                 val interfacesCount: Int,
+            objOffsets: ConstValue,
-                                 val methods: ConstValue,
+            objOffsetsCount: Int,
-                                 val methodsCount: Int,
+            interfaces: ConstValue,
-                                 val fields: ConstValue,
+            interfacesCount: Int,
-                                 val fieldsCount: Int,
+            methods: ConstValue,
-                                 val packageName: String?,
+            methodsCount: Int,
-                                 val relativeName: String?,
+            fields: ConstValue,
-                                 val writableTypeInfo: ConstPointer?) :
+            fieldsCount: Int,
+            packageName: String?,
+            relativeName: String?,
+            writableTypeInfo: ConstPointer?) :
+
             Struct(
                     runtime.typeInfoType,
 
+                    selfPtr,
+
                     name,
                     Int32(size),
 
@@ -162,8 +168,11 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
                 runtime.methodTableRecordType, methods)
 
         val reflectionInfo = getReflectionInfo(classDesc)
-
+        val typeInfoGlobal = llvmDeclarations.typeInfoGlobal
-        val typeInfo = TypeInfo(name, size,
+        val typeInfo = TypeInfo(
+                classDesc.typeInfoPtr,
+                name,
+                size,
                 superType,
                 objOffsetsPtr, objOffsets.size,
                 interfacesPtr, interfaces.size,
@@ -174,8 +183,6 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
                 llvmDeclarations.writableTypeInfoGlobal?.pointer
         )
 
-        val typeInfoGlobal = llvmDeclarations.typeInfoGlobal
-
         val typeInfoGlobalValue = if (!classDesc.typeInfoHasVtableAttached) {
             typeInfo
         } else {
@@ -265,8 +272,12 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
                     .also { it.setZeroInitializer() }
                     .pointer
         }
-
+        val vtable = vtable(superClass)
-        val typeInfo = TypeInfo(
+        val typeInfoWithVtableType = structType(runtime.typeInfoType, vtable.llvmType)
+        val typeInfoWithVtableGlobal = staticData.createGlobal(typeInfoWithVtableType, "", isExported = false)
+        val result = typeInfoWithVtableGlobal.pointer.getElementPtr(0)
+        val typeInfoWithVtable = Struct(TypeInfo(
+                selfPtr = result,
                 name = name,
                 size = size,
                 superType = superClass.typeInfoPtr,
@@ -277,12 +288,12 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
                 packageName = reflectionInfo.packageName,
                 relativeName = reflectionInfo.relativeName,
                 writableTypeInfo = writableTypeInfo
-        )
+              ), vtable)
 
-        val vtable = vtable(superClass)
+        typeInfoWithVtableGlobal.setInitializer(typeInfoWithVtable)
+        typeInfoWithVtableGlobal.setConstant(true)
 
-        return staticData.placeGlobal("", Struct(typeInfo, vtable))
+        return result
-                .pointer.getElementPtr(0)
     }
 
     private val OverriddenFunctionDescriptor.implementation get() = getImplementation(context)

runtime/src/main/cpp/Arrays.cpp

@@ -42,7 +42,7 @@ static inline void copyImpl(KConstRef thiz, KInt fromIndex,
 namespace {
 
 const ArrayHeader anEmptyArray = {
-  theArrayTypeInfo, /* permanent object */ 0, /* element count */ 0
+  const_cast<TypeInfo*>(theArrayTypeInfo), /* permanent object */ 0, /* element count */ 0
 };
 
 }  // namespace

runtime/src/main/cpp/Memory.cpp

@@ -39,9 +39,10 @@
 // Auto-adjust GC thresholds.
 #define GC_ERGONOMICS 1
 
-// TODO: ensure it it read-only.
+// TODO: ensure it is read-only.
 ContainerHeader ObjHeader::theStaticObjectsContainer = {
-  CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT
+  CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT,
+  0 /* Object count */
 };
 
 namespace {
@@ -416,7 +417,11 @@ RUNTIME_NORETURN void ThrowInvalidMutabilityException();
 }  // extern "C"
 
 inline void runDeallocationHooks(ObjHeader* obj) {
+  if (obj->has_meta_object()) {
+    ObjHeader::destroyMetaObject(&obj->typeInfoOrMeta_);
+  }
 #if KONAN_OBJC_INTEROP
+  // TODO: rewrite using meta-object.
   if (obj->type_info() == theObjCPointerHolderTypeInfo) {
     void* objcPtr =  *reinterpret_cast<void**>(obj + 1); // TODO: use more reliable layout description
     objc_release(objcPtr);
@@ -806,6 +811,29 @@ inline size_t containerSize(const ContainerHeader* container) {
 
 }  // namespace
 
+MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
+  MetaObjHeader* meta = konanConstructInstance<MetaObjHeader>();
+  TypeInfo* typeInfo = *location;
+  meta->typeInfo_ = typeInfo;
+#if KONAN_NO_THREADS
+  *location = reinterpret_cast<TypeInfo*>(meta);
+#else
+  TypeInfo* old = __sync_val_compare_and_swap(location, typeInfo, reinterpret_cast<TypeInfo*>(meta));
+  if (old->typeInfo_ != old) {
+    // Someone installed a new meta-object since the check.
+    konanFreeMemory(meta);
+    meta = reinterpret_cast<MetaObjHeader*>(old);
+  }
+#endif
+  return meta;
+}
+
+void ObjHeader::destroyMetaObject(TypeInfo** location) {
+  TypeInfo* meta = *location;
+  *location = nullptr;
+  konanFreeMemory(meta);
+}
+
 ContainerHeader* AllocContainer(size_t size) {
   auto state = memoryState;
 #if USE_GC
@@ -842,26 +870,26 @@ void FreeContainer(ContainerHeader* header) {
   }
 }
 
-void ObjectContainer::Init(const TypeInfo* type_info) {
+void ObjectContainer::Init(const TypeInfo* typeInfo) {
-  RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
+  RuntimeAssert(typeInfo->instanceSize_ >= 0, "Must be an object");
   uint32_t alloc_size =
-      sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_ + kObjectReservedTailSize;
+      sizeof(ContainerHeader) + sizeof(ObjHeader) + typeInfo->instanceSize_ + kObjectReservedTailSize;
   header_ = AllocContainer(alloc_size);
   if (header_) {
     // One object in this container.
     header_->setObjectCount(1);
      // header->refCount_ is zero initialized by AllocContainer().
-    SetMeta(GetPlace(), type_info);
+    SetHeader(GetPlace(), typeInfo);
     MEMORY_LOG("object at %p\n", GetPlace())
     OBJECT_ALLOC_EVENT(memoryState, type_info->instanceSize_, GetPlace())
   }
 }
 
-void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
+void ArrayContainer::Init(const TypeInfo* typeInfo, uint32_t elements) {
-  RuntimeAssert(type_info->instanceSize_ < 0, "Must be an array");
+  RuntimeAssert(typeInfo->instanceSize_ < 0, "Must be an array");
   uint32_t alloc_size =
       sizeof(ContainerHeader) + sizeof(ArrayHeader) -
-      type_info->instanceSize_ * elements + kObjectReservedTailSize;
+      typeInfo->instanceSize_ * elements + kObjectReservedTailSize;
   header_ = AllocContainer(alloc_size);
   RuntimeAssert(header_ != nullptr, "Cannot alloc memory");
   if (header_) {
@@ -869,10 +897,10 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
     header_->setObjectCount(1);
     // header->refCount_ is zero initialized by AllocContainer().
     GetPlace()->count_ = elements;
-    SetMeta(GetPlace()->obj(), type_info);
+    SetHeader(GetPlace()->obj(), typeInfo);
     MEMORY_LOG("array at %p\n", GetPlace())
     OBJECT_ALLOC_EVENT(
-        memoryState, -type_info->instanceSize_ * elements, GetPlace()->obj())
+        memoryState, -typeInfo->instanceSize_ * elements, GetPlace()->obj())
   }
 }
 
@@ -951,7 +979,7 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
   }
   OBJECT_ALLOC_EVENT(memoryState, type_info->instanceSize_, result)
   currentChunk_->asHeader()->incObjectCount();
-  setMeta(result, type_info);
+  setHeader(result, type_info);
   return result;
 }
 
@@ -964,7 +992,7 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun
   }
   OBJECT_ALLOC_EVENT(memoryState, -type_info->instanceSize_ * count, result->obj())
   currentChunk_->asHeader()->incObjectCount();
-  setMeta(result->obj(), type_info);
+  setHeader(result->obj(), type_info);
   result->count_ = count;
   return result;
 }
@@ -992,13 +1020,17 @@ void ReleaseRefFromAssociatedObject(const ObjHeader* object) {
 extern "C" {
 
 MemoryState* InitMemory() {
-  RuntimeAssert(offsetof(ArrayHeader, type_info_)
+  RuntimeAssert(offsetof(ArrayHeader, typeInfoOrMeta_)
                 ==
-                offsetof(ObjHeader,   type_info_),
+                offsetof(ObjHeader,   typeInfoOrMeta_),
                 "Layout mismatch");
-  RuntimeAssert(offsetof(ArrayHeader, container_offset_negative_)
+  RuntimeAssert(offsetof(ArrayHeader, containerOffsetNegative_)
                 ==
-                offsetof(ObjHeader  , container_offset_negative_),
+                offsetof(ObjHeader  , containerOffsetNegative_),
+                "Layout mismatch");
+  RuntimeAssert(offsetof(TypeInfo, typeInfo_)
+                ==
+                offsetof(MetaObjHeader, typeInfo_),
                 "Layout mismatch");
   RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers")
   RuntimeAssert(memoryState == nullptr, "memory state must be clear");

runtime/src/main/cpp/Memory.h

@@ -177,35 +177,34 @@ struct ContainerHeader {
 };
 
 struct ArrayHeader;
+struct MetaObjHeader;
 
 // Header of every object.
 struct ObjHeader {
-  const TypeInfo* type_info_;
+  TypeInfo* typeInfoOrMeta_;
-  container_offset_t container_offset_negative_;
+  container_offset_t containerOffsetNegative_;
 
   const TypeInfo* type_info() const {
-    // TODO: for moving collectors use meta-objects approach:
+    return typeInfoOrMeta_->typeInfo_;
-    //  - store tag in lower bit TypeInfo, which marks if meta-object is in place
-    //  - when reading type_info_ check if it is unaligned
-    //  - if it is, pointer points to the MetaObject
-    //  - otherwise this is direct pointer to TypeInfo
-    // Meta-object allows storing additional data associated with some objects,
-    // such as stable hash code.
-    return type_info_;
   }
 
-  void set_type_info(const TypeInfo* type_info) {
+  bool has_meta_object() const {
-    type_info_ = type_info;
+    return typeInfoOrMeta_ != typeInfoOrMeta_->typeInfo_;
+  }
+
+  MetaObjHeader* meta_object() {
+     return has_meta_object() ?
+        reinterpret_cast<MetaObjHeader*>(typeInfoOrMeta_) : createMetaObject(&typeInfoOrMeta_);
   }
 
   static ContainerHeader theStaticObjectsContainer;
 
   ContainerHeader* container() const {
-    if (container_offset_negative_ == 0) {
+    if (containerOffsetNegative_ == 0) {
       return &theStaticObjectsContainer;
     } else {
       return reinterpret_cast<ContainerHeader*>(
-          reinterpret_cast<uintptr_t>(this) - container_offset_negative_);
+          reinterpret_cast<uintptr_t>(this) - containerOffsetNegative_);
     }
   }
 
@@ -216,31 +215,23 @@ struct ObjHeader {
   inline bool permanent() const {
     return container()->permanent();
   }
+
+  static MetaObjHeader* createMetaObject(TypeInfo** location);
+  static void destroyMetaObject(TypeInfo** location);
 };
 
 // Header of value type array objects. Keep layout in sync with that of object header.
 struct ArrayHeader {
-  const TypeInfo* type_info_;
+  TypeInfo* typeInfoOrMeta_;
-  container_offset_t container_offset_negative_;
+  container_offset_t containerOffsetNegative_;
 
   const TypeInfo* type_info() const {
-    // TODO: for moving collectors use meta-objects approach:
+    return typeInfoOrMeta_->typeInfo_;
-    //  - store tag in lower bit TypeInfo, which marks if meta-object is in place
-    //  - when reading type_info_ check if it is unaligned
-    //  - if it is, pointer points to the MetaObject
-    //  - otherwise this is direct pointer to TypeInfo
-    // Meta-object allows storing additional data associated with some objects,
-    // such as stable hash code.
-    return type_info_;
-  }
-
-  void set_type_info(const TypeInfo* type_info) {
-    type_info_ = type_info;
   }
 
   ContainerHeader* container() const {
     return reinterpret_cast<ContainerHeader*>(
-        reinterpret_cast<uintptr_t>(this) - container_offset_negative_);
+        reinterpret_cast<uintptr_t>(this) - containerOffsetNegative_);
   }
 
   ObjHeader* obj() { return reinterpret_cast<ObjHeader*>(this); }
@@ -250,6 +241,12 @@ struct ArrayHeader {
   uint32_t count_;
 };
 
+// Header for the meta-object.
+struct MetaObjHeader {
+  // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
+  const TypeInfo* typeInfo_;
+};
+
 inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
   // Instance size is negative.
   return -obj->type_info()->instanceSize_ * obj->count_;
@@ -261,10 +258,10 @@ class Container {
   // Data where everything is being stored.
   ContainerHeader* header_;
 
-  void SetMeta(ObjHeader* obj, const TypeInfo* type_info) {
+  void SetHeader(ObjHeader* obj, const TypeInfo* type_info) {
-    obj->container_offset_negative_ =
+    obj->containerOffsetNegative_ =
         reinterpret_cast<uintptr_t>(obj) - reinterpret_cast<uintptr_t>(header_);
-    obj->set_type_info(type_info);
+    obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(type_info);
     RuntimeAssert(obj->container() == header_, "Placement must match");
   }
 };
@@ -340,10 +337,10 @@ class ArenaContainer {
 
   bool allocContainer(container_size_t minSize);
 
-  void setMeta(ObjHeader* obj, const TypeInfo* typeInfo) {
+  void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) {
-    obj->container_offset_negative_ =
+    obj->containerOffsetNegative_ =
         reinterpret_cast<uintptr_t>(obj) - reinterpret_cast<uintptr_t>(currentChunk_->asHeader());
-    obj->set_type_info(typeInfo);
+    obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
     RuntimeAssert(obj->container() == currentChunk_->asHeader(), "Placement must match");
   }
 

runtime/src/main/cpp/TypeInfo.h

@@ -45,6 +45,9 @@ struct FieldTableRecord {
 // This struct represents runtime type information and by itself is the compile time
 // constant.
 struct TypeInfo {
+    // Reference to self, to allow simple obtaining TypeInfo via meta-object.
+    const TypeInfo* typeInfo_;
+    // Hash of class name.
     ClassNameHash name_;
     // Negative value marks array class/string, and it is negated element size.
     int32_t instanceSize_;